scripts/butler-web-search-route-sim.sh

Rama main · réplica de lectura del repositorio original.

#!/usr/bin/env bash
# G1-12 / G1-12b: network search routing policy + optional handler sim + probe.
#
# Usage:
#   bash scripts/butler-web-search-route-sim.sh                      # policy only (~2s)
#   bash scripts/butler-web-search-route-sim.sh --handler            # + LLM handler (~60s, soft)
#   bash scripts/butler-web-search-route-sim.sh --handler --strict-handler  # 发版前：prefer 工具硬断言
#   bash scripts/butler-web-search-route-sim.sh --with-probe         # + web_search live probe
#
# Skip (exit 0): BUTLER_WEB_SEARCH_ROUTE_SIM=0
# Handler skip: BUTLER_WEB_SEARCH_ROUTE_HANDLER=0
#
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
export PYTHONPATH="$ROOT"

if [[ -f "$ROOT/.env" ]]; then
  # shellcheck source=scripts/lib/butler-source-env.sh
  source "$ROOT/scripts/lib/butler-source-env.sh"
  butler_source_env "$ROOT/.env" 2>/dev/null || true
fi

if [[ "${BUTLER_WEB_SEARCH_ROUTE_SIM:-1}" == "0" ]]; then
  echo "skip: BUTLER_WEB_SEARCH_ROUTE_SIM=0"
  exit 0
fi

WITH_PROBE=0
WITH_HANDLER=0
STRICT_HANDLER=0
for arg in "$@"; do
  case "$arg" in
    --with-probe) WITH_PROBE=1 ;;
    --handler) WITH_HANDLER=1 ;;
    --strict-handler) STRICT_HANDLER=1; WITH_HANDLER=1 ;;
  esac
done

exec python3 - "$WITH_PROBE" "$WITH_HANDLER" "$STRICT_HANDLER" <<'PY'
import json
import os
import sys

from butler.gateway.network_route_verify_runner import run_handler_route_cases
from butler.tools.network_route_verify import (
    run_web_search_probe,
)

with_probe = sys.argv[1] == "1"
with_handler = sys.argv[2] == "1"
strict_handler = sys.argv[3] == "1"
manifest = load_network_route_manifest()
if manifest is None:
    print("FAIL: network-search-routes.yaml not found")
    raise SystemExit(1)

print(f"network-route policy verify ({len(manifest.golden_cases)} cases)")
report = run_policy_golden_cases(manifest)
for case in report.cases:
    mark = "ok" if case.get("ok") else "FAIL"
    code = case.get("code")
    print(f"  [{mark}] {case['name']} tool={case['tool']} code={code!r}")
for err in report.errors:
    print(f"  error: {err}")

if not report.ok:
    print("policy verify: FAIL")
    raise SystemExit(1)
print("policy verify: ok")

handler_fail = 0
if with_handler:
    if os.getenv("BUTLER_WEB_SEARCH_ROUTE_HANDLER", "1").strip() == "0":
        print("\nhandler sim: skip (BUTLER_WEB_SEARCH_ROUTE_HANDLER=0)")
    else:
        has_llm = any(
            os.getenv(k, "").strip()
            for k in (
                "MINIMAX_API_KEY",
                "MINIMAX_CN_API_KEY",
                "DEEPSEEK_API_KEY",
                "OPENAI_API_KEY",
                "ANTHROPIC_API_KEY",
            )
        )
        if not has_llm:
            print("\nhandler sim: skip (no LLM API key)")
        else:
            mode = "strict" if strict_handler else "soft"
            print(f"\nhandler route sim ({len(manifest.handler_cases)} cases, {mode})")
            owner = os.getenv("BUTLER_OWNER_WECHAT_ID", "owner-route-handler-sim")
            project = os.getenv("BUTLER_WEB_SEARCH_ROUTE_PROJECT", "")
            hreport = run_handler_route_cases(
                manifest,
                owner_id=owner,
                project_name=project,
                strict=strict_handler,
            )
            for case in hreport.cases:
                mark = "ok" if case.get("ok") else "FAIL"
                tools = case.get("tools") or []
                print(f"  [{mark}] {case['name']} tools={tools}")
                for warn in case.get("warnings") or []:
                    print(f"    warn: {warn}")
            for err in hreport.errors:
                print(f"  error: {err}")
            for warn in hreport.warnings:
                print(f"  warn: {warn}")
            if hreport.ok:
                print(f"handler verify: ok ({mode})")
            else:
                print(f"handler verify: FAIL ({mode})")
                handler_fail = 1

if with_probe:
    print("")
    print(f"web_search probe: {manifest.probe_query!r}")
    probe_ok, payload = run_web_search_probe(manifest)
    print(json.dumps(payload, ensure_ascii=False, indent=2))
    if not probe_ok:
        print("probe: FAIL (network/proxy — non-fatal if policy ok)")
        raise SystemExit(2 if not handler_fail else 3)

if handler_fail:
    raise SystemExit(1)
PY