scripts/butler-coding-strict-pilot-multi.sh
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env bash
# G3 multi-category pilot — extend Phase B 单类 (deep) 覆盖到 3 个 PROD_PLAYBOOK 类别
# 3 categories × 2 cases = 6 pilots
# Case B-viol (fixture with violations): true-positive capture rate per category
# Case B-clean (fixture with no violations): false-positive baseline per category
# Plus 1 butler exec smoke: verify real LLM provider path is functional in this env.
#
# 与 scripts/butler-coding-strict-pilot.sh 区别：
# - 单类 (deep) → 3 类 (quick / deep / lingwen-drill) 结构性多样性
# - 加 Case B-clean (no violations) → 验 false-positive baseline
#
# 阈值：违例捕获率 ≥85%（spec §4 阶段5）
# 假阳性率目标：0 / 类（gate 在无违例时必须 pass）
set -uo pipefail
WFXM_HOME="${BUTFXM_HOME:-/home/ailearn/projects/WFXM}"
PYTHON_BIN="${PYTHON_BIN:-/home/ailearn/miniconda3/bin/python3}"
BUTLER_BIN="${BUTLER_BIN:-/home/ailearn/miniconda3/bin/butler}"
DATE="$(date -u +"%Y-%m-%d")"
TS="$(date -u +"%Y-%m-%dT%H-%M-%SZ")"
REPORT_DIR="$WFXM_HOME/docs/plans/pilot-reports"
REPORT="$REPORT_DIR/pilot-report-G3-$DATE-001.md"
TMPDIR_PREFIX="/tmp/pilot-g3-${DATE}-001"
mkdir -p "$REPORT_DIR"
rm -f "$TMPDIR_PREFIX"-*.json 2>/dev/null || true
cd "$WFXM_HOME"
cleanup_strict() {
"$WFXM_HOME/scripts/butler-coding-strict-opt-in.sh" off >/dev/null 2>&1 || true
}
trap cleanup_strict EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
# 类别形塑：类别名 → fixture violated 集合 + 真任务 preview
declare -A FIXTURE_VIOLATIONS=(
[quick]="T2"
[deep]="CA4,T8"
[lingwen-drill]="T7,T8"
)
declare -A FIXTURE_PREVIEW=(
[quick]="inspect repo modules (quick shallow)"
[deep]="refactor module X with CA4 invariants"
[lingwen-drill]="lingwen-drill patch-only idempotent fix"
)
CATEGORIES=(quick deep lingwen-drill)
RESULTS=()
echo "[g3-multi] enabling strict mode via opt-in..."
"$WFXM_HOME/scripts/butler-coding-strict-opt-in.sh" on >/dev/null 2>&1 || true
for cat in "${CATEGORIES[@]}"; do
violations="${FIXTURE_VIOLATIONS[$cat]}"
preview="${FIXTURE_PREVIEW[$cat]}"
json="$TMPDIR_PREFIX-$cat.json"
echo ""
echo "=== category: $cat ==="
echo " violated fixture: [$violations]"
echo " task preview: $preview"
# ---------- Case B-viol: violations 注入 → 期望 gate 阻断 ----------
set +e
out_v=$(BUTLER_CODING_STRICT=1 PILOT_VIOLATIONS="$violations" "$PYTHON_BIN" -c "
import json, os
from butler.tools.delegate_impl import apply_delegate_success_gates
violations_raw = os.environ['PILOT_VIOLATIONS']
violated = [v.strip() for v in violations_raw.split(',') if v.strip()]
dev_engine = {
'coding_knowledge': {
'mode': 'theorem_only',
'violated_theorems': violated,
},
'verify_output_tail': 'verify passed (auto-verify covers unit/integration only)',
'edits': 1,
'verify_passed': True,
}
ok, issues = apply_delegate_success_gates(
base=True, issues=[], category='$cat',
category_meta=None, project=None, role='dev',
dev_engine=dev_engine,
task='$preview', task_preview='$preview',
changes=[], messages=[],
summary='auto-verify pass; theorem violation remains',
)
print(json.dumps({'ok': ok, 'issues': issues, 'violated': violated}, ensure_ascii=False))
" 2>&1)
rc_v=$?
set -e
# ---------- Case B-clean: 无 violations → 期望 gate 不阻断 ----------
set +e
out_c=$(BUTLER_CODING_STRICT=1 "$PYTHON_BIN" -c "
import json, os
from butler.tools.delegate_impl import apply_delegate_success_gates
dev_engine = {
'coding_knowledge': {
'mode': 'theorem_only',
'violated_theorems': [],
},
'verify_output_tail': 'verify passed (clean run, no theorem violations)',
'edits': 1,
'verify_passed': True,
}
ok, issues = apply_delegate_success_gates(
base=True, issues=[], category='$cat',
category_meta=None, project=None, role='dev',
dev_engine=dev_engine,
task='$preview', task_preview='$preview',
changes=[], messages=[],
summary='clean run; no theorem violation',
)
print(json.dumps({'ok': ok, 'issues': issues, 'violated': []}, ensure_ascii=False))
" 2>&1)
rc_c=$?
set -e
# ---------- 解析 B-viol ----------
p_v=$(printf '%s\n' "$out_v" | tail -1 | "$PYTHON_BIN" -c "
import sys, json
try:
d = json.loads(sys.stdin.read().strip())
print(json.dumps({
'ok': d.get('ok'),
'issues': d.get('issues') or [],
'violated': sorted(set(d.get('violated') or [])),
}, ensure_ascii=False))
except Exception as e:
print(json.dumps({'ok': None, 'issues': ['PARSE_ERROR: ' + str(e)], 'violated': []}, ensure_ascii=False))
")
ok_v=$(printf '%s\n' "$p_v" | "$PYTHON_BIN" -c "import sys,json; print(json.load(sys.stdin).get('ok'))")
issues_v=$(printf '%s\n' "$p_v" | "$PYTHON_BIN" -c "import sys,json; print('\n'.join(json.load(sys.stdin).get('issues') or []))")
violated_set=$(printf '%s\n' "$p_v" | "$PYTHON_BIN" -c "import sys,json; print(','.join(json.load(sys.stdin).get('violated') or []))")
gate_blocked_v=$(printf '%s\n' "$issues_v" | grep -oE "CODING_STRICT_GATE[^{]*\(([^)]+)\)" | grep -oE "\([^)]+\)" | tr -d '()' | tr ',' '\n' | tr -d ' ' | sort -u | paste -sd, - || true)
gate_blocked_v="${gate_blocked_v:-}"
violated_set="${violated_set:-}"
# capture rate
captured_count=0
violated_count=0
if [ -n "$violated_set" ]; then
violated_count=$(echo "$violated_set" | tr ',' '\n' | grep -c .)
for id in $(echo "$violated_set" | tr ',' '\n'); do
[ -z "$id" ] && continue
if echo ",$gate_blocked_v," | grep -q ",$id,"; then
captured_count=$((captured_count + 1))
fi
done
fi
if [ "$violated_count" -gt 0 ]; then
capture_rate=$(awk -v c="$captured_count" -v n="$violated_count" 'BEGIN{printf "%.0f%%", (c/n)*100}')
else
capture_rate="N/A"
fi
# verdict (viol case)
if [ "$ok_v" = "True" ]; then
verdict_v="MISS"
elif [ "$ok_v" = "False" ] && [ "$captured_count" -eq "$violated_count" ]; then
verdict_v="MATCH"
elif [ "$ok_v" = "False" ] && [ "$captured_count" -gt 0 ]; then
verdict_v="PARTIAL"
else
verdict_v="UNKNOWN"
fi
# ---------- 解析 B-clean ----------
p_c=$(printf '%s\n' "$out_c" | tail -1 | "$PYTHON_BIN" -c "
import sys, json
try:
d = json.loads(sys.stdin.read().strip())
print(json.dumps({'ok': d.get('ok'), 'issues': d.get('issues') or []}, ensure_ascii=False))
except Exception as e:
print(json.dumps({'ok': None, 'issues': ['PARSE_ERROR: ' + str(e)]}, ensure_ascii=False))
")
ok_c=$(printf '%s\n' "$p_c" | "$PYTHON_BIN" -c "import sys,json; print(json.load(sys.stdin).get('ok'))")
issues_c=$(printf '%s\n' "$p_c" | "$PYTHON_BIN" -c "import sys,json; print('\n'.join(json.load(sys.stdin).get('issues') or []))")
has_csgate_c="no"
if echo "$issues_c" | grep -q "CODING_STRICT_GATE"; then
has_csgate_c="yes"
fi
# false positive = gate fired even with no violations
if [ "$ok_c" = "True" ] && [ "$has_csgate_c" = "no" ]; then
verdict_c="CLEAN_PASS"
elif [ "$ok_c" = "False" ] && [ "$has_csgate_c" = "yes" ]; then
verdict_c="FALSE_POSITIVE"
else
verdict_c="OTHER_BLOCKED"
fi
cat <<EOF > "$json"
{
"category": "$cat",
"case_viol": {
"violated_fixture": "$violated_set",
"gate_blocked": "$gate_blocked_v",
"captured_count": $captured_count,
"violated_count": $violated_count,
"capture_rate": "$capture_rate",
"verdict": "$verdict_v",
"ok": "$ok_v",
"rc": $rc_v
},
"case_clean": {
"ok": "$ok_c",
"has_csgate": "$has_csgate_c",
"verdict": "$verdict_c",
"rc": $rc_c
}
}
EOF
RESULTS+=("$cat $verdict_v $capture_rate $verdict_c")
echo " result: viol verdict=$verdict_v capture_rate=$capture_rate | clean verdict=$verdict_c"
done
# ---------- butler exec smoke ----------
echo ""
echo "=== butler exec smoke (real LLM provider path) ==="
SMOKE_JSON="$TMPDIR_PREFIX-smoke.json"
SMOKE_OUT_FILE="$TMPDIR_PREFIX-smoke.out"
set +e
timeout 60 "$BUTLER_BIN" exec "List the file paths under docs/plans/pilot-reports/ and report count. Do not edit any files." > "$SMOKE_OUT_FILE" 2>&1
smoke_rc=$?
set -e
smoke_has_csgate="no"
grep -q "CODING_STRICT_GATE" "$SMOKE_OUT_FILE" 2>/dev/null && smoke_has_csgate="yes"
# this is butler agent loop, not delegate — gate irrelevant for chat responses
# result is informational: confirms LLM provider is reachable
cat <<EOF > "$SMOKE_JSON"
{
"command": "butler exec <no-edit list task>",
"rc": $smoke_rc,
"has_csgate_in_output": "$smoke_has_csgate",
"note": "butler exec does not auto-invoke delegate_task for chat-style requests; output goes through butler agent loop, not the 4-gate chain. Evidence: real LLM provider reachable but delegate not auto-fired."
}
EOF
echo " result: butler exec rc=$smoke_rc (informational; not part of capture-rate computation)"
# ---------- 写报告 ----------
{
echo "# G3 Multi-Category Pilot Report (Batch 001)"
echo ""
echo "> **Date**: $DATE"
echo "> **Trigger**: G2-08 BUTLER_CODING_STRICT 默认升级 defer 决策（详见 \`docs/plans/decisions/butler-coding-strict-default-decision-2026-07-14.md\`）"
echo "> **Goal**: 累计 ≥3 任务类型 pilot run（触发升级条件之一），覆盖结构性多样性类别。"
echo "> **Stack**: 3 categories × 2 cases (viol + clean) = 6 fixture pilots + 1 butler exec smoke"
echo ""
echo "## 类别选择"
echo ""
echo "| 类别 | iterations | role | 工具限制 | 用途 |"
echo "|---|---|---|---|---|"
echo "| \`quick\` | 12 | dev | deny delegate_task / run_workflow | 短浅层 inspect 类任务 |"
echo "| \`deep\` | 24 | dev | 无 | 标准实现 |"
echo "| \`lingwen-drill\` | 24 | dev | deny write_file / delete_file | patch-only 强行约束 |"
echo ""
echo "三轴最大多样性：迭代范围、无政策 vs 有政策、写入限制；且三都属 \`PROD_PLAYBOOK_CATEGORIES\`（strict gate 实际触发范围）。"
echo ""
echo "## Per-Category 实证"
echo ""
for cat in "${CATEGORIES[@]}"; do
cat_json="$TMPDIR_PREFIX-$cat.json"
if [ -f "$cat_json" ]; then
echo "### $cat"
echo ""
echo '```json'
cat "$cat_json"
echo '```'
echo ""
fi
done
echo "## Capture Rate 总览"
echo ""
echo "| Category | Viol verdict | Capture rate | Clean verdict | False positive? |"
echo "|---|---|---|---|---|"
for r in "${RESULTS[@]}"; do
c=$(echo "$r" | awk '{print $1}')
v=$(echo "$r" | awk '{print $2}')
cap=$(echo "$r" | awk '{print $3}')
cln=$(echo "$r" | awk '{print $4}')
if [ "$cln" = "FALSE_POSITIVE" ]; then
fp="YES ❌"
else
fp="no ✅"
fi
echo "| \`$c\` | $v | $cap | $cln | $fp |"
done
echo ""
echo "## Butler exec smoke"
echo ""
echo '```json'
cat "$SMOKE_JSON"
echo '```'
echo ""
echo "## Verdict"
echo ""
all_match=true
any_fp=false
for r in "${RESULTS[@]}"; do
v=$(echo "$r" | awk '{print $2}')
cln=$(echo "$r" | awk '{print $4}')
if [ "$v" != "MATCH" ]; then
all_match=false
fi
if [ "$cln" = "FALSE_POSITIVE" ]; then
any_fp=true
fi
done
if $all_match && ! $any_fp; then
echo "**3/3 categories MATCH + 0 false positive** — G3 升级触发条件之一满足。"
elif $all_match; then
echo "**Partial: capture MATCH 但有 false positive** — 不满足升级条件，需 root cause。"
else
echo "**Non-MATCH：** 部分类别 capture rate < 85% — 见上表。"
fi
echo ""
echo "## 说明与限制"
echo ""
echo "- **Fixture-based 而非真 subagent terminal**：直接调 \`_tool_delegate_task\` 因循环导入（\`memory.diagnostics\`）在本 fresh-python env 无法走通；\`butler exec\` 走的是 butler agent loop（直接用 list_directory 等），不自动 invoke delegate_task。本批采用 category-shaped fixture + 真实 4-gate chain 端到端跑通（与 Phase B 同口径）。"
echo "- **butler exec smoke** 仅验证 LLM provider 路径可达；不计入 capture rate 计算。"
echo "- **真 per-category delegate 实证** 留待后续 shift：可通过构造强制 invoke delegate_task 的 test harness（绕过 circular import）实现。"
echo ""
echo "## 关联"
echo ""
echo "- 决策文档：\`docs/plans/decisions/butler-coding-strict-default-decision-2026-07-14.md\`"
echo "- 单一类 pilot runner：\`scripts/butler-coding-strict-pilot.sh\`（深类，Phase B 已跑）"
echo "- 4-gate 链入口：\`butler/tools/delegate_impl.py:281\`（\`apply_delegate_success_gates\`）"
echo "- PROD_PLAYBOOK 类别：\`butler/dev_engine/prod_delegate_bridge.py:30-38\`"
echo "- 类别定义：\`butler/delegate/delegate_categories.yaml\`"
} > "$REPORT"
echo ""
echo "[g3-multi] report written to $REPORT"
# ---------- 关闭严格 ----------
echo "[g3-multi] disabling strict mode..."
cleanup_strict
trap - EXIT INT TERM
exit 0