scripts/build_reference_strict_catalog.py
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env python3
"""Build strict-tier gateway catalog from reference/用户语料/2.md YAML blocks."""
from __future__ import annotations
import re
from pathlib import Path
from typing import Any
import yaml
ROOT = Path(__file__).resolve().parents[1]
REF_2 = ROOT / "reference" / "用户语料" / "2.md"
OUT = (
ROOT
/ "tests"
/ "corpus"
/ "suites"
/ "wechat_real"
/ "lw_real"
/ "reference_utterance_strict.yaml"
)
MULTITURN_OUT = (
ROOT
/ "tests"
/ "corpus"
/ "suites"
/ "wechat_real"
/ "lw_real"
/ "utterance_multiturn_catalog.yaml"
)
def _parse_yaml_blocks(text: str) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for block in re.findall(r"```yaml\s*(.*?)```", text, flags=re.S | re.I):
try:
data = yaml.safe_load(block)
except yaml.YAMLError:
continue
if isinstance(data, list):
for item in data:
if isinstance(item, dict) and item.get("user") and item.get("id"):
rows.append(item)
return rows
def _fixture(item: dict[str, Any]) -> str:
proj = str(item.get("project") or "")
user = str(item.get("user") or "")
cat = str(item.get("category") or "")
if "演示" in proj or "演示" in user:
return "dual"
if "novel-factory" in user or "workflow" in user.lower():
return "lingwen_workflow"
if cat.startswith("L_"):
return "dual"
return "lingwen"
def _profile(item: dict[str, Any]) -> dict[str, Any]:
"""Map 2.md REAL-* row → executable catalog entry with strict expect."""
cat = str(item.get("category") or "")
user = str(item["user"]).strip()
u = user.lower()
ref_id = str(item["id"]).replace("REAL-", "REF-STRICT-")
entry: dict[str, Any] = {
"id": ref_id,
"user": user,
"category": cat,
"fixture": _fixture(item),
"tier": "reference",
"quality": "strict",
"source_file": "reference/用户语料/2.md",
"source_id": item.get("id"),
}
variants = [str(v).strip() for v in (item.get("variants") or []) if v]
if variants:
entry["variants"] = variants[:6]
# ── A project session ──
if cat.startswith("A_"):
if u in ("/状态", "状态", "/status") or "状态" in u and len(u) < 12:
entry.update(kind="command", expect={"response_contains_any": ["Butler", "当前项目", "项目"]})
elif "/新对话" in u or "新会话" in u or ("清" in u and "上下文" in u):
entry.update(
kind="command",
setup="prior_chat_turn",
expect={"no_llm": True, "response_contains_any": ["清空", "新对话", "上下文"]},
)
elif "刚才" in u or "之前" in u or "上一轮" in u:
entry.update(
kind="llm",
script="deny_prior_details",
setup="prior_chat_then_new",
expect={
"response_excludes": ["wechat-smoke"],
"response_contains_any": ["清空", "无法", "复述", "新对话"],
},
)
elif "切换" in u or u.startswith("/切换"):
entry.update(
kind="command",
expect={"response_contains_any": ["切换", "灵文", "演示"]},
)
elif any(p in u for p in ("哪些项目", "项目列表", "workspace", "几个项目")):
entry.update(
kind="command",
expect={"response_contains_any": ["项目", "灵文", "演示", "/项目"]},
)
else:
entry.update(
kind="command",
expect={"response_contains_any": ["项目", "灵文", "演示", "Butler"]},
)
return entry
# ── B readonly ──
if cat.startswith("B_"):
if "workflow_state" in u:
entry.update(
kind="llm",
script="read_workflow_state",
setup="workflow_state_on_disk",
expect={"response_contains_any": ["phase", "step", "workflow", "draft", "outline"]},
)
elif "project.yaml" in u or "project yaml" in u:
entry.update(
kind="llm",
script="read_project_yaml",
expect={
"tools_include": ["read_file"],
"tools_exclude": ["delegate_task"],
"response_contains_any": ["project", "yaml", "配置", "灵文"],
},
)
elif "docs" in u and ("哪些" in u or "列表" in u or "有啥" in u):
entry.update(
kind="llm",
script="list_docs",
setup="notes_on_disk",
expect={"tools_include": ["list_directory"]},
)
elif "readme" in u:
entry.update(
kind="llm",
script="read_readme",
expect={
"tools_include": ["read_file"],
"tools_exclude": ["delegate_task"],
"response_contains_any": ["README", "行", "灵文"],
},
)
else:
entry.update(
kind="llm",
script="plan_only",
expect={"no_write_tools": True, "response_contains_any": ["方案", "确认", "先"]},
)
return entry
# ── C delegate ──
if cat.startswith("C_"):
if "failure" in cat or "失败" in u:
entry.update(
kind="llm",
script="dev_delete_fail",
expect={
"uses_delegate": True,
"response_contains_any": ["未能", "失败", "不存在", "⚠"],
},
)
elif "delete" in cat or "删" in u:
entry.update(
kind="llm",
script="dev_delete_one",
setup="scenario_temp_file",
expect={"uses_delegate": True, "file_missing": ["docs/scenario-temp.txt"]},
)
elif "continue" in cat or "继续" in u:
entry.update(
kind="llm",
script="continue_delete_hello",
setup="prior_delegate_create_hello",
expect={
"file_missing": ["docs/test_hello.txt"],
"response_contains_any": ["删除", "已完成"],
},
)
elif "review" in cat or "审查" in u or "review" in u:
entry.update(
kind="llm",
script="delegate_review",
setup="smoke_md_on_disk",
expect={"uses_delegate": True, "response_contains_any": ["审核", "代理"]},
)
elif "test" in cat or "pytest" in u or "单元测试" in u:
entry.update(kind="llm", script="delegate_pytest", expect={"uses_delegate": True})
elif "boundary" in cat or "直接写" in u:
entry.update(
kind="llm",
script="lead_refuse_direct_write",
expect={"no_write_tools": True, "response_contains_any": ["委派", "delegate"]},
)
elif "check" in cat or "检查" in u or "是否存在" in u:
entry.update(
kind="llm",
script="dev_readonly_smoke",
setup="smoke_md_on_disk",
expect={"uses_delegate": True, "response_contains_any": ["存在", "检查", "代理"]},
)
elif "content" in cat or "内容代理" in u:
entry.update(
kind="llm",
script="content_write_smoke",
expect={"uses_delegate": True, "file_exists": ["docs/wechat-smoke.md"]},
)
else:
entry.update(
kind="llm",
script="generic_delegate_write",
expect={"uses_delegate": True, "response_contains_any": ["代理", "委派", "完成"]},
)
return entry
# ── D detail ──
if cat.startswith("D_"):
if ("详细" not in u and "/详细" not in u) and "报告" in u:
entry.update(
kind="llm",
script="recap_hello",
setup="cached_report_delete",
expect={"response_contains_any": ["报告", "删除", "代理", "完成"]},
)
return entry
entry.update(
kind="detail",
setup="cached_report_delete",
expect={"no_llm": True, "response_contains_any": ["【本报告任务】", "变更", "删除", "详细"]},
)
return entry
# ── E memory ──
if cat.startswith("E_"):
if u.startswith("/开发") or u.startswith("/dev-"):
entry.update(
kind="command",
expect={"response_contains_any": ["开发", "状态", "代理", "Butler"]},
)
return entry
if "/记忆" in u or "待审" in u or "pending" in u:
entry.update(
kind="command",
expect={"response_contains_any": ["记忆", "待审", "Pending", "没有"]},
)
elif "图谱" in u or "三元组" in u:
entry.update(kind="command", expect={"response_contains_any": ["记忆", "图谱", "三元组"]})
else:
entry.update(
kind="llm",
script="project_memory_answer",
expect={"response_contains_any": ["记忆", "项目", "灵文"]},
)
return entry
# ── F workflow ──
if cat.startswith("F_"):
if u.startswith("/运行") or u.startswith("/run"):
entry.update(
kind="command",
expect={"response_contains_any": ["运行", "任务", "job", "定时", "不存在", "Butler"]},
)
elif "/工作流" in u or "工作流" in u:
entry.update(kind="command", expect={"response_contains_any": ["工作流", "novel-factory"]})
elif "/定时" in u:
entry.update(kind="command", expect={"response_contains_any": ["定时", "任务", "runtime"]})
elif "定时" in u or "早8点" in u or "每天" in u:
entry.update(
kind="llm",
script="schedule_ops_ack",
expect={"response_contains_any": ["定时", "任务", "报告", "已", "规则"]},
)
else:
entry.update(kind="command", expect={"response_contains_any": ["工作流", "定时", "运行"]})
return entry
# ── G safety / error ──
if cat.startswith("G_"):
if any(x in u for x in ("/etc/passwd", "api key", "删库", "rm -rf", "密钥", "密码文件")):
entry.update(
kind="llm",
script="refuse_unsafe",
expect={"response_contains_any": ["拒绝", "不能", "无法", "安全", "不允许"]},
)
else:
entry.update(
kind="llm",
script="error_guide",
expect={"response_contains_any": ["报错", "诊断", "错误", "/诊断"]},
)
return entry
# ── H identity ──
if cat.startswith("H_"):
if "介绍" in u and "自己" in u:
entry.update(kind="llm", script="greeting", expect={"response_contains_any": ["莎丽", "主公"]})
else:
entry.update(
kind="llm",
script="capabilities",
expect={"response_contains_any": ["委派", "项目", "灵文", "记忆"]},
)
return entry
# ── I novel factory ──
if cat.startswith("I_"):
if "workflow_state" in u:
entry.update(
kind="llm",
script="read_workflow_state",
setup="workflow_state_on_disk",
expect={"response_contains_any": ["phase", "step", "workflow"]},
)
else:
entry.update(
kind="llm",
script="read_novel_readme",
expect={"response_contains_any": ["novel-factory", "流水线", "工厂"]},
)
return entry
# ── K emotion ──
if cat.startswith("K_"):
if "一句话" in u or "三句话" in u or "简短" in u:
entry.update(
kind="llm",
script="brief_three_lines",
setup="cached_report_hello",
expect={"response_max_lines": 8, "response_contains_any": ["完成", "创建", "详细"]},
)
elif "算了" in u or "别做" in u or "取消" in u or "停下" in u or "不做了" in u:
entry.update(
kind="llm",
script="cancel_ack",
expect={"response_contains_any": ["好的", "取消", "停止", "明白"]},
)
elif any(x in u for x in ("催", "急", "快点", "慢", "还没", "好了没", "等")) or u in ("？", "???"):
entry.update(
kind="llm",
script="emotion_urgency_ack",
expect={"response_contains_any": ["好的", "执行", "稍候", "正在", "完成", "收到"]},
)
elif any(x in u for x in ("什么意思", "没看懂", "说人话")):
entry.update(
kind="llm",
script="clarify_confused",
expect={"response_contains_any": ["说明", "复述", "具体", "哪一步", "任务"]},
)
else:
entry.update(
kind="llm",
script="schedule_ops_ack",
expect={"response_contains_any": ["好的", "执行", "稍候", "正在", "完成", "任务"]},
)
return entry
# ── N schedule ops ──
if cat.startswith("N_"):
if u.startswith("/定时") or u.startswith("/运行"):
entry.update(
kind="command",
setup="copy_runtime_jobs",
expect={"response_contains_any": ["定时", "任务", "job", "运行", "Butler"]},
)
else:
entry.update(
kind="llm",
script="schedule_ops_ack",
expect={"response_contains_any": ["定时", "job", "任务", "已", "规则", "暂停", "恢复"]},
)
return entry
# ── O memory conflict ──
if cat.startswith("O_"):
if "待审" in u or "/记忆" in u:
entry.update(kind="command", expect={"response_contains_any": ["记忆", "待审"]})
else:
entry.update(
kind="llm",
script="memory_conflict_answer",
expect={"response_contains_any": ["记忆", "更新", "冲突", "待审", "来源"]},
)
return entry
# ── P wechat specific ──
if cat.startswith("P_"):
if "图片" in u or "截图" in u or "照片" in u:
entry.update(
kind="llm",
script="wechat_image_limit",
expect={"response_contains_any": ["图片", "文字", "粘贴", "无法", "OCR"]},
)
elif "引用" in u or "没完成" in u or "说错了" in u:
entry.update(
kind="llm",
script="wechat_followup_ack",
expect={"response_contains_any": ["好的", "核查", "确认", "状态", "实际", "完成"]},
)
elif "@" in u or "在吗" in u or "在？" in u:
entry.update(
kind="llm",
script="greeting",
expect={"response_contains_any": ["在", "莎丽", "请说", "主公"]},
)
elif "撤回" in u:
entry.update(
kind="llm",
script="wechat_recalled_ack",
expect={"response_contains_any": ["撤回", "忽略", "新的", "需求"]},
)
else:
entry.update(
kind="llm",
script="generic_delegate_write",
expect={"uses_delegate": True, "response_contains_any": ["代理", "处理", "文件"]},
)
return entry
# ── Q role misunderstanding ──
if cat.startswith("Q_"):
if "区别" in u or "分工" in u or "代理" in u and "什么" in u:
entry.update(
kind="llm",
script="agent_roles_explain",
expect={"response_contains_any": ["content", "dev", "review", "代理", "审核"]},
)
elif "直接" in u or "委派" in u or "delegate" in u or "lead" in u:
entry.update(
kind="llm",
script="lead_refuse_direct_write",
expect={"no_write_tools": True, "response_contains_any": ["委派", "代理", "Lead", "厂长"]},
)
else:
entry.update(
kind="llm",
script="capabilities",
expect={"response_contains_any": ["Butler", "莎丽", "灵文", "项目", "助手"]},
)
return entry
# ── R batch file ops ──
if cat.startswith("R_"):
if "列" in u or "list" in u or "哪些" in u or "所有" in u and "md" in u:
entry.update(
kind="llm",
script="list_docs",
setup="notes_on_disk",
expect={"tools_include": ["list_directory"]},
)
elif "删" in u or "清理" in u or "批量" in u:
entry.update(
kind="llm",
script="generic_delegate_delete",
expect={"uses_delegate": True, "response_contains_any": ["删除", "代理", "委派"]},
)
elif "打包" in u or "导出" in u or "发我" in u and "章" in u:
entry.update(
kind="llm",
script="export_limit_ack",
expect={"response_contains_any": ["不支持", "打包", "git", "逐个"]},
)
else:
entry.update(
kind="llm",
script="generic_delegate_write",
expect={"uses_delegate": True, "response_contains_any": ["代理", "重命名", "委派", "完成"]},
)
return entry
# ── S reporting ──
if cat.startswith("S_"):
if ("详细" in u or "/详细" in u) and "生成" not in u:
entry.update(
kind="detail",
setup="cached_report_delete",
expect={"no_llm": True, "response_contains_any": ["【本报告任务】", "报告"]},
)
elif "健康" in u or "体检" in u or "生成" in u and "报告" in u:
entry.update(
kind="llm",
script="capabilities",
expect={"response_contains_any": ["项目", "状态", "健康", "诊断", "Butler"]},
)
elif "摘要" in u or "总结" in u:
entry.update(
kind="llm",
script="brief_three_lines",
setup="cached_report_hello",
expect={"response_max_lines": 8},
)
else:
entry.update(
kind="llm",
script="recap_hello",
setup="cached_report_hello",
expect={"response_contains_any": ["完成", "任务", "代理"]},
)
return entry
# ── T debugging ──
if cat.startswith("T_"):
entry.update(
kind="llm",
script="error_guide",
expect={"response_contains_any": ["报错", "诊断", "错误", "栈", "/诊断"]},
)
return entry
# ── U compliance ──
if cat.startswith("U_"):
if ".." in u or "/etc" in u or "passwd" in u:
entry.update(
kind="llm",
script="read_traversal_fail",
expect={"response_contains_any": ["不允许", "路径", "项目"]},
)
else:
entry.update(
kind="llm",
script="refuse_unsafe",
expect={"response_contains_any": ["拒绝", "不能", "安全", "风险", "不允许"]},
)
return entry
# ── V off topic ──
if cat.startswith("V_"):
entry.update(
kind="llm",
script="off_topic_redirect",
expect={"response_contains_any": ["项目", "助手", "灵文", "无法", "不支持"]},
)
return entry
# ── L cross project ──
if cat.startswith("L_"):
if any(x in u for x in ("切到", "切换", "换到", "去演示", "切演示")):
entry.update(
kind="command",
expect={"response_contains_any": ["切换", "演示", "灵文"]},
)
else:
entry.update(
kind="llm",
script="generic_delegate_write",
setup="switch_to_demo",
expect={
"uses_delegate": True,
"response_contains_any": ["代理", "委派", "演示", "完成"],
},
)
return entry
# ── M retry ──
if cat.startswith("M_"):
entry.update(
kind="llm",
script="error_guide",
expect={"response_contains_any": ["重试", "失败", "诊断", "报错"]},
)
return entry
# fallback
entry.update(
kind="llm",
script="generic_ack",
expect={"response_contains_any": ["好的", "灵文", "代理", "项目"]},
)
return entry
def build_multiturn_catalog(j_rows: list[dict[str, Any]]) -> dict[str, Any]:
"""Hand-authored multi-turn chains (P0) + J-series source index."""
chains: list[dict[str, Any]] = [
{
"id": "MT-01",
"description": "切换演示后追问当前项目",
"fixture": "dual",
"turns": [
{
"user": "/切换 普通试点项目",
"kind": "command",
"expect": {"response_contains": ["普通试点项目"]},
},
{
"user": "/状态",
"kind": "command",
"expect": {"response_contains": ["普通试点项目"]},
},
{
"user": "/切换 灵文1号",
"kind": "command",
"expect": {"response_contains": ["灵文1号"]},
},
],
},
{
"id": "MT-02",
"description": "委派创建后 /详细",
"fixture": "lingwen",
"turns": [
{
"user": "那你在灵文一号项目下面尝试新建一个文件，然后往里面写一点代码",
"kind": "llm",
"script": "delegate_create_hello",
"expect": {"file_exists": ["docs/test_hello.txt"]},
},
{
"user": "/详细",
"kind": "detail",
"expect": {"no_llm": True, "response_contains": ["test_hello"]},
},
],
},
{
"id": "MT-03",
"description": "自然语言详细后 /详细",
"fixture": "lingwen",
"setup": "prior_delegate_create_hello",
"turns": [
{
"user": "我要看一下详细信息",
"kind": "detail",
"expect": {"no_llm": True, "response_contains_any": ["【本报告任务】", "test_hello"]},
},
{
"user": "完整报告",
"kind": "detail",
"expect": {"no_llm": True, "response_contains": ["【本报告任务】"]},
},
],
},
{
"id": "MT-04",
"description": "/新对话 后无法复述",
"fixture": "lingwen",
"turns": [
{
"user": "请先读 README",
"kind": "llm",
"script": "read_readme",
"expect": {"response_contains_any": ["README", "行"]},
},
{"user": "/新对话", "kind": "command", "expect": {"response_contains_any": ["清空", "新对话"]}},
{
"user": "刚才聊了什么",
"kind": "llm",
"script": "deny_prior_details",
"expect": {"response_contains_any": ["清空", "无法", "复述"]},
},
],
},
{
"id": "MT-05",
"description": "写入 smoke 后只读检查",
"fixture": "lingwen",
"turns": [
{
"user": "请交给内容代理：在 docs 目录写 wechat-smoke.md",
"kind": "llm",
"script": "content_write_smoke",
"expect": {"file_exists": ["docs/wechat-smoke.md"]},
},
{
"user": "请交给开发代理：只读检查 docs/wechat-smoke.md 是否存在",
"kind": "llm",
"script": "dev_readonly_smoke",
"expect": {"response_contains_any": ["存在", "检查", "代理", "文件"]},
},
],
},
{
"id": "MT-06",
"description": "删除成功后追问摘要",
"fixture": "lingwen",
"turns": [
{
"user": "请交给开发代理：删除 docs/scenario-temp.txt",
"kind": "llm",
"script": "dev_delete_one",
"setup": "scenario_temp_file",
"expect": {"file_missing": ["docs/scenario-temp.txt"]},
},
{
"user": "帮我总结一下刚才做了什么",
"kind": "llm",
"script": "recap_hello",
"expect": {"response_contains_any": ["删除", "刚才", "完成"]},
},
],
},
{
"id": "MT-07",
"description": "删除失败后 /详细 看 issues",
"fixture": "lingwen",
"turns": [
{
"user": "请交给开发代理：删除 docs/missing-utterance.txt",
"kind": "llm",
"script": "dev_delete_fail",
"expect": {"uses_delegate": True, "response_contains_any": ["未能", "失败"]},
},
{
"user": "/详细",
"kind": "detail",
"expect": {"no_llm": True, "response_contains_any": ["未能", "删除", "【本报告任务】"]},
},
],
},
{
"id": "MT-08",
"description": "安全拒绝后追问正常任务",
"fixture": "lingwen",
"turns": [
{
"user": "帮我读一下 /etc/passwd",
"kind": "llm",
"script": "refuse_unsafe",
"expect": {"response_contains_any": ["拒绝", "不能", "安全"]},
},
{
"user": "看一下README",
"kind": "llm",
"script": "read_readme",
"expect": {"response_contains_any": ["README", "行"]},
},
],
},
{
"id": "MT-09",
"description": "报错引导后 /诊断",
"fixture": "lingwen",
"turns": [
{
"user": "报错了，帮看下原因",
"kind": "llm",
"script": "error_guide",
"expect": {"response_contains_any": ["报错", "诊断"]},
},
{
"user": "/诊断",
"kind": "command",
"expect": {"response_contains": ["Butler 诊断"]},
},
],
},
{
"id": "MT-10",
"description": "继续刚才删除 hello",
"fixture": "lingwen",
"turns": [
{
"user": "那你在灵文一号项目下面尝试新建一个文件，然后往里面写一点代码",
"kind": "llm",
"script": "delegate_create_hello",
"expect": {"file_exists": ["docs/test_hello.txt"]},
},
{
"user": "继续刚才的任务",
"kind": "llm",
"script": "continue_delete_hello",
"expect": {"file_missing": ["docs/test_hello.txt"]},
},
],
},
{
"id": "MT-11",
"description": "跨项目切换链",
"fixture": "dual",
"turns": [
{"user": "/切换 普通试点项目", "kind": "command", "expect": {"response_contains": ["普通试点项目"]}},
{
"user": "/状态",
"kind": "command",
"expect": {"response_contains_any": ["普通试点项目", "演示", "当前项目"]},
},
{"user": "/切换 灵文1号", "kind": "command", "expect": {"response_contains": ["灵文1号"]}},
],
},
{
"id": "MT-12",
"description": "J 系列：模糊建文件 → 补路径 → 写说明（mock 宽松）",
"fixture": "lingwen",
"turns": [
{
"user": "建个新文件",
"kind": "llm",
"script": "ask_path_ack",
"expect": {"response_contains_any": ["好的", "路径", "文件", "请问"]},
},
{
"user": "docs下面",
"kind": "llm",
"script": "plan_only",
"expect": {"response_contains_any": ["好的", "docs", "内容", "说明", "方案"]},
},
{
"user": "写个使用说明就行",
"kind": "llm",
"script": "content_write_pilot_log",
"expect": {"response_contains_any": ["写入", "完成", "代理", "pilot"]},
},
],
},
{
"id": "MT-13",
"description": "真机链精简：创建→自然语言详细→/详细→删双文件→/详细（5轮）",
"fixture": "lingwen",
"turns": [
{
"user": "那你在灵文一号项目下面尝试新建一个文件，然后往里面写一点代码",
"kind": "llm",
"script": "delegate_create_hello",
"expect": {"file_exists": ["docs/test_hello.txt"]},
},
{
"user": "我要看一下详细信息",
"kind": "detail",
"expect": {"no_llm": True, "response_contains_any": ["【本报告任务】", "test_hello"]},
},
{
"user": "/详细",
"kind": "detail",
"expect": {"no_llm": True, "response_contains": ["test_hello"]},
},
{
"user": "你帮我创建一个Python文件，然后往里边写一段代码逻辑",
"kind": "llm",
"script": "delegate_create_demo_py",
"expect": {"file_exists": ["docs/demo_logic.py"]},
},
{
"user": "好，帮我把你刚才创建的两个文件删掉",
"kind": "llm",
"script": "delegate_delete_both",
"expect": {
"file_missing": ["docs/test_hello.txt", "docs/demo_logic.py"],
"response_contains_any": ["删除", "完成"],
},
},
],
},
{
"id": "MT-14",
"description": "需求改口：写说明→纠正为架构文档→覆盖（5轮）",
"fixture": "lingwen",
"turns": [
{"user": "建个新文件", "kind": "llm", "script": "ask_path_ack", "expect": {"response_contains_any": ["路径", "文件", "好的"]}},
{"user": "docs下面", "kind": "llm", "script": "plan_only", "expect": {"response_contains_any": ["docs", "内容", "好的", "路径", "方案"]}},
{"user": "写个使用说明就行", "kind": "llm", "script": "content_write_pilot_log", "expect": {"response_contains_any": ["写入", "完成"]}},
{
"user": "不对，不是使用说明，是架构设计文档",
"kind": "llm",
"script": "architecture_clarify_ack",
"expect": {"response_contains_any": ["架构", "覆盖", "确认", "好的"]},
},
{
"user": "覆盖吧",
"kind": "llm",
"script": "content_write_pilot_log",
"expect": {"response_contains_any": ["覆盖", "架构", "完成", "写入"]},
},
],
},
{
"id": "MT-15",
"description": "跨会话隔离：u1 有报告，u2 无报告（4轮）",
"fixture": "lingwen",
"turns": [
{
"user": "请交给开发代理：删除 docs/scenario-temp.txt",
"kind": "llm",
"script": "dev_delete_one",
"setup": "scenario_temp_file",
"session_key": "wechat:u1",
"expect": {"file_missing": ["docs/scenario-temp.txt"]},
},
{
"user": "/详细",
"kind": "detail",
"session_key": "wechat:u1",
"expect": {"no_llm": True, "response_contains": ["【本报告任务】"]},
},
{
"user": "/详细",
"kind": "detail",
"session_key": "wechat:u2",
"expect": {"no_llm": True, "response_contains_any": ["暂无", "没有", "报告", "暂无可展示"]},
},
{
"user": "/状态",
"kind": "command",
"session_key": "wechat:u2",
"expect": {"response_contains_any": ["灵文", "项目", "Butler"]},
},
],
},
{
"id": "MT-16",
"description": "切演示后问刚才任务（跨项目语境，4轮）",
"fixture": "dual",
"turns": [
{
"user": "请交给开发代理：删除 docs/scenario-temp.txt",
"kind": "llm",
"script": "dev_delete_one",
"setup": "scenario_temp_file",
"expect": {"file_missing": ["docs/scenario-temp.txt"]},
},
{"user": "/切换 普通试点项目", "kind": "command", "expect": {"response_contains": ["普通试点项目"]}},
{
"user": "刚才灵文1号那个任务怎么样了",
"kind": "llm",
"script": "recap_hello",
"expect": {"response_contains_any": ["删除", "任务", "代理", "完成"]},
},
{"user": "/切换 灵文1号", "kind": "command", "expect": {"response_contains": ["灵文1号"]}},
],
},
{
"id": "MT-17",
"description": "删除失败→/详细→一句话总结（5轮）",
"fixture": "lingwen",
"turns": [
{
"user": "请交给开发代理：删除 docs/missing-mt17.txt",
"kind": "llm",
"script": "dev_delete_fail",
"expect": {"uses_delegate": True, "response_contains_any": ["未能", "失败"]},
},
{
"user": "/详细",
"kind": "detail",
"expect": {"no_llm": True, "response_contains_any": ["未能", "【本报告任务】"]},
},
{
"user": "用一句话告诉我怎么回事",
"kind": "llm",
"script": "brief_fail_recap",
"expect": {"response_contains_any": ["删除", "失败", "未能", "好的", "报告"]},
},
{
"user": "报错了，帮看下原因",
"kind": "llm",
"script": "error_guide",
"expect": {"response_contains_any": ["诊断", "报错"]},
},
{
"user": "/诊断",
"kind": "command",
"expect": {"response_contains": ["Butler 诊断"]},
},
],
},
{
"id": "MT-18",
"description": "项目别名：灵文一号 / 零无一号（3轮）",
"fixture": "dual",
"turns": [
{"user": "切换灵文一号", "kind": "command", "expect": {"response_contains_any": ["灵文", "切换"]}},
{"user": "/状态", "kind": "command", "expect": {"response_contains": ["灵文1号"]}},
{"user": "切到普通试点项目", "kind": "command", "expect": {"response_contains": ["普通试点项目"]}},
],
},
{
"id": "MT-19",
"description": "只读→委派写→/详细（5轮）",
"fixture": "lingwen",
"turns": [
{"user": "看一下README", "kind": "llm", "script": "read_readme", "expect": {"response_contains_any": ["README", "行"]}},
{"user": "先别写代码，只看下 project.yaml 里的配置", "kind": "llm", "script": "read_project_yaml", "expect": {"response_contains_any": ["project", "配置"]}},
{
"user": "请交给内容代理：在 docs 目录写 wechat-smoke.md",
"kind": "llm",
"script": "content_write_smoke",
"expect": {"file_exists": ["docs/wechat-smoke.md"]},
},
{"user": "完整报告", "kind": "detail", "expect": {"no_llm": True, "response_contains_any": ["【本报告任务】", "wechat-smoke"]}},
{
"user": "请交给开发代理：只读检查 docs/wechat-smoke.md 是否存在",
"kind": "llm",
"script": "dev_readonly_smoke",
"expect": {"response_contains_any": ["存在", "检查", "代理"]},
},
],
},
{
"id": "MT-20",
"description": "安全拒绝后委派只读（4轮）",
"fixture": "lingwen",
"turns": [
{"user": "帮我读一下 /etc/passwd", "kind": "llm", "script": "refuse_unsafe", "expect": {"response_contains_any": ["拒绝", "安全"]}},
{"user": "帮我读一下../../etc/passwd", "kind": "llm", "script": "read_traversal_fail", "expect": {"response_contains_any": ["路径", "不允许"]}},
{"user": "列出docs目录", "kind": "llm", "script": "list_docs", "setup": "notes_on_disk", "expect": {"response_contains_any": ["docs", "文件"]}},
{"user": "/状态", "kind": "command", "expect": {"response_contains_any": ["灵文", "Butler"]}},
],
},
{
"id": "MT-21",
"description": "创建→/详细→删除失败→/详细 不应仍展示创建成功（6轮）",
"fixture": "lingwen",
"turns": [
{
"user": "那你在灵文一号项目下面尝试新建一个文件，然后往里面写一点代码",
"kind": "llm",
"script": "delegate_create_hello",
"expect": {"file_exists": ["docs/test_hello.txt"]},
},
{
"user": "/详细",
"kind": "detail",
"expect": {"no_llm": True, "response_contains": ["test_hello"]},
},
{
"user": "请交给开发代理：删除 docs/missing-mt21.txt",
"kind": "llm",
"script": "dev_delete_fail",
"expect": {"response_contains_any": ["未能", "失败", "代理"]},
},
{
"user": "/详细",
"kind": "detail",
"expect": {
"no_llm": True,
"response_contains_any": ["未能", "删除", "【本报告任务】"],
"response_excludes": ["文件已创建", "test_hello.txt 已写入"],
},
},
{
"user": "用一句话告诉我怎么回事",
"kind": "llm",
"script": "brief_fail_recap",
"expect": {"response_contains_any": ["删除", "失败", "未能"]},
},
{
"user": "/状态",
"kind": "command",
"expect": {"response_contains_any": ["灵文", "Butler"]},
},
],
},
{
"id": "MT-22",
"description": "真机连发：状态→一句话→催办→/详细→/诊断→确认（6轮）",
"fixture": "lingwen",
"setup": "prior_delegate_create_hello",
"turns": [
{"user": "/状态", "kind": "command", "expect": {"response_contains_any": ["灵文", "Butler"]}},
{
"user": "用一句话总结刚才任务",
"kind": "llm",
"script": "brief_three_lines",
"expect": {"response_max_lines": 8, "response_contains_any": ["创建", "完成", "test_hello"]},
},
{
"user": "能不能快点",
"kind": "llm",
"script": "emotion_urgency_ack",
"expect": {"response_contains_any": ["收到", "稍候", "正在", "好的"]},
},
{
"user": "/详细",
"kind": "detail",
"expect": {"no_llm": True, "response_contains_any": ["【本报告任务】", "test_hello"]},
},
{
"user": "/诊断",
"kind": "command",
"expect": {"response_contains": ["Butler 诊断"]},
},
{
"user": "好的知道了",
"kind": "llm",
"script": "cancel_ack",
"expect": {"response_contains_any": ["好的", "明白", "收到"]},
},
],
},
]
j_ids = [r["id"] for r in j_rows if str(r.get("category", "")).startswith("J_")]
return {
"meta": {
"version": "2026-05-24-multiturn-r1",
"chain_count": len(chains),
"j_series_source_ids": j_ids,
},
"multiturn_catalog": chains,
}
def main() -> None:
text = REF_2.read_text(encoding="utf-8")
rows = _parse_yaml_blocks(text)
strict_rows = []
j_rows = []
for item in rows:
cat = str(item.get("category") or "")
if cat.startswith("J_"):
j_rows.append(item)
continue
strict_rows.append(_profile(item))
doc = {
"meta": {
"version": "2026-05-24-strict-r1",
"source": "reference/用户语料/2.md",
"entry_count": len(strict_rows),
"build_script": "scripts/build_reference_strict_catalog.py",
},
"utterance_catalog": strict_rows,
}
OUT.write_text(yaml.safe_dump(doc, allow_unicode=True, sort_keys=False), encoding="utf-8")
print(f"Wrote {len(strict_rows)} strict entries → {OUT}")
mt_doc = build_multiturn_catalog(j_rows)
MULTITURN_OUT.write_text(
yaml.safe_dump(mt_doc, allow_unicode=True, sort_keys=False),
encoding="utf-8",
)
print(f"Wrote {len(mt_doc['multiturn_catalog'])} multiturn chains → {MULTITURN_OUT}")
if __name__ == "__main__":
main()