Ctrl K
ComunidadGuías…

scripts/build_vector_index.py

Rama main · réplica de lectura del repositorio original.

Archivo original ↗Preguntar a KiBot
#!/usr/bin/env python3
"""
Build vector index for x-knowledge-base semantic search.

Reads search_index.json, embeds each card's title + summary,
and saves vectors to vector_index.json in BOOKMARKS_DIR.

Usage:
    python3 scripts/build_vector_index.py
    python3 scripts/build_vector_index.py --incremental   # skip already-embedded cards
    python3 scripts/build_vector_index.py --dry-run       # show what would be done

Requires:
    EMBEDDING_PROVIDER=gemini|openai|ollama
    + corresponding API key (see .env.example)
"""

from __future__ import annotations

import argparse
import hashlib
import json
import math
import os
import re
import sys
from datetime import datetime, timezone
from pathlib import Path

# Allow running from scripts/ directory or skill root
sys.path.insert(0, str(Path(__file__).resolve().parent))
import xkb_paths
import xkb_console

# 這支腳本的輸出有 emoji,而它會被人在 Windows 主控台直接叫起來。
# 不轉的話,工作做完了卻死在回報那一行的 print。
xkb_console.use_utf8()

_SKILL_DIR = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(_SKILL_DIR))
from tools.embedding_providers import get_provider

WORKSPACE_DIR = xkb_paths.WORKSPACE
BOOKMARKS_DIR = xkb_paths.BOOKMARKS_DIR
INDEX_FILE = xkb_paths.INDEX_FILE
VECTOR_FILE = xkb_paths.VECTOR_FILE
# wiki/memory 的向量另外存一份小檔。召回時只需要這 ~1,900 個向量,
# 但它們混在 8,000 多個卡片向量裡,載入整個 62MB 檔案要 5.4 秒——
# 而召回是每句話都要跑的。分開存之後只需要載入十分之一。
SEMANTIC_FILE = BOOKMARKS_DIR / "semantic_index.json"
# 卡片向量另存一份。它不像 wiki/記憶檔那樣要整份掃過——
# 卡片是先由 gbrain 挑出候選,我們只需要驗證那幾張的相關度,
# 所以讀取端用 seek 取單筆,不必把整份載進來。
CARDS_FILE = BOOKMARKS_DIR / "cards_index.json"


# ── Text extraction ───────────────────────────────────────────────────────────

def _resolve_md_path(item: dict) -> Path | None:
    """索引項路徑解析:relative_path 是相對 workspace(如 memory/cards/x.md),
    舊程式誤用 BOOKMARKS_DIR 拼接導致永遠讀不到卡片(2026-07-14 修正)。"""
    abs_path = item.get("path") or ""
    if abs_path and Path(abs_path).exists():
        return Path(abs_path)
    rel = item.get("relative_path") or ""
    if not rel:
        return None
    for base in (WORKSPACE_DIR, BOOKMARKS_DIR):
        p = base / rel
        if p.exists():
            return p
    return None


def _extract_key_points_from_md(md_path: Path) -> str:
    """Extract the 關鍵論點/三個重點 section from a card markdown file.
    (2026-07-14:新版 9-section 卡片段落名為「關鍵論點」,一併支援舊名)"""
    if not md_path.exists():
        return ""
    try:
        content = md_path.read_text(encoding="utf-8")
        m = re.search(
            r'##\s+[^\n]*(?:關鍵論點|三個重點)[^\n]*\n(.+?)(?=\n##|\Z)',
            content, re.DOTALL
        )
        if m:
            text = re.sub(r'^[\-\*\•]\s*', '', m.group(1).strip(), flags=re.MULTILINE)
            return text.strip()
    except Exception:
        pass
    return ""


def _extract_key_point_list(md_path: Path) -> list[str]:
    """論點級 embedding(TODOS 2026-07-13):回傳「三個重點」逐條列表,
    每條將單獨成向量(鍵 relpath#kpN),召回粒度從卡片級升到論點級。"""
    text = _extract_key_points_from_md(md_path)
    if not text:
        return []
    points = [ln.strip() for ln in text.splitlines() if len(ln.strip()) >= 10]
    return points[:5]


def extract_card_text(item: dict) -> str:
    """Build embeddable text from a search index item.
    Uses title + summary (already extracted by build_search_index.sh).
    Falls back to reading the .md file if summary is empty.
    """
    title = (item.get("title") or "").strip()
    summary = (item.get("summary") or "").strip()
    key_points = ""

    md_path = _resolve_md_path(item)
    if md_path:
        if not summary:
            summary = _extract_summary_from_md(md_path)
        key_points = _extract_key_points_from_md(md_path)

    parts = [p for p in [title, summary, key_points] if p]
    return ". ".join(parts)[:900]  # cap at 900 chars (expanded for key points)


def _extract_summary_from_md(md_path: Path) -> str:
    if not md_path.exists():
        return ""
    try:
        content = md_path.read_text(encoding="utf-8", errors="ignore")
    except OSError:
        return ""

    # New format: ## 📝 一句話摘要 / Old format: ## 摘要
    for pattern in [
        r"##\s+[^\n]*一句[話话]摘要\s*\n(.+?)(?=\n##|\Z)",
        r"##\s+摘要\s*\n(.+?)(?=\n##|\Z)",
    ]:
        m = re.search(pattern, content, re.DOTALL)
        if m:
            text = m.group(1).strip().replace("\n", " ")
            text = re.sub(r"\s+", " ", text)
            return text[:300]
    return ""


# ── Cosine similarity (pure Python, no numpy) ─────────────────────────────────

def cosine_similarity(a: list, b: list) -> float:
    dot = sum(x * y for x, y in zip(a, b))
    norm_a = math.sqrt(sum(x * x for x in a))
    norm_b = math.sqrt(sum(y * y for y in b))
    denom = norm_a * norm_b
    return dot / denom if denom > 1e-9 else 0.0


# ── Load / save ───────────────────────────────────────────────────────────────

def load_vector_index(path: Path) -> dict:
    if path.exists():
        try:
            return json.loads(path.read_text(encoding="utf-8"))
        except (json.JSONDecodeError, OSError):
            pass
    return {"meta": {}, "vectors": {}}


def save_vector_index(data: dict, path: Path) -> None:
    path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")


# ── Wiki sections ─────────────────────────────────────────────────────────────

# 一段太短的內容嵌入後幾乎沒有訊息量,只會製造假命中
MIN_SECTION_CHARS = 80
# 一段太長會把主題稀釋掉,超過就截斷
MAX_SECTION_CHARS = 4000


def _chunks(header: str, body: str) -> list[tuple[str, str]]:
    """Split one section into as many vectors as it needs.

    A section longer than MAX_SECTION_CHARS used to be truncated, so the tail
    was written to the wiki and then silently dropped from the index — 56
    sections were losing their endings that way. Anything past the cap now
    becomes its own vector instead.

    The first chunk is byte-for-byte what truncation produced, so every vector
    already in the index keeps its hash and nothing is re-embedded to catch up;
    only the previously discarded remainder is new.
    """
    room = MAX_SECTION_CHARS - len(header)
    if room <= 0 or len(body) <= room:
        return [("", f"{header}{body}"[:MAX_SECTION_CHARS])]

    out = [("", f"{header}{body}"[:MAX_SECTION_CHARS])]
    start = room
    index = 2
    while start < len(body):
        end = min(start + room, len(body))
        if end < len(body):
            # Prefer a line break near the end of the window so a chunk does
            # not begin mid-sentence. Deterministic, so hashes stay stable.
            split = body.rfind("\n", start + int(room * 0.8), end)
            if split > start:
                end = split
        out.append((f"@{index}", header + body[start:end].strip()))
        start = end
        index += 1
    return out


def knowledge_section_docs() -> list[tuple[str, str, str]]:
    """把 wiki 主題頁與記憶檔切段,每段一個向量。回傳 [(key, text, hash)]。

    為什麼要做這件事:向量索引原本只涵蓋卡片。wiki——也就是使用者自己消化過、
    訊號密度最高的那一層——完全沒有語意搜尋,只能靠字串比對撈。
    中文沒有空格,字串比對本來就弱,於是「我們之前怎麼處理碳盤查的」會撈回
    一堆只命中「之前」「怎麼」「處理」的無關結果。那不是門檻調不好,
    是那一層根本沒有語意能力。

    切到「段」而不是整頁:一頁 680K 的主題頁嵌成單一向量,等於什麼都不像。
    """
    sources: list[tuple[Path, str]] = []
    if xkb_paths.WIKI_TOPICS_DIR.exists():
        sources += [(p, "wiki/topics") for p in sorted(xkb_paths.WIKI_TOPICS_DIR.glob("*.md"))]
    # 記憶檔也要語意化:hard trigger(「我們之前怎麼…」)查的就是這一層,
    # 而它原本同樣只有字串比對,靠功能詞就能命中。
    if xkb_paths.DATA_DIR.exists():
        sources += [(p, "memory") for p in sorted(xkb_paths.DATA_DIR.glob("*.md"))]
    if xkb_paths.MEMORY_MD.exists():
        sources.append((xkb_paths.MEMORY_MD, "memory"))

    docs: list[tuple[str, str, str]] = []
    for path, prefix in sources:
        try:
            content = path.read_text(encoding="utf-8")
        except OSError:
            continue
        content = re.sub(r"^---\n.*?\n---\n", "", content, flags=re.DOTALL)

        title = path.stem
        section = ""
        buffer: list[str] = []
        # A heading can repeat inside one file — memory/2026-05-09.md has
        # "教訓" fifteen times. Every copy produced the same key, so each
        # run kept a vector for the last of them and re-embedded the rest:
        # about a hundred sections that could never go incremental, paying
        # for an embedding every day. Later occurrences get a suffix; the
        # first keeps the original key so existing vectors stay valid.
        seen_keys: dict[str, int] = {}

        def flush(section_name: str, lines: list[str]) -> None:
            body = "\n".join(lines).strip()
            if len(body) < MIN_SECTION_CHARS:
                return
            key = f"{prefix}/{path.name}#{section_name or 'intro'}"
            occurrence = seen_keys.get(key, 0) + 1
            seen_keys[key] = occurrence
            if occurrence > 1:
                key = f"{key}~{occurrence}"
            for suffix, text in _chunks(f"{title} — {section_name}\n", body):
                docs.append((key + suffix, text,
                             hashlib.md5(text.encode("utf-8")).hexdigest()[:12]))

        for line in content.splitlines():
            if re.match(r"^#{1,3} .+", line):
                flush(section, buffer)
                section = re.sub(r"^#{1,3} ", "", line).strip()
                buffer = []
            else:
                buffer.append(line)
        flush(section, buffer)

    return docs


def write_semantic_index(vectors: dict[str, list[float]], path: Path) -> None:
    """寫出 <path>.bin(float32 單位向量)與 <path>(keys + dims 的 JSON)。"""
    import array

    keys = sorted(vectors)
    dims = len(vectors[keys[0]])
    packed = array.array("f")
    for key in keys:
        vec = vectors[key]
        norm = math.sqrt(sum(x * x for x in vec)) or 1.0
        packed.extend(x / norm for x in vec)

    payload = packed.tobytes()
    path.with_suffix(".bin").write_bytes(payload)
    # 記下位元組序、每個浮點數幾個位元組、總筆數:
    # 這三項任何一項對不上,切片出來的向量就是錯的,而且不會拋錯——
    # 只會安靜地變成空向量或錯位資料,讀取端據此驗證後才敢用。
    path.write_text(json.dumps({
        "dims": dims,
        "keys": keys,
        "count": len(keys),
        "normalized": True,
        "byteorder": sys.byteorder,
        "itemsize": packed.itemsize,
        "bytes": len(payload),
    }, ensure_ascii=False), encoding="utf-8")


# ── Main ──────────────────────────────────────────────────────────────────────

def main() -> int:
    parser = argparse.ArgumentParser(description="Build vector index for x-knowledge-base")
    parser.add_argument("--incremental", action="store_true",
                        help="Skip cards already in vector_index.json")
    parser.add_argument("--dry-run", action="store_true",
                        help="Show what would be done without calling embedding API")
    parser.add_argument("--batch-size", type=int, default=50)
    parser.add_argument("--index-file", default=str(INDEX_FILE))
    parser.add_argument("--vector-file", default=str(VECTOR_FILE))
    parser.add_argument("--env-file", help="dotenv file for credentials/config (process environment wins)")
    args = parser.parse_args()

    index_path = Path(args.index_file)
    vector_path = Path(args.vector_file)

    # Load search index
    if not index_path.exists():
        print(f"❌ search_index.json not found: {index_path}", file=sys.stderr)
        return 1

    raw = json.loads(index_path.read_text(encoding="utf-8"))

    # BM25 全文索引:跟向量索引同源,但**不依賴 embedding**。放在所有提早
    # return(dry-run / 沒東西要嵌入 / 憑證失敗)之前——不然索引缺了、又剛好
    # 沒有卡片變動,排程就永遠不重建它,BM25 腿靜默熄燈。
    # 一支腳本裡建一次,就不必在 6 支排程腳本裡各加一行、各自漂移。
    if not args.dry_run:
        fts_db = xkb_paths.BOOKMARKS_DIR / "fts_index.db"
        try:
            # 嚴格大於:同一秒重生的 index(1 秒 mtime 解析度)要能觸發重建。
            fresh = (fts_db.exists()
                     and fts_db.stat().st_mtime > index_path.stat().st_mtime)
        except OSError:
            fresh = False
        # build_fts_index 沒有 incremental:整包重讀、重斷詞、重建 FTS5、optimize、
        # 原子換檔。卡片沒動就不必每個 tick 都做一遍。mtime 比不到獨立改動的
        # wiki/memory 檔——那些靠每日全量管線補。
        if not fresh:
            try:
                import build_fts_index
                build_fts_index.build(index_path)
                if not fts_db.exists():
                    raise RuntimeError("build_fts_index 跑完但 fts_index.db 不存在")
            except Exception as e:  # noqa: BLE001
                # cron 把 stderr 丟掉,光印警告等於靜默。走 xkb_failures,健檢
                # (純 Python、不經 LLM)才看得到 BM25 索引連續建不起來。
                print(f"⚠️  BM25 索引沒建成(向量索引不受影響):{e}", file=sys.stderr)
                try:
                    import xkb_failures
                    xkb_failures.note("bm25 index build", e)
                except Exception:  # noqa: BLE001
                    pass
    items = raw.get("items", raw) if isinstance(raw, dict) else raw
    print(f"📚 Loaded {len(items)} cards from {index_path}")

    # Load existing vectors (for incremental mode)
    existing = load_vector_index(vector_path)
    existing_vectors: dict = existing.get("vectors", {})

    # Determine which cards to embed
    existing_hashes: dict = existing.get("text_hashes", {})

    to_embed = []
    # 這一次「看過」的每一個鍵,不管有沒有排進佇列。清理要靠它,不能靠
    # queued_keys:增量模式只是跳過『排進佇列』,來源一樣每個都走過。
    # 把兩者混為一談,就會把沒變動的段落當成消失了而刪掉。
    enumerated_keys: set[str] = set()

    # wiki 段落:把消化過的那一層也納入語意搜尋
    for wiki_key, wiki_text, wiki_hash in knowledge_section_docs():
        enumerated_keys.add(wiki_key)
        if not (args.incremental and wiki_key in existing_vectors
                and existing_hashes.get(wiki_key) == wiki_hash):
            to_embed.append((wiki_key, wiki_text, wiki_hash))
    print(f"📖 Wiki/memory sections queued: {len(to_embed)}")

    for item in items:
        key = item.get("relative_path") or item.get("path") or ""
        card_text = extract_card_text(item)
        if not card_text.strip():
            print(f"  ⚠️  No text for: {key}")
            continue
        text_hash = hashlib.md5(card_text.encode("utf-8")).hexdigest()[:12]
        md_path = _resolve_md_path(item)
        if md_path is not None:
            enumerated_keys.add(key)
        # md_path 是 None 時只是不把這份文件算成「走過」——它列不出論點鍵,
        # 算成走過的話既有的 #kpN 會全被當成消失而刪掉,下次路徑正常了再
        # 全部重新付費嵌入。但**要不要重新嵌入**還是交給下面的 hash 判斷:
        # 上一版在這裡 continue,於是卡片內文改了、markdown 暫時找不到,
        # 它就永遠停在舊向量上。
        if not (args.incremental and key in existing_vectors and existing_hashes.get(key) == text_hash):
            to_embed.append((key, card_text, text_hash))

        # 論點級 embedding(TODOS 2026-07-13):每條「關鍵論點」單獨成向量
        if md_path:
            title = (item.get("title") or "").strip()
            for pi, point in enumerate(_extract_key_point_list(md_path), 1):
                pkey = f"{key}#kp{pi}"
                enumerated_keys.add(pkey)
                ptext = (f"{title}. {point}" if title else point)[:500]
                phash = hashlib.md5(ptext.encode("utf-8")).hexdigest()[:12]
                if args.incremental and pkey in existing_vectors and existing_hashes.get(pkey) == phash:
                    continue
                to_embed.append((pkey, ptext, phash))

    # Alias for downstream use
    texts = [t for _, t, _ in to_embed]

    # 集合建一次就好。原本寫在生成式的條件裡,於是每一個既有鍵都重建一次
    # 一萬多個元素的集合——完整重跑時是一萬多次。
    queued_keys = {entry[0] for entry in to_embed}
    # 死鍵:這次走過的文件底下,索引裡有、但內容已經不存在的鍵。
    # 跟列舉放在一起算,因為它就是列舉的產物;放到下面去算,會落在
    # 「沒東西要嵌入就提早結束」的後面,而那正是平常的狀態。
    examined_docs = {key.split("#", 1)[0] for key in enumerated_keys}
    stale_keys = [key for key in existing_vectors
                  if key.split("#", 1)[0] in examined_docs
                  and key not in enumerated_keys]
    skipped = sum(1 for k in existing_vectors if k not in queued_keys)
    print(f"🔢 To embed: {len(to_embed)}  |  Skipped (incremental): {skipped}")

    if args.dry_run:
        print("\n[dry-run] First 3 cards that would be embedded:")
        for key, text, _text_hash in to_embed[:3]:
            print(f"  {key}")
            print(f"    → \"{text[:80]}...\"")
        return 0

    # 分區檔不完整就不能提早結束:一旦 semantic_index.bin 或 cards_index.bin
    # 被刪掉或寫到一半,之後每一次增量執行都會「沒有新東西」然後離開,
    # 而召回永遠停在慢的 JSON 路徑上——安靜地、永久地。
    #
    # write_semantic_index 會寫兩個檔(.json 的鍵表與 .bin 的向量),兩個都要在。
    def _partition_ok(path: Path) -> bool:
        return path.exists() and path.with_suffix(".bin").exists()

    # 要問寫入端「你會寫到哪」,不是問常數。下面寫檔時是看
    # XKB_SEMANTIC_INDEX / XKB_CARDS_INDEX 的,測試與備援路徑都會設它們;
    # 檢查常數等於在別的地方確認「檔案還在」,然後對真正在用的那份一無所知。
    _semantic_target = Path(os.getenv("XKB_SEMANTIC_INDEX", str(SEMANTIC_FILE)))
    _cards_target = Path(os.getenv("XKB_CARDS_INDEX", str(CARDS_FILE)))
    _partitions_ok = _partition_ok(_semantic_target) and _partition_ok(_cards_target)
    if stale_keys:
        print(f"🧹 {len(stale_keys)} 個索引鍵已經沒有對應內容,這一輪清掉")
    if not to_embed and _partitions_ok and not stale_keys:
        print("✅ Nothing to embed.")
        return 0
    if not to_embed:
        # 為什麼還要往下走,要說對。只有死鍵要清的時候說「分區索引檔不完整」
        # 是假的,而假的理由會讓下一個人去修一個不存在的問題。
        print("沒有新內容要嵌入,但分區索引檔不完整——從既有向量重新寫出。"
              if not _partitions_ok else
              "沒有新內容要嵌入,只清掉死鍵之後重新寫出。")

    # 沒有東西要嵌入就不要去要嵌入服務。純粹「清死鍵」的那一輪也會走到
    # 這裡,於是憑證不在時整支以 1 收場、而且一個死鍵都沒清掉——
    # 前一天它還只是安靜地印 Nothing to embed。
    provider = None
    if to_embed:
        try:
            provider = get_provider(env_file=args.env_file)
            print(f"🤖 Provider: {provider.__class__.__name__} / model: {getattr(provider, 'model', '?')}")
        except (EnvironmentError, ValueError) as e:
            print(f"❌ {e}", file=sys.stderr)
            return 1

    # Embed in batches
    keys = [k for k, _, _ in to_embed]
    hashes = [h for _, _, h in to_embed]
    vectors_list = []

    batch_size = args.batch_size
    total = len(texts)
    for i in range(0, total, batch_size):
        batch_texts = texts[i:i + batch_size]
        batch_keys = keys[i:i + batch_size]
        print(f"  Embedding {i+1}–{min(i+batch_size, total)}/{total}...", end=" ", flush=True)
        try:
            batch_vecs = provider.embed_batch(batch_texts, batch_size=batch_size)
            vectors_list.extend(zip(batch_keys, batch_vecs))
            print("✓")
        except Exception as e:
            print(f"\n❌ Failed at batch {i//batch_size + 1}: {e}", file=sys.stderr)
            return 1

    # Merge with existing vectors and hashes
    new_vectors = dict(existing_vectors)
    new_hashes = dict(existing_hashes)
    for (key, vec), h in zip(vectors_list, hashes):
        new_vectors[key] = vec
        new_hashes[key] = h

    # 索引原本只增不減。標題改名、記憶檔刪掉、卡片的關鍵論點從五條變三條——
    # 舊向量就永遠留著,照樣參與排序,然後顯示空白。實測 2,262 個 wiki 鍵
    # 裡有 80 個指向已經不存在的段落。
    #
    # 但「這次沒產生」不等於「不存在了」:增量執行本來就只看有變動的東西。
    # 安全的規則是:只清掉「這次有檢查過來源、而這次沒有產生」的鍵。沒被
    # 看過的來源,它的鍵一個都不動。
    # 清理沒有對應內容的索引鍵。這一段我寫錯過三次,每次都是同一個混淆的
    # 不同面向:
    #
    #   一  用 queued_keys 推出「檢查過的來源」。增量模式下,文件只要有一塊
    #       變了就算檢查過,於是它其他沒變的塊全被刪——改過的卡片會在論點鍵
    #       與卡片鍵之間永遠振盪,每晚重新付費嵌入。
    #   二  改成只在完整重建時清理。但每一條排程都帶 --incremental,於是清理
    #       從此不會執行。我把破壞性 bug 修成了一個關掉的功能。
    #   三  規則終於對了,卻放在「沒東西要嵌入就提早結束」的後面——而那正是
    #       平常的狀態。連跑三次,一次都沒清到。
    #
    # 真正的分辨是:**列舉 ≠ 排進佇列**。兩種模式都會走過每一個來源,
    # --incremental 只是跳過「把沒變的排進佇列」。所以要問「這次走到了哪些
    # 鍵」,而那個問題兩種模式的答案一樣好——這條規則因此不需要看模式,
    # 那正是它對的跡象。stale_keys 在上面列舉處算好,這裡只執行。
    #
    # 限定在「這次走過的文件」之內,順帶關掉另一個風險:卡片鍵只來自
    # --index-file,拿一份不完整的索引跑完整重建,沒走過的文件一根寒毛都不會動。
    #
    # 已知殘留:整頁被刪除時它一個段落都不會被列舉,於是它的鍵留著。
    # 這是安全的方向——寧可留著也不要誤刪。
    for key in stale_keys:
        new_vectors.pop(key, None)
        new_hashes.pop(key, None)

    # Save
    output = {
        "meta": {
            "provider": (provider.__class__.__name__.replace("Provider", "").lower()
                     if provider is not None
                     else existing.get("meta", {}).get("provider", "")),
            "model": (getattr(provider, "model", "") if provider is not None
                  else existing.get("meta", {}).get("model", "")),
            "dims": len(vectors_list[0][1]) if vectors_list else existing.get("meta", {}).get("dims", 0),
            "total": len(new_vectors),
            "point_vectors": sum(1 for k in new_vectors if "#kp" in k),
            "built_at": datetime.now(timezone.utc).isoformat(),
        },
        "vectors": new_vectors,
        "text_hashes": new_hashes,
    }
    save_vector_index(output, vector_path)

    # 另存召回用的索引,二進位格式。
    # 存 JSON 的話 1,868 個 3072 維向量要 76MB,光 json.loads 就要 5 秒——
    # 而召回是每句話都要跑的。float32 打包後只有 23MB,載入約 0.1 秒。
    # 順便先做單位化,查詢時只要點積,不必每次算平方根。
    semantic_vectors = {
        k: v for k, v in output.get("vectors", {}).items()
        if (k.startswith("wiki/topics/") or k.startswith("memory/"))
        and not k.startswith("memory/cards/")
    }
    if semantic_vectors:
        semantic_path = Path(os.getenv("XKB_SEMANTIC_INDEX", str(SEMANTIC_FILE)))
        write_semantic_index(semantic_vectors, semantic_path)
        print(f"✅ Saved {len(semantic_vectors)} semantic vectors → {semantic_path.with_suffix('.bin')}")

    card_vectors = {k: v for k, v in output.get("vectors", {}).items()
                    if k not in semantic_vectors}
    if card_vectors:
        cards_path = Path(os.getenv("XKB_CARDS_INDEX", str(CARDS_FILE)))
        write_semantic_index(card_vectors, cards_path)
        print(f"✅ Saved {len(card_vectors)} card vectors → {cards_path.with_suffix('.bin')}")

    print(f"\n✅ Saved {len(new_vectors)} vectors → {vector_path}")
    print(f"   Provider : {output['meta']['provider']} / {output['meta']['model']}")
    print(f"   Dims     : {output['meta']['dims']}")


    return 0


import xkb_usage  # noqa: E402  — 量測誰在跑,見 scripts/xkb_usage.py

if __name__ == "__main__":
    xkb_usage.record(__file__)
    raise SystemExit(main())