tests/test_index_is_derived.py
Rama main · réplica de lectura del repositorio original.
"""搜尋索引是衍生物：只有一個地方寫它，而且是從檔案重算。
2026-09-10 量到的狀態：八支程式各自對 search_index.json 做讀-改-寫。每一支都
是合併而不是覆蓋，所以沒有大規模掉資料，但它們對「這是同一個來源嗎」有不同
的認定（github-x / github_fork-x / github_star-x 被當成三個東西），也沒有共用
鎖。於是索引慢慢跟磁碟上的檔案說的不一樣。
把索引整份從檔案重建之後跟正在用的那份比：
現行 1680 筆 重建 1680 筆
重建後才出現的：3 筆（其中一張是當天健檢報出來的「孤兒卡」）
重建後會消失的：1 筆（同一份內容，被記成另一個路徑）
也就是說重建不只是等價，它比較準。修補才是漂移的來源。
這份測試守三件事：只有 builder 會寫索引、值得留下來的東西寫在檔案裡、
重建之後那些東西還在。
"""
from __future__ import annotations
import json
import os
import subprocess
import sys
import tempfile
import unittest
from unittest import mock
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SCRIPTS = ROOT / "scripts"
sys.path.insert(0, str(SCRIPTS))
import xkb_frontmatter # noqa: E402
BUILDER = SCRIPTS / "build_search_index.sh"
# 唯一可以寫索引的檔案。要新增的話，先問「這件事為什麼不能由檔案推導出來」。
ALLOWED_WRITERS = {"build_search_index.sh"}
CARD = """---
id: {stem}
type: knowledge-card
source_type: local
source_url: https://example.test/{stem}
category: 99-general
tags: [a, b]
---
# {title}
## 📌 一句話摘要
{summary}
"""
def _require_python3() -> None:
"""builder 用 python3 跑。Windows 上那是 Microsoft Store 的轉接器，
直接執行會以 49 結束、stderr 全空——看起來像 builder 壞了，其實它一行
都沒跑到。這種假失敗要說得出原因，不要讓人去查 build_search_index.sh。"""
try:
proc = subprocess.run(["python3", "-c", "pass"],
capture_output=True, timeout=30)
except (OSError, subprocess.SubprocessError):
raise unittest.SkipTest("這個環境沒有可執行的 python3")
if proc.returncode != 0:
raise unittest.SkipTest("python3 不是真的直譯器（Windows Store 轉接器）")
def _bash() -> str:
if os.name != "nt":
return "bash"
for candidate in ["C:/Program Files/Git/bin/bash.exe",
"C:/Program Files (x86)/Git/bin/bash.exe"]:
if Path(candidate).exists():
return candidate
raise unittest.SkipTest("no POSIX bash available")
class OnlyOneWriter(unittest.TestCase):
def test_nothing_else_writes_the_index(self):
offenders = []
for path in sorted(list(SCRIPTS.glob("*.py")) + list(SCRIPTS.glob("*.sh"))):
if path.name in ALLOWED_WRITERS:
continue
text = path.read_text(encoding="utf-8", errors="ignore")
if "INDEX_FILE.write_text" in text or "index_file.write_text" in text:
offenders.append(path.name)
self.assertEqual(
offenders, [],
"索引是衍生物，只能由 build_search_index.sh 寫。這些又開始自己修補了：\n "
+ "\n ".join(offenders))
def test_the_ingesters_go_through_the_rebuild(self):
for name in ["fetch_github_repos.py", "fetch_youtube_playlist.py",
"local_ingest.py", "sync_enriched_index.py"]:
text = (SCRIPTS / name).read_text(encoding="utf-8")
with self.subTest(script=name):
expected = ("xkb_index.rebuild()" if name == "sync_enriched_index.py"
else "batch.finish(")
self.assertIn(expected, text)
class Frontmatter(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.addCleanup(self._tmp.cleanup)
def _card(self, stem: str = "c1") -> Path:
path = self.root / f"{stem}.md"
path.write_text(CARD.format(stem=stem, title="標題", summary="一句話"),
encoding="utf-8")
return path
def test_marking_adds_the_flag_without_disturbing_the_rest(self):
card = self._card()
before = card.read_text(encoding="utf-8")
self.assertTrue(xkb_frontmatter.mark_excluded(card, "duplicate_source_url"))
after = card.read_text(encoding="utf-8")
self.assertTrue(xkb_frontmatter.is_excluded(card))
self.assertIn("excluded_reason: duplicate_source_url", after)
# 其他欄位與內文原樣保留——不重排、不重寫。
for line in ["id: c1", "type: knowledge-card", "category: 99-general",
"tags: [a, b]", "# 標題", "一句話"]:
self.assertIn(line, after)
self.assertEqual(before.count("---"), after.count("---"))
def test_marking_twice_does_not_rewrite(self):
card = self._card()
xkb_frontmatter.mark_excluded(card, "reason")
self.assertFalse(xkb_frontmatter.mark_excluded(card, "reason"))
def test_a_file_without_frontmatter_reports_failure(self):
"""標記不上跟標記了不一樣。回 False 讓呼叫端說得出來。"""
path = self.root / "plain.md"
path.write_text("# 沒有 frontmatter\n", encoding="utf-8")
self.assertFalse(xkb_frontmatter.mark_excluded(path, "reason"))
class TheFlagSurvivesARebuild(unittest.TestCase):
"""這是整個設計的關鍵：值得留下來的東西寫在檔案裡，所以重建洗不掉。
原本它寫在索引列上，於是實測正式索引 1680 筆裡帶 excluded 的有 0 筆——
兩支腳本的成果一次都沒留下來，而讀它的四個地方一直讀到 False。
"""
def test_an_excluded_card_is_still_excluded_after_a_full_rebuild(self):
bash = _bash()
_require_python3()
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
cards = root / "memory" / "cards"
bookmarks = root / "memory" / "bookmarks"
cards.mkdir(parents=True)
bookmarks.mkdir(parents=True)
index = bookmarks / "search_index.json"
keep = cards / "keep.md"
drop = cards / "drop.md"
keep.write_text(CARD.format(stem="keep", title="留著", summary="摘要"),
encoding="utf-8")
drop.write_text(CARD.format(stem="drop", title="重複", summary="摘要"),
encoding="utf-8")
xkb_frontmatter.mark_excluded(drop, "duplicate_source_url")
env = {**os.environ,
"WORKSPACE_DIR": str(root),
"BOOKMARKS_DIR": str(bookmarks),
"CARDS_DIR": str(cards),
"INDEX_FILE": str(index),
"PYTHONIOENCODING": "utf-8", "PYTHONUTF8": "1"}
proc = subprocess.run([bash, str(BUILDER)], env=env,
capture_output=True, text=True, encoding="utf-8", errors="replace")
self.assertEqual(proc.returncode, 0, proc.stderr)
rows = json.loads(index.read_text(encoding="utf-8"))["items"]
by_stem = {Path(r["relative_path"]).stem: r for r in rows}
self.assertTrue(by_stem["drop"].get("excluded"))
self.assertIn("duplicate_source_url", by_stem["drop"].get("excluded_reason", ""))
self.assertFalse(by_stem["keep"].get("excluded"))
class TheQualityScriptsActuallyRun(unittest.TestCase):
"""真的把它們跑一次。
2026-09-10：把 canonicalize_duplicates 改成標記檔案時，補上 _card_path
的那一步沒有跑到，於是它引用了一個不存在的名字。py_compile 過、既有測試
全綠、部署也成功——直到在 VPS 上真的執行才炸成 NameError。
當時的測試只檢查「原始碼裡有沒有那個字串」。字串在不在跟跑不跑得起來是
兩件事，而這個專案已經在別的地方吃過同一種虧。
"""
def _fixture(self, root: Path) -> Path:
cards = root / "memory" / "cards"
bookmarks = root / "memory" / "bookmarks"
cards.mkdir(parents=True)
bookmarks.mkdir(parents=True)
for stem in ("a", "b"):
(cards / f"{stem}.md").write_text(
CARD.format(stem=stem, title="標題", summary="一句話"),
encoding="utf-8")
index = bookmarks / "search_index.json"
# a 與 b 指向同一個來源：canonicalize 應該挑一個留下、標記另一個。
index.write_text(json.dumps({"version": "1.1", "items": [
{"path": str(cards / "a.md"), "relative_path": "memory/cards/a.md",
"title": "標題", "summary": "一句話", "tags": [], "category": "99-general",
"source_url": "https://example.test/same", "source_type": "local",
"enriched": True},
{"path": str(cards / "b.md"), "relative_path": "memory/cards/b.md",
"title": "2026-09-10", "summary": "", "tags": [], "category": "99-general",
"source_url": "https://example.test/same", "source_type": "local",
"enriched": True},
]}, ensure_ascii=False), encoding="utf-8")
return index
def _run(self, script: str, root: Path, index: Path):
env = {**os.environ,
"XKB_DATA_DIR": str(root / "memory"),
"CARDS_DIR": str(root / "memory" / "cards"),
"BOOKMARKS_DIR": str(root / "memory" / "bookmarks"),
"INDEX_FILE": str(index),
"PYTHONIOENCODING": "utf-8", "PYTHONUTF8": "1"}
return subprocess.run([sys.executable, str(SCRIPTS / script), "--dry-run"],
capture_output=True, text=True, encoding="utf-8", errors="replace", env=env)
def test_canonicalize_duplicates_runs(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
index = self._fixture(root)
proc = self._run("canonicalize_duplicates.py", root, index)
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertIn("duplicate groups", proc.stdout)
def test_normalize_index_quality_runs(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
index = self._fixture(root)
proc = self._run("normalize_index_quality.py", root, index)
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertIn("總項目數", proc.stdout)
class LocalIngestsAreNotLowQuality(unittest.TestCase):
"""本機擷取的出處是一個檔案路徑，不是網址——那不是品質問題。
2026-09-10：品質規則差一點把兩張本機匯入的好卡片排除掉，
「every-app/open-seo — Semrush/Ahrefs 開源替代方案」與那本 Obsidian 的書，
理由只是 source_url 不以 http 開頭。
"""
def setUp(self):
sys.path.insert(0, str(SCRIPTS))
import normalize_index_quality
self.reasons = normalize_index_quality.exclusion_reasons
def test_a_file_path_is_valid_provenance_for_a_local_ingest(self):
item = {"title": "一份本機文件", "summary": "有內容的摘要",
"source_type": "local-paper",
"source_url": "memory/bookmarks/github/stars/x.md"}
self.assertNotIn("invalid_source_url", self.reasons(item))
def test_a_broken_url_on_a_web_source_is_still_caught(self):
item = {"title": "測試書籤", "summary": "摘要",
"source_type": "x-bookmark",
"source_url": "https://x.com/i/status/test"}
self.assertIn("invalid_source_url", self.reasons(item))
def test_an_unenriched_bookmark_is_still_caught(self):
item = {"title": "Tweet 2073558354233094315", "summary": "",
"source_type": "x-bookmark",
"source_url": "https://x.com/i/status/2073558354233094315"}
self.assertIn("tweet_numeric_low_signal", self.reasons(item))
class GovernanceRunsTheTidyStep(unittest.TestCase):
def test_the_scheduled_job_calls_both_and_rebuilds(self):
"""排上去了才會跑。原本這兩支沒有任何排程在叫。"""
text = (SCRIPTS / "run_candidate_governance.sh").read_text(encoding="utf-8")
self.assertIn("canonicalize_duplicates", text)
self.assertIn("normalize_index_quality", text)
self.assertIn("xkb_index", text)
class DeduplicationMustLeaveOneCopy(unittest.TestCase):
"""去重不可以把整組都標掉——那不是去重，那是刪除。
2026-09-10：把「標記一個檔案」改成「標記同一份知識的所有檔案」時，用的是
source_url 去找。品質排除要的正是那個（空殼書籤的每一份副本都該排除），但
重複組本來就共用同一個來源，於是「要保留的那一份」也被撈進來標掉。七份全
排除，那份知識會從召回裡完全消失，而輸出只會說「已標記 6 張」。
"""
def setUp(self):
sys.path.insert(0, str(SCRIPTS))
import xkb_index
self.xkb_index = xkb_index
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
self.cards = self.root / "cards"
self.cards.mkdir()
self.addCleanup(self._tmp.cleanup)
patches = [
mock.patch.object(xkb_index.xkb_paths, "CARDS_DIR", self.cards),
mock.patch.object(xkb_index.xkb_paths, "BOOKMARKS_DIR", self.root / "bookmarks"),
mock.patch.object(xkb_index.xkb_paths, "WORKSPACE", self.root),
]
for patch in patches:
patch.start()
self.addCleanup(patch.stop)
xkb_index.reset_scan()
self.addCleanup(xkb_index.reset_scan)
def _card(self, stem: str, url: str) -> Path:
path = self.cards / f"{stem}.md"
path.write_text(
"---\n"
f"id: {stem}\n"
"type: knowledge-card\n"
f"source_url: {url}\n"
"category: 99-general\n"
"tags: [a]\n"
"---\n"
"\n"
f"# {stem}\n",
encoding="utf-8")
return path
def test_by_source_gathers_every_copy(self):
"""品質排除要的是這個：同一份知識的每一份副本。"""
url = "https://x.com/i/status/123456789012345678"
self._card("copy_a", url)
self._card("copy_b", url)
item = {"relative_path": "copy_a.md", "source_url": url}
found = self.xkb_index.files_for_item(item, by_source=True)
self.assertEqual({p.stem for p in found}, {"copy_a", "copy_b"})
def test_without_by_source_only_this_row_files(self):
"""去重要的是這個：不要把同組的其他成員撈進來。"""
url = "https://x.com/i/status/123456789012345678"
self._card("copy_a", url)
self._card("copy_b", url)
item = {"relative_path": "copy_a.md", "source_url": url}
found = self.xkb_index.files_for_item(item, by_source=False)
self.assertEqual({p.stem for p in found}, {"copy_a"})
def test_the_canonical_copy_is_never_marked(self):
"""跑完整支腳本，確認保留的那一份沒有被標。"""
url = "https://x.com/i/status/123456789012345678"
keep = self._card("keeper", url)
drop = self._card("dropper", url)
index = self.root / "search_index.json"
index.write_text(json.dumps({"version": "1.1", "items": [
{"path": str(keep), "relative_path": "keeper.md", "title": "好標題",
"summary": "夠長的摘要內容，超過十八個字元的門檻",
"tags": [], "category": "99-general", "source_url": url,
"source_type": "x-bookmark", "enriched": True},
{"path": str(drop), "relative_path": "dropper.md", "title": "dropper",
"summary": "", "tags": [], "category": "99-general", "source_url": url,
"source_type": "x-bookmark", "enriched": False},
]}, ensure_ascii=False), encoding="utf-8")
env = {**os.environ,
"XKB_DATA_DIR": str(self.root),
"CARDS_DIR": str(self.cards),
"BOOKMARKS_DIR": str(self.root / "bookmarks"),
"INDEX_FILE": str(index),
"PYTHONIOENCODING": "utf-8", "PYTHONUTF8": "1"}
proc = subprocess.run(
[sys.executable, str(SCRIPTS / "canonicalize_duplicates.py")],
capture_output=True, text=True, encoding="utf-8", errors="replace", env=env)
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertFalse(xkb_frontmatter.is_excluded(keep), "保留的那一份被標掉了")
self.assertTrue(xkb_frontmatter.is_excluded(drop))
def test_a_marking_can_be_undone(self):
"""把知識從召回裡拿掉的決定，必須有辦法反悔。"""
card = self._card("x", "https://example.test/x")
xkb_frontmatter.mark_excluded(card, "reason")
self.assertTrue(xkb_frontmatter.is_excluded(card))
self.assertTrue(xkb_frontmatter.unmark_excluded(card))
self.assertFalse(xkb_frontmatter.is_excluded(card))
self.assertFalse(xkb_frontmatter.unmark_excluded(card))
class ExclusionIsNotAOneWayDoor(unittest.TestCase):
"""排除要有退路，而退路要真的被某個管線步驟走過。
2026-09-10 審查抓到：旗標改成寫在檔案上之後，sync_enriched_index 裡那段
「卡片補上標題與摘要就清掉過期排除」只改索引列——而索引現在是衍生物，
下一次重建就把 excluded: true 原封不動讀回來。unmark_excluded() 除了測試
沒有任何呼叫者。
後果是排除變成單行道：一張被誤排除的卡（例如被「空欄位吞掉下一行」那個
解析 bug 害到的），即使後來補齊了內容也救不回來，只能手動改檔案。改成寫
在檔案上之前反而是安全的——那時候下一次重建就洗掉了。
"""
def test_the_pipeline_can_undo_a_quality_exclusion(self):
source = (SCRIPTS / "sync_enriched_index.py").read_text(encoding="utf-8")
self.assertIn("unmark_excluded", source)
self.assertIn("files_for_item", source)
def test_it_does_not_undo_a_duplicate_exclusion(self):
"""去重的標記不能被「補齊內容」撤掉——那會把副本放回召回裡。"""
source = (SCRIPTS / "sync_enriched_index.py").read_text(encoding="utf-8")
self.assertIn("duplicate_source_url", source)
def test_unmark_leaves_the_card_usable(self):
import xkb_frontmatter as fm
with tempfile.TemporaryDirectory() as tmp:
card = Path(tmp) / "c.md"
card.write_text(CARD.format(stem="c", title="標題", summary="摘要"),
encoding="utf-8")
fm.mark_excluded(card, "tweet_numeric_low_signal")
self.assertTrue(fm.is_excluded(card))
self.assertTrue(fm.unmark_excluded(card))
after = card.read_text(encoding="utf-8")
self.assertFalse(fm.is_excluded(card))
for line in ["id: c", "type: knowledge-card", "# 標題", "摘要"]:
self.assertIn(line, after)
def test_a_windows_path_in_the_reason_does_not_crash(self):
"""理由含 Windows 路徑時 re.sub 會把反斜線當跳脫序列。
canonicalize 的理由是 "duplicate_source_url; canonical={relative_path}"，
而那個路徑在 Windows 上是 memory\cards\abc.md。原本會拋
re.error: bad escape \c，而且是在已經改寫了一部分卡片之後才炸。
"""
import xkb_frontmatter as fm
with tempfile.TemporaryDirectory() as tmp:
card = Path(tmp) / "c.md"
card.write_text(CARD.format(stem="c", title="標題", summary="摘要"),
encoding="utf-8")
fm.mark_excluded(card, "first")
reason = "duplicate_source_url; canonical=" + chr(92).join(
["memory", "cards", "abc.md"])
self.assertTrue(fm.mark_excluded(card, reason) or True)
updated = fm.set_field(card.read_text(encoding="utf-8"),
"excluded_reason", reason)
self.assertIn("abc.md", updated)
if __name__ == "__main__":
unittest.main()