tests/test_benchmark_contract.py

Rama main · réplica de lectura del repositorio original.

import importlib.util
import subprocess
import sys
import tempfile
import unittest
from types import SimpleNamespace
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]
SPEC = importlib.util.spec_from_file_location("caveman_benchmark", ROOT / "benchmarks" / "run.py")
BENCHMARK = importlib.util.module_from_spec(SPEC)
assert SPEC.loader is not None
SPEC.loader.exec_module(BENCHMARK)


class BenchmarkContractTests(unittest.TestCase):
    def test_call_api_preserves_temperature_with_sdk_v2_signature(self):
        requests = []

        # SDK 2 removed temperature from the generated method signature. The
        # benchmark still needs the same wire setting for every comparison arm.
        def create(*, model, max_tokens, system, messages, extra_body):
            requests.append({
                "model": model, "max_tokens": max_tokens, "system": system,
                "messages": messages, **extra_body,
            })
            return SimpleNamespace(
                usage=SimpleNamespace(input_tokens=3, output_tokens=2),
                content=[SimpleNamespace(text="fixture reply")],
                stop_reason="end_turn",
            )

        client = SimpleNamespace(messages=SimpleNamespace(create=create))
        result = BENCHMARK.call_api(client, "claude-sonnet-4-6", "system", "prompt")
        self.assertEqual(requests, [{
            "model": "claude-sonnet-4-6", "max_tokens": 4096, "system": "system",
            "messages": [{"role": "user", "content": "prompt"}], "temperature": 0,
        }])
        self.assertEqual(result, {
            "input_tokens": 3, "output_tokens": 2,
            "text": "fixture reply", "stop_reason": "end_turn",
        })

    def test_dry_run_defaults_to_supported_model(self):
        result = subprocess.run(
            [sys.executable, str(ROOT / "benchmarks" / "run.py"), "--dry-run", "--trials", "1"],
            cwd=ROOT,
            capture_output=True,
            text=True,
            check=False,
        )
        self.assertEqual(result.returncode, 0, result.stderr)
        self.assertEqual(result.stdout.splitlines()[0], "Model:  claude-sonnet-4-6")

    def test_readme_has_one_replaceable_benchmark_region(self):
        readme = (ROOT / "README.md").read_text(encoding="utf-8")
        self.assertEqual(readme.count(BENCHMARK.BENCHMARK_START), 1)
        self.assertEqual(readme.count(BENCHMARK.BENCHMARK_END), 1)
        self.assertLess(readme.index(BENCHMARK.BENCHMARK_START), readme.index(BENCHMARK.BENCHMARK_END))

    def test_result_files_are_committable(self):
        ignored = (ROOT / ".gitignore").read_text(encoding="utf-8").splitlines()
        self.assertNotIn("benchmarks/results/*.json", ignored)

    def test_readme_publishes_no_uncommitted_output_percentage_or_accuracy(self):
        readme = (ROOT / "README.md").read_text(encoding="utf-8").lower()
        self.assertNotIn("65% average output", readme)
        self.assertNotIn("technical accuracy    ", readme)
        if not list((ROOT / "benchmarks" / "results").glob("*.json")):
            self.assertNotIn("**1214**", readme)
            for name in ("plugin.json", "marketplace.json"):
                manifest = (ROOT / ".claude-plugin" / name).read_text(encoding="utf-8")
                self.assertNotIn("65%", manifest)

    def test_docs_drop_unreproducible_compress_and_cavecrew_figures(self):
        # benchmark.py over tests/caveman-compress gives 33.2% total; 46% never
        # matched the fixtures, and no benchmark backs the cavecrew figures.
        banned = {
            "README.md": ["46% smaller"],
            "docs/HONEST-NUMBERS.md": ["~46%"],
            "skills/caveman-help/SKILL.md": ["~46%"],
            "skills/caveman-compress/README.md": ["46%", "59.6%"],
            "skills/cavecrew/SKILL.md": ["~700 tokens"],
            "agents/cavecrew-investigator.md": ["60% fewer"],
            "plugins/caveman/agents/cavecrew-investigator.md": ["60% fewer"],
        }
        for rel, needles in banned.items():
            text = (ROOT / rel).read_text(encoding="utf-8")
            for needle in needles:
                self.assertFalse(needle in text, f"{rel} still says {needle!r}")

    def test_chart_reads_terse_control_from_current_harness_table(self):
        spec = importlib.util.spec_from_file_location("benchmark_charts", ROOT / "benchmarks" / "render_charts.py")
        charts = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(charts)
        data = f"{BENCHMARK.BENCHMARK_START}\n" + (
            "| Task | Baseline (tokens) | Terse (tokens) | Caveman (tokens) | vs terse | vs baseline |\n"
            "| Fixture | 1000 | 100 | 75 | 25% | 92% |\n"
            "| **Average** | **1000** | **100** | **75** | **25%** | **92%** |\n"
        ) + BENCHMARK.BENCHMARK_END
        rows, average = charts.read_skill_rows(data)
        self.assertEqual(rows, [("Fixture", 100, 75, "-25%")])
        self.assertEqual(average, ("Average", 100, 75, "-25%"))
        self.assertEqual(charts.read_skill_rows(f"{BENCHMARK.BENCHMARK_START}\nNo result\n{BENCHMARK.BENCHMARK_END}"), ([], None))

    def test_update_readme_replaces_only_marker_body(self):
        with tempfile.TemporaryDirectory() as directory:
            path = Path(directory) / "README.md"
            path.write_text(f"before\n{BENCHMARK.BENCHMARK_START}\nold\n{BENCHMARK.BENCHMARK_END}\nafter\n", encoding="utf-8")
            original = BENCHMARK.README_PATH
            try:
                BENCHMARK.README_PATH = path
                BENCHMARK.update_readme("new table")
            finally:
                BENCHMARK.README_PATH = original
            self.assertEqual(
                path.read_text(encoding="utf-8"),
                f"before\n{BENCHMARK.BENCHMARK_START}\nnew table\n{BENCHMARK.BENCHMARK_END}\nafter\n",
            )


if __name__ == "__main__":
    unittest.main()