test-int/semantic/report.py
Rama main · réplica de lectura del repositorio original.
"""Rich CLI viewer for semantic search benchmark JSONL artifacts.
Usage:
python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl
python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl --sort-by avg_latency_ms
python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl --filter-combo sqlite
python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl --filter-suite paraphrase
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from rich.console import Console
from rich.table import Table
from rich.text import Text
from typing import Any
def load_benchmarks(path: Path) -> list[dict[str, Any]]:
"""Load benchmark records from a JSONL file."""
records = []
with path.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
records.append(json.loads(line))
return records
def _quality_cell(value: float) -> Text:
"""Color-code a quality metric value."""
text = f"{value:.3f}"
if value >= 0.90:
return Text(text, style="bold green")
if value >= 0.75:
return Text(text, style="green")
if value >= 0.50:
return Text(text, style="yellow")
if value > 0.0:
return Text(text, style="red")
return Text(text, style="dim red")
def _latency_cell(value: float) -> Text:
"""Color-code a latency value (ms)."""
text = f"{value:.1f}"
if value <= 5.0:
return Text(text, style="bold green")
if value <= 20.0:
return Text(text, style="green")
if value <= 100.0:
return Text(text, style="yellow")
return Text(text, style="red")
SORT_KEYS = {
"combo": lambda r: r["metrics"]["combo"],
"suite": lambda r: r["metrics"]["suite"],
"mode": lambda r: r["metrics"]["mode"],
"hit_at_1": lambda r: -r["metrics"].get("hit_at_1", 0),
"recall_at_5": lambda r: -r["metrics"].get("recall_at_5", 0),
"mrr_at_10": lambda r: -r["metrics"].get("mrr_at_10", 0),
"avg_latency_ms": lambda r: r["metrics"].get("avg_latency_ms", 0),
"total_time_ms": lambda r: r["metrics"].get("total_time_ms", 0),
}
def build_table(records: list[dict[str, Any]], title: str = "Semantic Search Benchmarks") -> Table:
"""Build a rich Table from benchmark records."""
table = Table(title=title, show_lines=False)
table.add_column("Combo", style="cyan", no_wrap=True)
table.add_column("Suite", style="magenta", no_wrap=True)
table.add_column("Mode", style="blue", no_wrap=True)
table.add_column("N", justify="right")
table.add_column("hit@1", justify="right", no_wrap=True)
table.add_column("R@5", justify="right", no_wrap=True)
table.add_column("MRR@10", justify="right", no_wrap=True)
table.add_column("avg ms", justify="right", no_wrap=True)
table.add_column("total ms", justify="right", no_wrap=True)
for rec in records:
m = rec["metrics"]
table.add_row(
m["combo"],
m["suite"],
m["mode"],
str(m["cases"]),
_quality_cell(m.get("hit_at_1", 0)),
_quality_cell(m.get("recall_at_5", 0)),
_quality_cell(m.get("mrr_at_10", 0)),
_latency_cell(m.get("avg_latency_ms", 0)),
_latency_cell(m.get("total_time_ms", 0)),
)
return table
def build_summary_table(records: list[dict[str, Any]]) -> Table:
"""Build a summary table comparing combos across suites."""
# Group by combo
combos: dict[str, dict[str, dict[str, Any]]] = {}
for rec in records:
m = rec["metrics"]
key = m["combo"]
if key not in combos:
combos[key] = {}
suite_mode = f"{m['suite']}/{m['mode']}"
combos[key][suite_mode] = m
table = Table(title="Summary: Best Recall@5 by Combo", show_lines=False)
table.add_column("Combo", style="cyan", no_wrap=True)
table.add_column("Lexical (best)", justify="right")
table.add_column("Paraphrase (best)", justify="right")
table.add_column("Avg Latency (best)", justify="right")
for combo_name, suite_modes in sorted(combos.items()):
# Find best recall@5 for lexical and paraphrase
lexical_best = max(
(v.get("recall_at_5", 0) for k, v in suite_modes.items() if k.startswith("lexical")),
default=0,
)
paraphrase_best = max(
(v.get("recall_at_5", 0) for k, v in suite_modes.items() if k.startswith("paraphrase")),
default=0,
)
# Find best (lowest) avg latency
avg_latencies = [
v.get("avg_latency_ms", 0)
for v in suite_modes.values()
if v.get("avg_latency_ms", 0) > 0
]
best_latency = min(avg_latencies) if avg_latencies else 0
table.add_row(
combo_name,
_quality_cell(lexical_best),
_quality_cell(paraphrase_best),
_latency_cell(best_latency),
)
return table
def main():
parser = argparse.ArgumentParser(description="View semantic search benchmark results")
parser.add_argument("path", type=Path, help="Path to JSONL benchmark artifact")
parser.add_argument(
"--sort-by", choices=list(SORT_KEYS.keys()), default="combo", help="Sort column"
)
parser.add_argument(
"--filter-combo", type=str, default=None, help="Filter by combo name (substring match)"
)
parser.add_argument(
"--filter-suite", type=str, default=None, help="Filter by suite name (substring match)"
)
parser.add_argument(
"--filter-mode", type=str, default=None, help="Filter by mode (fts, vector, hybrid)"
)
parser.add_argument("--no-summary", action="store_true", help="Skip the summary table")
args = parser.parse_args()
if not args.path.exists():
print(f"File not found: {args.path}", file=sys.stderr)
sys.exit(1)
records = load_benchmarks(args.path)
if not records:
print("No benchmark records found.", file=sys.stderr)
sys.exit(1)
# Apply filters
if args.filter_combo:
records = [r for r in records if args.filter_combo in r["metrics"]["combo"]]
if args.filter_suite:
records = [r for r in records if args.filter_suite in r["metrics"]["suite"]]
if args.filter_mode:
records = [r for r in records if args.filter_mode == r["metrics"]["mode"]]
if not records:
print("No records match the filters.", file=sys.stderr)
sys.exit(1)
# Sort
sort_fn = SORT_KEYS.get(args.sort_by, SORT_KEYS["combo"])
records.sort(key=sort_fn)
console = Console(width=max(120, Console().width))
# Timestamp from first record
timestamp = records[0].get("timestamp_utc", "unknown")
console.print(f"\n[dim]Benchmark run: {timestamp}[/dim]")
console.print(f"[dim]Records: {len(records)}[/dim]\n")
# Detail table
console.print(build_table(records))
# Summary table
if not args.no_summary:
console.print()
console.print(build_summary_table(records))
console.print()
if __name__ == "__main__":
main()