butler/eval_integration/sync_policy.py

Rama main · réplica de lectura del repositorio original.

"""Sink expectations and weak-consistency rules for ``butler eval sync``."""

from __future__ import annotations

from typing import Any

# required=True → missing sink yields warning; langfuse never required by default
SINK_EXPECTATIONS: dict[str, dict[str, bool]] = {
    "tcr": {"audit": True, "junit": True},
    "agent_weekly": {"audit": True},
    "capability": {"audit": True},
    "regression": {"audit": True},
    "wechat_corpus": {"audit": True},
    "memory_mb": {"audit": True},
    "b9_oracle": {"audit": True},
    "deepeval_agent": {"audit": True, "deepeval": True},
    "ragas_memory": {"audit": True, "ragas": True},
}


def _ok_from_row(row: dict[str, Any] | None) -> bool | None:
    if row is None:
        return None
    if "ok" in row:
        return bool(row["ok"])
    metrics = row.get("metrics")
    if isinstance(metrics, dict) and "ok" in metrics:
        return bool(metrics["ok"])
    return None


def evaluate_sync(suite_id: str, sink_rows: dict[str, dict[str, Any] | None]) -> dict[str, Any]:
    """Return sync report with ``ok``, ``warnings``, per-sink detail."""
    expectations = SINK_EXPECTATIONS.get(suite_id, {"audit": True})
    warnings: list[str] = []
    sinks_out: dict[str, Any] = {}

    for backend_id, row in sink_rows.items():
        present = row is not None
        ok_val = _ok_from_row(row)
        sinks_out[backend_id] = {
            "present": present,
            "ok": ok_val,
            "ts": row.get("ts") if row else None,
        }

    for backend_id, required in expectations.items():
        row = sink_rows.get(backend_id)
        if required and row is None:
            warnings.append(f"missing required sink: {backend_id}")

    ok_values = [v["ok"] for v in sinks_out.values() if v.get("present") and v.get("ok") is not None]
    if len(ok_values) >= 2 and len(set(ok_values)) > 1:
        warnings.append("sink ok status mismatch across backends")

    audit_ok = sinks_out.get("audit", {}).get("ok")
    junit_ok = sinks_out.get("junit", {}).get("ok")
    if audit_ok is not None and junit_ok is not None and audit_ok != junit_ok:
        warnings.append("audit vs junit ok mismatch")

    return {
        "suite_id": suite_id,
        "ok": len(warnings) == 0,
        "warnings": warnings,
        "sinks": sinks_out,
        "expectations": expectations,
    }