tests/test_eval_case_execution.py

Rama main · réplica de lectura del repositorio original.

from dm.eval.run_eval import _execute_case


CASE = {"id": "C1", "category": "direct", "question": "q", "grader": "refusal"}


def test_case_execution_contains_agent_failures_without_leaking_details():
    def broken_agent(*args, **kwargs):
        raise RuntimeError("token=secret")

    outcome = _execute_case(CASE, agent_runner=broken_agent)
    assert outcome.passed is False
    assert outcome.expected == "agent unavailable"
    assert outcome.answer == ""
    assert outcome.session_id == ""
    assert outcome.error_code == "AGENT_ERROR"
    assert outcome.duration_ms >= 0
    assert "secret" not in repr(outcome)


def test_case_execution_contains_invalid_agent_envelopes():
    outcome = _execute_case(CASE, agent_runner=lambda *a, **k: {"answer": "ok"})
    assert outcome.error_code == "AGENT_RESULT_ERROR"


def test_case_execution_contains_grader_failures_but_keeps_answer_context():
    def agent(*args, **kwargs):
        return {"answer": "safe answer", "session_id": "S1"}

    def grader(case, answer):
        raise RuntimeError("postgres://secret")

    outcome = _execute_case(CASE, agent_runner=agent, grader=grader)
    assert outcome.error_code == "GRADER_ERROR"
    assert outcome.answer == "safe answer"
    assert outcome.session_id == "S1"