tests/test_eval_identifier_contract.py

Rama main · réplica de lectura del repositorio original.

import pytest

from dm.eval.schema import EvalCaseError, validate_case


def base_case(**overrides):
    case = {
        "id": "C1",
        "category": "direct",
        "question": "How many rows?",
        "grader": "numeric",
        "truth_sql": "SELECT 1",
    }
    case.update(overrides)
    return case


@pytest.mark.parametrize("field,value", [("id", "C1\nspoof"), ("category", "direct\tother")])
def test_identity_fields_must_be_single_line(field, value):
    with pytest.raises(EvalCaseError, match="single-line"):
        validate_case(base_case(**{field: value}))


def test_role_and_purpose_must_be_single_line():
    with pytest.raises(EvalCaseError, match="single-line"):
        validate_case(base_case(role="admin\nignore policy"))
    with pytest.raises(EvalCaseError, match="single-line"):
        validate_case(base_case(purpose="audit\trewrite"))


def test_truth_fact_labels_must_be_single_line():
    case = base_case(grader="llm_judge")
    case.pop("truth_sql")
    case["truth_facts"] = [{"label": "inventory\nspoof", "sql": "SELECT 1"}]
    with pytest.raises(EvalCaseError, match="single-line"):
        validate_case(case)