tests/test_factoid_verification.py

Rama main · réplica de lectura del repositorio original.

"""
Tests for `_verify_report_factoids` and `_annotate_unverified_factoids`.

Acceptance:
  - factoids are extracted from the report
  - verified against positive extracts
  - refuted factoids (verified='false') are appended to the report
  - 'uncertain' is NOT marked inline (too many false positives)
  - without extracts: all factoids stay uncertain, no crash
  - on LLM failure: graceful, the report stays unchanged
"""

import unittest
from unittest.mock import MagicMock

from tests._helpers import MockLLM


def _make_orchestrator_stub(mode: str = "web_research"):
    from src.pipeline.orchestrator import ResearchOrchestrator
    stub = MagicMock(spec=ResearchOrchestrator)
    stub._mode = mode
    stub._classifier_calls = []
    stub.llm = MagicMock()
    return stub


def _patch_adapter_with(llm: MockLLM):
    from src.llm import classifier_adapter

    class _Pass:
        def __init__(self, _dual): pass
        async def complete(self, messages, max_tokens=None):
            return await llm.complete(messages, max_tokens=max_tokens)

    original = classifier_adapter.HarvestModelAdapter
    classifier_adapter.HarvestModelAdapter = _Pass
    return original


def _restore_adapter(original):
    from src.llm import classifier_adapter
    classifier_adapter.HarvestModelAdapter = original


class TestAnnotateUnverifiedFactoids(unittest.TestCase):
    """Test the static annotation logic in isolation."""

    def test_no_unverified_no_change(self):
        from src.pipeline.orchestrator import _annotate_unverified_factoids
        report = "## Bericht\n\nInhalt"
        verifications = [
            {"factoid": "x", "type": "claim", "verified": "true",
             "supporting_extract_id": "E1", "supporting_quote": "x",
             "confidence": 0.9, "fallback_used": False},
        ]
        out = _annotate_unverified_factoids(report, verifications)
        self.assertEqual(out, report)

    def test_only_uncertain_no_block(self):
        """'uncertain' is not marked in the report."""
        from src.pipeline.orchestrator import _annotate_unverified_factoids
        report = "## Bericht"
        verifications = [
            {"factoid": "x", "type": "claim", "verified": "uncertain",
             "supporting_extract_id": None, "supporting_quote": "",
             "confidence": 0.4, "fallback_used": True},
        ]
        out = _annotate_unverified_factoids(report, verifications)
        self.assertEqual(out, report)

    def test_low_conf_unverified_appended_to_report(self):
        """Low-confidence contradictions are listed in a discreet block at the
        end of the report (high-confidence cases go to
        ReportRevisionNode, not here)."""
        from src.pipeline.orchestrator import _annotate_unverified_factoids
        report = "## Bericht\n\nDie DGX kostet 500k USD und kam 2024."
        verifications = [
            # low confidence → goes into the notice block
            {"factoid": "DGX kostet 500k USD", "type": "numeric_spec",
             "verified": "false", "supporting_extract_id": "E1",
             "supporting_quote": "Listenpreis: $415k",
             "confidence": 0.65, "fallback_used": False},
            # verified → does not go there
            {"factoid": "kam 2024", "type": "date",
             "verified": "true", "supporting_extract_id": "E2",
             "supporting_quote": "released October 2024",
             "confidence": 0.95, "fallback_used": False},
        ]
        out = _annotate_unverified_factoids(report, verifications)
        # start of the report kept
        self.assertTrue(out.startswith(report))
        # notice block appended
        self.assertIn("Notes on source consistency", out)
        # only THE refuted factoid in it (true is not listed)
        self.assertIn("DGX kostet 500k USD", out.split("---")[-1])
        self.assertIn("Listenpreis: $415k", out)
        # the confidence is shown
        self.assertIn("0.65", out)

    def test_high_conf_unverified_NOT_in_report(self):
        """High-confidence contradictions (≥0.9) are NOT handled by
        _annotate_unverified_factoids — they go to ReportRevisionNode
        for a semantic correction."""
        from src.pipeline.orchestrator import _annotate_unverified_factoids
        report = "## Bericht\n\nDie DGX kostet 500k USD."
        verifications = [
            {"factoid": "DGX kostet 500k USD", "type": "numeric_spec",
             "verified": "false", "supporting_extract_id": "E1",
             "supporting_quote": "Listenpreis: $415k",
             "confidence": 0.95,  # high confidence!
             "fallback_used": False},
        ]
        out = _annotate_unverified_factoids(report, verifications)
        # the report stays completely unchanged — no annotation, no block
        self.assertEqual(out, report,
                         "high-confidence cases must no longer "
                         "be appended — they go to "
                         "ReportRevisionNode.")


if __name__ == "__main__":
    unittest.main()