tests/test_integration_product_query.py

Rama main · réplica de lectura del repositorio original.

"""
Integration test: a product-comparison query end to end through the
classifier + filter stack.

The scenario:
  - query: "Vergleich DGX B300 vs B200 — Preis-Leistungs-Verhältnis"
  - a capitalisation heuristic would extract 'Preis' / 'Leistungs' /
    'Verhältnis' as "person components"
  - the PersonHallucinationFilter would then discard all 528 extracts from
    78 sources
  - result: an empty report despite plenty of data

This test runs the pipeline:
  1. classify_query_anchor with the MockLLM, which returns a realistic
     classifier result
  2. PersonHallucinationFilter.applies_to with the returned anchor
  3. run_filter_pipeline on 528 simulated extracts

The protection lies here:
  - the `query_anchor` classifier with the confidence threshold 0.7 in is_person()
  - PersonHallucinationFilter.applies_to checks is_person()
  - if the classifier correctly says "no person" → filter inactive
  - if it is unsure (confidence < 0.7) → inactive as well

Expectation: filter inactive → all extracts survive → the report would not
be empty.

This complements the unit tests with an integration view close to what
happens in the orchestrator (without booting the orchestrator).
"""

import unittest

from src.pipeline.classifiers.query_anchor import classify_query_anchor
from src.pipeline.filters.base import run_filter_pipeline
from src.pipeline.filters.person_match import PersonHallucinationFilter
from tests._helpers import MockLLM, async_test


class _Ctx:
    def __init__(self, anchor):
        self.query_anchor = anchor


class TestProductQueryIntegration(unittest.TestCase):

    @async_test
    async def test_dgx_full_pipeline_no_extracts_lost(self):
        """End to end: product query → 528 extracts survive the filter pipeline.

        Runs through the WHOLE classifier + filter layer, the way the
        orchestrator calls it.
        """
        # ── Step 1: classify_query_anchor ──
        # realistic LLM result for the product query
        anchor_llm = MockLLM().queue_json({
            "anchor_type": "none",
            "target": "",
            "confidence": 0.94,
            "reasoning": (
                "Vergleichsanfrage zu Hardware-Spezifikationen. Die "
                "Query nennt zwei Produkte (DGX B300, DGX B200) und ein "
                "Bewertungs-Kriterium (Preis-Leistungs-Verhältnis). "
                "Keine Person und keine spezifische Organisation."
            ),
        })

        anchor, classifier_call = await classify_query_anchor(
            query="Vergleich DGX B300 vs B200 — Preis-Leistungs-Verhältnis",
            plan_summary=(
                "Strukturierter Vergleich der NVIDIA DGX-Modelle B300 "
                "und B200 hinsichtlich Performance, Stromaufnahme und "
                "Preispunkt."
            ),
            llm=anchor_llm,
        )

        # the classifier recognised correctly: no person
        self.assertEqual(anchor.type, "none")
        self.assertFalse(anchor.is_person())

        # ── Step 2: filter activation via is_person() ──
        # The only protection: anchor.is_person() is False, so the filter is
        # not activated. The confidence threshold 0.7 in is_person() is the
        # second line of defence — even if the classifier says
        # type="person" but confidence < 0.7, the filter does not run.
        person_filter_active = anchor.is_person()
        self.assertFalse(person_filter_active)

        # ── Step 3: filter pipeline ──
        # 528 extracts from 26 sources — realistic figures
        sources_by_url = {
            f"https://nvidia.com/dgx-{i}":
                f"DGX B300 vs B200 specifications: ... (Quelle {i})"
            for i in range(26)
        }
        extracts = []
        for src_idx in range(26):
            for ex_idx in range(528 // 26 + 1):
                extracts.append({
                    "source_url": f"https://nvidia.com/dgx-{src_idx}",
                    "fact": f"Spec-Detail {src_idx}-{ex_idx}",
                    "question_id": "F1",
                })
        extracts = extracts[:528]
        self.assertEqual(len(extracts), 528)

        ctx = _Ctx(anchor=anchor if person_filter_active else None)
        person_filter = PersonHallucinationFilter(
            source_content_lookup=sources_by_url.get,
        )

        kept, stats_map = await run_filter_pipeline(
            items=extracts, ctx=ctx, filters=[person_filter],
        )

        # ── Step 4: verification ──
        # CORE acceptance: all 528 extracts are kept
        self.assertEqual(len(kept), 528,
                         "product query: extracts lost!")
        self.assertFalse(stats_map["person_hallucination"].activated)
        self.assertEqual(stats_map["person_hallucination"].rejected, 0)

    @async_test
    async def test_low_confidence_hallucination_blocked(self):
        """Defence in depth: with low confidence the filter does NOT run.

        If the unsure classifier wrongly says "person" (confidence < 0.7),
        is_person() does not activate — so the filter does not run.
        """
        # the LLM returns "person" with low confidence
        bad_llm = MockLLM().queue_json({
            "anchor_type": "person",
            "target": "Preis Leistungs Verhältnis",
            "confidence": 0.55,  # below the threshold 0.7
            "reasoning": "Unsichere Klassifikation, vermutlich Fehler.",
        })

        anchor, _ = await classify_query_anchor(
            query="Vergleich DGX B300 vs B200 — Preis-Leistungs-Verhältnis",
            plan_summary="",
            llm=bad_llm,
        )

        # With confidence < 0.7 the classifier forces type → "none"
        # (safe default in classify_query_anchor itself).
        # is_person() is thus False in any case.
        self.assertEqual(anchor.type, "none")
        self.assertFalse(
            anchor.is_person(),
            "the confidence threshold 0.7 does not protect — extracts at risk!",
        )


if __name__ == "__main__":
    unittest.main()