evals/cases.json
Rama main · réplica de lectura del repositorio original.
{
"_about": {
"what": "Test set for the Job Match Radar scoring rubric (scoring-prompt.md). Each case is a real mis-score documented in the Scoring Calibration log (the project's private human-review record).",
"fields": {
"id": "stable case id",
"title/company/location": "real listing identity from the calibration log",
"description": "MINIMAL FIXTURE: a short stand-in JD body that embeds the exact trigger phrase quoted in the calibration log / scoring-prompt.md. The original full JDs are not archived; these fixtures exist only so a LIVE judge (--judge ollama) has body text to parse. The 'replay' judge does not use them.",
"gate": "the pre-flight gate (or rule) that SHOULD fire",
"type": "hard = pre-flight hard cap/auto-reject; soft = precision/soft-cap",
"expected_cap": "the binding (lowest) score the rubric should produce for this case",
"historical_score": "the score the model ACTUALLY gave at the time, per the calibration log (the documented failure)",
"provenance": "post-split = scored under the current Claude rubric (>=2026-04-25); pre-split = Gemini-era, NOT valid evidence about the current rubric, kept for the rule it documents",
"rule_status": "implemented = the gate is written into scoring-prompt.md; recommended = surfaced in calibration but not yet coded (expected to still fail = a known gap, not a regression)",
"source": "calibration-log date the case was documented"
},
"pass_rule": "A case PASSES when produced_score <= expected_cap (the gate fired). It FAILS when produced_score > expected_cap (the gate was skipped)."
},
"cases": [
{
"id": "company-a-tech-pm",
"title": "Senior Technical Product Manager",
"company": "Company A",
"location": "Berlin",
"description": "Senior Technical PM on our agentic-AI platform. Fluency in English & German is required. 5-8+ years of technical project management experience.",
"gate": "Q2 non-English fluency (German required)",
"type": "hard",
"expected_cap": 2,
"historical_score": 10,
"provenance": "post-split",
"rule_status": "implemented",
"source": "2026-05-09",
"note": "Also trips Q4 specialty seniority floor (5-8yr technical PM -> cap 4). Binding cap is Q2 = 2. Track B Core + AI-native + Watchlist must NOT stack past the hard ceiling."
},
{
"id": "company-b-organized-play",
"title": "Organized Play Manager",
"company": "Company B",
"location": "EU remote",
"description": "Manage organized play across EMEA. Fluency in Italian and/or Portuguese is required, plus regional gaming knowledge. 9-month fixed-term contract.",
"gate": "Q2 non-English fluency (Italian/Portuguese required)",
"type": "hard",
"expected_cap": 2,
"historical_score": 6,
"provenance": "post-split",
"rule_status": "implemented",
"source": "2026-05-09",
"note": "Drove the 2026-05-09 generalization of the language gate beyond German. Also trips the <12-month contract soft cap (6); binding cap is Q2 = 2."
},
{
"id": "company-i-community-mgr",
"title": "Community Manager",
"company": "Company I",
"location": "Berlin",
"description": "Own our Berlin community. Salary band 33,000-41,000 EUR/year. Core community management role.",
"gate": "Q5 posted comp below the comp floor",
"type": "hard",
"expected_cap": 2,
"historical_score": 6,
"provenance": "post-split",
"rule_status": "implemented",
"source": "2026-05-31",
"note": "Core CM title would otherwise score 8; posted band's upper bound (41k) is below the comp hard floor -> cap 2. Drove the addition of pre-flight Q5."
},
{
"id": "company-d-community-iberia",
"title": "Community Engagement Lead, Spain and Portugal",
"company": "Company D",
"location": "EU remote",
"description": "Lead community engagement for the Spain and Portugal market. Drive local-market community programs across Iberia.",
"gate": "Territory-named market title outside DACH/EU -> AVOID cap 4",
"type": "hard",
"expected_cap": 4,
"historical_score": 7,
"provenance": "post-split",
"rule_status": "implemented",
"source": "2026-06-05",
"note": "Scored 7 as Track A core Community Lead; a title naming a foreign market territory is market-localization, not a core community-lead role -> cap 4."
},
{
"id": "company-c-ai-innovation",
"title": "AI & Innovation Lead, Operations",
"company": "Company C",
"location": "Amsterdam",
"description": "Lead AI & innovation in Operations. We are office-first and do not offer remote-only roles. Based in our Amsterdam office.",
"gate": "Remote-policy parse: office-first + non-Berlin office -> cap 3",
"type": "hard",
"expected_cap": 3,
"historical_score": 7,
"provenance": "post-split",
"rule_status": "implemented",
"source": "2026-06-05",
"note": "Ideal AI+Ops axis scored 7; explicit no-remote-only language at a non-Berlin office is the real disqualifier -> cap 3."
},
{
"id": "n8n-senior-pm-enterprise",
"title": "Senior Product Manager (Enterprise)",
"company": "n8n",
"location": "Berlin / EU remote",
"description": "Senior PM for our Enterprise product at n8n, the AI-native workflow-automation platform.",
"gate": "Track B Senior PM at AI-dev-tooling -> seniority soft-cap at 8",
"type": "soft",
"expected_cap": 8,
"historical_score": 9,
"provenance": "post-split",
"rule_status": "implemented",
"source": "2026-06-17",
"note": "Right role surfaced; rubric over-claims near-perfect fit. Senior PM is a real seniority stretch -> cap 8, not 9. Domain modifier (+2) must not push past 8."
},
{
"id": "company-e-pm-paris-hybrid",
"title": "Product Manager II, AI & Data Security",
"company": "Company E",
"location": "Paris (hybrid)",
"description": "PM II on AI & Data Security. This is a hybrid role based in our Paris office (3 days/week on-site).",
"gate": "Hybrid at a non-Berlin EU location -> cap 3 (not EU-remote pass)",
"type": "hard",
"expected_cap": 3,
"historical_score": 6,
"provenance": "post-split",
"rule_status": "recommended",
"source": "2026-06-19",
"note": "Recommended 2026-06-19: hybrid non-Berlin is functionally relocation, not EU-remote. The remote-policy parse may partly cover 'hybrid (X days)'; flagged as a known gap until confirmed in the prompt."
},
{
"id": "company-f-senior-manager",
"title": "Senior Manager, Partner Field Operations",
"company": "Company F",
"location": "EU remote",
"description": "Senior Manager owning Partner Field Operations at Company F (5,000+ employees). Director-equivalent enterprise leadership role.",
"gate": "Enterprise title-inflation: >1000-employee 'Senior Manager' in a business-ops function -> seniority-floor cap",
"type": "soft",
"expected_cap": 4,
"historical_score": 6,
"provenance": "post-split",
"rule_status": "recommended",
"source": "2026-06-19",
"note": "Rejected as too senior. No implemented gate detects enterprise title-inflation yet -> known gap, expected to fail against the current prompt."
},
{
"id": "company-g-german-jd",
"title": "Community Manager",
"company": "Company G",
"location": "Berlin",
"description": "Wir suchen einen Community Manager fuer unser Berliner Team. Du betreust unsere Social-Media-Kanaele und entwickelst Content-Strategien fuer unsere Kunden.",
"gate": "Q1 JD body written in a non-English language (German)",
"type": "hard",
"expected_cap": 2,
"historical_score": 7,
"provenance": "pre-split",
"rule_status": "implemented",
"source": "2026-04-22",
"note": "Pre-2026-04-25 = Gemini-scored, NOT evidence about the current rubric. Kept for the Q1 rule it documents; the 7 is Gemini output."
},
{
"id": "company-h-german-fluency",
"title": "Social Media Manager",
"company": "Company H",
"location": "Berlin",
"description": "English-language JD. Requirements: fluent German (Muttersprache or C2) is required for this role.",
"gate": "Q2 non-English fluency (German required)",
"type": "hard",
"expected_cap": 2,
"historical_score": 8,
"provenance": "pre-split",
"rule_status": "implemented",
"source": "2026-04-22",
"note": "Pre-split Gemini-era. English body but explicit German fluency requirement -> Q2. Kept for the rule; the 8 is Gemini output."
},
{
"id": "nonsenior-smm",
"title": "Social Media Manager",
"company": "(representative)",
"location": "Berlin",
"description": "Social Media Manager (m/w/d). Own the content calendar across Instagram and TikTok.",
"gate": "Q3 'Social Media Manager' without a seniority prefix -> cap 6",
"type": "hard",
"expected_cap": 6,
"historical_score": 8,
"provenance": "pre-split",
"rule_status": "implemented",
"source": "2026-04-22",
"note": "Pre-split Gemini-era. Non-Senior SMM over-scored at 7-8; correct floor is 6. Validated post-split 2026-05-01 (Razer/Unframe accepted as 6)."
}
]
}