role-matcher.mjs

Rama main · réplica de lectura del repositorio original.

/**
 * role-matcher.mjs - Shared fuzzy role-title matching for tracker scripts.
 *
 * Both `merge-tracker.mjs` and `dedup-tracker.mjs` decide whether two
 * same-company tracker rows describe the same opening. Keeping this logic in
 * one module prevents the merge path from preserving rows that the later dedup
 * path would silently delete with weaker matching rules.
 */

export const SENIORITY_TOKENS = new Set([
  'junior', 'mid', 'middle', 'senior', 'staff', 'principal', 'lead', 'head',
  'chief', 'associate', 'intern', 'entry'
]);

// Seniority tokens that place a requisition BELOW the bare baseline title.
// "senior"/"principal" modify an ambiguous baseline and are routinely added or
// dropped when the same opening is re-posted, so seeing one on a single side is
// not evidence of a different job. These words are different in kind: they mean
// the req sits at a lower level than the unqualified title, with its own scope,
// comp band, and req ID. "Associate X" and a bare "X" at one company are two
// real openings, so a lone sub-baseline qualifier is a disagreement (#2009).
export const SUB_BASELINE_SENIORITY = new Set([
  'associate', 'junior', 'entry', 'intern',
]);

// A stated level — "Insurance Specialist II", "Registered Nurse 3", "Data
// Engineer Level 2" — is the employer's own statement that this requisition is
// not its sibling: a different pay band, scope and req number under one base
// title. The tokenizer cannot see it. `w.length > 3` drops every roman numeral
// up to VIII and every single digit, so "Insurance Specialist I" and
// "Insurance Specialist II" tokenize identically and score a perfect Jaccard
// ratio; merge-tracker then folds the second requisition into the first, keeps
// the first's title, and the second stops existing. Measured on a real
// 316-posting corpus: 15 titles (1 in 20) carry a level, and none of those
// postings stated a requisition number the Notes-column guard (#1524) could
// have caught instead.
//
// Roman and arabic forms fold onto one number so "Nurse II" and "Nurse 2" are
// the same statement. Bounded at six: beyond that the roman forms start to
// collide with real words and abbreviations, and no title needs them.
export const LEVEL_TOKENS = new Map([
  ['i', 1], ['ii', 2], ['iii', 3], ['iv', 4], ['v', 5], ['vi', 6],
  ['1', 1], ['2', 2], ['3', 3], ['4', 4], ['5', 5], ['6', 6],
]);

// A level token standing as its own word, optionally introduced by "level" /
// "grade" / "tier". The lookahead is what keeps "5G", "3.0" and "Web3" out —
// a digit glued to a letter or a point is not a level — and the boundary
// classes are what let "II (Remote)", "II, Days" and "II/III" read as levels
// despite the suffix. A slash bounds both sides, so a posting hiring at either
// of two levels states both.
const LEVEL_RE = /(?:^|[\s,(\/\-\u2013\u2014])(?:level|lvl|grade|tier)?\s*(i{1,3}|iv|vi?|[1-6])(?=$|[\s,)\/\-\u2013\u2014])/giu;

// Tokens that almost every role shares must not count as strong matching
// signal. This set covers seniority, work mode, contract shape, locations, and
// other words that frequently appear in titles without identifying the opening.
export const ROLE_STOPWORDS = new Set([
  // seniority / level
  'junior', 'mid', 'middle', 'senior', 'staff', 'principal', 'lead', 'head',
  'chief', 'associate', 'intern', 'entry', 'level',
  // contract / mode
  'remote', 'hybrid', 'onsite', 'contract', 'contractor', 'freelance',
  'fulltime', 'parttime', 'permanent', 'temporary', 'intern', 'internship',
  // generic job words
  'role', 'position', 'opportunity', 'team', 'based',
  // reposting/tracking annotations — meta noise, never part of the job itself
  'repost', 'reposted', 'relisted',
  // very common locations
  'bangalore', 'bengaluru', 'mumbai', 'delhi', 'hyderabad', 'pune', 'chennai',
  'london', 'berlin', 'paris', 'madrid', 'barcelona', 'amsterdam', 'dublin',
  'york', 'francisco', 'seattle', 'boston', 'austin', 'chicago', 'toronto',
  'tokyo', 'singapore', 'sydney', 'melbourne', 'lisbon', 'warsaw',
  // regions / countries
  'europe', 'emea', 'apac', 'latam', 'americas', 'india', 'spain', 'germany',
  'france', 'italy', 'canada', 'brazil', 'mexico', 'japan',
  // prepositions leaking through the length filter
  'with', 'from', 'into', 'over', 'this', 'that',
]);

// "Member of Technical Staff" (MTS) is a boilerplate level-prefix used by
// several companies for senior IC titles, not a content signal — e.g.
// "Member of Technical Staff, Connector Platform" vs "...Backend Platform"
// are different openings whose suffix should decide the match, not the
// prefix. Stripped as a literal phrase (not a blanket stopword on "member"/
// "technical") so those words keep their normal discriminating role in
// unrelated titles such as "Technical Program Manager" or "Team Member".
const MTS_PREFIX = /\bmember\s+of\s+technical\s+staff\b/g;

// Short specialty acronyms that are discriminating despite their length.
// Broad two-letter buckets such as AI/ML are intentionally excluded because
// they appear across many unrelated roles.
export const SHORT_SPECIALTY = new Set([
  'api', 'sre', 'sdk', 'cli', 'gpu', 'cpu',
  'ios', 'qa', 'ux', 'ui', 'ar', 'vr',
  'ocr', 'crm', 'erp',
]);

// Generic role-level descriptors. Two titles whose only overlap is in this set
// are not the same opening; they are merely written at the same role altitude.
export const BASELINE_TOKENS = new Set([
  'software', 'engineer', 'developer', 'manager', 'architect',
  'analyst', 'designer', 'consultant', 'specialist',
  'platform', 'systems', 'services',
  'backend', 'frontend', 'full', 'stack', 'fullstack',
  // 'product' alone cannot identify an opening: "Product Manager - Marketplace"
  // and "Product Manager - AI" must stay separate applications ("ai" is dropped
  // by the tokenizer, leaving only [product, manager] to match on).
  'product',
]);

/**
 * Lowercase a title and fold accented Latin letters onto their ASCII base.
 *
 * Every rule below matches against ASCII vocabulary, so an accent used to act
 * as a word separator rather than a letter: "Sênior" became ["s", "nior"],
 * leaving a phantom "nior" token that no stopword list covers. Folding first
 * keeps the accented spelling in the same vocabulary as the plain one.
 *
 * Only accents that NFD decomposes are folded. Letters with no canonical
 * decomposition (ø, ł, ß, đ) still reach the ASCII filter unchanged, exactly
 * as before.
 *
 * `\p{Mn}` (nonspacing mark), not `\p{Diacritic}`: the latter also matches
 * standalone characters such as "·", "^" and "`", which are separators in a
 * title. Deleting those would glue neighbouring words into one token.
 *
 * @param {unknown} value - Raw title, possibly not a string.
 * @returns {string} Lowercased, accent-folded text.
 */
function normalizeTitle(value) {
  const text = typeof value === 'string' ? value : String(value ?? '');
  return text
    .toLowerCase()
    .normalize('NFD')
    // Only marks sitting on an ASCII Latin base. Stripping EVERY \p{Mn} also
    // reached marks that carry meaning in other scripts: Devanagari matras
    // (कंपनी and कपनी became one token), Cyrillic breve (Йогурт -> иогурт) and
    // Japanese dakuten (バックエンド -> ハックエント, voiced kana folded onto
    // unvoiced). Latin accent-folding — the reason this function exists, per
    // the Sênior case — is unchanged, because those marks always follow an
    // ASCII base once the title is lowercased.
    .replace(/(?<=[a-z])\p{Mn}/gu, '')
    .normalize('NFC');
}

/**
 * Convert a role title into content tokens used for fuzzy matching.
 *
 * The tokenizer keeps long descriptive words and a narrow set of short
 * specialty acronyms, while dropping common stopwords. Baseline tokens are kept
 * in the result so they can contribute to the similarity ratio, but they cannot
 * be the only reason two titles match.
 *
 * @param {string} role - Raw role title from the tracker or TSV addition.
 * @returns {string[]} Ordered role-title tokens.
 */
export function roleTokens(role) {
  return normalizeTitle(role)
    // Replace with a generic baseline token, not empty space: a bare "Member
    // of Technical Staff" (no suffix) or a one-word-suffix MTS title (e.g.
    // "...Staff, Backend") would otherwise tokenize to 0 or 1 words, and
    // roleFuzzyMatch requires 2+ overlapping tokens — so even an exact
    // (punctuation-varying) repost of a short MTS title would fail to match
    // itself. "engineer" is already a BASELINE_TOKENS entry, so it pads the
    // token count without ever being the sole reason two titles match.
    .replace(MTS_PREFIX, ' engineer ')
    // Collapse slashed short acronyms into one token BEFORE punctuation is
    // stripped: "(CI/CD)" would otherwise become "ci cd" and both halves get
    // dropped by the length filter, making the qualifier invisible to the
    // matcher. A sibling req whose only qualifier is such an acronym (e.g.
    // "Senior SWE, Infrastructure (CI/CD)" vs "Senior SWE, Infrastructure")
    // tokenized identically to the bare title and got merged over it (#2165).
    // "cicd" / "tcpip" / "uiux" survive as content tokens.
    .replace(/\b([a-z0-9]{1,3})\/([a-z0-9]{1,3})\b/g, '$1$2')
    .replace(/[^\p{L}\p{M}\p{N}\s]/gu, ' ')
    .split(/\s+/)
    .filter(w => (w.length > 3 || SHORT_SPECIALTY.has(w)) && !ROLE_STOPWORDS.has(w));
}

/**
 * The levels a title states, as numbers, so roman and arabic forms compare equal.
 *
 * @param {string} title - Raw role title.
 * @returns {Set<number>} Empty when the title states no level.
 */
export function extractLevels(title) {
  const levels = new Set();
  for (const m of normalizeTitle(title).matchAll(LEVEL_RE)) {
    const level = LEVEL_TOKENS.get(m[1]);
    if (level !== undefined) levels.add(level);
  }
  return levels;
}

function extractSeniorities(title) {
  return new Set(
    normalizeTitle(title)
      .replace(/[^\p{L}\p{M}\p{N}\s]/gu, ' ')
      .split(/\s+/)
      .filter(w => SENIORITY_TOKENS.has(w))
  );
}

// Short acronyms are invisible to the tokenizer (it drops every word of three
// letters or fewer except SHORT_SPECIALTY), so "Enterprise Account Executive -
// DNB" and "Enterprise Account Executive - FSI" tokenize identically. Read them
// from the raw title instead, and only when written in capitals inside a title
// that is not itself all capitals: the capitals are what mark a segment, region
// or platform code ("FSI", "GCP", "DE") rather than a short common word. The
// noise list holds capitalised words that carry no requisition identity.
const SHORT_ACRONYM_NOISE = new Set([
  'ALL', 'AND', 'FOR', 'THE', 'NEW', 'FTE', 'PTE', 'LLC', 'INC', 'LTD',
  'SR', 'JR', 'II', 'III', 'IV', 'VI',
]);

export function extractShortAcronyms(title) {
  const text = String(title ?? '');
  // Any letter that is not upper case (lower case, or a script without case
  // such as Japanese or Arabic) means the title is not written in capitals,
  // whatever the script around the code.
  const letters = text.replace(/[^\p{L}]/gu, '');
  if (letters === '' || letters.replace(/\p{Lu}/gu, '') === '') return new Set();
  const found = text.match(/(?<![A-Za-z0-9])[A-Z]{2,3}(?![A-Za-z0-9])/g) ?? [];
  return new Set(found.filter(w => !SHORT_ACRONYM_NOISE.has(w)));
}

/**
 * Decide whether two role titles are likely the same opening.
 *
 * Matching requires at least two shared tokens, at least one shared token that
 * is not merely baseline job vocabulary, and a Jaccard overlap of 0.6 or more.
 * This preserves genuine reposts while keeping sibling roles such as
 * "Full Stack Engineer, Foundation" and "Full Stack Engineer, Guarded Releases"
 * as separate applications.
 *
 * @param {string} a - First role title.
 * @param {string} b - Second role title.
 * @returns {boolean} True when the titles are similar enough to deduplicate.
 */
export function roleFuzzyMatch(a, b) {
  // Identical titles (case/whitespace-insensitive) always match, even a bare
  // title that tokenizes to 0 words (e.g. a company posting just "Member of
  // Technical Staff" with no suffix) — tokenization can never be the reason
  // an exact repost fails to dedupe.
  const textA = String(a ?? '').trim().toLowerCase();
  const textB = String(b ?? '').trim().toLowerCase();
  if (textA && textA === textB) return true;

  const senA = extractSeniorities(a);
  const senB = extractSeniorities(b);

  // If both titles explicitly specify seniority, they MUST overlap in at least one seniority token.
  // e.g. "Senior" vs "Principal" -> differ, return false.
  // e.g. "Senior" vs "Senior Staff" -> overlap, proceed to Jaccard check.
  // e.g. "Engineer" vs "Senior Engineer" -> one lacks seniority, proceed to Jaccard check.
  if (senA.size > 0 && senB.size > 0) {
    const hasOverlap = [...senA].some(s => senB.has(s));
    if (!hasOverlap) return false;
  } else if (senA.size > 0 || senB.size > 0) {
    // Exactly one side states a seniority. The tokenizer drops seniority words
    // as stopwords, so "Associate Product Manager, Team" and "Product Manager,
    // Team" otherwise tokenize identically and score a perfect Jaccard ratio —
    // silently collapsing two real requisitions. A sub-baseline qualifier on
    // the lone side is a level disagreement, not a loose rewrite (#2009).
    const lone = senA.size > 0 ? senA : senB;
    if ([...lone].some(s => SUB_BASELINE_SENIORITY.has(s))) return false;
  }

  // A level stated on BOTH sides must agree, on the same terms as seniority
  // above: "Specialist I" vs "Specialist II" is two openings, while a level on
  // one side alone is a loose rewrite of one opening ("Engineer" vs "Senior
  // Engineer"), not evidence of two. Compared as sets, so a title that states
  // two levels ("Engineer II/III") still matches either of them.
  const lvlA = extractLevels(a);
  const lvlB = extractLevels(b);
  if (lvlA.size > 0 && lvlB.size > 0 && ![...lvlA].some(l => lvlB.has(l))) return false;

  const wordsA = [...new Set(roleTokens(a))];
  const wordsB = [...new Set(roleTokens(b))];
  if (wordsA.length === 0 || wordsB.length === 0) return false;

  const setB = new Set(wordsB);
  const overlap = wordsA.filter(w => setB.has(w));
  if (overlap.length < 2) return false;

  // Require at least one non-baseline token in the overlap. Roles that share
  // only generic descriptors like [software, engineer] or [full, stack,
  // engineer] are not the same opening.
  const discriminating = overlap.filter(w => !BASELINE_TOKENS.has(w));
  if (discriminating.length === 0) return false;

  // A generic base title carries no suffix of its own to counterbalance a
  // specialized sibling's extra word, so the shared tokens alone can cross the
  // Jaccard threshold even though that extra word is exactly the signal that
  // these are two different, separately-postable openings (e.g. "Senior
  // Analytics Engineer" vs "Senior Analytics Engineer, People Analytics").
  // When one title's token set is a strict subset of the other's, and the
  // superset's extra tokens contain a non-baseline word, treat that word as a
  // specialization marker and keep the titles distinct.
  const smaller = wordsA.length <= wordsB.length ? wordsA : wordsB;
  const larger = wordsA.length <= wordsB.length ? wordsB : wordsA;
  const isProperSubset = larger.length > smaller.length && overlap.length === smaller.length;
  if (isProperSubset) {
    const smallerSet = new Set(smaller);
    const extraTokens = larger.filter(w => !smallerSet.has(w));
    if (extraTokens.some(w => !BASELINE_TOKENS.has(w))) return false;
  }

  // A content word on EACH side that the other title lacks is two
  // disagreements, not a loose rewrite of one opening. The subset guard above
  // only sees a word on one side, and the Jaccard cut-off below cannot see
  // this at all: three shared words plus one unique word per side score
  // exactly 0.6. "Customer Success Manager, Enterprise" vs "Customer Success
  // Manager, Commercial" (and "Machine Learning Engineer, Vision" vs "...,
  // Speech") therefore matched, and merge-tracker folded the second
  // requisition into the first, overwriting its score, date and report link.
  //
  // #4058 added this guard for the one case where a level sat on one side
  // only ("Front Desk Assistant (Summer Housing)" vs "Administrative
  // Assistant II (Housing Front Desk)"); the vocabulary disagreement is the
  // signal, the level was only how that report found it. Mirrors the subset
  // guard: baseline-only differences do not split, and neither does a plural
  // ("Solution Architecture" vs "Solutions Architecture").
  const setA = new Set(wordsA);
  const isPluralOf = (x, y) => x === `${y}s` || y === `${x}s`;
  const uniqueA = wordsA.filter(w => !setB.has(w) && !BASELINE_TOKENS.has(w));
  const uniqueB = wordsB.filter(w => !setA.has(w) && !BASELINE_TOKENS.has(w));
  const unmatchedA = uniqueA.filter(x => !uniqueB.some(y => isPluralOf(x, y)));
  const unmatchedB = uniqueB.filter(y => !uniqueA.some(x => isPluralOf(x, y)));
  if (unmatchedA.length > 0 && unmatchedB.length > 0) return false;

  // A short acronym on EACH side that the other title lacks is the same
  // disagreement as a unique word on each side; see extractShortAcronyms.
  const acrA = extractShortAcronyms(a);
  const acrB = extractShortAcronyms(b);
  if ([...acrA].some(w => !acrB.has(w)) && [...acrB].some(w => !acrA.has(w))) return false;

  // Use a true set-based Jaccard ratio. Dividing by the smaller title inflates
  // matches for roles that share a long generic prefix but differ in specialty.
  const union = new Set([...wordsA, ...wordsB]).size;
  return overlap.length / union >= 0.6;
}