scan.mjs
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* scan.mjs — Zero-token portal scanner with a plugin-based provider layer.
*
* Providers live in providers/*.mjs and are loaded at startup. Each provider
* exports a default object with:
* - id: string — matched against `provider:` in portals.yml
* - detect(entry): {url}|null — optional auto-detection from careers_url
* - fetch(entry, ctx): [{title,url,company,location}] — required
*
* Files prefixed with _ are shared helpers (e.g. _http.mjs) and are never
* loaded as providers. Adding a new HTTP/API source = drop a *.mjs into
* providers/. Local executable parsers use `providers/local-parser.mjs` when
* `parser.command` + `parser.script` are set in portals.yml.
*
* A tracked_companies entry can set `provider:` explicitly to bypass
* URL-based auto-detection. The `transport:` field is reserved for future
* transports — Phase A only ships the http transport.
*
* Zero Claude API tokens — pure HTTP + JSON.
*
* Usage:
* node scan.mjs # scan all enabled companies
* node scan.mjs --dry-run # preview without writing files
* node scan.mjs --company Cohere # scan a single company
* node scan.mjs --verify # API-first liveness check; use Playwright when inconclusive
* node scan.mjs --verify --headed-fallback # retry anti-bot-blocked URLs in a headed browser (needs a display)
* node scan.mjs --verify --throttle # jittered ~5-10s gap between browser checks
* node scan.mjs --verify --throttle=8000 # custom base gap in ms (waits base..2*base)
* node scan.mjs --include-blacklisted # let data/blacklist.md matches through (annotated)
* node scan.mjs --since 7 # postings from the last 7 days
* node scan.mjs --posted-after 2026-07-01 # absolute lower bound on posting date
* node scan.mjs --posted-before 2026-08-01 # absolute upper bound on posting date
* node scan.mjs --rediscover-404 # re-verify tracked URLs that 404/410 (rides on --verify)
* node scan.mjs --quiet # suppress the manifesto footer
* node scan.mjs --help # print this usage block and exit
*/
import { buildWorkplacePolicy, isCountryKeyword } from './providers/_country.mjs';
import {
closeSync,
existsSync,
fchmodSync,
fsyncSync,
lstatSync,
mkdirSync,
openSync,
readFileSync,
realpathSync,
renameSync,
statSync,
unlinkSync,
writeFileSync,
} from 'fs';
import { randomUUID } from 'crypto';
import { fileURLToPath } from 'url';
import path from 'path';
import { loadYaml } from './lib/yaml-document.mjs';
import { makeHttpCtx } from './providers/_http.mjs';
import { formatHttpCacheWindow, httpCacheStats, resolveHttpCacheTtlMs, withHttpCache } from './providers/_http-cache.mjs';
import { buildTrustValidator } from './providers/_trust-validator.mjs';
import { loadProviders, resolveProvider } from './providers/_registry.mjs';
import { openProviderIdCache, PROVIDER_ID_CACHE_DIR } from './provider-id-cache.mjs';
import { mergeProviderPlugins } from './plugins/_engine.mjs';
import { classifyFetchError } from './verify-portals.mjs';
import { portalTopLevelTypos } from './validate-portals.mjs';
import { fingerprintText, findCrossListings } from './fingerprint-core.mjs';
import { computeListingFingerprint } from './listing-fingerprint.mjs';
import { resolveColumns, parseTrackerRow, normalizeTextKey, extractReqNumber, REQ_NUMBER_RE } from './tracker-parse.mjs';
import { workdayDedupKey, stripWorkdayRepostSuffix, isWorkdayJobUrl } from './providers/workday.mjs';
import { normalizeCompany, appendLedgerRows, companiesMatchIgnoringCorporateForm, LEGAL_SUFFIXES } from './tracker-utils.mjs';
import { normalizeCompanyName } from './invite-match.mjs';
import { withPipelineLock } from './pipeline-lock.mjs';
import { compileKeyword, compilePositiveKeyword, compileContentKeyword, buildTitleFilter, foldTitle, foldContentText, compileContentTraits } from './title-keywords.mjs';
import { flagValue, hasFlag, validateFlags } from './lib/cli-flags.mjs';
import { withPortalHealthLock } from './portal-health-lock.mjs';
import { localToday } from './lib/local-today.mjs';
import { printScanSummaryHeader } from './lib/scan-summary-marker.mjs';
import { isMainModule } from './lib/is-main-module.mjs';
import { OBSERVATIONAL_SCAN_HISTORY_STATUSES, SCAN_HISTORY_COLUMNS, parseScanHistoryLine, parsedScanHistory, scanHistoryDataLines } from './lib/scan-history-columns.mjs';
import { closesAtIso } from './lib/posting-close.mjs';
import { escapeFormulaCell } from './lib/tsv-formula-escape.mjs';
import { sectionKind } from './lib/pipeline-sections.mjs';
import { canonicalJobKey, normalizeUrl, LANGUAGE_PARAMS, languageSubtag, promoteKnownFragmentIdentity } from './url-key.mjs';
import { getCareerOpsRoot, getProfilePath, hasProfileOverride } from './path-resolver.mjs';
import { checkLivenessViaApi } from './liveness-api.mjs';
import { robotsAllows } from './robots-check.mjs';
import { resolveExperienceFilter, applyExperienceFilter } from './jd-years.mjs';
const CODE_ROOT = path.dirname(fileURLToPath(import.meta.url));
export const DATA_ROOT = getCareerOpsRoot();
// Preserve the actual shell environment before dotenv adds data-root values to
// process.env. The browser resolver uses this snapshot to distinguish an
// explicit shell path (which keeps cwd-relative behavior) from a path read from
// DATA_ROOT/.env (which must resolve relative to that file's directory). The
// helper also retains Windows' case-insensitive environment lookup when the
// result becomes an ordinary object.
export const BROWSER_PROCESS_ENV = snapshotBrowserProcessEnv();
try {
const { config } = await import('dotenv');
// quiet: dotenv's startup banner goes to stdout, which --json reserves for a
// single JSON object (#1906). Secrets are user-layer data, so a split
// checkout reads them beside the configured data root rather than from the
// caller's cwd (which may be the code checkout or an unrelated directory).
config({ path: path.join(DATA_ROOT, '.env'), quiet: true });
} catch {
// dotenv is optional — fall back to process.env if not installed
}
const parseYaml = loadYaml;
// ── Config ──────────────────────────────────────────────────────────
import { resolveTrackerPath } from './path-resolver.mjs';
import { browserLaunchOptions, snapshotBrowserProcessEnv } from './lib/browser-launch.mjs';
import { applyEntryResponseCap } from './lib/entry-response-cap.mjs';
import { applyEntryTimeout } from './lib/entry-timeout.mjs';
export { applyEntryResponseCap };
import { boardTruncation, describeTruncation } from './lib/board-truncation.mjs';
export const PORTALS_PATH = process.env.CAREER_OPS_PORTALS || path.join(DATA_ROOT, 'portals.yml');
// An explicit override wins; otherwise the root fixed above, since the .env just
// loaded may name another CAREER_OPS_ROOT that getProfilePath() would follow.
const PROFILE_PATH = hasProfileOverride() ? getProfilePath() : getProfilePath(DATA_ROOT);
// Overridable for the same reason the two inputs above are (#2271). A second
// search lane - a bridge/income track, a career-change track, a partner sharing
// the checkout - already gets its own portals.yml and profile, but without these
// two it still writes into the one inbox and the one dedup history. That is not
// just untidy: scan-history.tsv IS the dedup source, so a posting surfaced in
// lane A is silently counted as a duplicate in lane B and never shown at all.
// Exported because scan-ats-full.mjs, scan-interamt.mjs and scan-hn.mjs write to
// these same files through appendToPipeline/appendToScanHistory. They each used
// to carry their own bare-relative copy, so a sibling could check existence and
// create data/pipeline.md in the cwd, then append the actual results to the
// anchored one (#3510). One resolution, imported, cannot drift.
export const SCAN_HISTORY_PATH = process.env.CAREER_OPS_SCAN_HISTORY || path.join(DATA_ROOT, 'data/scan-history.tsv');
export const PIPELINE_PATH = process.env.CAREER_OPS_PIPELINE || path.join(DATA_ROOT, 'data/pipeline.md');
// The closing hint names the lane's inbox when CAREER_OPS_PIPELINE is set.
// `/career-ops pipeline` alone reads data/pipeline.md unless the agent's own
// environment carries the same variable, so a lane run that only says "run
// pipeline" sends the user to the other lane's inbox (#6379).
export function pipelineRunHint(lane = process.env.CAREER_OPS_PIPELINE) {
const file = lane?.trim();
return file
? `→ Run /career-ops pipeline ${file} to evaluate new offers (this lane's inbox, not data/pipeline.md).`
: '→ Run /career-ops pipeline to evaluate new offers.';
}
export const APPLICATIONS_PATH = resolveTrackerPath(DATA_ROOT);
const PROVIDERS_DIR = path.resolve(CODE_ROOT, 'providers');
// No directory creation at import time (#3159). Every writer below creates its
// own parent before its first write — scan-history (appendToScanHistory),
// scan-runs (appendScanRunSummary), portal-health (appendPortalHealth) — and
// pipeline.md goes through pipeline-lock.mjs, which creates data/ for the same
// reason. applications.md and blacklist.md are read-only here. Importing this
// module must stay side-effect free: a sibling that only reads a constant used
// to leave a stray data/ in whatever cwd it ran from.
// A board response can be tens of megabytes, and a request's timer also covers
// reading the body. More requests in flight can therefore make a healthy board
// miss its timeout on a slower connection. CAREER_OPS_SCAN_CONCURRENCY caps the
// whole company scan, and CAREER_OPS_SCAN_HOST_CONCURRENCY caps one hostname
// inside it. Verify has its own fixed limit. It hammers one URL at a time and is
// not the morning-scan wait.
export const DEFAULT_CONCURRENCY = 10;
/**
* A concurrency limit from an env value. Anything but a positive integer falls
* back to the default: a typo should not stop a scan, and parallelFetch never
* starts a task under a limit of 0.
*
* @param {unknown} raw - Raw env value.
* @param {number} [fallback=DEFAULT_CONCURRENCY] - Value used when `raw` is unusable.
* @returns {number}
*/
export function resolveConcurrency(raw, fallback = DEFAULT_CONCURRENCY) {
if (raw === undefined || raw === null || String(raw).trim() === '') return fallback;
const parsed = Number(raw);
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
}
export const SCAN_CONCURRENCY = resolveConcurrency(process.env.CAREER_OPS_SCAN_CONCURRENCY);
export const HOST_CONCURRENCY = resolveConcurrency(process.env.CAREER_OPS_SCAN_HOST_CONCURRENCY);
export const VERIFY_CONCURRENCY = 10;
export function companyFetchHost(company) {
try {
const declared = company?._provider?.fetchHost?.(company);
if (typeof declared === 'string' && declared.trim()) return declared.trim().toLowerCase();
const detected = company?._provider?.detect?.(company);
const raw = (detected && typeof detected.url === 'string' && detected.url)
|| (typeof company?.api === 'string' && company.api)
|| (typeof company?.careers_url === 'string' && company.careers_url)
|| '';
if (!raw) return '';
return new URL(raw).hostname;
} catch {
return '';
}
}
const PROGRESS_EVERY = 25;
export function isIgnorableDirectoryFsyncError(err, platform = process.platform) {
return ['EINVAL', 'ENOTSUP', 'ENOSYS'].includes(err?.code)
|| (platform === 'win32' && ['EACCES', 'EPERM'].includes(err?.code));
}
export function atomicWriteFile(filePath, text) {
const fileStat = lstatSync(filePath, { throwIfNoEntry: false });
const targetPath = fileStat?.isSymbolicLink() ? realpathSync(filePath) : filePath;
const tempPath = `${targetPath}.tmp-${process.pid}-${randomUUID()}`;
let fd = null;
let directoryFd = null;
try {
const existingMode = existsSync(targetPath) ? statSync(targetPath).mode & 0o7777 : null;
fd = openSync(tempPath, 'wx', 0o600);
if (existingMode !== null) fchmodSync(fd, existingMode);
writeFileSync(fd, text, 'utf-8');
fsyncSync(fd);
closeSync(fd);
fd = null;
renameSync(tempPath, targetPath);
try {
directoryFd = openSync(path.dirname(targetPath), 'r');
fsyncSync(directoryFd);
} catch (err) {
if (!isIgnorableDirectoryFsyncError(err)) throw err;
} finally {
if (directoryFd !== null) closeSync(directoryFd);
directoryFd = null;
}
} catch (err) {
if (fd !== null) closeSync(fd);
if (directoryFd !== null) closeSync(directoryFd);
try { unlinkSync(tempPath); } catch { /* best effort */ }
throw err;
}
}
export function emitJsonReceipt(receipt, exitCode) {
process.stdout.write(`${JSON.stringify(receipt)}\n`);
process.exitCode = exitCode;
}
// Provider loading + routing live in providers/_registry.mjs so the portal
// health check (verify-portals.mjs) can reuse the exact same layer without
// importing this module.
// ── Title filter ────────────────────────────────────────────────────
// How a keyword matches text lives in title-keywords.mjs, because
// openrouter-runner.mjs filters titles too and cannot import this file (scan.mjs
// creates data/ at import time). It called a second, hand-kept copy of this
// logic until the two drifted; there is now one implementation and this file
// re-exports it, so existing importers — scan-ats-full.mjs and test-all.mjs's
// sections 11b and 44 among them — keep resolving it from here.
// compileContentKeyword shares the `word:`/`stem:` prefix machinery but skips
// the title filter's short-acronym auto-anchor (#3274).
export { compileKeyword, compilePositiveKeyword, compileContentKeyword, buildTitleFilter, foldContentText, compileContentTraits };
// ── Declared-field whitelists (#3438) ──────────────────────────────
// A title whitelist cannot express "this posting is in an occupation I want"
// on a board that publishes an occupation code, because one title maps to
// several occupations and one occupation to unboundedly many titles.
//
// A target therefore declares WHICH FIELD its whitelist reads. Default is
// `title`, so a portals.yml that says nothing behaves byte-for-byte as before.
//
// field_filters: # top level, optional
// noc:
// positive: ["stem:22", "stem:13"]
// job_boards:
// - name: Job Bank — help desk
// filter_on: noc # string, or array (AND), default ["title"]
//
// No new matching semantics anywhere: each block is compiled by the same
// buildTitleFilter() as title_filter, so `word:`/`stem:`/substring and the
// word-boundary rules from #3103 stay identical across every field.
//
// `title` is deliberately not a key in field_filters: it routes to the
// existing top-level config.title_filter, the same compiled object as before.
/**
* @param {unknown} value - a target's `filter_on`: string, array, or absent.
* @returns {string[]} unique field names to gate on, in declared order,
* defaulting to ["title"]. Unique, because presence is counted once per
* field per job: a repeated field would double every count in the warning.
*/
export function normalizeFilterOn(value) {
const list = (Array.isArray(value) ? value : [value])
.filter(f => typeof f === 'string')
.map(f => f.trim())
.filter(Boolean);
return list.length > 0 ? [...new Set(list)] : ['title'];
}
// Key for the per-(target, field) presence counters. Keyed by the target's
// index in `targets`, not its name: two enabled targets may share a name
// (validate-portals only warns), and their counters must stay apart.
export function declaredFieldKey(targetId, field) {
return JSON.stringify([targetId, field]);
}
/**
* Whether a posting lacks a declared field. The gate cannot judge such a
* posting, so it passes and is counted — dropping it would be the same silent
* loss this feature exists to end, with the sign flipped.
* @param {unknown} value - `job[field]`
* @returns {boolean}
*/
export function isFieldAbsent(value) {
return value === undefined || value === null || String(value).trim() === '';
}
// Own scalar properties only: `filter_on: constructor` must not read
// Object.prototype, and an object value has no safe String() to judge.
export function declaredFieldValue(job, field) {
const value = Object.hasOwn(job, field) ? job[field] : undefined;
return ['string', 'number', 'boolean'].includes(typeof value) ? value : undefined;
}
// ── Title filter overrides (per-company broadened title net) ───────
// Optional. `title_filter_overrides` in portals.yml lets specific companies
// (matched by an explicit slug list — the company/tenant slug the scanner
// already derives from the job-board-aggregator dataset entry, e.g. the
// Workday tenant "uwaterloo") opt into a WIDER positive keyword net than the
// global `title_filter.positive`, without loosening the global filter for
// every other company in the sweep. Distinct from (and composes cleanly
// with) `content_filter.by_title_keyword`, which scopes a stricter
// description-level check to specific title keywords — this scopes a
// broader title-level net to specific companies.
//
// Shape:
// title_filter_overrides:
// - companies: ["uwaterloo", "ubc", "fanshawec"]
// positive_extra:
// - "Administrator"
// - "Coordinator"
//
// v1 keeps matching simple and explicit: a literal (case-insensitive) slug
// list, no fuzzy company-type inference, no domain heuristics.
export function buildTitleFilterOverrides(overrides) {
const map = new Map();
if (!Array.isArray(overrides)) return map;
for (const entry of overrides) {
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) continue;
const companies = Array.isArray(entry.companies) ? entry.companies : [];
const extraRaw = Array.isArray(entry.positive_extra) ? entry.positive_extra : [];
// compilePositiveKeyword (not compileKeyword) so positive_extra supports
// the same AND-groups / `word:`/`stem:` prefixes as title_filter.positive —
// it's an additive positive list, so it should behave like one.
const matchers = extraRaw
.filter(k => typeof k === 'string')
.map(k => k.trim().toLowerCase())
.filter(k => k.length > 0)
.map(compilePositiveKeyword);
if (matchers.length === 0) continue; // nothing to add — skip the entry entirely
for (const slug of companies) {
if (typeof slug !== 'string' || !slug.trim()) continue;
const key = slug.trim().toLowerCase();
map.set(key, (map.get(key) || []).concat(matchers));
}
}
return map;
}
// Wraps buildTitleFilter() with per-company overrides from
// buildTitleFilterOverrides(). Returns (title, companySlug) => boolean:
// - if the global title_filter already matches, pass (companySlug unused)
// - else, if companySlug has override entries, pass when any of its
// positive_extra keywords match AND no global negative keyword matches
// - a company with no override entry behaves EXACTLY like the plain
// buildTitleFilter(titleFilter) — the mechanism is a strict no-op for
// everyone not explicitly listed.
export function buildTitleFilterWithOverrides(titleFilter, overridesMap) {
const base = buildTitleFilter(titleFilter);
const overrides = overridesMap instanceof Map ? overridesMap : new Map();
if (overrides.size === 0) return (title) => base(title);
const normalize = (arr) => (Array.isArray(arr) ? arr : [])
.filter(k => typeof k === 'string')
.map(k => k.trim().toLowerCase())
.filter(k => k.length > 0)
.map(compileKeyword);
const negative = normalize(titleFilter?.negative);
return (title, companySlug) => {
if (base(title)) return true;
const extra = overrides.get(String(companySlug ?? '').trim().toLowerCase());
if (!extra || extra.length === 0) return false;
const lower = String(title ?? '').toLowerCase();
if (negative.some(m => m(lower))) return false;
return extra.some(m => m(lower));
};
}
// Compiled-matcher cache for matchedTitleKeywords(), keyed by the
// `title_filter.positive` array reference. The scan loop calls this once per
// job with the same titleFilter config object, so caching avoids recompiling
// every keyword (compileKeyword()) on every single job.
const compiledPositiveCache = new WeakMap();
function compiledPositiveMatchers(positiveList) {
if (compiledPositiveCache.has(positiveList)) return compiledPositiveCache.get(positiveList);
const compiled = positiveList
.filter(k => typeof k === 'string' && k.trim().length > 0)
.map(k => ({ raw: k, match: compilePositiveKeyword(foldTitle(k).trim()) }));
compiledPositiveCache.set(positiveList, compiled);
return compiled;
}
// Returns the raw (as-written in portals.yml) `title_filter.positive` keywords
// that matched a given title — used to scope `content_filter.by_title_keyword`
// overrides to only the categories that opted into a stricter content check.
// "Raw" includes a `word:` prefix if the entry carries one, so a
// `by_title_keyword` key must be written exactly as the positive entry is.
export function matchedTitleKeywords(title, titleFilter) {
const raw = Array.isArray(titleFilter?.positive) ? titleFilter.positive : [];
const lower = foldTitle(title || '');
return compiledPositiveMatchers(raw)
.filter(({ match }) => match(lower))
.map(({ raw: kw }) => kw);
}
// ── Location filter ─────────────────────────────────────────────────
// Optional. If `location_filter` is absent from portals.yml, all locations pass.
// Semantics (case-insensitive substring, in this order):
// - Empty / whitespace-only / non-string location AND no URL hint → pass
// (don't penalize missing or malformed provider data), UNLESS
// `location_filter.strict: true` and a restricting tier (`allow`, `block`,
// or `block_hard`) is configured — then reject, because a location-
// restricted sweep against a provider that does not return locations
// (iCIMS) otherwise silently inverts into "everything, plus matches from
// everywhere else" (#3276). Opt-in and default-unchanged.
// - `block_hard` matches → reject (the only tier `always_allow` cannot
// override; for country-level terms that are never a false rejection)
// - `always_allow` matches → pass (takes precedence over `block` — lets a
// multi-location string like "Remote, Belgium or France" through because
// the home region is an option, even though "france" is blocked). When
// always_allow names the US as a country (united states / usa / u.s. /
// u.s.a.), USPS state names and 2-letter codes are additional always_allow
// matches, so block: [Dublin] does not drop "Dublin, OH".
// - `block` matches → reject
// - `allow` empty → pass (already cleared block)
// - `allow` non-empty → must match a keyword, an exact worldwide label when
// non-strict and both allow-lists contain only countries (#6413), OR the
// TITLE carries an explicit remote marker (see titleSignalsRemote below),
// unless `remote_title_fallback: false` disables that final title rescue.
// Normalize a keyword list from portals.yml: tolerates a bare string
// (wrapped to a 1-item array), null/undefined (→ []), and non-string
// entries (filtered out). Survivors are lowercased, trimmed, and any
// resulting empty strings are dropped — an empty keyword would otherwise
// match every location via String.includes(''), silently bypassing the
// other tiers.
function normalizeKeywordList(value) {
if (value == null) return [];
const arr = Array.isArray(value) ? value : [value];
return arr
.filter(k => typeof k === 'string')
.map(k => k.toLowerCase().trim())
.filter(Boolean);
}
// Compile a location keyword into a word-boundary matcher.
//
// Plain String.includes() is wrong for location keywords because country and
// city names are prefixes of unrelated US place names. The motivating bug:
// blocking "india" also rejected "Indian Head, MD", "Indiana", and
// "Indianapolis" — real US locations, silently dropped from every scan.
// Likewise "china" would swallow "Chinatown" and "uk -" would swallow "Truck -".
//
// Lookarounds rather than \b so keywords that begin or end with punctuation
// (", IND", "UK -") still anchor correctly — \b is defined relative to word
// characters and behaves surprisingly at a punctuation edge.
// Letters, combining marks and numbers form words in every script; ASCII-only
// boundaries let "al," match inside "Montréal," (including decomposed accents).
// Note: distinct from compileKeyword() above, which serves the *title* filter and
// only boundary-anchors 2-3 letter acronyms. Location keywords need boundaries on
// every keyword, so they get their own compiler rather than changing title-matching
// behaviour. Returns a predicate, mirroring compileKeyword()'s shape.
// Edges are tested one code point at a time against a fully anchored class:
// V8 in Node 26 returns false for /[\p{L}]$/u against an astral letter at
// end-of-string (#4478), so "𐐀" lost its trailing boundary and matched "𐐀x".
const LOCATION_WORD_CP = /^[\p{L}\p{M}\p{N}]$/u;
function compileLocationKeyword(keyword) {
const escaped = keyword.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const codePoints = [...keyword];
const startsWord = LOCATION_WORD_CP.test(codePoints[0] ?? '');
const endsWord = LOCATION_WORD_CP.test(codePoints.at(-1) ?? '');
const prefix = startsWord ? '(?<![\\p{L}\\p{M}\\p{N}])' : '';
const suffix = endsWord ? '(?![\\p{L}\\p{M}\\p{N}])' : '';
const re = new RegExp(`${prefix}${escaped}${suffix}`, 'u');
return (lower) => re.test(lower);
}
function compileLocationKeywordList(value) {
// An entry written as two uppercase letters ("IN", "DE") is an ISO country
// code. Lowercased like every other keyword it would match the plain word
// "in" in "Anywhere in the World" (#6605), so it is matched against the
// original-case location instead. Lowercase and mixed-case entries keep the
// case-insensitive behavior. The lowercased URL hint carries no case, so a
// code never matches it.
// Each entry is judged on its own spelling, so ['IN', 'in'] compiles one
// case-sensitive code and one case-insensitive word.
const raw = value == null ? [] : (Array.isArray(value) ? value : [value]);
return raw
.filter(k => typeof k === 'string' && k.trim())
.map((entry) => {
const trimmed = entry.trim();
if (!/^[A-Z]{2}$/.test(trimmed)) return compileLocationKeyword(trimmed.toLowerCase());
const re = new RegExp(`(?<![\\p{L}\\p{M}\\p{N}])${trimmed}(?![\\p{L}\\p{M}\\p{N}])`, 'u');
return (lower, original) => typeof original === 'string' && re.test(original);
});
}
// Frozen USPS state-name + abbreviation table. Not a world gazetteer: only
// consulted when always_allow already names the United States as a country,
// so EU-targeted configs (no US token) keep their previous semantics.
const US_COUNTRY_ALWAYS_ALLOW = new Set(['united states', 'usa', 'u.s.', 'u.s.a.']);
const USPS_STATES = Object.freeze([
Object.freeze(['alabama', 'al']),
Object.freeze(['alaska', 'ak']),
Object.freeze(['arizona', 'az']),
Object.freeze(['arkansas', 'ar']),
Object.freeze(['california', 'ca']),
Object.freeze(['colorado', 'co']),
Object.freeze(['connecticut', 'ct']),
Object.freeze(['delaware', 'de']),
Object.freeze(['florida', 'fl']),
Object.freeze(['georgia', 'ga']),
Object.freeze(['hawaii', 'hi']),
Object.freeze(['idaho', 'id']),
Object.freeze(['illinois', 'il']),
Object.freeze(['indiana', 'in']),
Object.freeze(['iowa', 'ia']),
Object.freeze(['kansas', 'ks']),
Object.freeze(['kentucky', 'ky']),
Object.freeze(['louisiana', 'la']),
Object.freeze(['maine', 'me']),
Object.freeze(['maryland', 'md']),
Object.freeze(['massachusetts', 'ma']),
Object.freeze(['michigan', 'mi']),
Object.freeze(['minnesota', 'mn']),
Object.freeze(['mississippi', 'ms']),
Object.freeze(['missouri', 'mo']),
Object.freeze(['montana', 'mt']),
Object.freeze(['nebraska', 'ne']),
Object.freeze(['nevada', 'nv']),
Object.freeze(['new hampshire', 'nh']),
Object.freeze(['new jersey', 'nj']),
Object.freeze(['new mexico', 'nm']),
Object.freeze(['new york', 'ny']),
Object.freeze(['north carolina', 'nc']),
Object.freeze(['north dakota', 'nd']),
Object.freeze(['ohio', 'oh']),
Object.freeze(['oklahoma', 'ok']),
Object.freeze(['oregon', 'or']),
Object.freeze(['pennsylvania', 'pa']),
Object.freeze(['rhode island', 'ri']),
Object.freeze(['south carolina', 'sc']),
Object.freeze(['south dakota', 'sd']),
Object.freeze(['tennessee', 'tn']),
Object.freeze(['texas', 'tx']),
Object.freeze(['utah', 'ut']),
Object.freeze(['vermont', 'vt']),
Object.freeze(['virginia', 'va']),
Object.freeze(['washington', 'wa']),
Object.freeze(['west virginia', 'wv']),
Object.freeze(['wisconsin', 'wi']),
Object.freeze(['wyoming', 'wy']),
]);
// 2-letter codes: comma-state (", OH" / ",OH, USA") or a trailing token
// ("Dublin OH", Workday URL hint "dublin oh"). Not a generic word-boundary —
// English "in"/"or"/"me" in "Remote, Belgium or France" must not impersonate
// Indiana/Oregon/Maine. State *names* still use compileLocationKeyword.
// Unicode letters and marks are part of the token: "Montréal" is not "AL".
function compileUsStateAbbrev(abbr) {
const escaped = abbr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const re = new RegExp(`(?:,\\s*${escaped}(?![\\p{L}\\p{M}\\p{N}])|(?:^|[^\\p{L}\\p{M}\\p{N}])${escaped}[^\\p{L}\\p{M}\\p{N}]*$)`, 'u');
return (lower) => placesWithoutCountryCode(lower).some((place) => re.test(place));
}
// "City, Region, CC" ends in an ISO country code, and several of those are
// also USPS codes: "Bengaluru, Karnataka, IN" read as Indiana and "Toronto,
// ON, CA" as California, so always_allow beat the user's own block entry.
// A two-letter last part after two or more parts is the country slot. So is
// the last part of "Austin TX, US" or "Perth WA, AU", where a state code ends
// the city part. "Santa Fe, NM" and "Dublin, OH" keep their state. "LA, CA"
// is two state codes, so the last part is the state, not a country. So is
// a state code written twice, as in "Brooklyn, NY, NY". A place
// with a country slot other than US has no US state at all, so "Perth WA, AU"
// does not read as Washington. Providers join several places with "; "
// (Lever) or " · " (Jobvite), and each place gets its own check, so the TX in
// "Austin TX · Toronto, ON, CA" counts even though it is not at the end.
const USPS_CODES = new Set(USPS_STATES.map(([, abbr]) => abbr));
function placesWithoutCountryCode(lower) {
return lower.split(/[;·]/).flatMap((place) => {
const parts = place.split(',');
const code = parts.at(-1).trim();
if (parts.length < 2 || !/^\p{L}{2}$/u.test(code)) return [place];
if (parts.length === 2) {
const cityTail = /(?:^|[^\p{L}\p{M}\p{N}])(\p{L}{2})\s*$/u.exec(parts[0]);
if (!cityTail || !USPS_CODES.has(cityTail[1])) return [place];
if (USPS_CODES.has(code)) return [place];
} else if (USPS_CODES.has(code) && parts.at(-2).trim() === code) {
return [place];
}
return code === 'us' ? [parts.slice(0, -1).join(',')] : [];
});
}
// A state name that is also a country cannot beat the user's own `block`:
// "Tbilisi, Georgia" is the country, and reading "Georgia" as always_allow let
// it through a filter that blocked "Tbilisi". Such a name counts at the `allow`
// tier instead, so "Atlanta, Georgia" still passes and a blocked Georgian city
// does not. The 2-letter code is unambiguous and stays in always_allow, so
// "Dublin, GA" still beats a blocked "Dublin".
const US_STATE_NAMES_ALSO_COUNTRIES = new Set(['georgia']);
const US_STATE_ALWAYS_ALLOW_MATCHERS = USPS_STATES.flatMap(([name, abbr]) => [
...(US_STATE_NAMES_ALSO_COUNTRIES.has(name) ? [] : [compileLocationKeyword(name)]),
compileUsStateAbbrev(abbr),
]);
const US_STATE_ALLOW_MATCHERS = [...US_STATE_NAMES_ALSO_COUNTRIES].map(compileLocationKeyword);
// Some providers report a rolled-up display string ("5 Locations", "2 Locations")
// while the canonical URL still names the real primary location. Workday is the
// common case: .../job/Hyderabad-Telangana-India/Network-Engineer_R-65193-1 shows
// up as "5 Locations", so no `block` keyword can ever match the location field.
// Recover that signal by reading the path segment right after `/job/`.
//
// Deliberately narrow: only the post-`/job/` segment is inspected, never the whole
// URL. Scanning the full URL would match company slugs and ATS subdomains by
// accident (a "china" or "india" substring inside an unrelated path). Providers
// without the Workday hostname convention yield no hint and keep their previous
// behaviour exactly, even if their own routes also contain `/job/{id}`.
export function locationHintFromUrl(url) {
if (typeof url !== 'string' || url.trim() === '') return '';
let parsed;
try {
parsed = new URL(url);
} catch {
return '';
}
if (!parsed.hostname.toLowerCase().endsWith('.myworkdayjobs.com')) return '';
const segments = parsed.pathname.split('/').filter(Boolean);
const jobIdx = segments.lastIndexOf('job');
// Workday also emits /job/{Title}_{ReqId}; a location needs a title after it.
if (jobIdx === -1 || segments.length - jobIdx - 1 < 2) return '';
let segment = segments[jobIdx + 1];
try {
segment = decodeURIComponent(segment);
} catch {
// Malformed percent-encoding — fall back to the raw segment.
}
// "Hyderabad-Telangana-India" → "hyderabad telangana india" so multi-word
// block keywords like "united arab emirates" can still match.
return segment.replace(/[-_+]+/g, ' ').replace(/\s+/g, ' ').trim().toLowerCase();
}
// Some ATSs report the hiring office as the location even when the role is
// remote, and state the remoteness in the TITLE instead: Radancy/TalentBrew
// tenants return bare "City, State" strings, so
// "Program Manager - Remote" -> location "Las Vegas, Nevada"
// An `allow` list written in country/region terms ("united states", "remote")
// then rejects a genuinely remote US role. Live measurement on
// careers.unitedhealthgroup.com: 14 PM-family postings, 0 passed `allow`,
// 5 of them said "Remote" outright in the title.
//
// Only an unambiguous work-arrangement marker counts. A bare /remote/ test
// would admit domain compounds — "Remote Sensing Program Manager" is an
// on-site GIS role, and Esri (a tracked company) posts exactly those. So
// "remote" must be followed by end-of-string, a non-letter (")", ",", "-"),
// or " in …" as in "Remote in MO" — never by another word, which is what makes
// "remote sensing" / "remote monitoring" compounds.
export const REMOTE_TITLE_RE = /(?<![a-z])remote(?=$|\s*[^a-z\s]|\s+in\b)/;
// …and a negation before the word has to lose, which the marker regex alone
// cannot see: in "Non-Remote" / "Not Remote" the delimiter clears the lookbehind
// and the trailing position clears the lookahead, so an explicitly on-site role
// would bypass a non-empty `allow` list — the exact opposite of the intent.
// The separator class must be at least as broad as the marker's own delimiter
// lookahead, or the guard is trivially sidestepped. An ASCII-only `[\s-]*` let
// every non-ASCII dash through — "Non–Remote" (en dash), "Non‑Remote"
// (non-breaking hyphen), em dash, figure dash and minus all still read as
// remote. `[^a-z]*` matches the marker's breadth: it spans any run of
// non-letters, so no punctuation variant can slip between the negation and the
// word.
// It cannot over-reach, because it never crosses a letter: in "Nonprofit
// Program Manager - Remote" the run after "non" starts with "profit", so the
// negation cannot reach "remote". Same for "Not-for-Profit … - Remote",
// "Nordic … - Remote", "Notary … - Remote".
// A negation anywhere in the title disqualifies it. Over-rejecting here is the
// safe direction: this tier only ever *rescues* a posting, so a false negative
// restores the previous behavior while a false positive admits an on-site role.
export const REMOTE_NEGATED_RE = /\b(?:non|not|no)[^a-z]*remote/;
/** @param {unknown} title @returns {boolean} whether the title marks the role remote. */
export function titleSignalsRemote(title) {
if (typeof title !== 'string' || title.trim() === '') return false;
const lower = title.toLowerCase();
if (REMOTE_NEGATED_RE.test(lower)) return false;
return REMOTE_TITLE_RE.test(lower);
}
// Whole labels only: a substring must not promote qualified or negated
// geography ("Worldwide except Canada", "Not worldwide") to worldwide.
const WORLDWIDE_LOCATION_LABELS = new Set([
'worldwide',
'anywhere in the world',
'remote (worldwide)',
]);
// `url` and `title` are optional. Callers that omit them get the original
// location-only semantics, which is what the existing unit tests exercise.
export function buildLocationFilter(locationFilter) {
if (!locationFilter) return () => true;
const workplacePolicy = buildWorkplacePolicy(locationFilter.workplace_policy);
const alwaysAllowKeywords = normalizeKeywordList(locationFilter.always_allow);
const alwaysAllow = compileLocationKeywordList(locationFilter.always_allow);
// US-targeted configs list the country in always_allow and foreign cities
// in block. "Dublin, OH" does not contain "United States", so without this
// expansion block: [Dublin] rejects a real US job. Opt-in on the country
// token — configs with no US always_allow entry are unchanged.
const usExpansion = alwaysAllowKeywords.some(k => US_COUNTRY_ALWAYS_ALLOW.has(k));
if (usExpansion) {
alwaysAllow.push(...US_STATE_ALWAYS_ALLOW_MATCHERS);
}
const allowKeywords = normalizeKeywordList(locationFilter.allow);
const allow = compileLocationKeywordList(locationFilter.allow);
// Do not widen city-only or mixed city/country targeting. Reuse the shared
// country vocabulary rather than introducing a second country-name list.
const countryOnlyAllow = allowKeywords.length > 0
&& [...alwaysAllowKeywords, ...allowKeywords].every(isCountryKeyword);
// Only a list the user already filled: an empty `allow` means "everything not
// blocked passes", and adding a matcher to it would reject every other place.
if (usExpansion && allow.length > 0) {
allow.push(...US_STATE_ALLOW_MATCHERS);
}
const block = compileLocationKeywordList(locationFilter.block);
const blockHard = compileLocationKeywordList(locationFilter.block_hard);
// Opt-in: fail closed when there is nothing to judge on. Only meaningful when
// a restricting tier is configured — `{ strict: true }` alone restricts
// nothing and must not reject every location-less posting (#3276).
const strict = locationFilter.strict === true
&& (allow.length > 0 || block.length > 0 || blockHard.length > 0);
const filter = (location, url, title, offer) => {
if (!workplacePolicy(location, title, offer).eligible) return false;
const original = typeof location === 'string' ? location.trim() : '';
const lower = original.toLowerCase();
const hint = locationHintFromUrl(url);
// Nothing to judge on either field → pass (don't penalize missing data),
// unless the config opted into strict mode: a location-restricted sweep
// against a provider that never returns a location would otherwise let
// every out-of-region posting through (#3276).
if (lower === '' && hint === '') return !strict;
const matches = (m) => (lower !== '' && m(lower, original)) || (hint !== '' && m(hint));
// `block_hard` is the ONE tier always_allow cannot override. It exists because
// a European city name can be a whole word inside a non-European location, so
// word-boundary matching (#2087) does not catch it and always_allow's
// unconditional win silently discards the user's own block entry:
//
// "Porto Alegre, Rio Grande do Sul, Brazil" always_allow "Porto" beats block "Brazil"
// "USA - New York - Malta" always_allow "Malta" beats block "USA"
//
// Both configs already listed the country under `block`. Plain `block` cannot
// be promoted wholesale — always_allow was added in #650 precisely so a
// multi-location posting survives one blocked city ("Stockholm · London ·
// Madrid" must not die on a London entry) — so the user marks the entries
// that are country-level and therefore never a false rejection. Opt-in and
// additive: a config without `block_hard` behaves exactly as before.
if (blockHard.length > 0 && blockHard.some(matches)) return false;
// always_allow still wins over block, and may be satisfied by either field:
// a genuinely US role whose display string says "United States" is never
// rejected because of what its URL happens to contain.
if (alwaysAllow.length > 0 && alwaysAllow.some(matches)) return true;
if (block.length > 0 && block.some(matches)) return false;
if (allow.length === 0) return true;
if (allow.some(matches)) return true;
if (!strict && countryOnlyAllow && WORLDWIDE_LOCATION_LABELS.has(lower)) return true;
// Last resort only. Deliberately placed AFTER `block` so a remote title can
// never rescue a blocked location — "Program Manager - Remote" in Bengaluru
// stays rejected. This widens `allow`, never `block`.
return locationFilter.remote_title_fallback !== false && titleSignalsRemote(title);
};
// Detail readers may recheck this opt-in policy without repeating legacy
// location tiers. Arbitrary caller-provided filters have no such capability.
return Object.assign(filter, {
workplacePolicyOnly: (location, url, title, offer) => workplacePolicy(location, title, offer).eligible,
});
}
// ── Posting-age filter ──────────────────────────────────────────────
// Optional opt-in. If `max_posting_age_days` is absent (or not a positive
// integer) in portals.yml, every offer passes. An offer is skipped only when
// the provider supplied a postedAt (epoch ms) AND it is older than N days.
// Offers with no date always pass — same "don't penalize missing data"
// convention as the location filter. `now` is injectable for deterministic tests.
export function buildPostingAgeFilter(maxAgeDays, now = Date.now()) {
const max = Number(maxAgeDays);
if (!Number.isInteger(max) || max <= 0) return () => true;
const cutoff = now - max * 24 * 60 * 60 * 1000; // N days in ms, subtracted from now
return (postedAt) => {
if (typeof postedAt !== 'number' || !Number.isFinite(postedAt)) return true;
return postedAt >= cutoff;
};
}
// ── Posted-date lower bound (shared by the filter and the early-stop) ──
// --posted-after states a lower bound absolutely; --since <days> states the
// same thing relatively. They AND together with each other and with
// max_posting_age_days, so the NEWEST bound is what actually decides
// eligibility. Both consumers must agree on it: the downstream filter and the
// provider early-stop hint. Kept as one function so they cannot drift.
/**
* Parse and validate `--since <days>` from an argv slice.
*
* Shared by scan.mjs and scan-ats-full.mjs so ONE flag name cannot mean two
* different things. scan-ats-full.mjs used `Number(valueOf('--since')) || 3`,
* which silently swallowed every malformed operand: `--since abc` and
* `--since 0` both became 3 (the user believes they scanned the window they
* typed), `--since -5` produced a cutoff in the FUTURE so nothing was ever
* eligible (indistinguishable from "no new postings"), and `--since 1e400`
* became Infinity → an -Infinity cutoff, i.e. no window at all (#2498).
*
* Returns the day count, or null when the flag is absent — the DEFAULT is the
* caller's to choose (scan.mjs: no bound; scan-ats-full.mjs: 3 days), only the
* validation is shared. `error` is a ready-to-print message; callers print and
* exit rather than this throwing, so both CLIs fail the same way.
*
* @param {string[]} args - argv slice.
* @returns {{days: number|null, error: string|null}}
*/
export function parseSinceDays(args) {
// Every occurrence is collected, not just the first match of either form:
// picking one and ignoring the rest means `--since=7 --since` succeeds while
// an occurrence with no value goes unread.
const occurrences = args.filter((a) => a === '--since' || a.startsWith('--since='));
if (occurrences.length > 1) {
return { days: null, error: `--since given ${occurrences.length} times; pass it once` };
}
if (occurrences.length === 0) return { days: null, error: null };
const occ = occurrences[0];
const next = args[args.indexOf('--since') + 1];
const raw = occ.startsWith('--since=')
? occ.slice('--since='.length)
: (next != null && !next.startsWith('--') ? next : null);
const n = raw == null || raw === '' ? NaN : Number(raw);
// Number.isFinite also rejects Infinity and 1e309, which pass a bare `> 0`
// test and would yield an -Infinity cutoff (i.e. silently no window).
if (!Number.isFinite(n) || n <= 0) {
return { days: null, error: `--since expects a positive number of days, got ${raw == null || raw === '' ? '(no value)' : `"${raw}"`}` };
}
// Finite and positive is not enough: 1e300 days lands outside the ±8.64e15ms
// range a Date can represent, so the derived cutoff is an Invalid Date and
// toISOString() throws. Reject it here rather than let it surface as an
// unhandled "Invalid time value" mid-scan.
if (Number.isNaN(new Date(Date.now() - n * 86_400_000).getTime())) {
return { days: null, error: `--since ${raw} is too large to express as a date` };
}
return { days: n, error: null };
}
/**
* Collapse --posted-after and --since into a single absolute lower bound.
*
* @param {string|null} postedAfter - YYYY-MM-DD from --posted-after, or null.
* @param {number|null} sinceDays - Positive day count from --since, or null.
* @param {number} [now] - Injectable clock for tests.
* @returns {string|null} YYYY-MM-DD, the newer of the two, or null if neither.
*/
export function resolveEffectiveAfter(postedAfter, sinceDays, now = Date.now()) {
// Truncating --since to a date rather than an exact timestamp makes it
// marginally more permissive, which is the safe direction for a bound that
// also stops pagination.
// Guarded rather than assumed valid: this is exported and unit-tested, so it
// must not throw for any input. A day count large enough to push the cutoff
// outside the representable Date range yields an Invalid Date, and
// toISOString() would throw RangeError on it.
const cutoff = Number.isFinite(sinceDays) && sinceDays > 0 ? new Date(now - sinceDays * 86_400_000) : null;
const sinceIso = cutoff && !Number.isNaN(cutoff.getTime())
? cutoff.toISOString().slice(0, 10)
: null;
return [postedAfter, sinceIso].filter(Boolean).reduce((a, b) => (a > b ? a : b), null);
}
/**
* The oldest posting the filters would still accept — the early-stop floor.
*
* Stopping pagination any NEWER than this would leave eligible postings
* unfetched, which is the one thing the optimisation must never do. Returns
* null when no CLI window is active: max_posting_age_days constrains the floor
* but must not by itself switch early stopping on for configs that never asked.
*
* @param {string|null} effectiveAfter - Output of resolveEffectiveAfter.
* @param {*} maxAgeDays - config.max_posting_age_days (may be absent/invalid).
* @param {number} [now] - Injectable clock for tests.
* @returns {number|null} Epoch ms floor, or null to disable early stopping.
*/
export function resolveEarlyStopMs(effectiveAfter, maxAgeDays, now = Date.now()) {
if (!effectiveAfter) return null;
const max = Number(maxAgeDays);
const ageFloor = Number.isInteger(max) && max > 0 ? now - max * 86_400_000 : -Infinity;
return Math.max(Date.parse(`${effectiveAfter}T00:00:00Z`), ageFloor);
}
// ── Absolute posted-date filter ─────────────────────────────────────
// CLI-only (--posted-after / --posted-before), unlike the config-driven
// relative max_posting_age_days above. Both bounds optional and inclusive
// (before is treated as end-of-day). A job with no postedAt always passes —
// same "don't penalize missing data" convention as buildPostingAgeFilter.
export function buildPostedDateFilter(afterIso, beforeIso) {
const afterMs = afterIso ? Date.parse(afterIso) : NaN;
const beforeMs = beforeIso ? Date.parse(`${beforeIso}T23:59:59.999Z`) : NaN;
const hasAfter = Number.isFinite(afterMs);
const hasBefore = Number.isFinite(beforeMs);
if (!hasAfter && !hasBefore) return () => true;
return (postedAt) => {
if (typeof postedAt !== 'number' || !Number.isFinite(postedAt)) return true;
if (hasAfter && postedAt < afterMs) return false;
if (hasBefore && postedAt > beforeMs) return false;
return true;
};
}
// ── Content filter ──────────────────────────────────────────────────
// Optional. If `content_filter` is absent from portals.yml, all jobs pass.
// Filters on the job DESCRIPTION text to separate same-titled roles with
// different stacks (a "Software Engineer" listing that mentions "PHP" vs one
// that mentions "Rust"). Semantics (case-insensitive substring, in order):
// - Empty / whitespace-only / non-string description → PASS. The scanner is
// zero-token and only sees descriptions a provider already returns in its
// list payload; providers without one must never be silently dropped.
// - any `negative` keyword present → reject
// - `positive` empty → pass (already cleared negatives)
// - `positive` non-empty → at least one keyword must be present
//
// A keyword may opt in to boundary-anchored matching with a `word:` or `stem:`
// prefix (identical to `title_filter` — see title-keywords.mjs). Without a
// prefix an entry is a plain substring, so a bare negative `java` rejects every
// posting mentioning "JavaScript" and `ios` rejects "curiosity"; `word:java` /
// `stem:ios` fix that one entry while leaving the rest of the list untouched
// (#3274). The substring default is deliberate and unchanged: flipping it would
// silently narrow every configured install.
//
// `content_filter.by_title_keyword` (optional): scopes a stricter positive/
// negative pair to only the jobs whose title matched a specific
// `title_filter.positive` keyword, so e.g. an "AI Engineer" title-match can
// require the description to actually mention a concrete AI tool, without
// that requirement leaking onto unrelated categories like "Instructional
// Designer". When one or more of a job's matched title keywords has an
// override, the overrides govern (any override passing is enough); the
// global `positive`/`negative` pair is the fallback for jobs whose matched
// keyword(s) have no override entry.
//
// `min_match` (optional, the global pair and each by_title_keyword rule):
// require that many distinct positive entries, with " | " synonym groups. See
// "Trait matching" below. `rescue_title_misses` (optional, global only): let a
// posting the title filter would drop still pass on its description. See
// buildContentFilter.
//
// Provider support: `job.description` is populated only when the provider's
// list API returns the description body without a per-job request (the
// zero-token constraint). Providers that don't supply one leave it empty, and
// those jobs always pass this filter. The set shifts as providers are updated
// — check it with `grep -l 'description:' providers/*.mjs`.
// Normalize a keyword list (lowercase/trim/drop-empties) and compile each
// survivor into a matcher, so a `word:`/`stem:` prefix is honoured and a bare
// keyword keeps its substring behaviour. The `.length` checks downstream still
// read as "did the user configure any keyword here".
function compileContentKeywordList(value) {
return normalizeKeywordList(value).map(compileContentKeyword);
}
// ── Trait matching (#6020) ──────────────────────────────────────────
// `min_match` turns a positive list into a list of TRAITS: a posting passes
// when at least `min_match` distinct entries appear in its description, not
// just one. It exists because the same work hides under different titles, and
// one generic word ("demo", "customers") is a weak signal on its own while two
// or three together describe the role. Absent `min_match`, a rule keeps the
// any-one-keyword behaviour above, byte-for-byte.
//
// Only in a rule that sets `min_match`:
// - " | " (whitespace-delimited) separates alternative spellings of ONE
// trait, so "pre-sales | presales" counts once however many spellings
// appear. The separator needs the surrounding spaces, so a keyword such as
// "c|c++" stays one literal.
// - Entries are deduplicated case-insensitively, so "Demo" and "demo" are
// one trait rather than two cheap hits toward the threshold.
// - Both sides read the description folded the way title_filter folds a
// title (lowercase, accents removed, whitespace collapsed), plus the
// typographic hyphens JD text uses (U+2010 to U+2012) folded to "-" and
// soft hyphens dropped, so "customer‑facing" with a non-breaking hyphen
// still matches a configured "customer-facing".
// - Any `negative` entry present still rejects, whatever the count.
// Evaluate one description against compiled traits. `matched` is the verdict;
// the two lists say why, for tests and for any caller that wants to report it.
export function matchContentTraits(description, { positive = [], negative = [], minMatch = 1 } = {}) {
const folded = foldContentText(description);
const negativeMatches = negative.filter(t => t.match(folded)).map(t => t.original);
const matchedTraits = positive.filter(t => t.match(folded)).map(t => t.original);
return {
matched: negativeMatches.length === 0 && matchedTraits.length >= minMatch,
matchedTraits,
negativeMatches,
};
}
// Compile one positive/negative pair (the global one or a by_title_keyword
// rule). Throws on an unusable `min_match` rather than guessing: a threshold
// above the number of traits can never pass and would drop every posting.
function compileContentRule(rule, path) {
if (rule?.min_match === undefined) {
return {
traits: false,
positive: compileContentKeywordList(rule?.positive),
negative: compileContentKeywordList(rule?.negative),
};
}
const positive = compileContentTraits(rule.positive);
const negative = compileContentTraits(rule.negative);
const minMatch = rule.min_match;
if (!Number.isInteger(minMatch) || minMatch < 1 || minMatch > positive.length) {
throw new Error(positive.length === 0
? `${path}.min_match needs a non-empty ${path}.positive list`
: `${path}.min_match must be an integer from 1 to ${positive.length} (the number of distinct ${path}.positive entries)`);
}
return { traits: true, positive, negative, minMatch };
}
function contentRuleAccepts(rule, description) {
if (rule.traits) return matchContentTraits(description, rule).matched;
const lower = description.toLowerCase();
if (rule.negative.length > 0 && rule.negative.some(m => m(lower))) return false;
if (rule.positive.length === 0) return true;
return rule.positive.some(m => m(lower));
}
// `rescue_title_misses: true` (#6020) lets a posting that title_filter would
// drop still reach this check. scan.mjs consults it before booking a title
// miss; the returned filter is then called with `{ titleMissed: true }` and is
// stricter than for a posting whose title matched:
// - an empty or missing description FAILS (there is nothing to rescue on;
// the pass-on-empty rule exists so a title MATCH is never lost to a
// provider without descriptions, and does not apply to a title miss),
// - only the global positive/negative pair (with its min_match) decides:
// a title miss matched no title_filter.positive keyword, so no
// by_title_keyword rule is in scope,
// - a non-empty `positive` list is required, enforced when the filter is built.
// The flag is exposed as `filter.rescueTitleMisses` so a caller can tell
// whether to ask.
export function buildContentFilter(contentFilter) {
if (!contentFilter) return () => true;
const globalRule = compileContentRule(contentFilter, 'content_filter');
const rescue = contentFilter.rescue_title_misses;
if (rescue !== undefined && typeof rescue !== 'boolean') {
throw new Error('content_filter.rescue_title_misses must be true or false');
}
// Counted as traits, as validate-portals.mjs counts them, so a list holding
// only a bare `word:` or `stem:` is refused here too, not silently kept.
if (rescue === true && compileContentTraits(contentFilter.positive).length === 0) {
throw new Error('content_filter.rescue_title_misses needs a non-empty content_filter.positive list');
}
const byTitleKeyword = new Map();
if (contentFilter.by_title_keyword && typeof contentFilter.by_title_keyword === 'object' && !Array.isArray(contentFilter.by_title_keyword)) {
for (const [kw, rule] of Object.entries(contentFilter.by_title_keyword)) {
if (typeof kw !== 'string' || !kw.trim()) continue;
byTitleKeyword.set(kw.trim().toLowerCase(), compileContentRule(rule, `content_filter.by_title_keyword.${kw}`));
}
}
const filter = (description, matchedKeywords = [], { titleMissed = false } = {}) => {
const empty = typeof description !== 'string' || description.trim() === '';
if (titleMissed) {
if (rescue !== true || empty) return false;
return contentRuleAccepts(globalRule, description);
}
if (empty) return true;
const overrides = matchedKeywords
.filter(k => typeof k === 'string')
.map(k => byTitleKeyword.get(k.trim().toLowerCase()))
.filter(Boolean);
if (overrides.length > 0) {
return overrides.some(rule => contentRuleAccepts(rule, description));
}
return contentRuleAccepts(globalRule, description);
};
filter.rescueTitleMisses = rescue === true;
return filter;
}
// A content_filter that cannot be honoured (a min_match above its trait
// count, rescue_title_misses without a positive list) stops a scanner before
// any network call: guessing would drop every posting or keep all of them
// while the config reads as in force. Shared so every scanner that reads
// content_filter fails the same way.
export function buildContentFilterOrExit(contentFilter) {
try {
return buildContentFilter(contentFilter);
} catch (err) {
console.error(`Error: ${err.message}`);
process.exit(1);
}
}
// #6020. The one title decision for content_filter.rescue_title_misses, so
// every scanner that honours the key applies the same rule:
// 'match' — title_filter keeps the title; content_filter judges it as usual,
// 'rescue' — the title misses title_filter.positive and hits no
// title_filter.negative, and rescue_title_misses is on; the caller
// calls contentFilter(description, [], { titleMissed: true }) and
// books a failure there as a title rejection,
// 'reject' — a title rejection, as without the key.
// `titleFilter` defaults to buildTitleFilter(titleFilterConfig); a caller with
// its own compiled filter (per-company overrides, declared fields) passes it.
export function buildTitleGate(titleFilterConfig, contentFilter, titleFilter = buildTitleFilter(titleFilterConfig)) {
const rescue = contentFilter?.rescueTitleMisses === true;
const notVetoed = rescue ? buildTitleFilter({ negative: titleFilterConfig?.negative }) : null;
return (title) => {
if (titleFilter(title)) return 'match';
if (rescue && notVetoed(title)) return 'rescue';
return 'reject';
};
}
// ── Country-eligibility filter (#2093) ──────────────────────────────
// Optional, opt-in. If `country_eligibility_filter` is absent from
// portals.yml, all jobs pass — byte-identical to pre-#2093 behavior.
//
// Problem it solves: `location_filter` only reads the ATS provider's
// STRUCTURED location field (e.g. "Remote"), which many US companies use
// identically regardless of actual country eligibility. The real
// restriction — "US-based candidates only" vs. "US or Canada eligible" —
// often lives only in the JD DESCRIPTION body text, which this filter reads
// (same field `content_filter` already reads — `job.description`).
//
// It also reads `job.eligibilityText` (#6747): a source's own eligibility
// statement given outside the JD, such as the CryptocurrencyJobs feed
// teaser's "can be done remotely anywhere in the United States, or Canada".
// countryEligibilityText() below joins the two, and the rules apply to that
// text. content_filter and visa_filter never see `eligibilityText`, so a
// posting that carries one but no description still reads as having no JD.
//
// Semantics (case-insensitive substring), mirroring location_filter's
// "don't penalize missing data" discipline exactly:
// - Candidate's own `location.country` (config/profile.yml) is "United
// States" → always pass, unconditionally. An exclusionary "US only"
// phrase can never legitimately block a US-based candidate, so the
// filter no-ops entirely rather than special-casing every keyword check.
// - Empty / whitespace-only / non-string description → pass (no signal).
// - No `exclusionary` phrase matched → pass (ambiguous stays ambiguous,
// never guessed — this also means an `inclusive`-only match with no
// exclusionary wording present is a no-op pass, same as having no
// signal at all).
// - `exclusionary` phrase matched AND an `inclusive` phrase is also
// present → pass (the posting explicitly widens eligibility).
// - `exclusionary` phrase matched AND the candidate's own country is
// literally named in the JD text (e.g. a Canadian candidate scanning a
// posting that separately mentions "Canada" elsewhere) → pass.
// - `exclusionary` phrase matched, no `inclusive` phrase, and the
// candidate's own country isn't named → reject.
//
// Config shape (portals.yml):
// country_eligibility_filter:
// exclusionary: ["must be located in the united states", ...]
// inclusive: ["united states or canada", "north america", ...]
//
// Kept as a sibling block to `content_filter` rather than folded into its
// positive/negative shape: this filter cross-references
// `config/profile.yml`'s `location.country` and has its own three-way
// exclusionary/inclusive/candidate-country-named semantics, which doesn't
// fit content_filter's simpler two-list reject/require shape.
export function buildCountryEligibilityFilter(countryEligibilityFilter, candidateCountry) {
if (!countryEligibilityFilter) return () => true;
const candidateCountryLower = typeof candidateCountry === 'string'
? candidateCountry.toLowerCase().trim()
: '';
// A "US-based candidates only" restriction can never legitimately exclude
// a candidate who is themselves US-based — no-op the whole filter rather
// than relying on the literal-country-name check below (which would miss
// phrasing like "US-based candidates only" that never spells out "united
// states").
if (candidateCountryLower === 'united states') return () => true;
const exclusionary = normalizeKeywordList(countryEligibilityFilter.exclusionary);
const inclusive = normalizeKeywordList(countryEligibilityFilter.inclusive);
return (description) => {
if (typeof description !== 'string' || description.trim() === '') return true;
const lower = description.toLowerCase();
if (exclusionary.length === 0) return true;
if (!exclusionary.some(k => lower.includes(k))) return true;
if (inclusive.length > 0 && inclusive.some(k => lower.includes(k))) return true;
if (candidateCountryLower && lower.includes(candidateCountryLower)) return true;
return false;
};
}
/**
* The text country_eligibility_filter reads for a job: its description and
* its `eligibilityText` (#6747), each when it is a non-empty string, joined
* by a newline. '' when the job has neither, which the filter passes.
* @param {{ description?: unknown, eligibilityText?: unknown }} job
* @returns {string}
*/
export function countryEligibilityText(job) {
return [job?.description, job?.eligibilityText]
.filter(t => typeof t === 'string' && t.trim() !== '')
.join('\n');
}
/**
* The text visa_filter reads for a job: its description and its `visaText`,
* each when it is a non-empty string, joined by a newline. '' when the job
* has neither, which the filter treats as no mention.
* @param {{ description?: unknown, visaText?: unknown }} job
* @returns {string}
*/
export function visaFilterText(job) {
return [job?.description, job?.visaText]
.filter(t => typeof t === 'string' && t.trim() !== '')
.join('\n');
}
// ── Visa / work-authorization filter ────────────────────────────────
// Optional. If `visa_filter` is absent (or `enabled: false`), all jobs pass.
// Surfaces roles that sponsor a work visa — US (H-1B / H-1B1 / O-1),
// Singapore (Employment Pass / S Pass / ONE Pass / Work Pass), EU (Blue Card),
// UK (Skilled Worker), plus the generic "visa sponsorship" wording — and drops
// roles that explicitly refuse sponsorship. Like content_filter it reads the job
// DESCRIPTION text, so it only has signal for providers that populate
// job.description (see the content_filter header above); jobs without one fall
// back to the require_mention rule below.
//
// It also reads `job.visaText`: text a source gives outside the JD that may
// state sponsorship, such as the Startup Jobs feed teaser, which stays out of
// `description` because content_filter would read it as the JD.
// visaFilterText() below joins the two. content_filter and
// country_eligibility_filter never see `visaText`.
//
// Semantics (case-insensitive substring):
// - any `negative` keyword present → reject (an explicit "no sponsorship")
// - require_mention: false (default) → after clearing negatives, PASS —
// including jobs with no description. Use this to only weed out the
// explicit rejections while keeping everything unstated.
// - require_mention: true → keep only jobs whose description contains at least
// one `positive` keyword; a missing/empty description is rejected. Use this
// to surface *only* postings that actively advertise sponsorship.
//
// `positive` / `negative` default to a curated sponsorship vocabulary (US,
// Singapore, EU, UK work-visa wording) when omitted, so
// `visa_filter: { enabled: true }` works out of the box; supplying either list
// overrides that default.
export const DEFAULT_VISA_POSITIVE = [
'visa sponsorship',
'sponsor a visa',
'sponsor visas',
'will sponsor',
'sponsorship available',
'sponsorship is available',
'eligible for sponsorship',
'provide sponsorship',
'offer sponsorship',
'immigration support',
'h-1b',
'h1b',
'h-1b1',
'h1b1',
'o-1 visa',
// Singapore (Ministry of Manpower work passes). Multi-word forms only:
// bare 's pass' / 'one pass' false-positive on ordinary English
// ("Class Pass", "makes one pass"), so they are deliberately absent.
'employment pass',
's pass sponsorship',
's pass application',
's pass holder',
'one pass scheme',
'one pass application',
'one pass holder',
'work pass sponsorship',
'work pass application',
// EU Blue Card. Bare 'blue card' is kept: in employment text it is the EU
// permit far more often than anything else.
'eu blue card',
'blue card',
// UK Skilled Worker route.
'skilled worker visa',
'skilled worker sponsorship',
];
export const DEFAULT_VISA_NEGATIVE = [
'no visa sponsorship',
'no sponsorship',
'without sponsorship',
'unable to sponsor',
'not able to sponsor',
'cannot sponsor',
'do not sponsor',
'does not sponsor',
'not offer sponsorship',
'not provide sponsorship',
'sponsorship is not available',
'sponsorship not available',
'not offer visa sponsorship',
// Unambiguous sponsorship refusals phrased for non-US postings.
// (Relocation wording is deliberately excluded: relocation != sponsorship.)
'citizens and permanent residents only',
'permanent residents only',
];
export function buildVisaFilter(visaFilter) {
if (!visaFilter || visaFilter.enabled === false) return () => true;
const positive = visaFilter.positive != null
? normalizeKeywordList(visaFilter.positive)
: DEFAULT_VISA_POSITIVE.slice();
const negative = visaFilter.negative != null
? normalizeKeywordList(visaFilter.negative)
: DEFAULT_VISA_NEGATIVE.slice();
const requireMention = visaFilter.require_mention === true;
return (description) => {
const hasText = typeof description === 'string' && description.trim() !== '';
if (!hasText) return !requireMention;
const lower = description.toLowerCase();
if (negative.length > 0 && negative.some(k => lower.includes(k))) return false;
if (!requireMention) return true;
if (positive.length === 0) return true;
return positive.some(k => lower.includes(k));
};
}
// ── Salary filter ───────────────────────────────────────────────────
// Optional. If `salary_filter` is absent from portals.yml, all salaries pass.
// Semantics:
// - min/max are annual compensation filters (use annualized values)
// - max: 0 means "no upper limit"
// - If no salary data exists on a job, it passes (conservative behavior)
// - If both currencies are known and mismatch (e.g., USD filter, EUR job), it fails
// - Partial ranges (min only or max only) work correctly via overlap logic
// Uses null-safe checks (!= null, ??) to preserve 0 values correctly.
export function buildSalaryFilter(salaryFilter) {
if (!salaryFilter) return () => true;
// Coerce and validate bounds — malformed YAML must not silently mis-filter
const min = Number(salaryFilter.min ?? 0);
const max = Number(salaryFilter.max ?? 0);
const filterCurrency = (salaryFilter.currency || '').trim().toUpperCase();
if (!Number.isFinite(min) || !Number.isFinite(max) || min < 0 || max < 0) {
console.error('Warning: salary_filter.min/max must be non-negative numbers — salary filter disabled');
return () => true;
}
if (max > 0 && min > max) {
console.error('Warning: salary_filter.min cannot exceed salary_filter.max — salary filter disabled');
return () => true;
}
// If both min and max are 0, no filtering applied
if (min === 0 && max === 0) return () => true;
return (salary) => {
// If no salary data exists, pass (conservative - many providers don't expose salary)
if (!salary) return true;
const jobMin = salary.min ?? salary.max ?? null;
const jobMax = salary.max ?? salary.min ?? null;
// If we have no usable salary values, pass conservatively
if (jobMin == null && jobMax == null) return true;
// Currency handling - reject only if BOTH currencies exist and mismatch
const jobCurrency = (salary.currency || '').trim().toUpperCase();
if (filterCurrency && jobCurrency && filterCurrency !== jobCurrency) {
return false;
}
// Range overlap logic - reject ONLY if job is completely outside filter range
// Job entirely below user minimum
if (min > 0 && jobMax != null && jobMax < min) {
return false;
}
// Job entirely above user maximum
if (max > 0 && jobMin != null && jobMin > max) {
return false;
}
// Otherwise pass (overlap exists or no valid range to compare)
return true;
};
}
// CJK/Korean corporate-form markers (#2570), kept as a list local to this
// file rather than folded into invite-match.mjs's LEGAL_SUFFIXES. Two reasons
// they can't share one list: LEGAL_SUFFIXES strips a *trailing, space-
// delimited* word (`\s${suffix}$`), but 株式会社 is usually written unspaced,
// as a *prefix* (株式会社メルカリ) and sometimes as a suffix (メルカリ株式会社),
// so LEGAL_SUFFIXES's anchor fires on neither. (The spaced variant
// 株式会社 メルカリ is ordinary orthography too, and it already matched before
// this change — via the punctuation-to-space key and the containment
// fallback. The unspaced form is the actual gap.) And
// invite-match.mjs's normalizeCompanyName is deliberately stricter than
// companyMatch (loosely-quoted email text vs. an identity key that must never
// merge two different companies) — folding a CJK list into that stricter
// normalizer risks loosening its callers too, which the "splits, never
// merges" rule #2445/#2569 protects was written to prevent.
const CORPORATE_FORMS = [
'株式会社', '合同会社', '有限会社', // Japanese
'合名会社', '合資会社', '一般社団法人',
'股份有限公司', // Chinese (longer forms first)
'有限责任公司', '有限責任公司', // no 有限公司 suffix: 责任 sits between
'有限公司',
'주식회사', '유한회사', // Korean
];
// Split an already normalizeTextKey'd string into [forms, remainder], where
// `forms` names the corporate-form markers found at its leading and trailing
// edges, in that order, or null when there are none. Order is kept rather than
// sorted: 株式会社アカネ有限会社 and 有限会社アカネ株式会社 stay apart, the
// conservative reading when a name carries two forms.
// No space anchor: unlike LEGAL_SUFFIXES these forms are frequently written
// unspaced (株式会社メルカリ), which is exactly the case the \b-based approach
// cannot reach. The spaced variant (株式会社 メルカリ) already matched before
// #2570, through the punctuation-to-space key and the containment fallback
// below; the unspaced one is the gap. Longer forms are checked first
// (股份有限公司 before 有限公司) so a strip cannot leave a dangling 股份 behind.
//
// BOTH edges are inspected, not just the first form list order finds. A name
// carrying a form at each end (合同会社 アカネ株式会社) is rare, but reading
// only one of them let the other slip past the different-form check below:
// the shared 株式会社 was compared, the 合同会社 never was, and containment
// merged the pair.
//
// Returning the forms rather than just the remainder is what lets companyMatch
// tell "one side omitted the form" from "the two sides carry DIFFERENT forms".
// A bare remainder cannot express that difference, and collapsing it merges
// 株式会社アカネ with 合同会社アカネ — a KK and a GK are two different legal
// entities sharing a trade name, so that is a false merge, the one direction
// #2445/#2569's "splits, never merges" rule exists to forbid.
function stripCorporateForm(key) {
const forms = [];
let rest = key;
const prefix = CORPORATE_FORMS.find((form) => rest.startsWith(form));
if (prefix) {
forms.push(prefix);
rest = rest.slice(prefix.length);
}
const suffix = CORPORATE_FORMS.find((form) => rest.endsWith(form));
if (suffix) {
forms.push(suffix);
rest = rest.slice(0, -suffix.length);
}
return [forms.length ? forms.join('|') : null, rest];
}
// Apply the strip to a pair of keys, or return null: a verdict that the pair
// is NOT the same company, which companyMatch returns before any equality or
// containment check. Two cases earn it:
//
// - DIFFERENT explicit forms on the two sides are positive evidence of two
// different entities, the same way a mismatched req number is (#1524).
// Merely declining to strip was not enough: the raw keys still reached the
// containment fallback, where 株式会社アカネ is a bounded substring of
// 合同会社 株式会社アカネ.
// - Exactly one side is ONLY a marker ("株式会社" alone). That side carries no
// trade name to compare, and falling back to its raw key let it equal
// 株式会社株式会社 once that was stripped. Two identical bare markers still
// compare equal, since both then keep their raw keys.
//
// Otherwise strip, falling back to the raw key when the strip empties it, so
// neither the equality check nor the containment fallback is ever handed an
// empty "no signal" string.
function stripFormPair(rawA, rawB) {
const [formsA, restA] = stripCorporateForm(rawA);
const [formsB, restB] = stripCorporateForm(rawB);
if (formsA && formsB && formsA !== formsB) return null;
if (Boolean(formsA && !restA) !== Boolean(formsB && !restB)) return null;
return [restA || rawA, restB || rawB];
}
export function companyMatch(jobCompany, windowCompany) {
// Unicode-aware (#2393 family): the [a-z0-9] strip this used to carry erased
// non-Latin scripts outright, so 株式会社アカネ and 合同会社ゾロ both cleaned
// to '' and the equality check below reported two unrelated companies as the
// same one. The empty guard is part of the fix, not decoration — "no usable
// signal on either side" must never read as "identical".
//
// Corporate-form stripping (#2570) happens right here, before either the
// equality check or the containment fallback below, so both benefit. See
// stripFormPair: it returns null when the forms show two different
// entities, and that ends the comparison, so this stays a split, never a
// merge. The no-space key reaches every such verdict first in practice (an
// exhaustive search over prefix/suffix/separator combinations found no
// exception); the spaced key's null check is defensive, and keeps a null
// from ever being destructured.
const noSpaces = stripFormPair(
normalizeTextKey(jobCompany),
normalizeTextKey(windowCompany),
);
if (!noSpaces) return false;
const [c1NoSpaces, c2NoSpaces] = noSpaces;
if (c1NoSpaces && c1NoSpaces === c2NoSpaces) return true;
const withSpaces = stripFormPair(
normalizeTextKey(jobCompany, ' '),
normalizeTextKey(windowCompany, ' '),
);
if (!withSpaces) return false;
const [c1WithSpaces, c2WithSpaces] = withSpaces;
if (!c1WithSpaces || !c2WithSpaces) return false;
// Containment: a short window name should still match a longer official one
// ("Acme" vs "Acme Corp"), bounded so "Acme" does not match "Acmetric".
//
// The anchors are lookarounds, not \b: JS defines \b against ASCII \w even
// under the u flag. Keeping the accent (rather than stripping it to a space,
// as the [a-z0-9] filter did before) means '\bnestlé\b' can never hold —
// neither side of the trailing anchor is a word character — so Nestlé
// Deutschland vs Nestlé would silently stop matching. Same for Ørsted, Zoë
// and every other name whose first or last letter is non-ASCII.
//
// The anchor class is the one normalizeTextKey keeps, deliberately. An anchor
// class without \p{M} would treat a Devanagari matra as a boundary and split
// कंपनी mid-word — the key and its boundaries have to agree on what a letter
// is, or they drift the way #2397 and #2445 fixed elsewhere.
//
// 株式会社メルカリ vs メルカリ never reaches this containment fallback at
// all now (#2570): the corporate-form strip above already resolves it via
// the equality check, since 社 being a letter still means no anchor rule
// here could have matched it directly — Japanese isn't space-delimited, so
// no boundary exists for the lookbehind to find.
//
// compileLocationKeyword() above reached for lookarounds too, for a related
// reason ("\b behaves surprisingly at a punctuation edge"); its escape set is
// reused here because '\-' is an invalid identity escape under u. Both
// operands are already normalizeTextKey output — letters, marks, digits and
// spaces only — so the escape is defensive, not load-bearing.
const bounded = (name) => new RegExp(
`(?<![\\p{L}\\p{M}\\p{N}])${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?![\\p{L}\\p{M}\\p{N}])`,
'u',
);
return bounded(c2WithSpaces).test(c1WithSpaces) || bounded(c1WithSpaces).test(c2WithSpaces);
}
export function addDays(dateStr, days) {
const date = new Date(`${dateStr}T00:00:00Z`);
date.setUTCDate(date.getUTCDate() + days);
return date.toISOString().slice(0, 10);
}
// Reads config/profile.yml's `location.country` (already a documented
// profile field — see config/profile.example.yml) for the country-
// eligibility filter (#2093). Missing file, missing field, or a malformed
// profile all resolve to '' — buildCountryEligibilityFilter treats an empty
// candidate country the same as "not the candidate's own country's US
// no-op" and simply skips the literal-country-name pass-through, which is
// the same conservative "don't penalize missing data" default used
// throughout this file.
export function loadCandidateCountry(profilePath = PROFILE_PATH) {
if (!existsSync(profilePath)) return '';
try {
const raw = loadYaml(readFileSync(profilePath, 'utf-8')) || {};
const country = raw?.location?.country;
return typeof country === 'string' ? country.trim() : '';
} catch {
return '';
}
}
// config/profile.yml `candidate.years_of_experience`, the candidate's side of
// experience_filter. Unset reads as undefined, and resolveExperienceFilter then
// keeps the filter off and says why. A value that is set but not a number, or
// a profile that does not parse, is named here: "no years are set" would send
// the user looking for a key they already wrote.
export function loadCandidateYears(profilePath = PROFILE_PATH) {
if (!existsSync(profilePath)) return undefined;
let raw;
try {
raw = loadYaml(readFileSync(profilePath, 'utf-8')) || {};
} catch (err) {
console.warn(`⚠️ experience_filter could not read ${profilePath}: ${String(err?.message ?? err).split('\n')[0]}`);
return undefined;
}
const years = raw?.candidate?.years_of_experience;
if (years === undefined || years === null) return undefined;
if (typeof years === 'number' && Number.isFinite(years)) return years;
console.warn(`⚠️ candidate.years_of_experience in ${profilePath} must be a number, got ${JSON.stringify(years)}`);
return undefined;
}
export function loadReApplyWindows(profilePath = PROFILE_PATH) {
if (!existsSync(profilePath)) return {};
try {
const raw = loadYaml(readFileSync(profilePath, 'utf-8')) || {};
const windows = raw.re_apply_windows || {};
const validWindows = {};
for (const [company, win] of Object.entries(windows)) {
if (!win || typeof win !== 'object') continue;
const lastApplyDate = win.last_apply_date;
if (typeof lastApplyDate !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(lastApplyDate)) continue;
if (isNaN(Date.parse(lastApplyDate))) continue;
const sameRoleDays = win.same_role_days;
if (sameRoleDays !== undefined && (!Number.isInteger(sameRoleDays) || sameRoleDays < 0)) continue;
if (win.applied_to !== undefined && !Array.isArray(win.applied_to)) continue;
if (win.applied_to !== undefined && win.applied_to.some(x => typeof x !== 'string')) continue;
if (win.cross_role_bucket !== undefined && typeof win.cross_role_bucket !== 'string') continue;
validWindows[company] = win;
}
return validWindows;
} catch {
return {};
}
}
export function buildCooldownFilter(windows, today) {
if (!windows || Object.keys(windows).length === 0) {
return () => ({ skip: false });
}
const genericKeywords = new Set(['all', 'roles', 'role', 'family', 'bucket', 'group', 'team']);
return (job) => {
const jobCompany = job.company || '';
const jobTitleLower = (job.title || '').toLowerCase();
for (const [windowCompany, window] of Object.entries(windows)) {
if (companyMatch(jobCompany, windowCompany)) {
const lastApplyDate = window.last_apply_date;
const sameRoleDays = Number(window.same_role_days || 0);
if (!lastApplyDate) continue;
const cooldownUntil = addDays(lastApplyDate, sameRoleDays);
if (today >= cooldownUntil) {
continue;
}
if (Array.isArray(window.applied_to)) {
const matchesApplied = window.applied_to.some(role => {
const roleLower = role.toLowerCase();
return jobTitleLower.includes(roleLower);
});
if (matchesApplied) {
return { skip: true, reason: `cooldown:${windowCompany}:${cooldownUntil}`, cooldownUntil };
}
}
if (window.cross_role_bucket) {
const bucketKeywords = window.cross_role_bucket
.toLowerCase()
.split('_')
.filter(kw => kw && !genericKeywords.has(kw));
const matchesBucket = bucketKeywords.some(kw => {
if (kw === 'em') {
return /\bem\b/i.test(jobTitleLower) || jobTitleLower.includes('engineering manager');
}
return jobTitleLower.includes(kw);
});
if (matchesBucket) {
return { skip: true, reason: `cooldown:${windowCompany}:${cooldownUntil}`, cooldownUntil };
}
}
}
}
return { skip: false };
};
}
// ── URL rediscovery (--rediscover-404) ──────────────────────────────
// When a tracked company's job URL returns 404/410, the role may have just
// moved to a new URL (Workday/Greenhouse rotate URLs without closing roles).
// These helpers back an opt-in search-and-reverify fallback before giving up.
// extractCareersUrlDomain returns the hostname of a company's careers_url, or
// null when it's missing/unparseable. The presence of a domain is what gates
// the fallback — broad-discovery offers without a careers_url stay ineligible.
export function extractCareersUrlDomain(careersUrl) {
if (!careersUrl) return null;
try {
return new URL(careersUrl).hostname;
} catch {
return null;
}
}
// resolveSearchHref unwraps a DuckDuckGo HTML redirect (`/l/?uddg=<encoded>`)
// to its real destination, so domain matching sees the actual host instead of
// duckduckgo.com. Non-redirect hrefs pass through unchanged.
function resolveSearchHref(href) {
try {
const u = new URL(href, 'https://duckduckgo.com');
const isDdgHost = u.hostname === 'duckduckgo.com' || u.hostname.endsWith('.duckduckgo.com');
if (isDdgHost && u.pathname === '/l/') {
const target = u.searchParams.get('uddg');
if (target) return target;
}
} catch {
/* fall through to the raw href */
}
return href;
}
// pickRediscoveredUrl chooses the first result whose hostname *exactly* equals
// the careers domain (no substring/look-alike matches), unwrapping search-engine
// redirects first. Pure + exported so result-matching is unit-testable without
// driving a real browser. Returns null when nothing matches.
export function pickRediscoveredUrl(hrefs, domain) {
if (!domain || !Array.isArray(hrefs)) return null;
for (const raw of hrefs) {
const href = resolveSearchHref(raw);
let host;
try {
host = new URL(href).hostname;
} catch {
continue;
}
if (host === domain) return href;
}
return null;
}
// REDISCOVER_TIMEOUT_MS bounds the single fallback search so a slow or blocked
// search engine can't stall the sequential verify loop.
const REDISCOVER_TIMEOUT_MS = 10_000;
// searchForNewUrl runs one site-scoped search for a moved tracked role and
// returns a same-domain URL if found, else null. Every failure path returns
// null — the fallback must never throw into the verify loop. Leaves the page on
// a blank document so the next checkUrlLiveness call starts clean.
async function searchForNewUrl(page, offer) {
const domain = offer.careersUrlDomain;
if (!domain) return null;
const query = `"${offer.title}" "${offer.company}" site:${domain}`;
try {
await page.goto(
`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`,
{ waitUntil: 'domcontentloaded', timeout: REDISCOVER_TIMEOUT_MS },
);
const hrefs = await page.evaluate(() =>
Array.from(document.querySelectorAll('a.result__a'))
.map((a) => a.getAttribute('href'))
.filter(Boolean),
);
return pickRediscoveredUrl(hrefs, domain);
} catch {
return null;
} finally {
try {
await page.goto('about:blank');
} catch {
/* ignore — best-effort cleanup */
}
}
}
// ── Dedup ───────────────────────────────────────────────────────────
const PERMANENT_SCAN_HISTORY_STATUSES = new Set([
'skipped_invalid_url',
'skipped_blocked_host',
]);
/**
* Statuses the `scan_history.recheck_after_days` TTL is allowed to release.
*
* `added` is the row a live posting leaves behind. `skipped_expired` is the row
* a dead one leaves (#3891), and it belongs here because a posting can be
* relisted: pinning the retirement for good would put the recheck the user
* configured out of reach for exactly the URLs most likely to come back.
*
* `skipped_dup` says a scanner met a seen URL again, so it can pin no longer than
* whatever made the URL seen. As a permanent pin, the first repeat sighting kept
* an aged posting out for good, and scan-dayforce.mjs and scan-interamt.mjs write
* one on every scan that lists it. Its `first_seen` is the day of that repeat,
* not of the posting, so the TTL reads it only for a row with no original in
* scan-history; otherwise collectSeenUrls lets the row go with its original.
*
* Everything else keeps its permanent pin. The distinction is whether the status
* describes a POSTING, which can change, or the URL itself — an invalid URL does
* not become valid and a blocked host does not become reachable because a window
* elapsed.
*/
const RECHECKABLE_SCAN_HISTORY_STATUSES = new Set(['added', 'skipped_expired', 'skipped_dup']);
/**
* The offers not yet recorded under `status`, one per URL.
*
* The title, location and posting-age cuts run before dedup, and their rows never
* pin a URL (OBSERVATIONAL_SCAN_HISTORY_STATUSES), so without this every scan
* appended the same rows again. A URL already carrying a row with the same
* status is skipped, and so is a second listing of one URL within this scan.
* A different status still writes, so a posting whose verdict changes is
* recorded again.
*
* @param {Array<{url: string}>} offers
* @param {string} status
* @param {string} [scanHistoryText] - Full scan-history.tsv contents.
*/
export function unrecordedOffers(offers, status, scanHistoryText = '') {
const recorded = new Set();
for (const line of scanHistoryDataLines(scanHistoryText)) {
const [url, , , , , rowStatus] = line.split('\t');
if (url && rowStatus === status) recorded.add(seenJobKey(url));
}
return offers.filter((offer) => {
const key = seenJobKey(offer.url);
if (recorded.has(key)) return false;
recorded.add(key);
return true;
});
}
function daysBetweenIsoDates(start, end) {
if (!/^\d{4}-\d{2}-\d{2}$/.test(start) || !/^\d{4}-\d{2}-\d{2}$/.test(end)) return null;
const startDate = new Date(`${start}T00:00:00Z`);
const endDate = new Date(`${end}T00:00:00Z`);
// A month or day out of range (`2026-13-01`) is an Invalid Date, and
// toISOString() throws on one: a single such scan-history row aborted
// loadSeenUrls, and with it the scan. Read it as no date, as the check below
// reads `2026-02-30`.
if (Number.isNaN(startDate.getTime()) || Number.isNaN(endDate.getTime())) return null;
if (startDate.toISOString().slice(0, 10) !== start || endDate.toISOString().slice(0, 10) !== end) return null;
return Math.floor((endDate - startDate) / (1000 * 60 * 60 * 24));
}
// `today` defaults to the LOCAL calendar day, not the UTC one. This function
// gates a COOLDOWN (`today < cooldownUntil`) — the user asked not to see a
// posting until a date — and the UTC day is tomorrow for a west-of-Greenwich
// evening run, so the cooldown opened a day early (#3070). The recheck window
// below reads one day high the same way. Callers may still pass `today`
// explicitly; only the default moves.
export function shouldDedupScanHistoryRow({ firstSeen, status = 'added' }, { recheckAfterDays = null, today = localToday() } = {}) {
if (PERMANENT_SCAN_HISTORY_STATUSES.has(status)) return true;
if (OBSERVATIONAL_SCAN_HISTORY_STATUSES.has(status)) return false;
if (status.startsWith('cooldown:')) {
const parts = status.split(':');
const cooldownUntil = parts[parts.length - 1];
return today < cooldownUntil;
}
if (!RECHECKABLE_SCAN_HISTORY_STATUSES.has(status)) return true;
if (recheckAfterDays == null) return true;
const ageDays = daysBetweenIsoDates(firstSeen, today);
if (ageDays == null) return true;
return ageDays < recheckAfterDays;
}
/**
* The `scan_history` recheck policy portals.yml sets, in the shape
* collectSeenUrls and loadSeenUrls take. Exported so every scanner that builds
* a seen set releases the rows scan.mjs releases: a policy left out reads as
* "dedup forever", so a scanner that skipped it kept a URL seen after
* scan.mjs had let it back in.
*
* @param {object|null} [config] - Parsed portals.yml.
* @returns {{recheckAfterDays: number|null}}
*/
export function scanHistoryPolicy(config = {}) {
const raw = config?.scan_history?.recheck_after_days;
const parsed = Number.parseInt(raw, 10);
return {
recheckAfterDays: Number.isFinite(parsed) && parsed >= 0 ? parsed : null,
};
}
function historyPolicyFromPortals() {
try {
if (!PORTALS_PATH || !existsSync(PORTALS_PATH)) return {};
const raw = parseYaml(readFileSync(PORTALS_PATH, 'utf-8'));
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {};
return scanHistoryPolicy(raw);
} catch {
return {};
}
}
/**
* Whether the company+role dedupe key includes location.
*
* True when `scan_history.dedup_include_location` is true. An explicit
* `false` stays off. When the flag is missing, a non-array
* `location_filter.workplace_policy` turns it on.
*
* @param {object} [config] - Parsed portals.yml.
* @returns {boolean} Whether the location joins the company+role dedupe key.
*/
export function resolveDedupIncludeLocation(config = {}) {
if (config.scan_history?.dedup_include_location === true) return true;
if (config.scan_history?.dedup_include_location === false) return false;
const policy = config.location_filter?.workplace_policy;
return !!policy && typeof policy === 'object' && !Array.isArray(policy);
}
/**
* Read the opt-in `scan_history.dedup_include_language` switch.
*
* An employer can publish one requisition in more than one language, with the
* same title and location in each. The company+role key collapses those
* versions into whichever one the provider returned first — correct for a
* candidate who reads every language, wrong for one who only applies to
* postings in some of them: the survivor may be the version they discard.
* With the switch on, two postings whose languages are both known and differ
* are not duplicates. An unknown language on either side keeps the default
* answer.
*
* Strict boolean `true` only, for the same reason as
* {@link resolveDedupIncludeLocation}.
*
* @param {object} [config] - Parsed portals.yml.
* @returns {boolean} Whether differing posting languages keep postings apart.
*/
export function resolveDedupIncludeLanguage(config = {}) {
return config.scan_history?.dedup_include_language === true;
}
/**
* Read the top-level `preferred_languages` list: the languages the candidate
* wants a posting in, best first.
*
* Each entry is reduced the way posting languages are (languageFormsForDedup),
* so `en-GB` and `en` name one language. A single string counts as a one-entry
* list; anything that is not a non-empty string is skipped, and a repeated
* language keeps its first place.
*
* @param {object} [config] - Parsed portals.yml.
* @returns {string[]} Language forms, best first; empty when unset.
*/
export function resolvePreferredLanguages(config = {}) {
const raw = config.preferred_languages;
const list = typeof raw === 'string' ? [raw] : Array.isArray(raw) ? raw : [];
return [...new Set(list.flatMap(entry => languageFormsForDedup(entry)))];
}
/**
* Rank a posting language against the preferred list: its place in the list,
* or -1 when it is not there or unknown.
*
* Handed to providers as `ctx.preferredLanguageRank`, so a provider compares
* its own codes by the same rules without its own copy of them.
*
* @param {string[]} preferred - From resolvePreferredLanguages.
* @returns {(language: unknown) => number}
*/
export function preferredLanguageRanker(preferred) {
return (language) => {
const [form] = languageFormsForDedup(language);
return form === undefined ? -1 : preferred.indexOf(form);
};
}
/**
* Order one provider's jobs so that postings in a preferred language come
* first, best language first; the rest keep the provider's order.
*
* The sort is stable. Where several language versions of one posting come
* back, the dedup below keeps the first one it meets when dedup_include_language
* is off, so this is what makes it keep the preferred version; when it is on,
* every version is kept, in this order. The provider's own order — each
* posting's default language first — is what the language-aware URL dedup
* resolves seen rows against, so siblingsByUrl is built from the unsorted list.
*
* @template {{language?: unknown}} T
* @param {T[]} jobs
* @param {(language: unknown) => number} rank - From preferredLanguageRanker.
* @returns {T[]} A new array; `jobs` is left as it was.
*/
export function orderByPreferredLanguage(jobs, rank) {
const keyOf = (job) => {
const r = rank(job?.language);
return r < 0 ? Number.POSITIVE_INFINITY : r;
};
return jobs
.map((job, index) => ({ job, index, key: keyOf(job) }))
.sort((a, b) => a.key - b.key || a.index - b.index)
.map(({ job }) => job);
}
// Query params that carry no identity information for a job posting. Safe to
// strip when computing the URL-shaped dedup key. An allowlist on purpose:
// several ATSes key the posting off a query param (Greenhouse's gh_jid), so a
// blanket strip would collapse distinct roles.
const DEDUP_STRIP_PARAMS = new Set([
// The language a page renders in: the language versions of one posting
// share a key, and languageFormsFromUrl reads the param back to tell them apart.
...LANGUAGE_PARAMS,
'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'ref', 'src', 'source', 'gh_src', 'lever-origin', 'lever-source',
'rltr', // StepStone: regenerated per request, so one posting returns as new every scan
]);
/**
* Normalize a job posting URL into a stable, URL-shaped comparison key.
*
* Strips cosmetic query params (locale/tracking), promotes recognized
* hash-route job IDs before dropping other fragments, drops a trailing slash,
* and lowercases scheme, host, and path. Callers keep writing the original URL.
*
* Query values keep their original casing. Those can be identity-bearing
* (Greenhouse's gh_jid), which is why DEDUP_STRIP_PARAMS is an allowlist.
* This function does not turn a Greenhouse URL into `greenhouse:{id}`. That
* question is {@link seenJobKey}.
*
* Falls back to the raw string when the URL is malformed.
*
* @param {string} url
* @returns {string}
*/
export function normalizeUrlForDedup(url) {
if (typeof url !== 'string' || !url) return url;
let parsed;
try {
parsed = new URL(url);
} catch {
return url;
}
for (const param of Array.from(parsed.searchParams.keys())) {
if (DEDUP_STRIP_PARAMS.has(param.toLowerCase())) {
parsed.searchParams.delete(param);
}
}
promoteKnownFragmentIdentity(parsed);
parsed.hash = '';
parsed.pathname = parsed.pathname.replace(/\/+$/, '').toLowerCase() || '/';
return parsed.toString();
}
/** Same-job key for a seen-set. Short ATS ids win; otherwise the URL-shaped key. */
export function seenJobKey(url) {
if (typeof url !== 'string' || !url) return url;
return canonicalJobKey(url) || normalizeUrlForDedup(url);
}
/**
* The posting language a URL asks for, as dedup forms; none when it names none.
*
* {@link normalizeUrlForDedup} drops these params, so the language versions of
* one posting share a URL key. Reading the param back tells those versions
* apart on rows that carry no `language` cell of their own (pipeline.md,
* applications.md, scan-history rows written without one).
*
* @param {unknown} url
* @returns {string[]} Zero or one form, as {@link languageFormsForDedup}.
*/
export function languageFormsFromUrl(url) {
if (typeof url !== 'string' || !url) return [];
let parsed;
try {
parsed = new URL(url);
} catch {
return [];
}
for (const [name, value] of parsed.searchParams) {
if (LANGUAGE_PARAMS.includes(name.toLowerCase())) return languageFormsForDedup(value);
}
return [];
}
/**
* Title form for telling apart the language versions of one posting: the same
* provider wrote both sides, so only case and whitespace are folded.
*
* @param {unknown} title
* @returns {string}
*/
export function variantTitleKey(title) {
return String(title ?? '').normalize('NFKC').trim().replace(/\s+/g, ' ').toLowerCase();
}
/**
* Record one seen version of a posting under its URL key, in the shape
* {@link isUnseenLanguageVersion} reads.
*
* @param {Map<string, Array<{languages: string[], title: string}>>} variantsByUrl
* @param {string} key - From {@link normalizeUrlForDedup}.
* @param {string[]} languages - Language forms; empty when unknown.
* @param {unknown} [title] - Raw title; omitted when the source records none.
*/
export function recordUrlVariant(variantsByUrl, key, languages, title) {
let rows = variantsByUrl.get(key);
if (!rows) variantsByUrl.set(key, (rows = []));
rows.push({ languages, title: variantTitleKey(title) });
}
/**
* Whether a candidate whose URL key was already seen is a language version of
* that posting which none of the seen rows was in.
*
* Some ATSes serve every language version of a posting at one URL with a
* language param, which {@link normalizeUrlForDedup} drops, so the versions
* share a URL key. A seen row's language is its own when it recorded one.
* Otherwise it is resolved against `siblings`, this scan's versions of the same
* posting in provider order: the one sibling whose title matches the row's,
* else the first sibling — a provider emits a posting's own default language
* first. True only when every seen row resolves to a language and none is the
* candidate's; a candidate of unknown language stays a duplicate.
*
* @param {Array<{languages: string[], title: string}>|undefined} seenRows - From {@link recordUrlVariant}.
* @param {{languages: string[], title: unknown}} candidate
* @param {Array<{languages: string[], title: unknown}>} siblings - Including the candidate.
* @returns {boolean}
*/
export function isUnseenLanguageVersion(seenRows, candidate, siblings = []) {
if (!Array.isArray(seenRows) || seenRows.length === 0) return false;
if (!candidate.languages.length) return false;
const known = siblings.filter(sibling => sibling.languages.length > 0);
const resolve = (row) => {
if (row.languages.length) return row.languages;
const titled = row.title ? known.filter(sibling => variantTitleKey(sibling.title) === row.title) : [];
if (titled.length === 1) return titled[0].languages;
return known[0]?.languages ?? [];
};
return seenRows.every((row) => {
const languages = resolve(row);
return languages.length > 0 && !candidate.languages.some(language => languages.includes(language));
});
}
/**
* The leading checkbox marker of a `data/pipeline.md` entry.
*
* Only ` ` and `x` are recognized, matching the pre-existing gates: `- [!]`
* marks a URL that could not be fetched, which is not evidence a posting was
* surfaced.
*
* Anchored, because the URL it guards is then matched anywhere in the entry: a
* hand-written note that happens to contain checkbox syntax mid-sentence and a
* link would otherwise seed that link and silently bury a live posting. Leading
* whitespace is tolerated so an indented entry still dedupes, as it did when the
* URL had to sit immediately after the checkbox.
*/
const PIPELINE_CHECKBOX_RE = /^\s*- \[[ x]\]\s+/;
/**
* As `PIPELINE_CHECKBOX_RE`, but rejecting indentation.
*
* The company/role gate stays exactly as strict as the `^- \[` anchor it
* replaces: widening it would seed *more* role keys, and one role key suppresses
* every other posting that shares it.
*/
const PIPELINE_CHECKBOX_STRICT_RE = /^- \[[ x]\]\s+/;
/**
* A labeled trailing segment of a pipeline entry, as written by the two writers
* that emit one: `formatPipelineOffer` here (`posted:`, `trust:`, `yrs:`, `note:`) and
* `appendRankAnnotation` in `rank-pipeline.mjs` (`rank:`).
*
* Labeled segments are appended after the positional columns, so one slides into
* the location column whenever an offer has neither a location nor a
* compensation. Used only to reject such a cell as a location.
*
* The label set is an ALLOW-LIST, not the shape `\p{L}[\p{L}\p{N} _-]*:` it
* replaces. "Any word followed by a colon" is not a property that separates
* metadata from locations: a location is free text on the board's side, and
* `Remote: EMEA` / `Remote: Southeast US` (live Neo4j postings) are ordinary
* values of that field. Reading one as metadata drops it to the bare wildcard
* key, which then suppresses every city-specific variant of the same role —
* exactly the collapse `dedup_include_location` exists to prevent, reintroduced
* through the parser instead of the key.
*
* Deliberately NOT in the list: `score:`. `rank-pipeline.mjs` uses that word
* only inside the prompt it sends an LLM ("score: 0-5, where 5 is an excellent
* match"); the segment it actually writes to pipeline.md is `rank: {n}/5 — …`.
* Prompt text is not a serialization format. Extend this list only alongside a
* writer that really emits the label.
*
* Case-insensitive because the shape it replaces was (`\p{L}` spans both cases)
* and because the two directions are not symmetric: failing to recognize a real
* segment invents a city for a role and lets it resurface, while a genuine
* location beginning `Posted: ` / `Rank: ` does not occur.
*/
const PIPELINE_LABELED_SEGMENT_RE = /^(?:posted|trust|yrs|note|rank):\s/iu;
/**
* The `~~…~~` wrapper an expired entry is written with.
*
* Matched only at the start of the entry body, never searched for line-wide.
* `~` is a legal URL character and a documented `note:` column may carry its own
* strikethrough, so treating `~` as a global signal both truncates real URLs and
* strips live entries of their pair. Expiry is a property of the entry, so it is
* read at the entry boundary.
*/
const PIPELINE_STRIKETHROUGH_RE = /^~~([\s\S]*?)~~/;
/**
* A URL inside a pipeline entry.
*
* Terminates on whitespace and `|` only. `|` cannot appear unencoded in a URL
* and is this format's cell separator, so it is the one safe boundary; every
* other character stays legal. `~` (RFC 3986 unreserved) and `)` (a sub-delim)
* in particular must not terminate the match — excluding them truncated `~user`
* paths and parenthesised region suffixes, and a truncated URL both stops
* deduping and seeds a bare-origin key that everything else on that host then
* false-matches against.
*
* `local:` entries are deliberately not matched — the gates this feeds have
* always been http(s)-only.
*/
const PIPELINE_URL_RE = /https?:\/\/[^\s|]+/;
/**
* Split a `data/pipeline.md` checkbox line into its entry body and expiry state.
*
* The body is whatever follows the checkbox, unwrapped when the entry is struck
* out, so every caller parses the same cell sequence either way. An unclosed
* wrapper still reads as expired: the opening `~~` is the marker.
*
* @param {string} line - One raw line of `data/pipeline.md`.
* @param {RegExp} checkboxRe - Which checkbox anchor this gate accepts.
* @returns {{body: string, expired: boolean}|null} Null when the line is not an
* entry at all.
*/
function pipelineEntry(line, checkboxRe) {
const checkbox = line.match(checkboxRe);
if (!checkbox) return null;
const rest = line.slice(checkbox[0].length);
if (!rest.startsWith('~~')) return { body: rest, expired: false };
const closed = rest.match(PIPELINE_STRIKETHROUGH_RE);
return { body: closed ? closed[1] : rest.slice(2), expired: true };
}
/**
* Extract the job URL from a `data/pipeline.md` checkbox line, wherever it sits.
*
* Six line shapes are documented across the modes, and only the one
* `appendToPipeline` writes leads with the URL. The others lead with a report
* number (`#NNN`, `modes/pipeline.md`), a report link
* (`[NNN](reports/…)`, `reconcile-pipeline.mjs`), a pre-screen marker (`#--`,
* `modes/pipeline.md`), or a strikethrough (`~~…~~`, `modes/pipeline.md` and
* `modes/oferta.md`). Anchoring the URL to the checkbox missed all five.
*
* An expired entry still seeds a URL key; only its company/role pair is withheld
* (see `extractPipelineCompanyRole`).
*
* @param {string} line - One raw line of `data/pipeline.md`.
* @returns {string|null} The URL, or null when the line carries none.
*/
function extractPipelineUrl(line) {
const entry = pipelineEntry(line, PIPELINE_CHECKBOX_RE);
if (!entry) return null;
const match = entry.body.match(PIPELINE_URL_RE);
return match ? match[0] : null;
}
/**
* Extract the company/role pair from a `data/pipeline.md` checkbox line.
*
* Company and role are the two cells *after* the URL cell, not cells 1 and 2 —
* see `extractPipelineUrl` for why the URL is not always first.
*
* Two shapes deliberately yield nothing, mirroring the `status !== 'added'`
* rule the scan-history branch of `collectSeenCompanyRoles` already applies:
*
* - **Expired entries** (`~~…~~`). Strikethrough is how the pipeline records the
* same state scan-history records as `skipped_expired`; seeding a dead
* posting's role key would let a dead SF URL bury a live NY req. Read at the
* entry boundary, so a `note:` column containing its own strikethrough leaves
* a live entry's pair intact.
* - **Pre-screen discards** (`#-- | {url} | skipped (…)`). The cell after the
* URL is a discard reason, not a company.
*
* The third cell after the URL is reported as `location` only where it really is
* one, because that column is positional and means different things per shape:
*
* - Only in the URL-first shape (`urlIndex === 0`) — the one `appendToPipeline`
* writes — is it the location. A report-led processed entry (`#143 | {url} |
* … | 4.2/5 | PDF ✅`) puts the SCORE in that position, and keying on it would
* invent a city for an already-processed role.
* - A labeled segment (`posted:` / `trust:` / `yrs:` / `note:` / `rank:`) slides into that
* position when the offer carried no location and no compensation, so it is
* skipped too. Only those five labels count — see
* {@link PIPELINE_LABELED_SEGMENT_RE} for why a bare `word:` prefix does not.
*
* Both fall back to `''` (location unknown), which `companyRoleDedupKey` reads as
* "matches every city" — the conservative direction.
*
* @param {string} line - One raw line of `data/pipeline.md`.
* @returns {{company: string, role: string, location: string}|null} The pair plus
* its location (`''` when unknown), or null when the line has none to contribute.
*/
export function extractPipelineCompanyRole(line) {
const entry = pipelineEntry(line, PIPELINE_CHECKBOX_STRICT_RE);
if (!entry || entry.expired) return null;
const cells = entry.body.split('|').map(cell => cell.trim());
if (cells[0].startsWith('#--')) return null;
const urlIndex = cells.findIndex(cell => PIPELINE_URL_RE.test(cell));
if (urlIndex === -1) return null;
const [company = '', role = '', third = ''] = cells.slice(urlIndex + 1);
const location = urlIndex === 0 && !PIPELINE_LABELED_SEGMENT_RE.test(third) ? third : '';
return { company, role, location, url: cells[urlIndex] };
}
/**
* Build the seen-URL set from already-read source texts. An absent file is
* passed as '' (the readIfExists convention shared with
* `collectSeenCompanyRoles`) — every parse below yields nothing on ''.
*
* `extraTokensFor(url, portal)` (optional) lets a caller add provider-scoped
* dedup tokens alongside the plain normalized-URL one — e.g. a Workday
* requisition served under several tenant sites (#3439): a historical row
* recorded under site A's URL wouldn't otherwise match a fresh job fetched
* from site B, since normalizeUrlForDedup compares URLs verbatim. Only
* scan-history.tsv rows carry a `portal` column (the pipeline.md/
* applications.md sources don't record which scanner/provider produced a
* URL), so the hook only fires there. Return a string, an array of strings,
* or a falsy value for "nothing extra".
*/
function scanHistoryRows(text, rows) {
return rows ?? parsedScanHistory(text);
}
// Heading depth decides which pipeline rows are finished work. `## Processed`
// and a checked box are finished. A deeper heading is a subdivision, and a
// shallower heading ends the section. collectSeenUrls and the pipeline write
// share this so a row one of them releases is a row the other can write.
const PIPELINE_SECTION_LEVEL = 2;
function processedSectionAfter(line, inProcessed) {
const heading = line.match(/^(#+)\s+(.*)$/);
if (!heading || heading[1].length > PIPELINE_SECTION_LEVEL) return inProcessed;
return heading[1].length === PIPELINE_SECTION_LEVEL
&& /^processed\b/i.test(heading[2].trim());
}
function pipelineRowIsFinished(line, inProcessed) {
return inProcessed || /^\s*- \[x\]/i.test(line);
}
export function collectSeenUrls(sources = {}, policy = {}, { extraTokensFor } = {}) {
const { scanHistoryText = '', pipelineText = '', applicationsText = '', historyRows = null } = sources;
const seen = new Set();
// Which language versions each seen URL key stands for — read by
// isUnseenLanguageVersion under scan_history.dedup_include_language.
const variants = new Map();
const identityPromotableUrls = new Map();
// Rows the age policy has released. Held rather than counted here: the two
// sources parsed below carry no age policy of their own, so a row the TTL has
// just freed can be re-pinned a few lines later. The count is taken at the end,
// against the finished set, so it reports what is actually rescannable.
const recheckCandidates = new Set();
const recheckListingTokens = new Map();
const addIdentityPromotableUrl = (urlToken, listingKey = '') => {
if (!identityPromotableUrls.has(urlToken)) identityPromotableUrls.set(urlToken, new Set());
identityPromotableUrls.get(urlToken).add(listingKey || null);
};
const pinHistoryRow = (urlToken, url, title, language) => {
seen.add(urlToken);
const recorded = languageFormsForDedup(language);
recordUrlVariant(variants, urlToken, recorded.length ? recorded : languageFormsFromUrl(url), title);
};
const dupRows = [];
const agedOrphanDups = new Set();
// scan-history.tsv
// Line 0 only when it is the header; a headerless legacy file starts with a data row.
// `historyRows` is that parse already done, so a snapshot does not repeat it.
for (const row of scanHistoryRows(scanHistoryText, historyRows)) {
const { url, first_seen: firstSeen, portal, status: rawStatus, title, language, listing_key: listingKey } = row;
const status = rawStatus || 'added';
if (!url) continue;
// Not pinned, no listing identity, and not a recheck candidate either: the
// row records a config rejection or an inconclusive Apply-control check,
// and the URL was never queued, so counting it as "eligible again" would
// overstate what the TTL released. See OBSERVATIONAL_SCAN_HISTORY_STATUSES.
if (OBSERVATIONAL_SCAN_HISTORY_STATUSES.has(status)) continue;
// Under a recheck window a repeat sighting takes its verdict from the URL's
// other rows, which can sit anywhere in the file, so it waits for all of
// them (next loop). With no window it pins like any other row.
if (status === 'skipped_dup' && policy.recheckAfterDays != null) {
dupRows.push({ url, firstSeen, title, language });
continue;
}
if (shouldDedupScanHistoryRow({ firstSeen, status }, policy)) {
const urlToken = seenJobKey(url);
pinHistoryRow(urlToken, url, title, language);
// Only accepted or intentionally deferred posting rows can teach this
// scan a strong identity. A verification rejection such as an expired
// URL may pin that URL, but must not suppress a different live alias
// with the same provider identity. (A missing Apply control never gets
// here: its row is observational and pins nothing, see above.)
const identityMayPromote = status === 'added' || status.startsWith('cooldown:');
if (identityMayPromote) {
addIdentityPromotableUrl(urlToken, listingKey);
}
if (extraTokensFor && identityMayPromote) {
for (const token of [].concat(extraTokensFor(url, portal, listingKey) || [])) {
if (token) seen.add(token);
}
}
} else {
const urlToken = seenJobKey(url);
recheckCandidates.add(urlToken);
if (status === 'added' && listingKey) {
recheckListingTokens.set(urlToken, `listing:${listingKey}`);
}
}
}
// A `skipped_dup` row is dated the day of the repeat, and a scanner writes one
// on every run that lists the URL, so reading that date would restart the
// clock each time. The URL's own rows decide instead, whatever the repeat's
// age: one still pinned keeps it seen, and once they are all released the
// repeat goes with them.
//
// Only a row with no original here falls back on its own `first_seen`: the
// agent workflow in modes/scan.md writes these for a URL whose original lives
// in the tracker or pipeline. An aged one stops pinning and nothing more. It
// stays out of recheckCandidates, which the pipeline pass below reads as
// "scan-history released this URL": a row repeating a `- [x]` entry would
// otherwise un-pin the very entry it repeats.
for (const { url, firstSeen, title, language } of dupRows) {
const urlToken = seenJobKey(url);
if (!seen.has(urlToken) && recheckCandidates.has(urlToken)) continue;
if (seen.has(urlToken) || shouldDedupScanHistoryRow({ firstSeen, status: 'skipped_dup' }, policy)) {
pinHistoryRow(urlToken, url, title, language);
} else {
agedOrphanDups.add(urlToken);
}
}
// pipeline.md — extract URLs from checkbox lines, wherever the URL sits in the
// line (see extractPipelineUrl: five of the six documented shapes lead with a
// report number, a report link, or a strikethrough rather than the URL).
//
// This loop carried no age policy, which is what made
// `scan_history.recheck_after_days` a no-op: on an install that has been
// scanning for a while every URL the TTL releases above is listed here too, so
// it was re-pinned immediately and the window never freed anything.
//
// What may be released is decided by whether the row is still ACTIONABLE. A
// `- [ ]` row is a line the user can still pull from, and re-scanning it would
// append a SECOND copy of a job already on the list. A `- [x]` row, or any row
// under `## Processed`, is finished work: no queue entry can be duplicated, so
// the scan-history TTL is allowed to govern it alone. Release also requires the
// URL to be a recheck candidate, so a pipeline-only URL is never un-pinned.
// Sections are `##` in PIPELINE_SKELETON — `# Pipeline` is the document title,
// `## Pending` and `## Processed` are the sections. Heading depth decides what
// a heading does, in both directions:
//
// deeper than a section (`###`) a subdivision INSIDE it; changes nothing,
// so `### August` under `## Processed` stays
// released and `### Processed leftovers`
// under `## Pending` releases nothing
// at section level (`##`) ends the previous section and opens this
// one; released only if it is `Processed`
// shallower (`#`) outranks a section, so it ends it too — a
// `# Backlog` after `## Processed` must not
// inherit the released state
//
// Both failures are the same failure: a row that is still queued gets handed
// back to the scanner, which appends a second copy of a job already on the
// list. That duplication is what this function exists to prevent.
let inProcessed = false;
for (const line of pipelineText.split('\n')) {
inProcessed = processedSectionAfter(line, inProcessed);
const url = extractPipelineUrl(line);
if (!url) continue;
const key = seenJobKey(url);
if (pipelineRowIsFinished(line, inProcessed) && recheckCandidates.has(key)) continue;
seen.add(key);
recordUrlVariant(variants, key, languageFormsFromUrl(url), extractPipelineCompanyRole(line)?.role);
// A stale history row is still pinned while its actionable pipeline entry
// exists. Keep its strong identity pinned for the same period, or an ATS
// alias could slip through while the original URL remains queued.
const listingToken = recheckListingTokens.get(key);
if (listingToken) {
seen.add(listingToken);
addIdentityPromotableUrl(key, recheckListingTokens.get(key)?.slice('listing:'.length));
}
}
// applications.md — extract URLs from report links and any inline URLs
for (const match of applicationsText.matchAll(/https?:\/\/[^\s|)]+/g)) {
const key = seenJobKey(match[0]);
seen.add(key);
recordUrlVariant(variants, key, languageFormsFromUrl(match[0]));
}
// Counted against the finished set: a released row that applications.md or an
// actionable pipeline row pinned again is not eligible, and saying so keeps the
// number the scanners print honest.
let recheckEligible = 0;
for (const key of [...recheckCandidates, ...agedOrphanDups]) if (!seen.has(key)) recheckEligible++;
return { seen, recheckEligible, identityPromotableUrls, variants };
}
// Path options mirror mergeIntoPipeline's seam below: the defaults are the
// CAREER_OPS_ROOT-anchored module constants, and a caller with its own lane
// (or a test with a fixture) passes explicit paths. Before CAREER_OPS_ROOT the
// defaults were cwd-relative strings, so callers could retarget them by
// chdir'ing; an anchored default needs a real parameter instead.
export function loadSeenUrls(policy = {}, {
scanHistoryPath = SCAN_HISTORY_PATH,
pipelinePath = PIPELINE_PATH,
applicationsPath = APPLICATIONS_PATH,
extraTokensFor,
} = {}) {
return collectSeenUrls({
scanHistoryText: readIfExists(scanHistoryPath),
pipelineText: readIfExists(pipelinePath),
applicationsText: readIfExists(applicationsPath),
}, policy, { extraTokensFor });
}
/**
* Normalize a company label when no alias map is configured.
*
* This deliberately does only the pre-existing behavior: trim and lowercase the
* raw company name. `buildCompanyCanonicalizer` wraps this with the optional
* alias map so installs without `company_aliases` keep byte-for-byte dedupe
* semantics.
*
* @param {unknown} name - Raw company value from a tracker row or provider job.
* @returns {string} Lowercased, trimmed company key.
*/
function defaultCompanyNormalizer(name) {
return String(name ?? '').trim().toLowerCase();
}
/**
* Build a company-name canonicalizer from `config.company_aliases`.
*
* The map is `{ CanonicalName: [alias, ...] }`; every alias and the canonical
* name itself resolve to the lowercased canonical name. This closes the gap
* where an ATS org name, for example Greenhouse "Intercom", differs from the
* tracker/brand label, for example "Fin". Without it, the company+role dedupe
* key never matches the tracker and the same role is re-scanned every run.
*
* Unknown names pass through as plain lowercased text, so behavior is unchanged
* for companies with no alias entry.
*
* Canonical names always keep their own identity when an alias collides with
* one. An alias claimed by multiple canonical companies also passes through
* unchanged so malformed config cannot silently merge unrelated companies.
*
* @param {Record<string, unknown>|undefined|null} aliases - Optional canonical
* company name to alias list map.
* @returns {(name: unknown) => string} Canonicalizer for tracker and scan-side
* company labels.
*/
export function buildCompanyCanonicalizer(aliases) {
const map = new Map();
if (aliases && typeof aliases === 'object' && !Array.isArray(aliases)) {
const entries = Object.entries(aliases);
const canonicalKeys = new Set();
// Canonical names always own their identity, independent of YAML key order.
for (const [canonical] of entries) {
const canon = defaultCompanyNormalizer(canonical);
if (!canon) continue;
map.set(canon, canon);
canonicalKeys.add(canon);
}
const aliasTargets = new Map();
for (const [canonical, list] of entries) {
const canon = defaultCompanyNormalizer(canonical);
if (!canon) continue;
const arr = Array.isArray(list) ? list : [list];
for (const a of arr) {
const alias = defaultCompanyNormalizer(a);
if (!alias || canonicalKeys.has(alias)) continue;
if (!aliasTargets.has(alias)) aliasTargets.set(alias, new Set());
aliasTargets.get(alias).add(canon);
}
}
// Ambiguous aliases fail open as their raw normalized label. This may allow
// a duplicate through, but it cannot silently suppress another company.
for (const [alias, targets] of aliasTargets) {
if (targets.size === 1) map.set(alias, targets.values().next().value);
}
}
/**
* Canonicalize one raw company label through the alias map.
*
* @param {unknown} name - Raw company value from a tracker row or provider job.
* @returns {string} Canonical lowercased company key.
*/
return function canonicalizeCompany(name) {
const key = defaultCompanyNormalizer(name);
return map.get(key) ?? key;
};
}
const ROLE_LOCATION_SUFFIXES = new Set([
'amer',
'americas',
'amsterdam',
'apac',
'austin',
'barcelona',
'bay area',
'belgium',
'berlin',
'boston',
'brussels',
'budapest',
'canada',
'chicago',
'copenhagen',
'dublin',
'emea',
'eu',
'europe',
'finland',
'france',
'frankfurt',
'germany',
'hamburg',
'helsinki',
'india',
'ireland',
'italy',
'la',
'latin america',
'lisbon',
'london',
'los angeles',
'madrid',
'melbourne',
'milan',
'montreal',
'munich',
'netherlands',
'new york',
'north america',
'nyc',
'on site',
'onsite',
'oslo',
'paris',
'poland',
'porto',
'prague',
'remote',
'rome',
'san francisco',
'seattle',
'sf',
'singapore',
'spain',
'stockholm',
'sydney',
'tokyo',
'toronto',
'uk',
'united kingdom',
'united states',
'us',
'usa',
'vancouver',
'vienna',
'warsaw',
'zurich',
]);
const ROLE_REMOTE_SUFFIXES = new Set([
'distributed',
'hybrid',
'on site',
'onsite',
'remote',
'wfh',
'work from home',
]);
/**
* Normalize bracket text before checking whether it is a location suffix.
*
* @param {unknown} tag - Text from a trailing parenthetical or bracket suffix.
* @returns {string} Lowercased, punctuation-normalized suffix text.
*/
function normalizeRoleSuffixTag(tag) {
return String(tag ?? '')
.toLowerCase()
.replace(/[^a-z0-9]+/g, ' ')
.trim()
.replace(/\s+/g, ' ');
}
/**
* Decide whether a trailing role-title suffix is a location/remote tag.
*
* Only known remote/location suffixes are stripped. Seniority, discipline, team,
* and product qualifiers are intentionally preserved so distinct role variants
* do not collapse to the same scanner dedupe key.
*
* @param {unknown} tag - Text from a trailing parenthetical or bracket suffix.
* @returns {boolean} True when the suffix is safe to remove for dedupe.
*/
function isRoleLocationSuffix(tag) {
const normalized = normalizeRoleSuffixTag(tag);
if (!normalized) return false;
if (ROLE_LOCATION_SUFFIXES.has(normalized)) return true;
const raw = String(tag ?? '').toLowerCase();
const parts = raw
.split(/[,/|;]+|\s+(?:and|or)\s+/g)
.map(normalizeRoleSuffixTag)
.filter(Boolean);
if (parts.length > 1 && parts.every(part => ROLE_LOCATION_SUFFIXES.has(part))) return true;
for (const remote of ROLE_REMOTE_SUFFIXES) {
const prefix = `${remote} `;
if (normalized.startsWith(prefix) && ROLE_LOCATION_SUFFIXES.has(normalized.slice(prefix.length))) {
return true;
}
}
return false;
}
// "(m/w/d)", "(f/m/x)", "(H/F)", "(all genders)": the letter order differs by
// board, and the tag never tells two roles apart. "h" (homme) only counts
// next to "f": "(H/W)" is hardware, a discipline that does tell roles apart.
const ROLE_GENDER_TAG_RE = /^(?:[mwfdx](?:\s*\/\s*(?:[mwfdx]|div|divers)){1,3}|(?:h\s*\/\s*f|f\s*\/\s*h)(?:\s*\/\s*[xd])?|all genders?)$/;
/**
* Normalize a role title for stable scan-time duplicate identity.
*
* Equivalent tracker/provider titles should collapse to one key when a company
* splits a role per location with a trailing tag like "(Berlin)". Requisition
* IDs live in URLs rather than titles, so this identity remains URL-agnostic.
*
* The normalizer lowercases the title, strips trailing location/remote
* parenthetical/bracketed tags such as "(Berlin)" and "[Remote]", then
* collapses punctuation and whitespace so em dash vs hyphen or double spaces do
* not split a key.
*
* This helper does not infer posting churn or detect repost clusters. Those
* post-tracking facts remain the responsibility of detect-reposts.mjs and the
* company-history `postingChurn` axis.
*
* @param {unknown} role - Raw role title from a tracker row or provider job.
* @returns {string} Normalized role key.
*/
export function normalizeRoleForDedup(role) {
// NFKC up front so full-width brackets fold to their ASCII forms while the
// suffix loop can still see them: "Engineer （Remote）" now strips the same
// way "Engineer (Remote)" always did.
//
// This does NOT make the loop understand non-Latin suffixes — the tag itself
// is still matched against the English-only ROLE_LOCATION_SUFFIXES set via
// normalizeRoleSuffixTag(), which carries its own [a-z0-9] strip. So
// "エンジニア（東京）" and "エンジニア（大阪）" remain two keys. Teaching the
// suffix vocabulary other scripts is a separate change (new vocabulary, not
// a key fix) and is deliberately out of scope here.
let title = String(role ?? '').normalize('NFKC').toLowerCase();
while (true) {
// Bracketed: "Engineering Manager (Remote)", "… [US]".
const bracketed = title.match(/\s*[\[(]([^[\]()]+)[\])]\s*$/);
if (bracketed && (isRoleLocationSuffix(bracketed[1]) || ROLE_GENDER_TAG_RE.test(bracketed[1].trim()))) {
title = title.slice(0, bracketed.index).trimEnd();
continue;
}
// Pipe-delimited: "Senior Engineering Manager, Grafana Frontend | USA | Remote".
// Grafana Labs posts one requisition per country and packs the place into
// the TITLE rather than only the location field, so the same role arrives
// as six titles. Without this, the tracker's clean title never matches the
// scanned one and an already-evaluated role is re-added on the next scan.
// Only the trailing segment is considered, and only when it is a known
// location suffix, so "Engineering Manager | Payments" keeps its qualifier.
const piped = title.match(/\s*\|\s*([^|]+?)\s*$/);
if (piped && isRoleLocationSuffix(piped[1])) {
title = title.slice(0, piped.index).trimEnd();
continue;
}
break;
}
// Unicode-aware (#2393 family): the [a-z0-9] strip this used to carry keyed
// every non-Latin title to '', so バックエンドエンジニア and フロントエンド
// エンジニア at one company shared a dedupe key and the scan dropped the
// second as already-seen. Space separator keeps the word-collapsing shape.
return normalizeTextKey(title, ' ');
}
/**
* The separators a provider or a board uses to pack SEVERAL places into the one
* free-text location field.
*
* The field is not reliably a single place, and the packing is not done one way.
* Sampled across live Greenhouse boards, a multi-location value uses `;`, the
* word `or`, `|` or `/`, and a single value mixes two of them — Anthropic ships
* `"Boston, MA; Remote-Friendly (Travel-Required) | San Francisco, CA | Seattle,
* WA | New York City, NY; Washington, DC"`. The scanner's own providers add a
* fifth: greenhouse/ashby/eightfold/gem/ibm/echojobs all fold a multi-site role's
* extra cities into the string themselves, `' · '`-joined, in whatever order the
* upstream array happened to arrive in.
*
* `,` is deliberately NOT a separator. It is the city/region delimiter INSIDE a
* place ("London, UK"), so splitting on it would shatter every ordinary location
* into fragments and make "London, UK" and "Dublin, UK" share the fragment `uk`.
*
* Used only by {@link normalizeLocationForDedup}; nothing else parses the field.
*/
const LOCATION_LIST_SEPARATOR_RE = /\s*(?:[;|\u00b7/]|\bor\b)\s*/iu;
/**
* Normalize a posting location into a dedupe-key component.
*
* Each place is keyed by the same rule as the role (`normalizeTextKey`,
* space-separated) so "London, UK" and "London UK" are one city and no second
* private strip has to exist. A missing, blank or non-string location yields ''
* — see {@link companyRoleDedupKey} for what that means.
*
* A value carrying several places is reduced to the SET of them, sorted, rather
* than keyed as the verbatim string. Keying the string verbatim is stable only
* while the employer lists the cities in a constant order, and nothing holds
* that order still: the value is free text on the board's side, and on ours the
* providers listed above build it from an upstream array. Re-order the list and
* the verbatim key changes, so a posting already in scan-history reads as new
* and re-enters the pipeline — the same duplicate-per-scan failure the location
* key exists to avoid, arriving from the other direction. Sorting the set makes
* the key depend on WHICH places a posting names and not on the order it names
* them in, and deduplicating it absorbs the boards that repeat a city.
*
* A single-place value is unaffected: it splits into one segment and joins back
* to exactly the string this function returned before, so keys already seeded
* from single-place rows keep matching.
*
* @param {unknown} location - Raw location label from a provider or a data file.
* @returns {string} Normalized location key, or '' when unknown.
*/
export function normalizeLocationForDedup(location) {
if (typeof location !== 'string') return '';
const places = new Set();
for (const segment of location.split(LOCATION_LIST_SEPARATOR_RE)) {
const place = normalizeTextKey(segment, ' ');
if (place) places.add(place);
}
// `+` cannot appear inside a component: normalizeTextKey keeps only letters,
// marks and digits, so the join is unambiguous and two different sets can
// never render as one string.
return [...places].sort().join('+');
}
/**
* Build the canonical company+role dedupe key.
*
* This shared helper is used by both the tracker-side load and the scan-side
* check so those two code paths cannot drift. `canonicalize` defaults to plain
* lowercase/trim behavior when no alias map is configured.
*
* `location` is the opt-in fourth component (`scan_history.dedup_include_location`,
* see `resolveDedupIncludeLocation`). Callers that omit it — every caller before
* the flag existed — get the identical `company::role` string they always did,
* which is what makes the flag inert by default.
*
* A key with NO location is deliberately a wildcard rather than a fourth
* component that happens to be empty: the seed sources do not all record a
* location (applications.md usually has no Location column, and a report-led
* pipeline row puts a score where a pending row puts the city), so a bare key
* has to keep matching every city. Losing that would let a role the user has
* already applied to resurface once per city the moment the flag went on.
*
* The location component is the canonical SET of the places the posting names
* (see {@link normalizeLocationForDedup}), not the provider's display string, so
* a board that re-orders a multi-city list does not turn one posting into two.
*
* @param {unknown} company - Raw company label.
* @param {unknown} role - Raw role title.
* @param {(name: unknown) => string} [canonicalize] - Company canonicalizer.
* @param {unknown} [location] - Optional posting location. Blank/absent/non-string
* → the bare two-component key.
* @returns {string} Stable dedupe key: `company::role`, or `company::role@@places`.
*/
export function companyRoleDedupKey(company, role, canonicalize = defaultCompanyNormalizer, location = undefined) {
const base = `${canonicalize(company)}::${normalizeRoleForDedup(role)}`;
const places = normalizeLocationForDedup(location);
// `@@` and not a third `::`: normalizeTextKey strips punctuation, so neither
// the role nor the location component can ever contain the separator, and a
// company alias carrying one cannot forge a located key out of a bare one.
return places ? `${base}@@${places}` : base;
}
/**
* Marker for a seeded company+role row whose requisition is unknown (in the
* language sets: whose language is unknown). A single such row keeps the key a
* plain duplicate, exactly as if neither signal were read at all.
*/
export const ANY_REQUISITION = '*';
/**
* Canonical requisition IDs for company+role dedupe — every form the source
* could name; empty when none is known.
*
* Two same-titled postings at one employer are not always one role: UBC ran two
* "Programmer Analyst I" requisitions at once (JR25919 and JR25853, different
* departments), and the second was dropped as a duplicate of the first. The
* requisition is the one signal that tells them apart. It is read from:
*
* - a requisition id the provider read from a dedicated ATS field
* (`Job.requisitionId`, e.g. SmartRecruiters `refNumber`), or the same value
* recorded in scan-history's `requisition_id` column. Not parsed: it is the
* employer's own identifier, not text, so it counts even without a digit.
* It is compared case-folded, with whitespace flattened the way the
* scan-history writer flattens it, so a live id meets its copy read back
* from the file. Forms read from a URL or from text must contain a digit;
* - a Workday URL, via `workdayDedupKey` (the same parse the provider uses for
* cross-site dedupe, #3439, including its `-N` repost-suffix stripping);
* - labelled free text (tracker Notes, a posting title) via the tracker's own
* `extractReqNumber`, the vocabulary merge-tracker.mjs already relies on to
* keep same-title rows apart (#1524).
*
* Only these explicit forms count. A generic board's numeric posting ID is not
* read: a repost gets a new one, and treating it as a requisition would disable
* the company+role key it is layered on.
*
* A labelled ID with a trailing `-N` is ambiguous. On Workday the suffix is a
* cross-site repost disambiguator and `JR25919-1` IS JR25919 (the URL path
* strips it via `workdayDedupKey`); on Lever or Greenhouse `ABC123-1` and
* `ABC123-2` are two requisitions. The source decides how many forms come
* back:
*
* - Workday URL: one form, the suffix stripped.
* - Known non-Workday URL: one form, the label kept whole.
* - No URL (a tracker note on a layout without a URL column): BOTH forms, as
* labelled and suffix-stripped. Nothing is guessed. A seeded row records
* every form and a candidate is distinct only when NONE of its forms was
* seen, so the ambiguous note matches whichever board the posting turns out
* to live on: note `req JR25919-1` recognises Workday `_JR25919`, and note
* `req ABC123-1` recognises a Lever title carrying `req ABC123-1` (whose
* own single form is `ABC123-1`), so neither applied posting is re-queued.
* Guessing one form was wrong in both directions: stripping changed the
* Lever ID, while keeping only the suffix-bearing form missed Workday.
*
* The suffix rule (`stripWorkdayRepostSuffix`) only strips a one- or
* two-digit tail, so Walmart's `R-2593225` is one form on every path.
*
* Comparison ignores case only: prefixes and punctuation identify distinct
* requisitions. Bare JR/R_ tokens retain the prefix consumed as a label by
* the shared tracker parser, including label separators such as `JR: 25919`.
* Glued punctuation (`JR-25919`) remains part of the identifier. Numeric-only
* text (`Req #25919`) may omit a prefix, so it supplies no proof of a distinct
* requisition. A numeric ID extracted from a Workday URL is authoritative.
*
* @param {{url?: unknown, text?: unknown, requisitionId?: unknown}} [source] -
* Posting URL, free text, and/or a provider-supplied requisition id. The id
* wins when present.
* @returns {string[]} Canonical requisition IDs, as-labelled form first.
*/
export function requisitionIdsForDedup({ url, text, requisitionId } = {}) {
const suppliedId = typeof requisitionId === 'string' ? requisitionId.trim() : '';
const workdayKey = !suppliedId && typeof url === 'string' ? workdayDedupKey({ url }) : null;
if (suppliedId) return [normalizeScanScalar(suppliedId).toUpperCase()];
let raws;
if (workdayKey) {
// `workday:{hostname}:{reqId}` — a hostname has no colon, so the ID is
// everything after the second one.
raws = [workdayKey.split(':').slice(2).join(':')];
} else {
const match = String(text ?? '').match(REQ_NUMBER_RE);
const separatedPrefix = match?.[0].match(/^(JR|R_)[\s:#]+/i);
const labelled = match && /^(?:JR[-_]?|R_)\d/i.test(match[0])
? match[0].toUpperCase()
: separatedPrefix && /^\d/.test(match[1])
? `${separatedPrefix[1]}${match[1]}`.toUpperCase()
: extractReqNumber(text);
if (!labelled) return [];
if (/^[\d-]+$/.test(labelled)) return [];
const workdayUrl = isWorkdayJobUrl(url);
raws = workdayUrl === true
? [stripWorkdayRepostSuffix(labelled)]
: workdayUrl === false ? [labelled] : [labelled, stripWorkdayRepostSuffix(labelled)];
}
const forms = [];
for (const raw of raws) {
const id = String(raw ?? '').toUpperCase();
if (/\d/.test(id) && !forms.includes(id)) forms.push(id);
}
return forms;
}
/**
* The source's requisition ID as labelled — the first form of
* {@link requisitionIdsForDedup} — or null. A convenience for callers that
* want one ID to print; the dedupe decision itself compares every form.
*
* @param {{url?: unknown, text?: unknown}} [source]
* @returns {string|null}
*/
export function requisitionIdForDedup(source) {
return requisitionIdsForDedup(source)[0] ?? null;
}
/**
* Whether a candidate that matched a seen company+role key is nevertheless a
* different requisition.
*
* True only when every seeded row for the key named its requisition and none of
* the candidate's forms was seen. Any unknown on either side keeps the
* historical answer — a duplicate — so the check can only let through postings
* the company itself labelled as distinct. An ambiguous source contributes
* every form it could mean (see {@link requisitionIdsForDedup}), and one hit
* on any of them is a duplicate.
*
* @param {Set<string>|undefined} seededRequisitions - Requisitions seen for matching keys.
* @param {string[]|string|null} candidateRequisitions - From {@link requisitionIdsForDedup}.
* @returns {boolean}
*/
export function isDistinctRequisition(seededRequisitions, candidateRequisitions) {
return unseenOnEveryRow(seededRequisitions, candidateRequisitions);
}
/**
* The posting language as dedup forms, or none when the source did not say:
* the one form languageSubtag (url-key.mjs) reduces it to, shared with the
* tracker's URL comparison.
*
* @param {unknown} language - `Job.language`, or a scan-history `language` cell.
* @returns {string[]} Zero or one form.
*/
export function languageFormsForDedup(language) {
const form = languageSubtag(language);
return form ? [form] : [];
}
/**
* Whether a candidate that matched a seen company+role key is nevertheless
* another language version of a seeded posting.
*
* Languages are compared per requisition, so a requisition seen on one posting
* and a language seen on another never add up to a duplicate. Only seeded rows
* that could be the candidate's requisition take part: rows naming one of its
* forms, rows of unknown requisition, and every row when the candidate names
* none. True only when at least one such row exists, every one of them named
* its language, and none of those is the candidate's. A row with no known
* language keeps the duplicate.
*
* @param {Map<string, Set<string>>|undefined} seededLanguages - Languages seen
* for matching keys, by requisition form ({@link ANY_REQUISITION} for a row
* that named none), as recorded by {@link recordLanguages}.
* @param {string[]} candidateLanguages - From {@link languageFormsForDedup}.
* @param {string[]|string|null} [candidateRequisitions] - From {@link requisitionIdsForDedup}.
* @returns {boolean}
*/
export function isDistinctLanguage(seededLanguages, candidateLanguages, candidateRequisitions = []) {
if (!(seededLanguages instanceof Map)) return false;
const requisitions = toRequisitionForms(candidateRequisitions);
const overlapping = [...seededLanguages]
.filter(([requisition]) => requisitions.length === 0
|| requisition === ANY_REQUISITION
|| requisitions.includes(requisition))
.map(([, languages]) => languages);
return overlapping.length > 0
&& overlapping.every(languages => unseenOnEveryRow(languages, candidateLanguages));
}
function unseenOnEveryRow(seeded, candidate) {
const forms = toRequisitionForms(candidate);
return forms.length > 0
&& seeded instanceof Set
&& seeded.size > 0
&& !seeded.has(ANY_REQUISITION)
&& forms.every(form => !seeded.has(form));
}
function toRequisitionForms(requisitions) {
if (Array.isArray(requisitions)) return requisitions.filter(Boolean);
return requisitions ? [requisitions] : [];
}
/** Match only the requisition and language sets belonging to keys that match
* this location. A locationless candidate overlaps every located row, but a
* located candidate overlaps only its exact key and genuinely locationless
* wildcard rows. A seen key stops being a duplicate when the candidate is a
* different requisition OR (with language-aware dedup, i.e. a non-empty
* `candidateLanguages`) a language version no seeded posting of its
* requisition was in.
*/
export function matchesSeenCompanyRole({ key, baseKey, seen, requisitions, locatedRequisitions, languages = new Map(), locatedLanguages = new Map() }, candidate, candidateLanguages = []) {
if (key === null) return false;
const distinct = (requisitionSets, languageSets, k) =>
isDistinctRequisition(requisitionSets.get(k), candidate)
|| isDistinctLanguage(languageSets.get(k), candidateLanguages, candidate);
if (seen.has(key) && !distinct(requisitions, languages, key)) return true;
if (key !== baseKey && seen.has(baseKey)
&& !distinct(requisitions, languages, baseKey)) return true;
return key === baseKey && locatedRequisitions.has(baseKey)
&& !distinct(locatedRequisitions, locatedLanguages, baseKey);
}
/** Record a row's forms (requisitions or languages) under a key, or the
* {@link ANY_REQUISITION} marker when the row named none. */
function recordForms(formsByKey, key, forms) {
let seen = formsByKey.get(key);
if (!seen) formsByKey.set(key, (seen = new Set()));
const list = toRequisitionForms(forms);
if (list.length === 0) seen.add(ANY_REQUISITION);
for (const form of list) seen.add(form);
}
/** Record a row's languages under a key, filed under each of its requisition
* forms ({@link ANY_REQUISITION} when it named none) — the shape
* {@link isDistinctLanguage} reads. */
function recordLanguages(languagesByKey, key, requisitions, languages) {
let byRequisition = languagesByKey.get(key);
if (!byRequisition) languagesByKey.set(key, (byRequisition = new Map()));
const list = toRequisitionForms(requisitions);
for (const requisition of list.length > 0 ? list : [ANY_REQUISITION]) {
recordForms(byRequisition, requisition, languages);
}
}
/**
* Build the seen-role set from the same three sources as `loadSeenUrls`.
*
* Existing rows are canonicalized with the same company aliasing and role-title
* normalization used for freshly scanned jobs. That lets URL-new duplicates match
* older entries instead of being evaluated again.
*
* Seeding from applications.md alone made the key effectively intra-run: a role
* added by a prior scan lives in scan-history and pipeline, and does not reach
* applications.md until the user evaluates and applies. Companies that open one req
* per city therefore leaked one city variant per scan — run 1 added the SF req
* (marking the key in memory only), run 2 re-seeded from applications.md, found the
* key absent, and the NY req cleared both the URL check and the role check.
*
* Two deliberate semantics on the scan-history source:
*
* - Only `added` rows seed a key. `skipped_expired` / `skipped_invalid_url` /
* `skipped_blocked_host` are URL-level failures, not evidence the role was
* surfaced; seeding from them would let a dead SF URL bury a live NY req. Because
* an expired posting is recorded as `skipped_expired` rather than `added`, this
* self-heals: when the canonical posting dies, its city variants become eligible
* again on the next scan.
* - Seeding honours `scan_history.recheck_after_days` via the existing
* `shouldDedupScanHistoryRow` predicate, so the role key cannot outlive the URL
* key it mirrors.
*
* @param {{applicationsText?: string, scanHistoryText?: string, pipelineText?: string}} sources
* Raw text of each dedupe source; absent sources default to empty.
* @param {{recheckAfterDays?: number|null, today?: string}} [policy] - Scan-history
* recheck policy, shared with `loadSeenUrls`.
* @param {(name: unknown) => string} [canonicalize=defaultCompanyNormalizer] -
* Company canonicalizer shared with scan-side dedupe.
* @param {{includeLocation?: boolean, locatedBases?: Set<string>|null}} [options] -
* When `includeLocation` is true (`scan_history.dedup_include_location`), each
* row seeds a location-qualified key wherever its source records a location, and
* the bare wildcard key where it does not. Default false = the historical keys,
* byte for byte. `locatedBases`, when a Set is supplied, additionally collects
* the BARE key of every row that seeded a located one — see
* {@link loadDedupSnapshot} for what reads it. `requisitionsByBase`, when a Map
* is supplied, collects every requisition seen per actual dedup key (or
* {@link ANY_REQUISITION} for a row that named none) — see
* {@link isDistinctRequisition}. `locatedRequisitionsByBase` separately
* aggregates located rows for matching a locationless candidate; it never
* acts as a bare wildcard against a located candidate. `languagesByBase` and
* `locatedLanguagesByBase` do the same for posting languages, filed per key
* under each requisition form of the row that carried them (see
* {@link isDistinctLanguage}).
*
* Requisition id and language are recorded per posting URL in scan-history
* (`requisition_id`, `language`). A tracker or pipeline row carries neither, so
* it inherits them from the scan-history row with the same URL; a row whose URL
* scan-history never recorded them for stays unknown.
* @returns {Set<string>} Existing company+role dedupe keys.
*/
export function collectSeenCompanyRoles(sources = {}, policy = {}, canonicalize = defaultCompanyNormalizer, { includeLocation = false, locatedBases = null, requisitionsByBase = null, locatedRequisitionsByBase = null, languagesByBase = null, locatedLanguagesByBase = null } = {}) {
const { applicationsText = '', scanHistoryText = '', pipelineText = '', historyRows = null } = sources;
const seen = new Set();
const postingAttributes = collectPostingAttributes(scanHistoryText, historyRows);
const attributesFor = (url) => (typeof url === 'string' && url
? postingAttributes.get(seenJobKey(url.trim())) ?? {}
: {});
const add = (company, role, location, requisition = null, language = null) => {
const c = String(company ?? '').trim();
const r = String(role ?? '').trim();
if (!c || !r) return;
// Header and markdown-separator cells are not roles.
if (c.toLowerCase() === 'company') return;
if (/^[-:]+$/.test(c) || /^[-:]+$/.test(r)) return;
const key = companyRoleDedupKey(c, r, canonicalize, includeLocation ? location : undefined);
seen.add(key);
// Reverse index, recorded where the key is BUILT rather than by splitting a
// finished key back apart. `defaultCompanyNormalizer` only trims and
// lowercases, so a company label may legitimately contain the `@@`
// separator; a parsed base would then be wrong for exactly the rows that
// matter. Skipped entirely when the flag is off — no key can carry a
// location, so the set stays empty and every reader of it is inert.
if (locatedBases && includeLocation) {
const base = companyRoleDedupKey(c, r, canonicalize);
if (key !== base) locatedBases.add(base);
}
if (requisitionsByBase) {
recordForms(requisitionsByBase, key, requisition);
}
if (languagesByBase) {
recordLanguages(languagesByBase, key, requisition, language);
}
const base = companyRoleDedupKey(c, r, canonicalize);
if (locatedRequisitionsByBase && key !== base) {
recordForms(locatedRequisitionsByBase, base, requisition);
}
if (locatedLanguagesByBase && key !== base) {
recordLanguages(locatedLanguagesByBase, base, requisition, language);
}
};
// applications.md — header-aware parse (tracker-parse.mjs, #954). The old
// positional regex captured the wrong cells on customized layouts (e.g. with a
// Location column), so the seen-set keyed on garbage and dedup misfired.
//
// `row.location` is present only when the user's tracker actually has a
// Location column; the default layout has none, so an applied role seeds the
// wildcard and no city variant of it can resurface.
if (applicationsText) {
const lines = applicationsText.split('\n');
const colmap = resolveColumns(lines);
for (const line of lines) {
const row = parseTrackerRow(line, colmap);
if (!row) continue;
const attributes = attributesFor(row.url);
add(row.company, row.role, row.location,
requisitionIdsForDedup({ url: row.url, text: row.notes, requisitionId: attributes.requisitionId }),
languageFormsForDedup(attributes.language));
}
}
// scan-history.tsv — url, first_seen, portal, title, company, status, location,
// and (trailing) requisition_id + language. This is the source that carries a
// location for every scanned posting, and so the one that makes the opt-in key
// discriminate between two cities of one role.
for (const row of scanHistoryRows(scanHistoryText, historyRows)) {
const status = row.status || 'added';
if (!row.url) continue;
if (status !== 'added') continue;
if (!shouldDedupScanHistoryRow({ firstSeen: row.first_seen, status }, policy)) continue;
add(row.company, row.title, row.location,
requisitionIdsForDedup({ url: row.url, requisitionId: row.requisition_id }),
languageFormsForDedup(row.language));
}
// pipeline.md — company/title are the two cells after the URL cell, plus
// optional trailing columns (location, compensation, posted:/trust:/note:
// segments). The URL is not always first, and expired/pre-screen shapes
// contribute no pair at all — see extractPipelineCompanyRole. Same failure the
// applications.md branch above fixed in #954: a positional regex read the
// wrong cells, so the seen-set keyed on garbage.
for (const line of pipelineText.split('\n')) {
const pair = extractPipelineCompanyRole(line);
if (!pair) continue;
const attributes = attributesFor(pair.url);
add(pair.company, pair.role, pair.location,
requisitionIdsForDedup({ url: pair.url, requisitionId: attributes.requisitionId }),
languageFormsForDedup(attributes.language));
}
return seen;
}
/**
* Requisition id and language per posting URL, from every scan-history row that
* recorded them (whatever its status: these describe the posting, not whether it
* was surfaced). Keyed by {@link seenJobKey}, the same form the URL
* dedup uses, so a tracker or pipeline row spelling the URL differently still
* finds its posting. One URL can have several rows (a re-add, a verify
* outcome); they merge field by field, a later non-empty value replacing an
* earlier one and an empty cell keeping it.
*
* @param {string} scanHistoryText - Full scan-history.tsv contents.
* @param {Array<Record<string, string>>|null} [historyRows] - Rows from
* {@link parsedScanHistory} when the caller already parsed the file.
* @returns {Map<string, {requisitionId?: string, language?: string}>}
*/
export function collectPostingAttributes(scanHistoryText, historyRows = null) {
const attributes = new Map();
for (const row of scanHistoryRows(scanHistoryText, historyRows)) {
const requisitionId = row.requisition_id.trim();
const language = row.language.trim();
if (!row.url || (!requisitionId && !language)) continue;
const key = seenJobKey(row.url.trim());
attributes.set(key, {
...attributes.get(key),
...(requisitionId ? { requisitionId } : {}),
...(language ? { language } : {}),
});
}
return attributes;
}
function readIfExists(filePath) {
return existsSync(filePath) ? readFileSync(filePath, 'utf-8') : '';
}
/**
* Load company+role keys already surfaced by a prior scan or tracked by the user.
*
* Thin filesystem wrapper over {@link collectSeenCompanyRoles}, mirroring the
* source list `loadSeenUrls` already reads.
*
* The two leading positional parameters are unchanged, so existing callers keep
* working. The extra sources are injectable via the trailing options object: the
* module-level paths are relative to `process.cwd()`, so a test that passes only a
* sandbox tracker would otherwise pick up the developer's real scan-history and
* pipeline (CI only avoids this because those files are gitignored).
*
* @param {string} [appsPath=APPLICATIONS_PATH] - Applications tracker path.
* @param {(name: unknown) => string} [canonicalize=defaultCompanyNormalizer] -
* Company canonicalizer shared with scan-side dedupe.
* @param {object} [options] - Additional sources and policy.
* @param {{recheckAfterDays?: number|null, today?: string}} [options.policy] -
* Scan-history recheck policy, shared with `loadSeenUrls`.
* @param {string} [options.scanHistoryPath=SCAN_HISTORY_PATH] - Scan-history path.
* @param {string} [options.pipelinePath=PIPELINE_PATH] - Pipeline inbox path.
* @param {boolean} [options.includeLocation=false] - Opt-in location-aware keys,
* forwarded to {@link collectSeenCompanyRoles}.
* @param {Set<string>|null} [options.locatedBases=null] - Optional reverse-index
* sink, forwarded to {@link collectSeenCompanyRoles} so the two seed paths
* cannot drift.
* @returns {Set<string>} Existing company+role dedupe keys.
*/
export function loadSeenCompanyRoles(
appsPath = APPLICATIONS_PATH,
canonicalize = defaultCompanyNormalizer,
{ policy = {}, scanHistoryPath = SCAN_HISTORY_PATH, pipelinePath = PIPELINE_PATH, includeLocation = false, locatedBases = null } = {},
) {
return collectSeenCompanyRoles({
applicationsText: readIfExists(appsPath),
scanHistoryText: readIfExists(scanHistoryPath),
pipelineText: readIfExists(pipelinePath),
}, policy, canonicalize, { includeLocation, locatedBases });
}
// ── Pipeline writer ─────────────────────────────────────────────────
function normalizeScanScalar(value) {
return String(value ?? '')
.replace(/[\r\n\t]+/g, ' ')
.replace(/ {2,}/g, ' ')
.trim();
}
function normalizeScanUrl(value) {
return String(value ?? '').trim().split(/\s+/)[0] || '';
}
const MARKDOWN_ESCAPE_CHARS = {
'\\': '\\\\',
'[': '\\[',
']': '\\]',
};
export function sanitizeMarkdownField(value) {
return normalizeScanScalar(value)
.replace(/[\\[\]]/g, char => MARKDOWN_ESCAPE_CHARS[char])
.replace(/\|/g, '/');
}
function sanitizePipelineUrl(value) {
return normalizeScanUrl(value)
.replace(/[\\[\]]/g, char => MARKDOWN_ESCAPE_CHARS[char])
.replace(/\|/g, '%7C');
}
// One TSV cell as scan-history stores it: tabs and line breaks flattened to
// spaces (the format has no escape for them), then reversible formula
// escaping. parseScanHistoryLine undoes the escaping, so a reader gets back
// normalizeScanScalar(value).
export function sanitizeTsvField(value) {
return escapeFormulaCell(normalizeScanScalar(value));
}
// Format an offer's parsed compensation (the annualized {min,max,currency} that
// providers like Ashby attach as `offer.salary`) into a compact, sanitized cell
// such as `120000-160000 USD`. Returns '' when there is no usable salary data.
// Non-positive bounds are dropped (a 0 min/max is meaningless comp data, not "$0").
export function formatCompensation(salary) {
if (!salary || typeof salary !== 'object') return '';
const num = (n) => (Number.isFinite(n) && n > 0 ? String(Math.round(n)) : null);
const lo = num(salary.min);
const hi = num(salary.max);
const range = lo && hi && lo !== hi ? `${lo}-${hi}` : (lo || hi || '');
if (!range) return '';
const currency = typeof salary.currency === 'string' ? salary.currency.trim() : '';
return sanitizeMarkdownField(currency ? `${range} ${currency}` : range);
}
// Trust/legitimacy signal (#1743): the scanner sets offer.trustScore (0-100) +
// offer.trustFlags on every job (see buildTrustValidator). Surface it only when
// it's meaningful — a score below 100 means the validator penalized the posting
// (e.g. missing_apply_url, invalid_url, suspicious_domain). A clean posting
// (score 100) or a scan without trust_filter configured stays byte-identical
// (empty), exactly like the posted:/note: segments.
export function trustIsFlagged(offer) {
return typeof offer.trustScore === 'number' && Number.isFinite(offer.trustScore) && offer.trustScore < 100;
}
function trustFlagList(offer) {
return Array.isArray(offer.trustFlags)
? offer.trustFlags.filter((f) => typeof f === 'string' && f.trim())
: [];
}
// Labeled pipeline segment, e.g. `trust: 60 missing_apply_url,suspicious_domain`.
// '' when the posting isn't flagged, so an unflagged offer produces no segment.
export function formatTrustSegment(offer) {
if (!trustIsFlagged(offer)) return '';
const flags = trustFlagList(offer);
const body = flags.length ? `${offer.trustScore} ${flags.join(',')}` : String(offer.trustScore);
return sanitizeMarkdownField(`trust: ${body}`);
}
export function formatPipelineOffer(offer) {
const url = sanitizePipelineUrl(offer.url);
const company = sanitizeMarkdownField(offer.company);
const title = sanitizeMarkdownField(offer.title);
// Optional trailing columns, each sanitized like every other field:
// 4th = location, 5th = compensation.
// Gate location on an actual string so malformed provider data (a number or
// object) degrades to the 3-column form instead of stringifying into a
// spurious column. The columns are positional, so a present compensation
// forces the (possibly empty) location cell to keep comp in column 5.
// loadSeenUrls dedups on the URL and ignores trailing columns (backward-compatible).
const location = typeof offer.location === 'string' ? sanitizeMarkdownField(offer.location) : '';
const compensation = formatCompensation(offer.salary);
const base = `- [ ] ${url} | ${company} | ${title}`;
let line = base;
if (compensation) line = `${base} | ${location} | ${compensation}`;
else if (location) line = `${base} | ${location}`;
// Optional labeled posting-date segment (like note:) — keeps the positional
// 1/3/4/5-column contract in modes/pipeline.md intact.
const posted = postedAtIsoDate(offer.postedAt);
if (posted) line = `${line} | posted: ${posted}`;
// Labeled trust/legitimacy segment (#1743) — rides like posted:/note:, emitted
// only when the scanner flagged the posting (score < 100). Ordered after
// posted:, before note:, for a stable serialization.
const trust = formatTrustSegment(offer);
if (trust) line = `${line} | ${trust}`;
// Labeled years-of-experience segment (experience_filter): the JD's stated
// requirement ("5+", "3-5", "none stated", "unread"). Absent when the filter
// is off, so such a row stays byte-identical. After trust:, before note:.
const yrs = typeof offer.years?.label === 'string' ? sanitizeMarkdownField(offer.years.label) : '';
if (yrs) line = `${line} | yrs: ${yrs}`;
// Optional free-text ranking signal (e.g. a curated-list flag an importer
// attaches). Labeled — not positional like location/compensation — so it can
// ride on any row shape (bare URL, 3-, 4-, or 5-column) without a reader
// confusing it for a positional cell, and it stays generic: nothing here is
// source-specific, and an offer without `note` produces byte-identical output.
const note = typeof offer.note === 'string' ? sanitizeMarkdownField(offer.note) : '';
return note ? `${line} | note: ${note}` : line;
}
// postedAt arrives as epoch ms (or absent). Convert to 'YYYY-MM-DD', or '' when missing.
function postedAtIsoDate(postedAt) {
if (typeof postedAt !== 'number' || !Number.isFinite(postedAt) || postedAt <= 0) return '';
return new Date(postedAt).toISOString().slice(0, 10);
}
export function formatScanHistoryRow(offer, date, status = 'added') {
const record = {
url: normalizeScanUrl(offer.url),
first_seen: date,
portal: offer.source,
title: offer.title,
company: offer.company,
status,
location: offer.location || '',
// JD-content fingerprint (#1597): 16 hex chars when the provider's list
// API shipped a usable description, '' otherwise. Lets later scans flag
// the same body re-posted under a different company (agency cross-listing)
// without storing the body.
fingerprint: offer.fingerprint ?? fingerprintText(offer.description),
posted_at: postedAtIsoDate(offer.postedAt),
// Trust/legitimacy signal (#1743): score (only when the scanner flagged the
// posting, i.e. < 100) + comma-joined flags; a clean posting or a scan
// without trust_filter leaves both empty.
trust_score: trustIsFlagged(offer) ? String(offer.trustScore) : '',
trust_flags: trustIsFlagged(offer) ? trustFlagList(offer).join(',') : '',
// Normalized company key (#2093): the canonical company form shared across
// the tracker (normalizeCompanyName — lowercased, punctuation/whitespace
// folded, trailing legal-entity suffixes stripped) so "Acme Inc.",
// "Acme, Inc." and "ACME Inc" all key to `acme`. Stored at write time so
// repost/name-matching never has to route through executing a script, and
// the raw display `company` stays faithful to what the provider returned.
normalized_company: normalizeCompanyName(offer.company || ''),
// Requisition id and posting language, as the provider reported them
// (Job.requisitionId / Job.language); '' when it didn't. They let
// company+role dedup tell apart two same-titled postings with different
// requisitions or language variants, across runs.
requisition_id: typeof offer.requisitionId === 'string' ? offer.requisitionId : '',
language: typeof offer.language === 'string' ? offer.language : '',
listing_key: typeof offer.listingKey === 'string' ? offer.listingKey : '',
closes_at: closesAtIso(offer.closesAt) || closesAtIso(offer),
};
return SCAN_HISTORY_COLUMNS.map((name) => sanitizeTsvField(record[name])).join('\t');
}
// The provider must explicitly supply all three ATS-native identity fields;
// a URL or title never fills a missing component. This v1 token shares the
// scanner's seen set with normalized URL tokens but remains type-prefixed.
export function listingIdentityToken(offer) {
if (!offer?.listingIdentity) return null;
try {
const record = computeListingFingerprint({ strong: offer.listingIdentity });
return record.listing_key ? `listing:${record.listing_key}` : null;
} catch {
// Malformed or incomplete provider identity is not strong evidence; the
// existing URL and company/role rules remain responsible for that offer.
return null;
}
}
/**
* Replace any provider-supplied listing key with the key derived from the
* validated identity, or clear it when the identity is incomplete/malformed.
* The scanner calls this before a job can reach history output.
*/
export function refreshListingKey(offer) {
const token = listingIdentityToken(offer);
offer.listingKey = token ? token.slice('listing:'.length) : '';
return token;
}
export function isOfferSeen(offer, seenUrls, identityPromotableUrls = null) {
const urlToken = seenJobKey(offer?.url);
const identityToken = listingIdentityToken(offer);
if (seenUrls.has(urlToken)) {
const historicalKeys = identityPromotableUrls?.get(urlToken);
const currentKey = identityToken?.slice('listing:'.length);
if (identityToken && historicalKeys
&& (historicalKeys.has(null) || historicalKeys.has(currentKey))) seenUrls.add(identityToken);
return true;
}
return Boolean(identityToken && seenUrls.has(identityToken));
}
export function markOfferSeen(offer, seenUrls, { includeIdentity = true } = {}) {
const urlToken = seenJobKey(offer?.url);
const identityToken = listingIdentityToken(offer);
seenUrls.add(urlToken);
if (identityToken && includeIdentity) {
offer.listingKey = identityToken.slice('listing:'.length);
seenUrls.add(identityToken);
}
}
/**
* Keep the first live offer for each strong ATS identity and add its token to
* the run's seen set. Call only with offers accepted by liveness verification.
*/
export function retainVerifiedListingIdentities(offers, seenUrls, verificationStatusByOffer = new Map()) {
const selectedByIdentity = new Map();
const withoutIdentity = [];
for (const offer of offers) {
const token = listingIdentityToken(offer);
if (!token) {
withoutIdentity.push(offer);
continue;
}
const status = verificationStatusByOffer.get(offer);
const rank = status === 'active' ? 1 : 0;
const current = selectedByIdentity.get(token);
if (!current || rank > current.rank) selectedByIdentity.set(token, { offer, rank });
}
const retained = [...selectedByIdentity.values()].map(({ offer }) => offer);
const retainedSet = new Set(retained);
// Keep input order for unrelated offers while replacing an uncertain alias
// with an active one for the same identity, regardless of which was fetched
// first.
const ordered = offers.filter((offer) => !listingIdentityToken(offer) || retainedSet.has(offer));
const uniqueOrdered = [];
const seenObjects = new Set();
for (const offer of ordered) {
if (seenObjects.has(offer)) continue;
seenObjects.add(offer);
uniqueOrdered.push(offer);
}
for (const offer of [...withoutIdentity, ...retained]) {
const token = listingIdentityToken(offer);
if (!token) continue;
seenUrls.add(token);
offer.listingKey = token.slice('listing:'.length);
}
return uniqueOrdered;
}
/**
* Move a verified posting to a rediscovered URL. The source URL's ATS identity
* cannot describe the target until its provider resolves that target explicitly.
*/
export function migrateOfferToUrl(offer, url) {
const { listingIdentity: _sourceIdentity, listingKey: _sourceKey, ...sourceFields } = offer;
return { ...sourceFields, url, previousUrl: offer.url };
}
/**
* Parse scan-history.tsv rows that carry a fingerprint, for the cross-listing
* check. Older rows without the 8th column simply never match. Takes the file
* text ('' for an absent file), like its `collect*` siblings.
*
* @param {string} [scanHistoryText] - Full scan-history.tsv contents.
* @param {Array<Record<string, string>>|null} [historyRows] - Rows from
* {@link parsedScanHistory} when the caller already parsed the file.
* @returns {Array<{url: string, dateStr: string, company: string, title: string, fingerprint: string}>}
*/
export function collectFingerprintHistory(scanHistoryText = '', historyRows = null) {
const rows = [];
for (const row of scanHistoryRows(scanHistoryText, historyRows)) {
// The header's url cell is the word "url". A real row never is.
if (row.url === 'url') continue;
if (!row.fingerprint.trim()) continue;
rows.push({
url: row.url.trim(),
dateStr: row.first_seen.trim(),
title: row.title.trim(),
company: row.company.trim(),
fingerprint: row.fingerprint.trim(),
});
}
return rows;
}
/**
* Filesystem wrapper over {@link collectFingerprintHistory}.
*
* @param {string} [historyPath] - Override for tests.
*/
export function loadFingerprintHistory(historyPath = SCAN_HISTORY_PATH) {
return collectFingerprintHistory(readIfExists(historyPath));
}
/**
* Read the three dedup sources once and derive every per-run dedup structure
* from that single read (#2382). A scan run used to parse scan-history.tsv
* three times and pipeline.md/applications.md twice each — at 50k history rows
* that is ~600 ms of redundant parsing per run.
*
* The snapshot is deliberately per-run: callers hold the returned object in
* run-scoped locals and nothing is cached at module level, so a later run
* always re-reads the files. Dedup state is therefore frozen at run start;
* rows appended by a concurrent process mid-run are picked up by the next run
* (the previous re-read at the cross-listing step could not safely observe
* them anyway — scan-history appends are not locked).
*
* @param {{recheckAfterDays?: number|null, today?: string}} [policy] -
* Scan-history recheck policy, shared by the URL and company+role sets.
* @param {(name: unknown) => string} [canonicalize=defaultCompanyNormalizer] -
* Company canonicalizer for the role keys.
* @returns {{seen: Set<string>, recheckEligible: number, seenCompanyRoles: Set<string>, seenCompanyRoleBases: Set<string>, seenCompanyRoleRequisitions: Map<string, Set<string>>, locatedRequisitionsByBase: Map<string, Set<string>>, seenCompanyRoleLanguages: Map<string, Set<string>>, locatedLanguagesByBase: Map<string, Set<string>>, fingerprintHistory: Array<{url: string, dateStr: string, company: string, title: string, fingerprint: string}>}}
*/
// Same path seam as loadSeenUrls/appendToPipeline: anchored defaults, explicit
// paths for a caller with its own lane or a test with a fixture.
export function loadDedupSnapshot(policy = {}, canonicalize = defaultCompanyNormalizer, {
scanHistoryPath = SCAN_HISTORY_PATH,
pipelinePath = PIPELINE_PATH,
applicationsPath = APPLICATIONS_PATH,
includeLocation = false,
} = {}) {
const scanHistoryText = readIfExists(scanHistoryPath);
const pipelineText = readIfExists(pipelinePath);
const applicationsText = readIfExists(applicationsPath);
// One parse. The four readers below walk these rows instead of splitting
// the file again (seen URLs, posting attributes, company roles, fingerprints).
const historyRows = parsedScanHistory(scanHistoryText);
const { seen, recheckEligible, identityPromotableUrls, variants: seenUrlVariants } = collectSeenUrls(
{ scanHistoryText, pipelineText, applicationsText, historyRows },
policy,
{ extraTokensFor: (_url, _portal, listingKey) => listingKey ? `listing:${listingKey}` : null },
);
// Preserve the exported snapshot field for existing callers. The scanner's
// decision uses locatedRequisitionsByBase, not this legacy set.
const seenCompanyRoleBases = new Set();
// Keep exact/wildcard keys separate from the aggregate of located rows.
// Only locationless candidates consult the latter.
const seenCompanyRoleRequisitions = new Map();
const locatedRequisitionsByBase = new Map();
const seenCompanyRoleLanguages = new Map();
const locatedLanguagesByBase = new Map();
const seenCompanyRoles = collectSeenCompanyRoles({ applicationsText, scanHistoryText, pipelineText, historyRows }, policy, canonicalize, {
includeLocation,
locatedBases: seenCompanyRoleBases,
requisitionsByBase: seenCompanyRoleRequisitions,
locatedRequisitionsByBase,
languagesByBase: seenCompanyRoleLanguages,
locatedLanguagesByBase,
});
const fingerprintHistory = collectFingerprintHistory(scanHistoryText, historyRows);
return { seen, identityPromotableUrls, recheckEligible, seenUrlVariants, seenCompanyRoles, seenCompanyRoleBases, seenCompanyRoleRequisitions, locatedRequisitionsByBase, seenCompanyRoleLanguages, locatedLanguagesByBase, fingerprintHistory };
}
// Standard skeleton created on fresh install — matches the format documented
// in modes/pipeline.md and expected by /career-ops pipeline.
const PIPELINE_SKELETON = `# Pipeline — Pending URLs
Paste job URLs below as \`- [ ] {url}\` then run \`/career-ops pipeline\`.
## Pending
## Processed
`;
function sameJobWriteKey(url) {
if (typeof url !== 'string' || !url) return '';
return canonicalJobKey(url) || url;
}
// Keys whose history rows no longer pin. A still-pinning row of the same
// posting wins, so an older released row cannot un-pin a newer one.
function releasedHistoryKeys(historyText, policy) {
const pinned = new Set();
const released = new Set();
for (const row of scanHistoryRows(historyText)) {
const status = row.status || 'added';
if (!row.url || OBSERVATIONAL_SCAN_HISTORY_STATUSES.has(status)) continue;
const key = sameJobWriteKey(row.url);
if (!key) continue;
if (shouldDedupScanHistoryRow({ firstSeen: row.first_seen, status }, policy)) pinned.add(key);
else released.add(key);
}
for (const key of pinned) released.delete(key);
return released;
}
function keysInPipeline(text, released = new Set()) {
const keys = new Set();
let inProcessed = false;
for (const line of String(text ?? '').split('\n')) {
inProcessed = processedSectionAfter(line, inProcessed);
const url = extractPipelineUrl(line);
if (!url) continue;
const key = sameJobWriteKey(url);
if (!key) continue;
// An open `- [ ]` stays pinned even when history released the posting.
// Releasing it would queue a second copy of a job still on the list.
if (pipelineRowIsFinished(line, inProcessed) && released.has(key)) continue;
keys.add(key);
}
return keys;
}
function offersMissingFromPipeline(offers, pipelineText, historyText = '', policy = {}) {
const keys = keysInPipeline(pipelineText, releasedHistoryKeys(historyText, policy));
const fresh = [];
for (const offer of offers) {
const key = sameJobWriteKey(offer?.url);
if (!key || keys.has(key)) continue;
keys.add(key);
fresh.push(offer);
}
return fresh;
}
function keysInScanHistory(text, status, policy = {}) {
const keys = new Set();
// `added` and `skipped_expired` stop blocking once the recheck window
// releases them. Every other status blocks on any earlier row, so a skip
// bucket and a permanent rejection stay one row.
const recheckable = RECHECKABLE_SCAN_HISTORY_STATUSES.has(status);
for (const line of scanHistoryDataLines(text)) {
const row = parseScanHistoryLine(line);
if (row.status !== status || !row.url) continue;
if (recheckable && !shouldDedupScanHistoryRow({ firstSeen: row.first_seen, status }, policy)) continue;
const key = sameJobWriteKey(row.url);
if (key) keys.add(key);
}
return keys;
}
function offersMissingFromHistory(offers, status, historyText, policy = {}) {
const keys = keysInScanHistory(historyText, status, policy);
const fresh = [];
for (const offer of offers) {
const key = sameJobWriteKey(offer?.url);
if (!key || keys.has(key)) continue;
keys.add(key);
fresh.push(offer);
}
return fresh;
}
// Locked (pipeline-lock.mjs) so scan.mjs, scan-ats-full.mjs, and plugins.mjs
// (pipeline mode) — the three current callers — can never interleave their
// read-modify-write and silently drop each other's offers.
// Same seam as loadSeenUrls above: the default is the CAREER_OPS_ROOT-anchored
// module constant; a caller with its own lane (or a fixture) passes the path.
//
// The read that decides "already here" happens inside the lock. Two scans can
// both decide a posting is new before either writes; the second write sees the
// first's row and skips that key.
export async function appendToPipeline(offers, {
pipelinePath = PIPELINE_PATH,
scanHistoryPath = SCAN_HISTORY_PATH,
historyPolicy = null,
} = {}) {
if (offers.length === 0) return;
await withPipelineLock(pipelinePath, async () => {
// Auto-create with standard skeleton if missing (fresh-install guard).
let text = existsSync(pipelinePath)
? readFileSync(pipelinePath, 'utf-8')
: PIPELINE_SKELETON;
const policy = historyPolicy ?? historyPolicyFromPortals();
// A directory at the history path is not a history. readFileSync throws
// EISDIR, and that throw used to land here, before the pipeline write,
// so a failed history update discarded the jobs about to be saved.
let historyText = '';
try {
historyText = readIfExists(scanHistoryPath);
} catch (err) {
if (err?.code !== 'EISDIR') throw err;
}
const fresh = offersMissingFromPipeline(offers, text, historyText, policy);
if (fresh.length === 0) return;
const headings = [...text.matchAll(/^##[ \t].*$/gm)];
const pending = headings.find(h => sectionKind(h[0]) === 'pending');
if (!pending) {
// No Pending section found — insert one before Processed (or at end)
const processed = headings.find(h => sectionKind(h[0]) === 'processed');
const insertAt = processed ? processed.index : text.length;
const block = `\n## Pending\n\n` + fresh.map(formatPipelineOffer).join('\n') + '\n\n';
text = text.slice(0, insertAt) + block + text.slice(insertAt);
} else {
// Find the end of existing Pending content (next ## or end)
const afterMarker = pending.index + pending[0].length;
const nextSection = text.indexOf('\n## ', afterMarker);
const insertAt = nextSection === -1 ? text.length : nextSection;
const block = '\n' + fresh.map(formatPipelineOffer).join('\n') + '\n';
text = text.slice(0, insertAt) + block + text.slice(insertAt);
}
atomicWriteFile(pipelinePath, text);
});
}
// data/scan-history.tsv has exactly the same set of concurrent writers as
// data/pipeline.md — scan.mjs, scan-ats-full.mjs, scan-interamt.mjs and
// plugins.mjs — so it takes the same lock appendToPipeline does, on its own
// path. Unlocked, two writers race in two places: the create branch below is a
// check-then-write, and its writeFileSync truncates, so a scanner that loses
// the race erases rows the winner already appended; and a multi-row
// appendFileSync is not atomic, so a concurrent append can interleave mid-line.
// Both surface as rows that silently stop counting, because every reader skips
// a malformed line quietly.
//
// `alreadyLocked` is for the callers whose APPEND is decided by a READ of this
// same file — check-liveness's expired-verdict recorder and
// appendUnrecordedToScanHistory below — which must hold the lock across both
// or their plan is a check-then-act (a concurrent scanner appending in between
// makes them write a row that is no longer needed). The lock is not reentrant,
// so such a caller cannot simply wrap this call; it takes the lock itself and
// passes true. Everything else must leave it alone: true with no lock actually
// held reintroduces both races described above.
// `historyPath` is for a caller that resolves its own data root at call time,
// as the LinkedIn mail drop does. SCAN_HISTORY_PATH is fixed when this module
// first loads. The dedup read, the append, and the lock share that path.
export async function appendToScanHistory(offers, date, status = 'added', {
alreadyLocked = false,
historyPolicy = null,
historyPath = SCAN_HISTORY_PATH,
} = {}) {
const write = () => {
// Ensure file + header exist. The header is SCAN_HISTORY_COLUMNS, the same
// list the row writer (formatScanHistoryRow) emits in order. Written ONLY
// on fresh-file creation; existing files are never rewritten.
if (!existsSync(historyPath)) {
mkdirSync(path.dirname(historyPath), { recursive: true });
atomicWriteFile(historyPath, `${SCAN_HISTORY_COLUMNS.join('\t')}\n`);
}
// Re-read under the lock (the caller holds it, or withPipelineLock does).
// A same-status row still blocks a duplicate while it pins. `added` and
// `skipped_expired` stop blocking once the recheck window releases them,
// so a later scan can record the posting coming back or closing again.
const policy = historyPolicy ?? historyPolicyFromPortals();
const fresh = offersMissingFromHistory(offers, status, readIfExists(historyPath), policy);
if (fresh.length === 0) return 0;
const lines = fresh.map(o => formatScanHistoryRow(o, date, status)).join('\n') + '\n';
appendLedgerRows(historyPath, lines);
return fresh.length;
};
if (alreadyLocked) return write();
else return await withPipelineLock(historyPath, write);
}
/**
* appendToScanHistory for a skip bucket a scanner refills on every run: only
* the offers not yet recorded under `status` are written (unrecordedOffers),
* so the file grows with new postings rather than with scans (#2695). This is
* how scan.mjs records its own skips; the scanners beside it appended theirs
* straight, re-recording every skipped posting on every run.
*
* @param {Array<object>} offers
* @param {string} date
* @param {string} status
* @returns {Promise<number>} rows written
*/
export async function appendUnrecordedToScanHistory(offers, date, status) {
if (offers.length === 0) return 0;
return withPipelineLock(SCAN_HISTORY_PATH, async () => {
const fresh = unrecordedOffers(offers, status, readIfExists(SCAN_HISTORY_PATH));
if (fresh.length === 0) return 0;
// The inner write re-reads under this same lock and may drop an alias the
// pre-check still treated as new. The count is rows that landed.
return appendToScanHistory(fresh, date, status, { alreadyLocked: true });
});
}
// ── Company blacklist (#1742) ───────────────────────────────────────
// User Layer, so it follows the data root like every other input this file reads
// (#3510). It was a bare relative string, i.e. resolved against process.cwd(),
// which meant a user with a data root configured got whatever blacklist happened
// to sit in the directory they ran from — usually none, so their do-not-apply
// list was silently empty while the run reported no filtering at all.
const BLACKLIST_PATH = path.join(DATA_ROOT, 'data/blacklist.md');
function normalizeBlacklistDomain(domain) {
return String(domain || '').trim().toLowerCase().replace(/\.$/, '');
}
/**
* Parse the user's do-not-apply list (data/blacklist.md, user layer, opt-in).
*
* The file is a small markdown table the user owns:
* `| Company | Since | Scope | Reason |`. Nothing here ever creates or writes
* it — an absent file means no filtering. Companies are keyed with the same
* normalization every tracker writer shares (normalizeCompany, #1460), so a
* blacklist row "Acme Corp." still catches an ATS feed that says "acme corp".
*
* @param {string} text - Raw data/blacklist.md content.
* @returns {Map<string, {company: string, since: string, scope: 'company'|'domain', reason: string}>}
* Normalized company key or domain:<hostname> → entry. First row wins on duplicate keys.
*/
export function parseBlacklist(text) {
const entries = new Map();
for (const line of String(text ?? '').replace(/\r/g, '').split('\n')) {
if (!line.trim().startsWith('|')) continue;
const cells = line.split('|').map(s => s.trim());
const company = cells[1] || '';
if (!company || /^[-: ]+$/.test(company)) continue; // separator row
if (company.toLowerCase() === 'company') continue; // header row
const scope = (cells[3] || 'company').toLowerCase();
const value = scope === 'domain' ? normalizeBlacklistDomain(company) : normalizeCompany(company);
const key = scope === 'domain' ? `domain:${value}` : value;
if (!value || entries.has(key)) continue;
entries.set(key, {
company,
since: cells[2] || '',
// A blank or unsupported scope keeps the long-standing company-name
// behavior. Only the documented `domain` value enables host matching.
scope: scope === 'domain' ? 'domain' : 'company',
reason: cells[4] || '',
});
}
return entries;
}
/**
* Find the blacklist entry that applies to one posting.
*
* `company` is the established default: compare the feed's company label with
* the normalized table value. `domain` is opt-in: the table's Company cell is
* a hostname suffix, so `ibm.com` matches `jobs.ibm.com` but not `notibm.com`.
* This deliberately does not infer parent/subsidiary ownership from a URL.
*
* @param {Map<string, {company: string, since: string, scope?: string, reason: string}>} blacklist
* @param {string} company - Feed-provided company label.
* @param {string} url - Posting URL.
* @returns {{company: string, since: string, scope?: string, reason: string}|null}
*/
export function findBlacklistEntry(blacklist, company, url) {
if (!blacklist || blacklist.size === 0) return null;
const companyEntry = blacklist.get(normalizeCompany(company || ''));
if (companyEntry && companyEntry.scope !== 'domain') return companyEntry;
let hostname;
try {
hostname = normalizeBlacklistDomain(new URL(url).hostname);
} catch {
return null;
}
for (const entry of blacklist.values()) {
if (entry.scope !== 'domain') continue;
const suffix = normalizeBlacklistDomain(entry.company);
if (suffix && (hostname === suffix || hostname.endsWith(`.${suffix}`))) return entry;
}
return null;
}
/**
* Load data/blacklist.md if the user opted in. Absent file = empty Map = no
* filtering anywhere — the scan stays byte-identical to a pre-#1742 run.
*
* @param {string} [filePath] - Override for tests.
* @returns {Map<string, {company: string, since: string, scope: string, reason: string}>}
*/
export function loadBlacklist(filePath = BLACKLIST_PATH) {
if (!existsSync(filePath)) return new Map();
return parseBlacklist(readFileSync(filePath, 'utf-8'));
}
/**
* Parse data-static/aggregator-domains.txt into a Map keyed by domain.
* Format: `domain.com # reason`
* Skips blank lines and lines starting with `#`.
*
* @param {string} text - Raw data-static/aggregator-domains.txt content.
* @returns {Map<string, {domain: string, reason: string}>}
*/
export function parseAggregatorDomains(text) {
const entries = new Map();
for (let line of String(text ?? '').replace(/\r/g, '').split('\n')) {
line = line.trim();
if (!line || line.startsWith('#')) continue;
const hashIdx = line.indexOf('#');
let domain = line;
let reason = '';
if (hashIdx !== -1) {
domain = line.slice(0, hashIdx);
reason = line.slice(hashIdx + 1);
}
domain = domain.trim().toLowerCase();
reason = reason.trim();
if (!domain) continue;
entries.set(domain, { domain, reason });
}
return entries;
}
export const AGGREGATOR_DOMAINS_PATH = process.env.CAREER_OPS_AGGREGATOR_DOMAINS || path.join(CODE_ROOT, 'data-static/aggregator-domains.txt');
/**
* Load data-static/aggregator-domains.txt dataset.
*
* @param {string} [filePath] - Override for tests.
* @returns {Map<string, {domain: string, reason: string}>}
*/
export function loadAggregatorDomains(filePath = AGGREGATOR_DOMAINS_PATH) {
if (!existsSync(filePath)) return new Map();
return parseAggregatorDomains(readFileSync(filePath, 'utf-8'));
}
/**
* Check if an offer's URL hostname matches a known aggregator domain.
* Uses the same `new URL(offer.url).hostname` pattern as `extractCareersUrlDomain()`.
*
* @param {{url: string}} offer - Offer object with a url property.
* @param {Map<string, {domain: string, reason: string}>} [domainsMap] - Optional map of aggregator domains.
* @returns {{domain: string, reason: string}|null} Matched entry or null.
*/
export function checkAggregatorRepost(offer, domainsMap = loadAggregatorDomains()) {
if (!offer || !offer.url || !domainsMap || domainsMap.size === 0) return null;
let hostname;
try {
hostname = new URL(offer.url).hostname.toLowerCase();
} catch {
return null;
}
if (hostname.endsWith('.')) {
hostname = hostname.slice(0, -1);
}
if (!hostname) return null;
for (const [domain, entry] of domainsMap) {
if (hostname === domain || hostname.endsWith('.' + domain)) {
return entry;
}
}
return null;
}
// ── Scan-run persistence (#1604) ────────────────────────────────────
// Anchored for the same reason (#3510), and with a reader to agree with:
// stats.mjs:36 reads join(DATA_ROOT, 'data', 'scan-runs.tsv'). While this was
// cwd-relative the writer and the reader could name different files, and the
// trend stats simply under-reported whatever landed elsewhere.
const SCAN_RUNS_PATH = path.join(DATA_ROOT, 'data/scan-runs.tsv');
// One row of run counters per non-dry scan — today these numbers are printed
// once in the summary and lost when the terminal scrolls. Full ISO timestamp
// (two scans in one day must not collapse). `status` is 'completed' for a
// finished run; a run that dies after the sweep starts records 'failed' via
// writeRunFailureRow (#2643) so trend stats can exclude survivorship bias.
// Consumers MUST parse by header name, never by position — columns may be
// appended in later versions.
export const SCAN_RUNS_HEADER = 'timestamp\tstatus\tcompanies\tboards\tfound\tfiltered_title\tfiltered_tier\tfiltered_location\tfiltered_posting_age\tfiltered_salary\tfiltered_content\tfiltered_cooldown\tdupes\tnew_added\terrors\tfiltered_blacklist\tfiltered_visa\tfiltered_posted_date\tfiltered_country_eligibility\n';
// Failure-path writes (#2643). main() registers a snapshot closure once the
// sweep's counters exist (never on --dry-run, never before the sweep starts —
// a config error is not a run). The fatal catch and the SIGINT handler both
// call writeRunFailureRow; the snapshot is consumed on first use so the two
// signals can never double-write. Best-effort by design: a failure to record
// the failure must not mask the original error, so everything is swallowed.
let runFailureSnapshot = null;
export function registerRunFailureSnapshot(fn) {
runFailureSnapshot = typeof fn === 'function' ? fn : null;
}
// NOT locked, deliberately. This runs from the SIGINT handler and the fatal
// path, which call process.exit() with nothing able to await first, and every
// lock entry point in pipeline-lock.mjs is async. It is already best-effort and
// swallows its own errors. So a failure row written while a migration is
// swapping the file can still be lost; the completed-run path above is the one
// that is covered. Stated rather than implied, because the alternative is a
// comment claiming a guarantee this function does not provide.
export function writeRunFailureRow(status = 'failed', filePath = SCAN_RUNS_PATH) {
const snapshot = runFailureSnapshot;
runFailureSnapshot = null;
if (!snapshot) return false;
try {
appendScanRunSummary({ ...snapshot(), status }, filePath);
return true;
} catch {
return false;
}
}
export function appendScanRunSummary(c, filePath = SCAN_RUNS_PATH) {
mkdirSync(path.dirname(filePath), { recursive: true });
if (!existsSync(filePath)) writeFileSync(filePath, SCAN_RUNS_HEADER, 'utf-8');
// The header is written only on first creation, so a release that appends or inserts a counter
// leaves existing files with a header that no longer describes the rows below it. Nothing
// migrates it and nothing notices: stats.mjs reads by column NAME, so it silently returns a
// neighbouring counter. Surface the mismatch here rather than papering over it — rewriting the
// header in place would misalign every historical row instead.
if (!existsSync(filePath)) {
atomicWriteFile(filePath, SCAN_RUNS_HEADER);
} else {
const onDisk = (readFileSync(filePath, 'utf-8').split('\n', 1)[0] || '') + '\n';
if (onDisk !== SCAN_RUNS_HEADER) {
console.error(
`Warning: ${filePath} header has ${onDisk.trim().split('\t').length} columns but this build writes `
+ `${SCAN_RUNS_HEADER.trim().split('\t').length}. Rows below the header are positionally offset and `
+ `stats.mjs will exclude them. Move ${filePath} aside to start a fresh file — deleting only the header does NOT recover it, because the file still exists and the next run would read the first data row as the header.`,
);
}
}
const row = [
c.timestamp, c.status ?? 'completed', c.companies, c.boards, c.found,
c.filteredTitle, c.filteredTier, c.filteredLocation, c.filteredPostingAge,
c.filteredSalary, c.filteredContent, c.filteredCooldown, c.dupes, c.newAdded, c.errors,
// filtered_blacklist (#1742) appended at the END, per the header-name
// contract above: files created with an older header keep parsing (the
// extra trailing cell is simply not named there).
c.filteredBlacklist ?? 0,
// filtered_visa appended at the END for the same reason.
c.filteredVisa ?? 0,
// filtered_posted_date appended at the END for the same reason.
c.filteredPostedDate ?? 0,
// filtered_country_eligibility (#2093) appended at the END for the same reason.
c.filteredCountryEligibility ?? 0,
].join('\t') + '\n';
appendLedgerRows(filePath, row);
}
// ── Portal health persistence (#1744) ───────────────────────────────
// Anchored to the data root (#3510), read by stats.mjs:39 at the same anchor.
//
// This path has moved twice. It was dirname(fileURLToPath(import.meta.url)) —
// the script's own directory — until 96c578b made it cwd-relative, because a
// sandboxed run was writing fixture rows into the live data/portal-health.tsv of
// whatever checkout owned scan.mjs. That problem was real; the mechanism traded
// one unanchored path for another, and left this file with two different rules
// for where user data lives. Isolation now comes from CAREER_OPS_ROOT, which is
// how the rest of the suite already sandboxes writes — see
// tests/portal-health-path.test.mjs, which still asserts the checkout's own data
// directory is never touched.
const PORTAL_HEALTH_PATH = path.join(DATA_ROOT, 'data/portal-health.tsv');
export const PORTAL_HEALTH_HEADER = 'timestamp\tcompany\tstatus\n';
// Locked (portal-health-lock.mjs) so a concurrent read-modify-write of this
// same file — e.g. tests/portal-health-guard.mjs's regression-cleanup path —
// can never interleave with this append and silently discard one side.
export async function appendPortalHealth(healthRecords, filePath = PORTAL_HEALTH_PATH) {
await withPortalHealthLock(filePath, async () => {
mkdirSync(path.dirname(filePath), { recursive: true });
if (!existsSync(filePath)) atomicWriteFile(filePath, PORTAL_HEALTH_HEADER);
let lines = '';
for (const r of healthRecords) {
lines += [r.timestamp, r.company, r.status].join('\t') + '\n';
}
if (lines) appendLedgerRows(filePath, lines);
});
}
export function loadPortalHealth(filePath = PORTAL_HEALTH_PATH) {
if (!existsSync(filePath)) return [];
const lines = readFileSync(filePath, 'utf-8').split('\n');
const records = [];
for (let i = 1; i < lines.length; i++) {
const line = lines[i].trim();
if (!line) continue;
const parts = line.split('\t');
if (parts.length >= 3) {
records.push({ timestamp: parts[0], company: parts[1], status: parts[2] });
}
}
return records;
}
export function computeConsecutiveFailures(healthRecords) {
const streaks = new Map();
for (const r of healthRecords) {
// Healthy statuses reset the streak; every other status counts toward it.
// Inverted (vs. listing failure statuses) so the newer error kinds
// (auth/server/unknown) can't silently fall outside the streak again.
// 'empty' is deliberately healthy: a live board with 0 jobs is reachable.
// 'partial' is reachable too: the board answered, it was only cut short,
// often by a cap the user set on purpose. verify-portals and a new slug
// cannot fix that, so it has its own streak (computePartialStreaks).
if (r.status === 'reachable' || r.status === 'empty' || r.status === 'partial') {
streaks.set(r.company, 0);
} else {
streaks.set(r.company, (streaks.get(r.company) || 0) + 1);
}
}
return streaks;
}
// Consecutive `partial` runs per company: any other status resets it.
export function computePartialStreaks(healthRecords) {
const streaks = new Map();
for (const r of healthRecords) {
streaks.set(r.company, r.status === 'partial' ? (streaks.get(r.company) || 0) + 1 : 0);
}
return streaks;
}
export function emptyTargetStatus(observation) {
// A provider outside this HTTP context (local-parser or a keyed plugin)
// gives no transport evidence. Preserve its previous empty classification.
return observation.requests > 0 && observation.successfulResponses === 0
? 'unverified_zero' : 'empty';
}
// ── Parallel fetch with concurrency limit ───────────────────────────
export async function parallelFetch(tasks, limit, onDone = () => {}, { hostOf = null, hostLimit = limit } = {}) {
// Verify and any other caller that does not name a host keeps the plain
// worker pool. Completion order of the returned array stays what it was.
if (typeof hostOf !== 'function') {
const results = [];
let i = 0;
async function next() {
while (i < tasks.length) {
const task = tasks[i++];
results.push(await task());
onDone();
}
}
const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => next());
await Promise.all(workers);
return results;
}
const n = tasks.length;
if (n === 0 || !(limit > 0)) return [];
const perHost = Math.max(1, hostLimit);
const hosts = tasks.map((task, index) => {
try {
const host = hostOf(task, index);
return typeof host === 'string' ? host : '';
} catch {
return '';
}
});
const results = new Array(n);
const pending = hosts.map((_, index) => index);
const hostActive = new Map();
let active = 0;
let finished = 0;
let firstError = null;
function takeReady() {
for (let k = 0; k < pending.length; k++) {
const index = pending[k];
const host = hosts[index];
// An empty host is not a bucket. Unknown URLs must not share one cap.
if (host && (hostActive.get(host) || 0) >= perHost) continue;
pending.splice(k, 1);
return index;
}
return -1;
}
await new Promise((resolve, reject) => {
const launch = () => {
while (active < limit) {
const index = takeReady();
if (index < 0) break;
active += 1;
const host = hosts[index];
if (host) hostActive.set(host, (hostActive.get(host) || 0) + 1);
Promise.resolve()
.then(() => tasks[index]())
.then(
(value) => {
results[index] = value;
try {
onDone();
} catch (err) {
if (firstError === null) firstError = err;
}
},
(err) => {
if (firstError === null) firstError = err;
},
)
.finally(() => {
active -= 1;
finished += 1;
if (host) {
const left = (hostActive.get(host) || 1) - 1;
if (left <= 0) hostActive.delete(host);
else hostActive.set(host, left);
}
launch();
if (finished === n) {
if (firstError !== null) reject(firstError);
else resolve();
}
});
}
};
launch();
});
return results;
}
// stderr, so it shows under --quiet and never mixes into the --json receipt.
export function progressTicker(total, { every = PROGRESS_EVERY, write = (line) => process.stderr.write(line), now = () => new Date() } = {}) {
let done = 0;
return () => {
done++;
if (done % every === 0 || done === total) {
write(`[${now().toTimeString().slice(0, 8)}] scan: ${done}/${total} sources done\n`);
}
};
}
// ── Main ────────────────────────────────────────────────────────────
export async function verifyOffers(offers, { headedFallback = false, throttleBaseMs = 0, rediscover = false } = {}, {
checkApi = checkLivenessViaApi,
loadPlaywright = () => import('playwright'),
// Consulted before each headed retry (#5776); see robots-check.mjs.
robotsGate = robotsAllows,
} = {}) {
// These helpers do not import Playwright. In particular, apply the same URL
// guard before the API rung: an embedded ATS id must not bless a private URL.
const {
rejectPrivateOrInvalid, checkUrlLiveness, checkUrlLivenessWithFallback,
createHeadedPageProvider, newLivenessPage, installLivenessRouteGuard, jitteredDelayMs, sleep,
takeSuppressedBlockCount,
} = await import('./liveness-browser.mjs');
// The HTTP rung can run concurrently; each provider retains its own rate
// limit inside checkLivenessViaApi. Index results so completion order never
// reorders the output or the subsequent sequential browser checks.
const verdicts = new Array(offers.length);
await parallelFetch(offers.map((offer, i) => async () => {
const guard = rejectPrivateOrInvalid(offer.url);
if (guard) {
verdicts[i] = { result: 'uncertain', ...guard };
return;
}
try {
verdicts[i] = await checkApi(offer.url);
} catch (err) {
console.warn(` ⚠️ API verification failed for ${offer.url}; falling back to Playwright: ${err.message}`);
verdicts[i] = null;
}
}), VERIFY_CONCURRENCY);
let browser = null, page = null, headed = null;
let hadBrowserOffer = false;
async function browserForOffer() {
if (!browser) {
let chromium;
try {
({ chromium } = await loadPlaywright());
} catch (err) {
throw new Error(
`--verify requires Playwright with Chromium (run "npx playwright install chromium"): ${err.message}`,
{ cause: err },
);
}
try {
browser = await chromium.launch(
browserLaunchOptions(
{ headless: true },
{ projectRoot: DATA_ROOT, processEnv: BROWSER_PROCESS_ENV },
),
);
} catch (err) {
throw new Error(
`--verify could not launch Chromium (run "npx playwright install chromium" or re-run without --verify): ${err.message}`,
{ cause: err },
);
}
page = await newLivenessPage(browser);
await installLivenessRouteGuard(page);
headed = headedFallback ? createHeadedPageProvider(chromium, {
projectRoot: DATA_ROOT,
processEnv: BROWSER_PROCESS_ENV,
}) : null;
}
// Space only browser work, including a search after an API 404 and the
// offer after a successful migration. API-only verdicts need no WAF delay.
const wait = hadBrowserOffer ? jitteredDelayMs(throttleBaseMs) : 0;
if (wait) await sleep(wait);
hadBrowserOffer = true;
return page;
}
const checkPage = (page, url) => headed
? checkUrlLivenessWithFallback(page, url, { getHeadedPage: () => headed.get(), robotsGate })
: checkUrlLiveness(page, url);
// Two permanent buckets, one observational, one transient passthrough:
// verified → active pages, transient nav errors and a still-empty page
// (empty_page)
// expired → classifier-confirmed dead postings (HTTP 4xx, redirect markers,
// body patterns, listing pages)
// dropped → page loaded but classifier saw no Apply control, including a
// page too short to read (insufficient_content). --verify is an
// opt-in stricter filter; keeping these defeats the purpose. Not
// proof the posting is closed, so the row never pins the URL and
// the next --verify scan loads the page again (#4832).
// invalid → up-front URL guard rejections (malformed / non-http / private)
const verified = [];
const verificationStatusByOffer = new Map();
const expired = [];
const dropped = [];
const invalid = [];
const migrated = [];
try {
// Sequential — project rule: never Playwright in parallel
for (let i = 0; i < offers.length; i++) {
const offer = offers[i];
let offerPage = null;
let verdict = verdicts[i];
// Non-null API `uncertain` is an explicit conservative conclusion, just
// as in check-liveness.mjs. Only an inconclusive null needs the browser.
if (!verdict) {
offerPage = await browserForOffer();
verdict = await checkPage(offerPage, offer.url);
}
const { result, code, reason } = verdict;
if (result === 'expired') {
// 404/410 on a tracked company may just be a moved role — run one
// search + re-verify before giving up (opt-in via --rediscover-404).
// Only HTTP 404/410 qualifies, from either rung; soft-expiry signals
// (redirect/body/listing) are real closures, not URL moves.
const httpGone = code === 'http_gone' || /^[a-z][a-z0-9-]*_api_gone$/.test(code);
if (rediscover && httpGone && offer.tracked && offer.careersUrlDomain) {
offerPage ??= await browserForOffer();
const newUrl = await searchForNewUrl(offerPage, offer);
if (newUrl) {
// Mirror the primary check: without the headed fallback, a
// challenge-prone domain would flag the rediscovered URL as
// expired just because the recheck hit the same anti-bot wall.
const recheck = await checkPage(offerPage, newUrl);
// Require a *confirmed* live page before migrating. A transient
// 'uncertain' (timeout/DNS/5xx) must not commit an unverified URL —
// fall through to expired (the original 404/410 is a real closure).
if (recheck.result === 'active') {
const migratedOffer = migrateOfferToUrl(offer, newUrl);
migrated.push(migratedOffer);
verificationStatusByOffer.set(migratedOffer, 'active');
console.log(` 🔄 migrated ${offer.company} | ${offer.title} → ${newUrl}`);
continue;
}
}
}
expired.push({ ...offer, reason });
console.log(` ❌ expired ${offer.company} | ${offer.title} (${reason})`);
} else if (result === 'uncertain' && GUARD_CODES.has(code)) {
// Guard failures are permanent (not transient like a timeout) — record them
// separately so they don't end up in pipeline.md but DO appear in scan-history
// with a precise status, dedup-blocking them on subsequent scans.
invalid.push({ ...offer, code, reason });
console.log(` ⛔ invalid ${offer.company} | ${offer.title} (${reason})`);
} else if (result === 'uncertain' && (code === 'no_apply_control' || code === 'insufficient_content')) {
// Page loaded but classifier could not find an Apply control (or too
// little text to hold one: insufficient_content). Drop from
// pipeline and record in scan-history for visibility only: the row does
// not pin the URL (OBSERVATIONAL_SCAN_HISTORY_STATUSES), so the next
// --verify scan checks it again rather than losing a live posting whose
// button the classifier does not recognise (#4832).
dropped.push({ ...offer, reason });
console.log(` ⚠️ no-apply ${offer.company} | ${offer.title} (${reason})`);
} else {
// 'active' or 'uncertain' due to navigation_error (transient — retry next scan)
verified.push(offer);
verificationStatusByOffer.set(offer, result);
const icon = result === 'active' ? '✅' : '⚠️';
console.log(` ${icon} ${result.padEnd(9)} ${offer.company} | ${offer.title}`);
}
}
} finally {
try {
if (headed) await headed.close();
} finally {
if (browser) await browser.close();
}
}
// The route guard counts, rather than logs, subresources a DNS filter blocks.
const quietBlocks = typeof takeSuppressedBlockCount === 'function' ? takeSuppressedBlockCount() : 0;
if (quietBlocks) console.warn(`--verify: ${quietBlocks} tracker/analytics requests skipped (no usable address, e.g. a DNS ad-blocker); not counted against any posting`);
return { verified, expired, dropped, invalid, migrated, verificationStatusByOffer };
}
// Stable codes from liveness-browser's up-front URL guard. Routing dispatches
// on these codes (not on regex over reason strings) so wording can change
// without breaking the pipeline.
const GUARD_CODES = new Set(['invalid_url', 'unsupported_protocol', 'blocked_host']);
// guardStatusFor maps a guard code to the canonical scan-history status string.
function guardStatusFor(code) {
if (code === 'blocked_host') return 'skipped_blocked_host';
// invalid_url and unsupported_protocol both surface as malformed input
return 'skipped_invalid_url';
}
// ── CLI args ────────────────────────────────────────────────────────
// #2270: `node scan.mjs --help` used to run a full live scan and write to
// pipeline.md/scan-history.tsv instead of printing usage — the flag was
// never checked at all. Same shape as scan-ats-full.mjs (#1633/#1635),
// reply-watch.mjs (#2743/#2745) and dedup-tracker.mjs (#2744/#2746), shared
// via lib/cli-flags.mjs's validateFlags() (#2775).
const KNOWN_FLAGS = [
'--dry-run', '--verify', '--headed-fallback', '--throttle', '--rediscover-404',
'--include-blacklisted', '--company', '--posted-after', '--posted-before',
'--since', '--quiet', '--json', '--help', '-h',
];
// Flags whose space-separated value is the NEXT argv token (the `--flag=value`
// form is self-contained and never needs this). --throttle is deliberately
// excluded: only its bare and `--throttle=<ms>` forms are read below, so a
// following token is never its value.
const VALUE_FLAGS = ['--company', '--posted-after', '--posted-before', '--since'];
const USAGE = `Usage:
node scan.mjs # scan all enabled companies
node scan.mjs --dry-run # preview without writing files
node scan.mjs --company Cohere # scan a single company
node scan.mjs --verify # API-first liveness check; use Playwright when inconclusive
node scan.mjs --verify --headed-fallback # retry anti-bot-blocked URLs in a headed browser (needs a display)
node scan.mjs --verify --throttle # jittered ~5-10s gap between browser checks
node scan.mjs --verify --throttle=8000 # custom base gap in ms (waits base..2*base)
node scan.mjs --rediscover-404 # re-verify tracked URLs that 404/410 (rides on --verify)
node scan.mjs --include-blacklisted # let data/blacklist.md matches through (annotated)
node scan.mjs --since 7 # postings from the last 7 days
node scan.mjs --posted-after 2026-07-01 # absolute lower bound on posting date
node scan.mjs --posted-before 2026-08-01 # absolute upper bound on posting date
node scan.mjs --json # emit one machine-readable receipt on stdout
node scan.mjs --quiet # suppress the manifesto footer
node scan.mjs --help # print this usage block and exit
Exit status: 0 when the scan ran, 1 on a configuration or fatal error, 2 when
every target it tried failed (with --json: when any target failed).`;
async function main() {
const args = process.argv.slice(2);
// --throttle=<ms> cannot join VALUE_FLAGS (see above), and validateFlags
// refuses `=` on any flag outside that list, so the documented form is checked
// here: a positive number passes as the bare flag, anything else stays refused.
const flagsToValidate = args.map((a) => (/^--throttle=0*[1-9]\d*$/.test(a) ? '--throttle' : a));
validateFlags(flagsToValidate, KNOWN_FLAGS, USAGE, { valueFlags: VALUE_FLAGS });
const dryRun = args.includes('--dry-run');
const jsonMode = args.includes('--json');
if (jsonMode) console.log = console.error.bind(console);
const verify = args.includes('--verify');
// Opt-in: on an anti-bot challenge (e.g. pracuj.pl Cloudflare wall), retry the
// URL in a headed browser. Off by default — headed Chromium needs a display, so
// scheduled/unattended scans should not rely on it.
const headedFallback = args.includes('--headed-fallback');
// --throttle or --throttle=<ms>: jittered gap between --verify checks to stay
// under rate-based WAF limits (pracuj.pl flags the session after a few rapid
// hits). Default base 5000ms. Off by default — most ATS feeds don't need it.
const throttleArg = args.find((a) => a === '--throttle' || a.startsWith('--throttle='));
const throttleBaseMs = throttleArg ? (Number(throttleArg.split('=')[1]) || 5000) : 0;
// --rediscover-404: when a tracked company's URL 404/410s, search for the
// moved role and re-verify before marking it expired. Opt-in; rides on --verify.
const rediscover = args.includes('--rediscover-404');
// --include-blacklisted: bypass the data/blacklist.md filter for auditing.
// Matching postings flow through annotated instead of being counted out.
const includeBlacklisted = args.includes('--include-blacklisted');
// flagValue reads both `--flag value` and `--flag=value`; a bare indexOf misses
// the second form entirely and silently falls back to the unfiltered default.
//
// flagValue alone cannot tell an ABSENT flag from one passed with no operand —
// both give undefined — so it is paired with hasFlag, per cli-flags.mjs's own
// guidance. Without that, a trailing `--posted-after` would fall back to "no
// bound" and scan everything: the same silent-default failure this fixes.
const requireValue = (flag) => {
const value = flagValue(args, flag);
if (value === undefined || value === '') {
if (hasFlag(args, flag)) {
console.error(`Error: ${flag} requires a value`);
process.exit(1);
}
return null;
}
return value;
};
const filterCompanyRaw = requireValue('--company');
const filterCompany = filterCompanyRaw?.toLowerCase() ?? null;
// --company may ignore a legal suffix. It must not ignore a place name.
const legalSuffixes = new Set(LEGAL_SUFFIXES);
// --posted-after / --posted-before <YYYY-MM-DD>: absolute-date bounds on the
// employer's real posting date (job.postedAt), gated against a typo since a
// silently-ignored bound would look like "no jobs matched" instead of an error.
const isValidIsoDate = (s) => {
if (!/^\d{4}-\d{2}-\d{2}$/.test(s)) return false;
const d = new Date(`${s}T00:00:00Z`);
return !Number.isNaN(d.getTime()) && d.toISOString().slice(0, 10) === s;
};
const postedAfter = requireValue('--posted-after');
const postedBefore = requireValue('--posted-before');
if (postedAfter != null && !isValidIsoDate(postedAfter)) {
console.error(`Error: --posted-after expects YYYY-MM-DD, got "${postedAfter}"`);
process.exit(1);
}
if (postedBefore != null && !isValidIsoDate(postedBefore)) {
console.error(`Error: --posted-before expects YYYY-MM-DD, got "${postedBefore}"`);
process.exit(1);
}
// --since <days>: a RELATIVE lower bound on the employer's posting date —
// the same thing --posted-after expresses absolutely, and it filters exactly
// like it does. Matches scan-ats-full.mjs, which has always treated --since
// as a filter; one flag name should not mean two different things.
//
// It additionally unlocks an optimisation. providers/workday.mjs returns
// postings newest-first and can stop paginating once a page is entirely past
// the window, but that only fires when ctx carries sinceMs — and scan.mjs
// built a bare makeHttpCtx(), so every Workday tenant paginated to its
// max_pages cap on every run however stale the deep pages were.
//
// Flag presence, operand validity, duplicate occurrences and the
// out-of-Date-range case are all handled by the SHARED parseSinceDays(), so
// scan-ats-full.mjs cannot disagree about what --since means (#2498).
const since = parseSinceDays(args);
if (since.error) {
console.error(`Error: ${since.error}`);
process.exit(1);
}
const sinceDays = since.days;
const effectiveAfter = resolveEffectiveAfter(postedAfter, sinceDays);
// 1. Load providers
const providers = await loadProviders(PROVIDERS_DIR);
// Opt-in: merge enabled keyed/auth-gated provider plugins. Returns immediately
// (no discovery, no dotenv, no process.env mutation) when config/plugins.yml is
// absent — so a plain scan with no plugins configured stays byte-identical.
await mergeProviderPlugins(providers, { root: path.dirname(PROVIDERS_DIR), dataRoot: DATA_ROOT });
if (providers.size === 0) {
console.error('Error: no providers loaded from providers/');
process.exit(1);
}
// 2. Read portals.yml
if (!existsSync(PORTALS_PATH)) {
console.error('Error: portals.yml not found. Run onboarding first.');
process.exit(1);
}
let rawConfig;
try {
rawConfig = parseYaml(readFileSync(PORTALS_PATH, 'utf-8'));
} catch (err) {
console.error(`Error: failed to parse ${PORTALS_PATH}: ${err.message}`);
process.exit(1);
}
const config = rawConfig && typeof rawConfig === 'object' ? rawConfig : {};
const typos = portalTopLevelTypos(config);
if (typos.length > 0) {
for (const typo of typos) {
console.error(
`Error: ${PORTALS_PATH}: unknown top-level key "${typo.key}" — did you mean "${typo.suggestion}"? The misspelled key is ignored, so this scan would report success without applying it.`,
);
}
console.error('Example: node scan.mjs --dry-run --json');
process.exit(1);
}
const companies = Array.isArray(config.tracked_companies) ? config.tracked_companies : [];
const boards = Array.isArray(config.job_boards) ? config.job_boards : [];
const titleFilter = buildTitleFilter(config.title_filter);
// Seniority tier classifier integration
let classifyTier = null;
const TIERS = ['intern', 'entry', 'mid', 'senior'];
const tierList = list => list.filter(t => typeof t === 'string').map(t => t.toLowerCase());
const skipTiers = Array.isArray(config.skip_tiers) ? tierList(config.skip_tiers) : [];
// #5778. An entry's own skip_tiers replaces the global list for that entry;
// [] turns the tier filter off there.
const entrySkipTiers = entry => (Array.isArray(entry.skip_tiers) ? tierList(entry.skip_tiers) : skipTiers);
const locationFilter = buildLocationFilter(config.location_filter);
const postingAgeFilter = buildPostingAgeFilter(config.max_posting_age_days);
const postedDateFilter = buildPostedDateFilter(effectiveAfter, postedBefore);
// Same bound the filter above uses, widened by max_posting_age_days when set.
// Derived by the same helper so the hint and the filter cannot disagree.
const earlyStopSinceMs = resolveEarlyStopMs(effectiveAfter, config.max_posting_age_days);
const salaryFilter = buildSalaryFilter(config.salary_filter);
const trustValidator = buildTrustValidator(config.trust_filter);
const contentFilter = buildContentFilterOrExit(config.content_filter);
// #6020. Postings that title_filter.positive missed (and no title_filter
// negative vetoed) still reach the content check, and pass it only on their
// description. See the title gate in the scan loop.
const rescueTitleMisses = contentFilter.rescueTitleMisses === true;
const titleGate = buildTitleGate(config.title_filter, contentFilter, titleFilter);
const candidateCountry = loadCandidateCountry();
const countryEligibilityFilter = buildCountryEligibilityFilter(config.country_eligibility_filter, candidateCountry);
const visaFilter = buildVisaFilter(config.visa_filter);
const visaEnabled = Boolean(config.visa_filter) && config.visa_filter.enabled !== false;
// Stated years of experience (jd-years.mjs). Runs on the new offers only,
// after every other cut, so it costs at most one JD request per posting
// that would otherwise reach pipeline.md. Independent of skip_tiers: that
// reads the title, this reads the JD body.
// The profile is read only when an enabled block needs its years, so a
// profile problem is never reported for a filter that is off.
const experienceBlock = config.experience_filter;
const needsProfileYears = experienceBlock !== null && typeof experienceBlock === 'object'
&& !Array.isArray(experienceBlock) && (experienceBlock.enabled === undefined || experienceBlock.enabled === true)
&& experienceBlock.years == null;
const experienceFilter = resolveExperienceFilter(experienceBlock, needsProfileYears ? loadCandidateYears() : undefined);
if (!experienceFilter.enabled && experienceFilter.reason) console.warn(`⚠️ ${experienceFilter.reason}`);
// 3. Resolve a provider for each enabled company / board
const targets = [];
let skippedCount = 0;
let boardCount = 0;
let companyFilterMatched = filterCompanyRaw == null;
const resolveErrors = [];
const agentHandoff = [];
const unclaimed = [];
const enabledWarnings = [];
/**
* Processes a list of configuration entries, resolves their appropriate data providers,
* and appends valid entries to the global scanning targets list.
* @param {Array<{ name?: string, enabled?: boolean, [key: string]: unknown }>} entries - List of entries.
* @param {{ isBoard?: boolean }} [options={}] - Configuration options.
*/
function resolveEntries(entries, { isBoard = false } = {}) {
for (const entry of entries) {
if (!entry || typeof entry !== 'object') continue;
if (typeof entry.name !== 'string' || !entry.name.trim()) {
// A disabled nameless row was already skipped before this warning.
if (entry.enabled !== false) {
console.error(`⚠️ Skipping entry — missing or non-string 'name' field: ${JSON.stringify(entry)}`);
}
continue;
}
// Substring, the tracker's normalized equality ("Acme Labs Inc" for
// "Acme Labs, Inc."), plus legal suffixes only. "Acme Labs, Inc."
// selects a row named "Acme Labs". A place or descriptor such as
// "Canada" does not, so "Acme Canada" does not select "Acme". Tracker
// dedup still allows those descriptors. A disabled row still counts,
// so --company does not say the company is missing when it is only
// turned off.
if (filterCompany && !entry.name.toLowerCase().includes(filterCompany)
&& normalizeCompany(entry.name) !== normalizeCompany(filterCompanyRaw)
&& !companiesMatchIgnoringCorporateForm(entry.name, filterCompanyRaw, legalSuffixes)) continue;
if (filterCompanyRaw != null) companyFilterMatched = true;
if (entry.enabled === false) continue;
// Only a real boolean false turns an entry off. The string "false" and
// YAML's old `no` stay in the run, and the count alone does not say which
// company. validate-portals already rejects this shape; say it here too,
// with the name, because scan does not run that validator.
if (entry.enabled !== undefined && typeof entry.enabled !== 'boolean') {
enabledWarnings.push(
`${entry.name}: enabled is ${JSON.stringify(entry.enabled)}, not a boolean. Only boolean false turns a company off, so this one stays in the run and is scanned if a provider matches it.`,
);
}
const resolved = resolveProvider(entry, providers);
if (!resolved) {
skippedCount++;
unclaimed.push(entry);
if (entry.scan_method === 'websearch') {
agentHandoff.push({
company: entry.name,
method: 'websearch',
query: entry.scan_query || entry.search_query || entry.careers_url || '',
});
}
continue;
}
if (resolved.error) {
resolveErrors.push({ company: entry.name, error: resolved.error });
continue;
}
targets.push({ ...entry, _provider: resolved.provider, _isBoard: isBoard, _targetId: targets.length });
if (isBoard) boardCount++;
}
}
resolveEntries(companies);
resolveEntries(boards, { isBoard: true });
if (filterCompanyRaw != null && !companyFilterMatched) {
console.error(`Error: no tracked company matches "${filterCompanyRaw}"`);
process.exit(1);
}
// #3438. Startup checks for field_filters / filter_on, before any network
// call. scan.mjs does not run validatePortalsConfig, so every rule that
// decides whether a declared whitelist actually filters is enforced here as
// well; tests/scan-field-filters-parity.test.mjs holds the two rule sets
// together. Each shape rejected below would otherwise leave the whitelist
// narrower than written, or compile to "no positive constraint" and pass
// every posting while the config looks in force.
const exitOnConfigError = (message) => {
console.error(`Error: ${message}`);
process.exit(1);
};
const isPlainObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
const FIELD_FILTER_KEYS = ['positive', 'negative'];
if (config.field_filters !== undefined && !isPlainObject(config.field_filters)) {
exitOnConfigError('field_filters must be an object keyed by field name');
}
for (const [name, block] of Object.entries(config.field_filters ?? {})) {
// `title` routes to the top-level title_filter, so a block under this key
// is never read — and with no title_filter present, the scan would pass
// every title while this list looks like a whitelist in force.
if (name === 'title') {
exitOnConfigError('field_filters.title is never read - filter_on "title" uses the top-level title_filter. Move these keywords there.');
}
if (!isPlainObject(block)) {
exitOnConfigError(`field_filters.${name} must be an object with positive and/or negative lists`);
}
for (const key of Object.keys(block)) {
if (!FIELD_FILTER_KEYS.includes(key)) {
exitOnConfigError(`field_filters.${name}.${key} is not a recognized key - expected one of ${FIELD_FILTER_KEYS.join(', ')}`);
}
}
// buildTitleFilter silently drops a list written as a bare string and any
// non-string or blank entry. title_filter keeps that leniency for existing
// configs; field_filters is new, so it has none to preserve.
let keywordCount = 0;
for (const key of FIELD_FILTER_KEYS) {
const list = block[key];
if (list === undefined || list === null) continue;
if (!Array.isArray(list)) {
exitOnConfigError(`field_filters.${name}.${key} must be a list of strings - a bare string is ignored`);
}
if (list.some(k => typeof k !== 'string' || k.trim() === '')) {
exitOnConfigError(`field_filters.${name}.${key} entries must be non-empty strings`);
}
keywordCount += list.length;
}
if (keywordCount === 0) {
exitOnConfigError(`field_filters.${name} has no usable keyword in positive or negative - it would match every posting`);
}
}
// One compiled predicate per declared field, built by the same compiler as
// titleFilter so no field gets its own matching dialect.
const fieldFilters = new Map(
Object.entries(config.field_filters ?? {}).map(([name, block]) => [name, buildTitleFilter(block)]),
);
for (const target of targets) {
if (target.filter_on !== undefined) {
const declared = Array.isArray(target.filter_on) ? target.filter_on : [target.filter_on];
if (declared.length === 0) {
exitOnConfigError(`${target.name}: filter_on must not be an empty list - omit the key to gate on title`);
}
if (declared.some(f => typeof f !== 'string' || f.trim() === '')) {
exitOnConfigError(`${target.name}: filter_on must be a non-empty string or a list of them`);
}
}
for (const field of normalizeFilterOn(target.filter_on)) {
if (field !== 'title' && !fieldFilters.has(field)) {
exitOnConfigError(`${target.name}: filter_on "${field}" has no field_filters.${field} block in portals.yml`);
}
}
if (target.skip_tiers !== undefined
&& (!Array.isArray(target.skip_tiers)
|| target.skip_tiers.some(t => typeof t !== 'string' || !TIERS.includes(t.toLowerCase())))) {
exitOnConfigError(`${target.name}: skip_tiers must be a list of ${TIERS.join(', ')}, or [] to turn the tier filter off`);
}
}
const tierFilterActive = skipTiers.length > 0 || targets.some(t => entrySkipTiers(t).length > 0);
if (tierFilterActive) {
const mod = await import('./classify-tier.mjs');
classifyTier = mod.classifyTier || mod.default;
}
const localParserCount = targets.filter(t => t._provider.id === 'local-parser').length;
const companyCount = targets.length - boardCount;
const parts = [`${companyCount} companies`];
if (boardCount > 0) parts.push(`${boardCount} job boards`);
parts.push(`${localParserCount} local parser`);
parts.push(`${skippedCount} skipped — no provider matched`);
console.log(`Scanning ${parts.join('; ')} via providers`);
if (dryRun) console.log('(dry run — no files will be written)\n');
for (const warning of enabledWarnings) console.error(`Warning: ${warning}`);
if (unclaimed.length > 0) {
console.log('No provider matched these entries. Set provider to an id this scanner loads, point careers_url at a board a provider detects, or set scan_method: websearch so the handoff list names the company:');
for (const entry of unclaimed) {
const where = typeof entry.careers_url === 'string' && entry.careers_url
? entry.careers_url
: (typeof entry.api === 'string' && entry.api ? entry.api : '(no careers_url)');
const method = entry.scan_method ? ` (scan_method: ${entry.scan_method})` : '';
console.log(` • ${entry.name} — ${where}${method}`);
}
}
// 3.5. Load the user's do-not-apply list (#1742). Opt-in: absent file =
// empty Map = the filter below never fires.
const blacklist = loadBlacklist();
// 4. Load dedup sets — one read per source file for the whole run (#2382).
const historyPolicy = scanHistoryPolicy(config);
const canonicalizeCompany = buildCompanyCanonicalizer(config.company_aliases);
const dedupIncludeLocation = resolveDedupIncludeLocation(config);
const dedupIncludeLanguage = resolveDedupIncludeLanguage(config);
const preferredLanguages = resolvePreferredLanguages(config);
const preferredLanguageRank = preferredLanguages.length ? preferredLanguageRanker(preferredLanguages) : null;
const dedupSnapshot = loadDedupSnapshot(historyPolicy, canonicalizeCompany, { includeLocation: dedupIncludeLocation });
const seenUrls = dedupSnapshot.seen;
const seenUrlVariants = dedupSnapshot.seenUrlVariants ?? new Map();
const seenCompanyRoles = dedupSnapshot.seenCompanyRoles;
const seenCompanyRoleRequisitions = dedupSnapshot.seenCompanyRoleRequisitions ?? new Map();
const locatedRequisitionsByBase = dedupSnapshot.locatedRequisitionsByBase ?? new Map();
const seenCompanyRoleLanguages = dedupSnapshot.seenCompanyRoleLanguages ?? new Map();
const locatedLanguagesByBase = dedupSnapshot.locatedLanguagesByBase ?? new Map();
// 5. Fetch from each target
// LOCAL day. This one value does two things that both care which day it is:
// it is the `today` buildCooldownFilter compares against, and it is the
// firstSeen date written into scan-history.tsv. On the UTC day a
// west-of-Greenwich evening scan opened cooldowns early AND stamped history
// rows with tomorrow, which then read one day old on the next recheck (#3070).
const date = localToday();
const windows = loadReApplyWindows();
const cooldownFilter = buildCooldownFilter(windows, date);
let totalFilteredCooldown = 0;
const cooldownOffers = [];
// Config-rejected offers, kept so the scan-history row can say what the
// summary counter only counts. All three lists stay empty under --dry-run,
// the same as every other history write below.
const titleFilteredOffers = [];
const locationFilteredOffers = [];
const ageFilteredOffers = [];
let totalFound = 0;
let totalFilteredTitle = 0;
let totalFilteredTier = 0;
// #3438: rejections by a declared non-title field, kept apart from
// totalFilteredTitle so the summary says which whitelist did the work.
let totalFilteredDeclaredField = 0;
// Jobs that cleared the declared-field gate with a declared field absent,
// i.e. passed without that whitelist ever judging them.
let totalPassedFieldAbsent = 0;
const declaredFieldSeen = new Map();
const declaredFieldAbsent = new Map();
let totalFilteredLocation = 0;
let totalFilteredPostingAge = 0;
let totalFilteredPostedDate = 0;
let totalFilteredSalary = 0;
let totalFilteredContent = 0;
// #6020. Title misses that content_filter.rescue_title_misses kept on their
// description. Summary line only: the receipt and scan-runs.tsv are unchanged.
let totalRescuedByContent = 0;
let totalFilteredCountryEligibility = 0;
let totalFilteredBlacklist = 0;
let annotatedBlacklisted = 0;
let totalFilteredVisa = 0;
let totalDupes = 0;
const newOffers = [];
const verifyDedupKeys = new Map();
const errors = [...resolveErrors];
// Targets whose fetch threw. Not errors.length: that also holds resolve
// errors and a local parser rescued by its API fallback, which succeeded.
let failedTargets = 0;
const emptyTargets = [];
const unverifiedZeroTargets = [];
// Boards a provider read only partly (page cap, 429, fetch error), keyed by
// company: see lib/board-truncation.mjs. Logged `partial`, never `reachable`.
/** @type {Map<string, import('./lib/board-truncation.mjs').BoardTruncation>} */
const partialTargets = new Map();
// Arm the failure-path row (#2643) now that the sweep is about to start and
// every counter it reads is in scope. new_added is hardcoded 0 on a failed
// run even if the sweep added postings before dying (the count isn't settled
// mid-sweep). Excluded from trend averages so it can't skew them, but a
// raw-TSV reader should treat that 0 as a sentinel, not a true count.
if (!dryRun) {
registerRunFailureSnapshot(() => ({
timestamp: new Date().toISOString(),
companies: targets.filter(t => !t._isBoard).length,
boards: targets.filter(t => t._isBoard).length,
found: totalFound, filteredTitle: totalFilteredTitle, filteredTier: totalFilteredTier,
filteredLocation: totalFilteredLocation, filteredPostingAge: totalFilteredPostingAge,
filteredSalary: totalFilteredSalary, filteredContent: totalFilteredContent,
filteredCooldown: totalFilteredCooldown, dupes: totalDupes, newAdded: 0,
errors: errors.length, filteredBlacklist: totalFilteredBlacklist,
filteredVisa: totalFilteredVisa, filteredPostedDate: totalFilteredPostedDate,
filteredCountryEligibility: totalFilteredCountryEligibility,
}));
// Ctrl-C mid-sweep is the common abort. Best effort: record, then die
// with the conventional SIGINT code.
process.once('SIGINT', () => {
writeRunFailureRow('failed');
process.exit(130);
});
}
// Values providers resolve once and reuse on later scans (provider-id-cache.mjs).
const idCaches = openProviderIdCache(path.join(DATA_ROOT, PROVIDER_ID_CACHE_DIR));
// Fetch every target concurrently, but filter board copies only after all
// employers have made their location decisions for this scan's URLs.
const deferredBoards = [];
const employerLocationRejected = new Set();
const employerLocationAccepted = new Set();
// Boards whose fetch failed on a transient error, retried once after the
// sweep (see retryTransientFailures below).
const TRANSIENT_KINDS = new Set(['network', 'server']);
const TRANSIENT_RETRY_GAP_MS = 3_000;
/** @type {Array<{ company: any, entry: any }>} */
const transientFailures = [];
const retriedTargets = new Set();
const tasks = targets.map(company => async () => {
let provider = company._provider;
const observation = { requests: 0, successfulResponses: 0, lastStatus: null };
// includeUndated is deliberately ALWAYS true, independent of the window.
// It does not mean "include undated postings in the results" — scan.mjs
// already decides that downstream, where buildPostedDateFilter passes a
// posting with no parseable date. It means "provider, do not pre-empt that
// decision": without it, workday.mjs's no-date-skip returns page 0 only for
// any tenant whose CXS payload omits postedOn entirely, silently dropping
// postings this scanner would have kept.
//
// It covers the all-undated tenant, not the mixed one. workday.mjs's
// pageIsPastWindow reads dated postings only, so on a page mixing stale
// dated postings with undated ones the early-stop still fires and undated
// postings on later pages go unfetched. Documented in modes/scan.md; the
// fix belongs in workday.mjs, where closing it costs the optimisation on
// every tenant that mixes.
const ctx = {
...makeHttpCtx({
onRequest: () => { observation.requests++; },
onResponse: status => {
observation.lastStatus = status;
if (status >= 200 && status < 300) observation.successfulResponses++;
},
}),
sinceMs: earlyStopSinceMs,
includeUndated: true,
locationHints: config.location_filter,
// A provider that serves extra language versions of a posting at extra
// cost fetches them only when the scan will keep them.
dedupIncludeLanguage,
// Present only when portals.yml sets preferred_languages: a provider that
// returns language versions selectively also returns the preferred ones.
...(preferredLanguageRank ? { preferredLanguageRank } : {}),
// This scan's own title decision, for a provider that pays extra per
// posting (Workday's multi-location detail GETs) and can skip postings
// the title filter below will drop. Absent when this company's filter_on
// leaves the title out, since then no title is dropped.
...(normalizeFilterOn(company.filter_on).includes('title') ? { titleWanted: (title) => titleGate(title) === 'match' } : {}),
idCache: idCaches.forProvider(provider.id),
};
applyEntryResponseCap(ctx, company);
applyEntryTimeout(ctx, company);
let sourceName = provider.id === 'local-parser' ? 'local-parser' : `${provider.id}-api`;
const recordError = (err, { retryable = false } = {}) => {
failedTargets++;
const entry = {
company: company.name,
error: err.message,
kind: classifyFetchError(err),
status: err.status ?? observation.lastStatus,
};
errors.push(entry);
if (retryable && TRANSIENT_KINDS.has(entry.kind) && !retriedTargets.has(company)) {
transientFailures.push({ company, entry });
}
};
try {
let jobs;
try {
jobs = await provider.fetch(company, ctx);
} catch (parserErr) {
if (provider.id !== 'local-parser') throw parserErr;
const fallback = resolveProvider(company, providers, { skipIds: ['local-parser'] });
if (!fallback || fallback.error) throw parserErr;
provider = fallback.provider;
sourceName = `${provider.id}-api`;
ctx.idCache = idCaches.forProvider(provider.id);
jobs = await provider.fetch(company, ctx);
errors.push({
company: company.name,
error: `local parser failed, used API fallback: ${parserErr.message}`,
});
}
if (!Array.isArray(jobs)) {
throw new Error(`${provider.id}: fetch() did not return an array`);
}
totalFound += jobs.length;
const truncation = boardTruncation(jobs);
// Boards (job_boards: aggregators) too: unlike an empty result, which is
// routine for a keyword search, a capped or failed walk is a real gap.
if (truncation) partialTargets.set(company.name, truncation);
if (!company._isBoard && jobs.length === 0) {
if (emptyTargetStatus(observation) === 'empty') emptyTargets.push(company.name);
else unverifiedZeroTargets.push(company.name);
}
const processJobs = async () => {
// This fetch's language versions of each posting, in provider order and
// before any filter: a seen row is matched against every version, also
// those a filter rejects (a German title can fail a title filter that its
// English version passes).
const siblingsByUrl = new Map();
if (dedupIncludeLanguage) {
for (const job of jobs) {
const languages = languageFormsForDedup(job?.language);
if (!languages.length) continue;
const key = seenJobKey(job.url);
if (!siblingsByUrl.has(key)) siblingsByUrl.set(key, []);
siblingsByUrl.get(key).push({ languages, title: job.title });
}
}
const declaredFields = normalizeFilterOn(company.filter_on);
const companySkipTiers = entrySkipTiers(company);
const orderedJobs = preferredLanguageRank ? orderByPreferredLanguage(jobs, preferredLanguageRank) : jobs;
for (const job of orderedJobs) {
const listingToken = refreshListingKey(job);
// #3438. Presence accounting only — no verdict, no rejection. It runs
// before every filter below, including the blacklist skip, because it
// answers "does this provider publish this field at all", which no
// later filter's opinion can change.
for (const field of declaredFields) {
if (field === 'title') continue;
const key = declaredFieldKey(company._targetId, field);
declaredFieldSeen.set(key, (declaredFieldSeen.get(key) || 0) + 1);
if (isFieldAbsent(declaredFieldValue(job, field))) {
declaredFieldAbsent.set(key, (declaredFieldAbsent.get(key) || 0) + 1);
}
}
// Trust enrichment — runs before filters, never drops
const trustResult = trustValidator(job);
job.trustScore = trustResult.score;
job.trustFlags = trustResult.flags;
job.trustLevel = trustResult.level;
// Company blacklist (#1742) — the user's own do-not-apply decision,
// checked first: it's company-level, not a per-posting signal. Never
// silent: skips are counted and reported in the run summary, and
// --include-blacklisted lets the posting through annotated instead.
if (blacklist.size > 0) {
const blEntry = findBlacklistEntry(blacklist, job.company || company.name || '', job.url);
if (blEntry) {
if (!includeBlacklisted) {
totalFilteredBlacklist++;
continue;
}
annotatedBlacklisted++;
job.blacklisted = true;
const label = `blacklisted${blEntry.reason ? `: ${blEntry.reason}` : ''}`;
job.note = typeof job.note === 'string' && job.note.trim()
? `${label} — ${job.note}`
: label;
}
}
// #3438. Absent filter_on → normalizeFilterOn returns ["title"] and
// this is the same titleFilter(job.title) call as before, against the
// same compiled object. Declared fields are ANDed, and a rejection is
// booked to the field that failed: with filter_on: [title, noc], a
// matching title and a rejected noc is a field rejection.
let failedField = null;
let sawAbsentField = false;
// #6020. With content_filter.rescue_title_misses, a title that misses
// title_filter.positive (and hits no title_filter.negative) is not
// rejected here: the decision moves to the content check below, where
// the description alone can keep it. If that check fails, the miss is
// booked as a title rejection (and a skipped_title row) as before. A
// filter between the two that drops it first books it as its own.
let titleMissed = false;
for (const field of declaredFields) {
if (field === 'title') {
const verdict = titleGate(job.title);
if (verdict === 'rescue') { titleMissed = true; continue; }
if (verdict === 'reject') { failedField = field; break; }
} else {
const value = declaredFieldValue(job, field);
if (isFieldAbsent(value)) sawAbsentField = true;
else if (!fieldFilters.get(field)(String(value))) { failedField = field; break; }
}
}
if (failedField === 'title') {
totalFilteredTitle++;
if (!dryRun) titleFilteredOffers.push({ ...job, source: sourceName });
continue;
}
if (failedField !== null) {
totalFilteredDeclaredField++;
continue;
}
if (sawAbsentField) totalPassedFieldAbsent++;
if (classifyTier && companySkipTiers.includes(classifyTier(job.title))) {
totalFilteredTier++;
continue;
}
// job.title is passed so a role whose remoteness is stated in the title
// ("Program Manager - Remote") isn't rejected for a city-only location.
const locationUrl = normalizeUrlForDedup(job.url);
const employerRejected = company._isBoard && employerLocationRejected.has(locationUrl)
&& !employerLocationAccepted.has(locationUrl);
if (employerRejected || !locationFilter(job.location, job.url, job.title, job)) {
if (!company._isBoard) employerLocationRejected.add(locationUrl);
totalFilteredLocation++;
if (!dryRun) locationFilteredOffers.push({ ...job, source: sourceName });
continue;
}
if (!company._isBoard) employerLocationAccepted.add(locationUrl);
if (!postingAgeFilter(job.postedAt)) {
totalFilteredPostingAge++;
if (!dryRun) ageFilteredOffers.push({ ...job, source: sourceName });
continue;
}
if (!postedDateFilter(job.postedAt)) {
totalFilteredPostedDate++;
continue;
}
if (!salaryFilter(job.salary)) {
totalFilteredSalary++;
continue;
}
if (titleMissed) {
if (!contentFilter(job.description, [], { titleMissed: true })) {
totalFilteredTitle++;
if (!dryRun) titleFilteredOffers.push({ ...job, source: sourceName });
continue;
}
totalRescuedByContent++;
} else if (!contentFilter(job.description, matchedTitleKeywords(job.title, config.title_filter))) {
totalFilteredContent++;
continue;
}
if (!countryEligibilityFilter(countryEligibilityText(job))) {
totalFilteredCountryEligibility++;
continue;
}
if (!visaFilter(visaFilterText(job))) {
totalFilteredVisa++;
continue;
}
const dedupUrl = seenJobKey(job.url);
const language = languageFormsForDedup(job.language);
// A seen URL key passes only as a language version of that posting no
// seen row was in; a match on the provider's listing identity stays a
// duplicate. Such a version skips the company+role check below as
// well: the row it would match there is the same posting's, already
// resolved here, and an unknown-language row would hold it back again.
let newLanguageVersion = false;
if (isOfferSeen(job, seenUrls, dedupSnapshot.identityPromotableUrls)) {
newLanguageVersion = dedupIncludeLanguage && seenUrls.has(dedupUrl) && isUnseenLanguageVersion(
seenUrlVariants.get(dedupUrl),
{ languages: language, title: job.title },
siblingsByUrl.get(dedupUrl),
);
if (!newLanguageVersion) {
totalDupes++;
continue;
}
}
// Compare requisitions only in overlapping locations: the exact key,
// truly locationless wildcard rows, and (for a locationless candidate)
// the prebuilt aggregate of located rows. An unknown ID keeps the
// historical duplicate decision. Aggregators use URL dedup only.
const baseKey = companyRoleDedupKey(job.company, job.title, canonicalizeCompany);
// No employer, no company+role identity. A board that cannot name a
// posting's employer returns `company: ''` (providers/_types.js allows
// it; builtin, platsbanken, justjoin and vdab do it), and `::{title}`
// then stood for every unnamed employer at once: a second "Senior
// Software Engineer" from a different company was dropped as a
// duplicate of the first. Seeding already skips such rows
// (collectSeenCompanyRoles), so they dedup by URL only, as an
// aggregator's do.
const key = company.aggregator === true || !canonicalizeCompany(job.company)
? null
: (dedupIncludeLocation
? companyRoleDedupKey(job.company, job.title, canonicalizeCompany, job.location)
: baseKey);
const requisition = requisitionIdsForDedup({ url: job.url, text: job.title, requisitionId: job.requisitionId });
if (!newLanguageVersion && matchesSeenCompanyRole({ key, baseKey, seen: seenCompanyRoles,
requisitions: seenCompanyRoleRequisitions, locatedRequisitions: locatedRequisitionsByBase,
languages: seenCompanyRoleLanguages, locatedLanguages: locatedLanguagesByBase },
requisition, dedupIncludeLanguage ? language : [])) {
totalDupes++;
continue;
}
const cooldownResult = cooldownFilter(job);
if (cooldownResult.skip) {
markOfferSeen(job, seenUrls);
recordUrlVariant(seenUrlVariants, dedupUrl, language, job.title);
totalFilteredCooldown++;
cooldownOffers.push({
job: { ...job, source: sourceName },
status: cooldownResult.reason,
});
continue;
}
// Mark as seen to avoid intra-scan dupes. The index is maintained in the
// same breath as the set it indexes, so a role first surfaced with a
// city THIS run also suppresses a locationless twin later in the run —
// not only across runs.
// Verify mode can reject this URL as expired or invalid later. Pin the
// URL now for same-run URL dedup, but wait to pin its cross-URL identity
// until liveness verification accepts an offer.
markOfferSeen(job, seenUrls, { includeIdentity: !verify });
recordUrlVariant(seenUrlVariants, dedupUrl, language, job.title);
if (key !== null && !verify) {
seenCompanyRoles.add(key);
recordForms(seenCompanyRoleRequisitions, key, requisition);
recordLanguages(seenCompanyRoleLanguages, key, requisition, language);
if (key !== baseKey) {
recordForms(locatedRequisitionsByBase, baseKey, requisition);
recordLanguages(locatedLanguagesByBase, baseKey, requisition, language);
}
}
// Tag with the company's careers domain so verify can offer a 404/410
// rediscovery fallback. A null domain (no careers_url) marks the offer
// as broad-discovery — ineligible for the fallback, per the issue scope.
const careersUrlDomain = extractCareersUrlDomain(company.careers_url);
const offerForVerification = {
...job,
source: sourceName,
tracked: Boolean(careersUrlDomain),
careersUrlDomain,
};
newOffers.push(offerForVerification);
if (verify) verifyDedupKeys.set(offerForVerification, {
key, baseKey, requisition, language, newLanguageVersion,
});
}
};
if (company._isBoard) {
// Never await an employer barrier inside a fetch worker: enough fast
// boards could occupy every worker before the employer can start.
deferredBoards.push(async () => {
try { await processJobs(); } catch (err) { recordError(err); }
});
} else {
await processJobs();
}
} catch (err) {
recordError(err, { retryable: true });
}
});
// Second pass for boards that failed on a TRANSIENT error (timeout, reset,
// 5xx) during the concurrent sweep. A provider already retries each request
// with backoff, but those retries run inside the same burst that caused the
// failure: on 2026-11-07 Marvell's first page timed out four times while
// other Workday boards on the same data center were paging, then read in
// full in 26 s on its own. Re-running the failed boards one at a time after
// the sweep removes the contention instead of repeating into it: the same
// deferred sequential retry scan-ats-full.mjs runs, and the AWS Builders'
// Library advice to retry with the load reduced rather than amplify it
// ("Timeouts, retries, and backoff with jitter"). Bounded: one extra attempt
// per board, never for a 4xx (a 403 or 404 will not change in seconds).
const retryTransientFailures = async () => {
if (transientFailures.length === 0) return;
const pending = transientFailures.splice(0);
process.stderr.write(`scan: retrying ${pending.length} board(s) that failed on a transient error, one at a time: ${pending.map((p) => p.company.name).join(', ')}\n`);
for (const { company, entry } of pending) {
retriedTargets.add(company);
const at = errors.indexOf(entry);
if (at !== -1) errors.splice(at, 1);
failedTargets--;
await new Promise((r) => setTimeout(r, TRANSIENT_RETRY_GAP_MS));
await tasks[targets.indexOf(company)]();
}
};
const companyFetches = async () => {
await parallelFetch(tasks, SCAN_CONCURRENCY, progressTicker(tasks.length), {
hostLimit: HOST_CONCURRENCY,
hostOf: (_task, index) => companyFetchHost(targets[index]),
});
await retryTransientFailures();
};
if (process.env.CAREER_OPS_HTTP_CACHE === '0') {
await companyFetches();
} else {
const ttlMs = resolveHttpCacheTtlMs(process.env.CAREER_OPS_HTTP_CACHE_TTL_MS);
await withHttpCache({
dir: path.join(DATA_ROOT, 'data', 'cache', 'http'),
ttlMs,
write: !dryRun,
}, async () => {
await companyFetches();
const stats = httpCacheStats();
if (!stats || (stats.hits === 0 && stats.revalidated === 0)) return;
const parts = [];
if (stats.hits > 0) parts.push(`reused ${stats.hits} board responses saved in the last ${formatHttpCacheWindow(ttlMs)}`);
if (stats.revalidated > 0) parts.push(`rechecked ${stats.revalidated} older board responses`);
process.stderr.write(`scan: ${parts.join(', and ')}. Set CAREER_OPS_HTTP_CACHE=0 to always download.\n`);
});
}
for (const processBoard of deferredBoards) await processBoard();
// --dry-run leaves no trace, this cache included.
if (!dryRun) {
try {
await idCaches.save();
} catch (err) {
// Losing the cache costs a lookup on the next scan, not this scan's results.
console.warn(`⚠️ could not save the provider id cache: ${err.message}`);
}
}
// 5.5. Optional liveness verification — drop expired and guard-rejected postings
let verifiedOffers = newOffers;
let expiredOffers = [];
let droppedOffers = [];
let invalidOffers = [];
let migratedOffers = [];
if (verify && newOffers.length > 0) {
console.log(`\nVerifying liveness of ${newOffers.length} new offer(s) via API, with sequential Playwright fallback...`);
const result = await verifyOffers(newOffers, { headedFallback, throttleBaseMs, rediscover });
verifiedOffers = result.verified;
expiredOffers = result.expired;
droppedOffers = result.dropped;
invalidOffers = result.invalid;
migratedOffers = result.migrated;
// Migrated offers re-enter the pipeline at their newly discovered URL.
if (migratedOffers.length > 0) {
verifiedOffers = [...verifiedOffers, ...migratedOffers];
for (const migrated of migratedOffers) {
const previous = newOffers.find((offer) => offer.url === migrated.previousUrl);
const priorKeys = previous && verifyDedupKeys.get(previous);
if (priorKeys) verifyDedupKeys.set(migrated, {
...priorKeys,
requisition: requisitionIdsForDedup({ url: migrated.url, text: migrated.title, requisitionId: migrated.requisitionId }),
language: languageFormsForDedup(migrated.language),
});
}
}
// Several URL aliases can reach verification before liveness is known.
// Keep the first accepted offer for each strong identity, and only then
// publish the identity token to this run's dedup set. Rejected aliases do
// not hide a later live URL.
const countBeforeIdentityDedup = verifiedOffers.length;
verifiedOffers = retainVerifiedListingIdentities(verifiedOffers, seenUrls, result.verificationStatusByOffer);
totalDupes += countBeforeIdentityDedup - verifiedOffers.length;
const acceptedOffers = [];
for (const offer of verifiedOffers) {
const keys = verifyDedupKeys.get(offer);
// A language version the URL check let through skips the role check
// here too, for the same reason as in the scan loop.
if (keys?.key !== null && keys?.key !== undefined && !keys.newLanguageVersion && matchesSeenCompanyRole({
key: keys.key,
baseKey: keys.baseKey,
seen: seenCompanyRoles,
requisitions: seenCompanyRoleRequisitions,
locatedRequisitions: locatedRequisitionsByBase,
languages: seenCompanyRoleLanguages,
locatedLanguages: locatedLanguagesByBase,
}, keys.requisition, dedupIncludeLanguage ? keys.language : [])) {
totalDupes++;
continue;
}
if (keys?.key !== null && keys?.key !== undefined) {
seenCompanyRoles.add(keys.key);
recordForms(seenCompanyRoleRequisitions, keys.key, keys.requisition);
recordLanguages(seenCompanyRoleLanguages, keys.key, keys.requisition, keys.language);
if (keys.key !== keys.baseKey) {
recordForms(locatedRequisitionsByBase, keys.baseKey, keys.requisition);
recordLanguages(locatedLanguagesByBase, keys.baseKey, keys.requisition, keys.language);
}
}
acceptedOffers.push(offer);
}
verifiedOffers = acceptedOffers;
}
// 5.6. Experience filter: read each new offer's stated years. Only a required
// overall bar whose lowest reading is beyond the band drops (action: drop);
// an unread JD, or one with no such bar, is kept and labelled.
let experienceDropped = [];
if (experienceFilter.enabled && verifiedOffers.length > 0) {
console.log(`\nReading stated years of experience for ${verifiedOffers.length} new offer(s)...`);
// A first scan with the filter on can read hundreds of JDs; say how far along.
const { kept, dropped } = await applyExperienceFilter(verifiedOffers, experienceFilter, {
onProgress: (done, total) => {
if (done % PROGRESS_EVERY === 0 || done === total) process.stderr.write(`[${new Date().toTimeString().slice(0, 8)}] scan: ${done}/${total} JDs read\n`);
},
});
verifiedOffers = kept;
experienceDropped = dropped;
}
// 5.7. Cross-listing check (#1597): fingerprint each new offer's JD body and
// compare against recent history rows from a DIFFERENT company — the same
// requirements text under two names is usually an agency re-post of a direct
// listing (or vice versa), which URL and company+role dedup both miss.
// Fingerprints are computed once here and reused by appendToScanHistory.
for (const offer of verifiedOffers) {
offer.fingerprint = fingerprintText(offer.description);
}
// History rows come from the run-start snapshot: nothing has appended to
// scan-history.tsv yet at this point in the run (all writes happen below),
// so this sees the same bytes a re-read would — minus the third full parse.
const crossListings = findCrossListings(verifiedOffers, dedupSnapshot.fingerprintHistory);
// 6. Write results
if (!dryRun && verifiedOffers.length > 0) {
await appendToPipeline(verifiedOffers);
await appendToScanHistory(verifiedOffers, date);
}
// Postings the experience filter dropped: recorded for visibility, no dedup
// weight (OBSERVATIONAL_SCAN_HISTORY_STATUSES), so changing your years or
// stretch_years hands them back on the next scan. Once per posting.
if (!dryRun && experienceDropped.length > 0) {
const newRows = unrecordedOffers(experienceDropped, 'skipped_experience', readIfExists(SCAN_HISTORY_PATH));
if (newRows.length > 0) await appendToScanHistory(newRows, date, 'skipped_experience');
}
if (!dryRun && cooldownOffers.length > 0) {
const cooldownGroups = {};
for (const item of cooldownOffers) {
if (!cooldownGroups[item.status]) {
cooldownGroups[item.status] = [];
}
cooldownGroups[item.status].push(item.job);
}
for (const [status, group] of Object.entries(cooldownGroups)) {
await appendToScanHistory(group, date, status);
}
}
// Expired postings — plus the old URLs of migrated offers — are recorded as
// skipped_expired so subsequent scans dedup-skip the dead URLs.
const expiredForHistory = [
...expiredOffers,
...migratedOffers.map(o => ({ ...o, url: o.previousUrl })),
];
if (!dryRun && expiredForHistory.length > 0) {
await appendToScanHistory(expiredForHistory, date, 'skipped_expired');
}
// Offers the title, location and posting-age cuts removed: recorded for
// visibility, never added to pipeline.md. All three are
// OBSERVATIONAL_SCAN_HISTORY_STATUSES, so the rows carry no dedup weight —
// the setting that rejected them is one the user edits, and a row written
// under the old setting must not suppress the same posting once it moves.
// Each posting is recorded once per status, not once per scan, so the file
// grows with new postings rather than with scans (#2695).
if (!dryRun && (titleFilteredOffers.length > 0 || locationFilteredOffers.length > 0 || ageFilteredOffers.length > 0)) {
const historyText = readIfExists(SCAN_HISTORY_PATH);
const newTitleRows = unrecordedOffers(titleFilteredOffers, 'skipped_title', historyText);
const newLocationRows = unrecordedOffers(locationFilteredOffers, 'skipped_location', historyText);
const newAgeRows = unrecordedOffers(ageFilteredOffers, 'skipped_age', historyText);
if (newTitleRows.length > 0) await appendToScanHistory(newTitleRows, date, 'skipped_title');
if (newLocationRows.length > 0) await appendToScanHistory(newLocationRows, date, 'skipped_location');
if (newAgeRows.length > 0) await appendToScanHistory(newAgeRows, date, 'skipped_age');
}
// Pages that loaded but had no Apply control: recorded for visibility and
// never added to pipeline.md. The row carries no dedup weight
// (OBSERVATIONAL_SCAN_HISTORY_STATUSES), because "no visible Apply control"
// is not proof the posting is closed; the next scan verifies it again.
// Each posting is recorded once, not once per scan.
if (!dryRun && droppedOffers.length > 0) {
const newDroppedRows = unrecordedOffers(droppedOffers, 'skipped_no_apply_control', readIfExists(SCAN_HISTORY_PATH));
if (newDroppedRows.length > 0) await appendToScanHistory(newDroppedRows, date, 'skipped_no_apply_control');
}
// Guard-rejected URLs (invalid / unsupported protocol / blocked host) are
// recorded with a precise status so subsequent scans dedup-skip them via
// loadSeenUrls, but they never reach pipeline.md.
if (!dryRun && invalidOffers.length > 0) {
// Group by code so the TSV reflects the actual reason category.
const byStatus = new Map();
for (const o of invalidOffers) {
const status = guardStatusFor(o.code);
if (!byStatus.has(status)) byStatus.set(status, []);
byStatus.get(status).push(o);
}
for (const [status, group] of byStatus) {
await appendToScanHistory(group, date, status);
}
}
// 7. Print summary
printScanSummaryHeader('Portal Scan', date);
const summaryCompanies = targets.filter(t => !t._isBoard).length;
const summaryBoards = targets.filter(t => t._isBoard).length;
console.log(`Companies scanned: ${summaryCompanies}`);
if (summaryBoards > 0) console.log(`Job boards scanned: ${summaryBoards}`);
console.log(`Total jobs found: ${totalFound}`);
if (config.title_filter || totalFilteredTitle > 0) {
console.log(`Filtered by title: ${totalFilteredTitle} removed`);
}
if (fieldFilters.size > 0) {
console.log(`Filtered by field: ${totalFilteredDeclaredField} removed`);
console.log(`Passed, field absent: ${totalPassedFieldAbsent} ungated`);
}
if (tierFilterActive) {
console.log(`Filtered by tier: ${totalFilteredTier} removed`);
}
if (config.location_filter || totalFilteredLocation > 0) {
console.log(`Filtered by location: ${totalFilteredLocation} removed`);
}
if (config.max_posting_age_days != null || totalFilteredPostingAge > 0) {
console.log(`Filtered by age: ${totalFilteredPostingAge} removed`);
}
// effectiveAfter, not postedAfter — --since sets a lower bound too, and a
// scan that filtered by date should say so regardless of which flag set it.
if (effectiveAfter || postedBefore) {
console.log(`Filtered by posted date: ${totalFilteredPostedDate} removed`);
}
if (config.salary_filter || totalFilteredSalary > 0) {
console.log(`Filtered by salary: ${totalFilteredSalary} removed`);
}
if (config.content_filter || totalFilteredContent > 0) {
console.log(`Filtered by content: ${totalFilteredContent} removed`);
}
if (rescueTitleMisses) {
console.log(`Title misses kept by content: ${totalRescuedByContent}`);
}
if (config.country_eligibility_filter || totalFilteredCountryEligibility > 0) {
console.log(`Filtered by country eligibility: ${totalFilteredCountryEligibility} removed`);
}
if (visaEnabled) {
console.log(`Filtered by visa: ${totalFilteredVisa} removed`);
}
if (Object.keys(windows).length > 0 || totalFilteredCooldown > 0) {
console.log(`Filtered by cooldown: ${totalFilteredCooldown} removed`);
}
if (experienceFilter.enabled) {
const band = `${experienceFilter.years} yrs + ${experienceFilter.stretchYears} stretch`;
console.log(experienceFilter.action === 'drop'
? `Filtered by experience: ${experienceDropped.length} removed (band: JD minimum ≤ ${band})`
: `Experience filter: label only, nothing removed (band: JD minimum ≤ ${band})`);
}
console.log(`Duplicates: ${totalDupes} skipped`);
if (blacklist.size > 0) {
if (includeBlacklisted) {
console.log(`Blacklisted: ${annotatedBlacklisted} let through annotated (--include-blacklisted)`);
} else {
console.log(`Blacklisted: ${totalFilteredBlacklist} skipped (blacklist)`);
}
}
if (experienceDropped.length > 0) {
console.log(`\nRemoved by experience_filter (the JD's lowest stated requirement is above your band):`);
for (const offer of experienceDropped) {
console.log(` - ${offer.company} — ${offer.title}: requires ${offer.years.req?.display}`);
console.log(` "${offer.years.req?.mentions.find(m => m.kind === 'required' && m.scope === 'overall' && m.floor === offer.years.req?.min)?.quote ?? ''}"`);
console.log(` ${offer.url}`);
}
}
if (experienceFilter.enabled && verifiedOffers.length > 0) {
const counts = { in: 0, out: 0, unknown: 0 };
for (const o of verifiedOffers) if (o.years) counts[o.years.verdict]++;
const unread = verifiedOffers.filter(o => o.years && !o.years.req).length;
console.log(`Stated years (kept): ${counts.in} in band, ${counts.unknown} unknown (${unread} unread)${counts.out ? `, ${counts.out} above band (labelled)` : ''}`);
}
if (crossListings.length > 0) {
console.log(`\n⚠️ Possible cross-listings (same JD text, different company) — warn only, nothing was dropped:`);
for (const { offer, row, score } of crossListings) {
console.log(` - ${offer.company} — ${offer.title}`);
console.log(` ≈ ${Math.round(score * 100)}% of ${row.company} — ${row.title} (seen ${row.dateStr})`);
console.log(` ${offer.url}`);
console.log(` vs ${row.url}`);
}
console.log(` If one side is an agency, apply through ONE channel only — a double submission burns both (#1596).`);
}
const aggregatorMap = loadAggregatorDomains();
if (aggregatorMap.size > 0 && verifiedOffers.length > 0) {
const aggregatorMatches = [];
for (const offer of verifiedOffers) {
const match = checkAggregatorRepost(offer, aggregatorMap);
if (match) {
aggregatorMatches.push({ offer, match });
}
}
if (aggregatorMatches.length > 0) {
console.log(`\n⚠️ Possible aggregator reposts (listed on a known aggregator domain) — warn only, nothing was dropped:`);
for (const { offer, match } of aggregatorMatches) {
console.log(` - ${offer.company} — ${offer.title}`);
console.log(` ${offer.url}`);
console.log(` (${match.domain}: ${match.reason || 'known aggregator'})`);
}
console.log(` Aggregators often scrape primary boards — consider applying directly on the employer's career site (#3577).`);
}
}
// #3438. A declared field that never appeared on a single posting means
// the provider does not supply it: the whitelist silently passed everything
// for that target. That is precisely the failure this feature exists to
// surface, so it is reported even though nothing was dropped.
const deadDeclarations = [];
for (const target of targets) {
for (const field of normalizeFilterOn(target.filter_on)) {
if (field === 'title') continue;
const key = declaredFieldKey(target._targetId, field);
const seen = declaredFieldSeen.get(key) || 0;
const absent = declaredFieldAbsent.get(key) || 0;
if (seen > 0 && absent === seen) deadDeclarations.push({ name: target.name, field, seen });
}
}
if (deadDeclarations.length > 0) {
console.log(`
⚠️ Declared field never observed — that whitelist is effectively OFF:`);
for (const d of deadDeclarations) {
console.log(` ${d.name}: "${d.field}" absent on all ${d.seen} job(s) from this target`);
}
console.log(` Check the field name and whether the provider supplies it.`);
}
if (historyPolicy.recheckAfterDays != null) {
console.log(`Recheck eligible: ${dedupSnapshot.recheckEligible} old scan-history URL(s)`);
}
if (verify) {
console.log(`Expired (verified): ${expiredOffers.length} dropped`);
console.log(`Rediscovered (moved): ${migratedOffers.length} migrated`);
console.log(`No apply control: ${droppedOffers.length} dropped`);
console.log(`Invalid (guarded): ${invalidOffers.length} dropped`);
}
console.log(`New offers added: ${verifiedOffers.length}`);
// Trust validation summary (only when trust_filter is configured)
if (config.trust_filter && config.trust_filter.enabled !== false && verifiedOffers.length > 0) {
const trustHigh = verifiedOffers.filter(o => o.trustLevel === 'high').length;
const trustMedium = verifiedOffers.filter(o => o.trustLevel === 'medium').length;
const trustLow = verifiedOffers.filter(o => o.trustLevel === 'low').length;
console.log(`Trust validation: ${trustHigh} high, ${trustMedium} medium, ${trustLow} low`);
// Flag breakdown
/** @type {Record<string, number>} */
const flagCounts = {};
for (const o of verifiedOffers) {
for (const f of (o.trustFlags || [])) {
flagCounts[f] = (flagCounts[f] || 0) + 1;
}
}
if (Object.keys(flagCounts).length > 0) {
const parts = Object.entries(flagCounts).map(([k, v]) => `${k}: ${v}`);
console.log(`Trust flags: ${parts.join(', ')}`);
}
}
if (agentHandoff.length > 0) {
console.log(`Agent/WebSearch handoff: ${agentHandoff.length} compan${agentHandoff.length === 1 ? 'y' : 'ies'} not handled by zero-token providers`);
for (const item of agentHandoff.slice(0, 25)) {
const hint = item.query ? ` — ${item.query}` : '';
console.log(` • ${item.company} (${item.method})${hint}`);
}
if (agentHandoff.length > 25) {
console.log(` … ${agentHandoff.length - 25} more omitted; narrow with --company or inspect portals.yml`);
}
}
const unreachableTargets = errors.filter((e) => e.kind === 'slug_gone');
const networkTargets = errors.filter((e) => e.kind === 'network');
const otherErrors = errors.filter((e) => e.kind !== 'slug_gone' && e.kind !== 'network');
const STREAK_THRESHOLD = config.portal_health_threshold || 3;
const nowStr = new Date().toISOString();
const healthRecords = [];
// Record each errored target under its real classifyFetchError kind. Before
// this, only slug_gone/network were recorded and auth (401/403), server
// (5xx), and unknown fell through to 'reachable' — so a portal WAF-403ing
// every run was logged as healthy forever and never reached the 🚨 streak
// escalation. The TSV status vocabulary is additive: auth/server/unknown
// join the existing reachable|slug_gone|network|empty.
const errorKindByCompany = new Map(
errors.filter((e) => e.kind).map((e) => [e.company, e.kind])
);
for (const t of targets) {
const isEmpty = emptyTargets.includes(t.name);
const isUnverifiedZero = unverifiedZeroTargets.includes(t.name);
let status = errorKindByCompany.get(t.name) || 'reachable';
// A board read only partly is not healthy coverage: logged `partial`, never
// `reachable`, with its own repeat warning below (not the 🚨 dead-board
// streak). Checked before `empty`, so a tagged empty read cannot pass as
// healthy.
if (status === 'reachable' && partialTargets.has(t.name)) status = 'partial';
if (status === 'reachable' && isEmpty) status = 'empty';
if (status === 'reachable' && isUnverifiedZero) status = 'unverified_zero';
healthRecords.push({ timestamp: nowStr, company: t.name, status });
}
const pastHealth = loadPortalHealth();
const currentStreaks = computeConsecutiveFailures([...pastHealth, ...healthRecords]);
const partialStreaks = computePartialStreaks([...pastHealth, ...healthRecords]);
const persistentlyDead = [];
const newlyDeadSlug = [];
const newlyDeadNetwork = [];
// All error kinds can reach the 🚨 persistent list (auth/server/unknown
// included — a WAF that 403s the scanner every run is coverage decay too).
// Below threshold, only slug_gone/network keep their dedicated warnings;
// auth/server/unknown stay in the one-off `Errors (N):` print below.
for (const e of [...unreachableTargets, ...networkTargets, ...otherErrors.filter((x) => x.kind),
...unverifiedZeroTargets.map(company => ({ company, kind: 'unverified_zero' }))]) {
const streak = currentStreaks.get(e.company) || 1;
if (streak >= STREAK_THRESHOLD) {
if (!persistentlyDead.includes(e.company)) persistentlyDead.push(e.company);
} else if (e.kind === 'slug_gone') {
if (!newlyDeadSlug.some(x => x.company === e.company)) newlyDeadSlug.push(e);
} else if (e.kind === 'network') {
newlyDeadNetwork.push(e);
}
}
if (persistentlyDead.length > 0) {
console.log(`\n🚨 FIX NEEDED: ${persistentlyDead.length} target(s) have been unreachable for ${STREAK_THRESHOLD}+ runs:`);
console.log(` ${persistentlyDead.join(', ')}`);
console.log(` Run: node verify-portals.mjs to check if the ATS migrated, or update their board slugs.`);
}
if (newlyDeadSlug.length > 0) {
const names = newlyDeadSlug.map(x => x.company).join(', ');
console.log(`\n⚠️ ${newlyDeadSlug.length} target(s) unreachable (slug?): ${names} — run: node verify-portals.mjs`);
}
if (emptyTargets.length > 0) {
console.log(`🟡 ${emptyTargets.length} target(s) live but empty: ${emptyTargets.join(', ')}`);
}
if (partialTargets.size > 0) {
const parts = [...partialTargets].map(([company, t]) => describeTruncation(company, t));
console.log(`\n⚠️ ${partialTargets.size} board(s) only PARTLY read — postings beyond the cut were not scanned:`);
for (const p of parts) console.log(` • ${p}`);
// A repeat is not a dead board: the board answers every run. The usual
// cause is a cap on the entry, which only the entry can change.
const repeated = [...partialTargets.keys()].filter((c) => (partialStreaks.get(c) || 0) >= STREAK_THRESHOLD);
if (repeated.length > 0) {
console.log(` Partly read for ${STREAK_THRESHOLD}+ runs in a row: ${repeated.join(', ')}`);
console.log(` For a page cap, raise max_pages on the entry or narrow its search; a fetch error or rate limit that repeats points at the board's host.`);
}
}
if (unverifiedZeroTargets.length > 0) {
console.log(`⚠️ ${unverifiedZeroTargets.length} target(s) returned zero jobs without a successful HTTP response: ${unverifiedZeroTargets.join(', ')}`);
}
if (newlyDeadNetwork.length > 0) {
console.log(`\nNetwork errors (${newlyDeadNetwork.length}):`);
for (const e of newlyDeadNetwork) {
console.log(` ✗ ${e.company}: ${e.error}`);
}
}
if (otherErrors.length > 0) {
console.log(`\nErrors (${otherErrors.length}):`);
for (const e of otherErrors) {
console.log(` ✗ ${e.company}: ${e.error}`);
}
}
if (verifiedOffers.length > 0) {
console.log('\nNew offers:');
for (const o of verifiedOffers) {
const trustSuffix = o.trustScore != null && o.trustScore < 100
? ` [Trust: ${o.trustScore}/100${o.trustFlags?.length ? ' — ' + o.trustFlags.join(', ') : ''}]`
: '';
const blacklistSuffix = o.blacklisted ? ' [BLACKLISTED — on your do-not-apply list]' : '';
console.log(` + ${o.company} | ${o.title} | ${o.location || 'N/A'}${trustSuffix}${blacklistSuffix}`);
}
if (dryRun) {
console.log('\n(dry run — run without --dry-run to save results)');
} else {
console.log(`\nResults saved to ${PIPELINE_PATH} and ${SCAN_HISTORY_PATH}`);
}
}
// Persist this run's counters (#1604) — guarded exactly like the other
// writes; a --dry-run must leave no trace.
if (!dryRun) {
await appendPortalHealth(healthRecords);
// Locked on the same file the migration locks, for the reason the sibling
// append at appendToScanHistory already gives: two writers interleaving a
// read-modify-write on an append-only TSV silently drop each other's rows.
// migrate-scan-runs.mjs rewrites this file wholesale, so without the lock a
// run finishing mid-migration is written into a file that is then replaced
// by a snapshot taken before it, and the row is gone from the live file and
// from the .bak. Measured at ~9ms of exposure, and reproduced.
await withPipelineLock(SCAN_RUNS_PATH, () => appendScanRunSummary({
timestamp: new Date().toISOString(), status: 'completed',
companies: summaryCompanies, boards: summaryBoards, found: totalFound,
filteredTitle: totalFilteredTitle, filteredTier: totalFilteredTier,
filteredLocation: totalFilteredLocation, filteredPostingAge: totalFilteredPostingAge,
filteredSalary: totalFilteredSalary,
filteredContent: totalFilteredContent, filteredCooldown: totalFilteredCooldown,
dupes: totalDupes, newAdded: verifiedOffers.length, errors: errors.length,
filteredBlacklist: totalFilteredBlacklist,
filteredVisa: totalFilteredVisa,
filteredPostedDate: totalFilteredPostedDate,
filteredCountryEligibility: totalFilteredCountryEligibility,
}));
}
// The run completed (or was a dry run) — disarm the failure row.
registerRunFailureSnapshot(null);
console.log(`\n${pipelineRunHint()}`);
console.log('→ Share results and get help: https://discord.gg/8pRpHETxa4');
if (jsonMode) {
const filtered = totalFilteredTitle + totalFilteredDeclaredField + totalFilteredTier + totalFilteredLocation
+ totalFilteredPostingAge + totalFilteredPostedDate + totalFilteredSalary
+ totalFilteredContent + totalFilteredCountryEligibility + totalFilteredBlacklist
+ totalFilteredVisa + totalFilteredCooldown;
emitJsonReceipt({
version: 'careerops.scan.receipt@1',
date,
scanned: targets.length,
skipped: skippedCount,
found: totalFound,
filtered,
duplicates: totalDupes,
added: verifiedOffers.length,
added_urls: verifiedOffers.map(offer => offer.url),
errors: errors.map(({ company, error }) => ({ company, error })),
unverified_zero: unverifiedZeroTargets,
// Boards read only partly (lib/board-truncation.mjs), so automation can
// tell a short read from a complete one without parsing stdout.
partial: [...partialTargets].map(([company, { source: _source, ...t }]) => ({ company, ...t })),
dry_run: dryRun,
}, errors.length > 0 ? 2 : 0);
}
// A run in which every target it tried failed read nothing, and exiting 0
// told cron and the web scheduler it succeeded with zero new offers. Exit 2,
// the receipt's failure code, so the outage is visible. Only when all failed:
// a few dead boards are routine, and every run would turn red. An empty
// config tried nothing and still exits 0. This runs after the bookkeeping
// above, so the scan-runs and portal-health rows are already written; the
// message goes to stderr last, where a scheduler reads the cause.
const attempted = targets.length + resolveErrors.length;
const everyTargetFailed = attempted > 0 && failedTargets + resolveErrors.length === attempted;
if (everyTargetFailed && !jsonMode) {
console.error(`Error: every scan target failed (${attempted} of ${attempted}); no board could be read. See the errors above.`);
process.exitCode = 2;
}
}
// Only run main() when invoked directly (`node scan.mjs`), not when imported by tests.
// `|| ''` guards the case where Node is invoked without a script arg (e.g. `node -e`).
if (isMainModule(import.meta.url)) {
main().catch(err => {
console.error('Fatal:', err.message);
writeRunFailureRow('failed');
process.exitCode = 1;
});
}