check-liveness.mjs
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* check-liveness.mjs — Playwright job link liveness checker
*
* Tests whether job posting URLs are still active or have expired.
* Uses the same detection logic as scan.md step 7.5.
* Zero Claude API tokens. Two rungs: a free public-API check first
* (liveness-api.mjs, no browser), then Playwright for everything else.
*
* An `expired` verdict is recorded back into data/scan-history.tsv as a
* `skipped_expired` row, so a posting confirmed dead once stops resurfacing
* (#3891). See recordExpiredVerdicts below for why that is default-on.
*
* Usage:
* node check-liveness.mjs <url1> [url2] ...
* node check-liveness.mjs --file urls.txt
*
* Exit code: 0 if all active, 1 if any expired or uncertain
*/
import { readFile } from 'fs/promises';
import {
checkUrlLivenessWithFallback,
createHeadedPageProvider,
newLivenessPage,
createHostSpacer,
takeSuppressedBlockCount,
sleep,
} from './liveness-browser.mjs';
import { checkLivenessViaApi } from './liveness-api.mjs';
import { robotsAllows } from './robots-check.mjs';
import { planExpiredHistoryRows, trackerLivenessTargets, DEFAULT_TRACKER_STATUSES, planExpiredDiscards, postingClosedNote, DEFAULT_DISCARD_STATUSES, discardableExpiredUrls } from './liveness-core.mjs';
import { appendToScanHistory, seenJobKey, SCAN_HISTORY_PATH } from './scan.mjs';
import { withPipelineLock } from './pipeline-lock.mjs';
import { isMainModule } from './lib/is-main-module.mjs';
import { localToday } from './lib/local-today.mjs';
import { existsSync, readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { flagValue, hasFlag, validateFlags } from './lib/cli-flags.mjs';
import { execFileSync } from 'child_process';
import { createHash } from 'crypto';
import { getCareerOpsRoot, resolveTrackerPath } from './path-resolver.mjs';
import { browserLaunchOptions } from './lib/browser-launch.mjs';
const USAGE = `Usage:
node check-liveness.mjs [--no-fallback] [--throttle[=ms]] [--no-record] <url1> [url2] ...
node check-liveness.mjs [--no-fallback] [--throttle[=ms]] [--no-record] --file urls.txt
node check-liveness.mjs --tracker [--status=Evaluated,Applied] # postings of tracker rows in these statuses
node check-liveness.mjs --tracker --discard-expired [--apply] # dry-run move expired Evaluated rows to Discarded; --apply writes
node check-liveness.mjs --no-record # check only; write nothing to scan-history
node check-liveness.mjs --help # print this usage block and exit
node check-liveness.mjs -h # alias for --help`;
const KNOWN_FLAGS = [
'--no-fallback', '--throttle', '--no-record', '--file', '--tracker', '--status',
'--discard-expired', '--apply', '--help', '-h',
];
/**
* `--throttle` alone means the 5000ms default. `--throttle 1000` and
* `--throttle=1000` both mean 1000. A non-numeric equals value stays on the
* default rather than becoming a URL to check.
*
* @param {string[]} args
* @returns {number}
*/
function throttleBaseMsFrom(args) {
if (!hasFlag(args, '--throttle')) return 0;
const eq = args.find((a) => typeof a === 'string' && a.startsWith('--throttle='));
const raw = eq !== undefined ? eq.slice('--throttle='.length) : (() => {
const idx = args.indexOf('--throttle');
const next = idx === -1 ? undefined : args[idx + 1];
return typeof next === 'string' && /^\d+$/.test(next) ? next : undefined;
})();
const n = Number(raw);
return Number.isFinite(n) && n > 0 ? n : 5000;
}
/**
* Tokens that are not flags and not a flag's operand. Those are the URLs.
* `--dry-run` is not a flag this script accepts, so it never reaches here:
* validateFlags rejects it first. A numeric `--throttle` operand is not a URL.
*
* @param {string[]} args
* @returns {string[]}
*/
function urlPositionals(args) {
const skip = new Set();
for (let i = 0; i < args.length; i++) {
const a = args[i];
if (typeof a !== 'string') continue;
if (a.startsWith('--file=') || a.startsWith('--status=') || a.startsWith('--throttle=')) {
skip.add(i);
continue;
}
if (a === '--file' || a === '--status') {
skip.add(i);
if (args[i + 1] !== undefined) skip.add(i + 1);
continue;
}
if (a === '--throttle') {
skip.add(i);
const next = args[i + 1];
if (typeof next === 'string' && /^\d+$/.test(next)) skip.add(i + 1);
continue;
}
if (a === '-h' || a.startsWith('--')) skip.add(i);
}
return args.filter((_, i) => !skip.has(i));
}
/**
* Is this run allowed to record its verdicts?
*
* Default-ON, opt-OUT. An opt-IN flag would be the bug wearing a fix: the
* scheduled routines pass no flags, so a gated recording never runs from the
* one caller that produces most of these verdicts — which is precisely how the
* expensive `expired` result came to be computed and discarded (#3891).
* `--no-record` covers the real need for the opposite: a read-only spot check
* that leaves no trace.
*
* @param {string[]} args - process.argv.slice(2).
* @returns {boolean}
*/
export function recordingEnabled(args = []) {
return !args.includes('--no-record');
}
/**
* Write the run's `expired` verdicts into data/scan-history.tsv.
*
* Everything about WHICH rows get written is decided by planExpiredHistoryRows
* (only expired, only URLs the history knows, once each); this is the effect.
* A history file that does not exist yet means no URL is known, so there is
* nothing to retire and nothing to create.
*
* The row is stamped with localToday(), like every other scan-history writer:
* a UTC day is the wrong day for most of the world for part of every day.
*
* The read and the append share ONE critical section. What to append is decided
* by what the file already says, so reading outside the lock is a check-then-act:
* a scanner appending the same `skipped_expired` row in between would leave two
* death certificates for one posting. The lock is not reentrant, hence
* `alreadyLocked` on the append rather than a second acquisition.
*
* @param {{url: string, result: string}[]} verdicts
* @returns {Promise<number>} How many rows were written.
*/
export async function recordExpiredVerdicts(verdicts) {
// This one check is deliberately OUTSIDE the lock, unlike the read below.
// No history means no URL is known, so there is nothing to retire and no
// reason to pay for a lock — and locking it would buy nothing, because the
// scenario it looks like it prevents is not preventable. A scanner that
// creates the file and surfaces this URL a microsecond after the check is
// indistinguishable from one that does it a second after the whole function
// returns: either way the posting was discovered after the verdict, and the
// next sweep is what covers it. (The file is created by atomicWriteFile,
// which renames into place, so this can never see a half-written history.)
if (!existsSync(SCAN_HISTORY_PATH)) return 0;
return withPipelineLock(SCAN_HISTORY_PATH, async () => {
const rows = planExpiredHistoryRows(
readFileSync(SCAN_HISTORY_PATH, 'utf-8'),
verdicts,
seenJobKey,
);
// The writer drops an alias when another spelling of that posting already
// has a pinning skipped_expired row. The number below is rows that landed.
if (rows.length > 0) return await appendToScanHistory(rows, localToday(), 'skipped_expired', { alreadyLocked: true });
return 0;
});
}
const CODE_ROOT = dirname(fileURLToPath(import.meta.url));
function runSetStatus(args, trackerPath) {
try {
return JSON.parse(execFileSync(process.execPath, [join(CODE_ROOT, 'set-status.mjs'), ...args], {
encoding: 'utf-8',
env: { ...process.env, CAREER_OPS_TRACKER: trackerPath },
stdio: ['ignore', 'pipe', 'pipe'],
}));
} catch (err) {
try {
return JSON.parse(err.stdout);
} catch {
return { error: (typeof err.stderr === 'string' && err.stderr.trim()) || err.message };
}
}
}
/**
* Move planned rows to Discarded through set-status.mjs (locked atomic write).
* Dry-run when apply is false: resolve only, write nothing.
*
* The liveness sweep can take minutes, so each row is planned again against
* the tracker as it is now, and set-status refuses the write under its lock
* if the tracker changed after that re-plan.
*
* @param {{num: number, company: string, role: string, status: string, url: string}[]} rows
* @param {{apply: boolean, note: string, trackerPath: string, reportsRoot: string}} options
* @returns {{num: number, company: string, role: string, url: string, changed: boolean, dryRun: boolean, error?: string, skipped?: string}[]}
*/
export function discardExpiredRows(rows, { apply, note, trackerPath, reportsRoot }) {
return rows.map((row) => {
const base = { num: row.num, company: row.company, role: row.role, url: row.url, changed: false, dryRun: !apply };
const current = readFileSync(trackerPath, 'utf-8');
if (!planExpiredDiscards(current, [row.url], { reportsRoot }).some((r) => r.num === row.num)) {
return { ...base, skipped: 'no longer Evaluated with this posting' };
}
const args = ['--row', String(row.num), 'Discarded', '--note', note, '--json',
'--expect-tracker', createHash('sha256').update(current, 'utf8').digest('hex')];
if (!apply) args.push('--dry-run');
const out = runSetStatus(args, trackerPath);
if (out.error) return { ...base, error: out.error };
return { ...base, changed: Boolean(out.changed), dryRun: Boolean(out.dryRun) || !apply };
});
}
async function main() {
const args = process.argv.slice(2);
// Unknown flags used to survive into the URL list: `--dry-run --file urls.txt`
// checked three strings and never opened the file, and `--help` ran before
// that filter so `--help --bogus` exited 0. validateFlags rejects the unknown
// flag first, then prints help. --throttle's operand is optional (bare
// `--throttle` is the 5000ms default), and --status keeps the message the
// status-flag test locks, so the missing-operand check stays here rather
// than in requireOperand. See tests/validate-flags-value-operand.test.mjs.
validateFlags(args, KNOWN_FLAGS, USAGE, {
valueFlags: ['--file', '--status', '--throttle'],
});
if (hasFlag(args, '--file')) {
const filePath = flagValue(args, '--file');
if (typeof filePath !== 'string' || !filePath.trim() || filePath.startsWith('--')) {
console.error('Error: --file requires a value');
process.exit(1);
}
}
// Portals like pracuj.pl serve a Cloudflare anti-bot wall to headless Chromium.
// On a challenge we retry once in a headed browser (which clears it); pass
// --no-fallback to stay fully headless (e.g. on a machine with no display).
const noFallback = args.includes('--no-fallback');
// --throttle, --throttle <ms>, or --throttle=<ms>: wait base..2*base ms
// (jittered) between browser checks of the same host (createHostSpacer in
// liveness-browser.mjs) to stay under rate-based WAF limits. pracuj.pl's
// Cloudflare flags the session after ~2 rapid hits, so a bulk run needs
// spacing. Bare --throttle uses a 5000ms base. The space form used to be
// ignored (the number was checked as a URL) and the base stayed 5000.
const throttleBaseMs = throttleBaseMsFrom(args);
const record = recordingEnabled(args);
// `--status Rejected` as well as `--status=Rejected`. Only the `=` form was
// read, so the space form, and any `--status` with no value, fell back to
// the default statuses without a word, and `--discard-expired --apply`
// then rewrote Evaluated and Applied rows the user never asked about.
// A --status with no value is a usage error (the hasFlag + flagValue
// pairing lib/cli-flags.mjs documents for this).
const statusGiven = hasFlag(args, '--status');
const statusValue = statusGiven ? flagValue(args, '--status') : undefined;
if (statusGiven && (typeof statusValue !== 'string' || !statusValue.trim() || statusValue.startsWith('--'))) {
console.error('--status needs a value, e.g. --status=Evaluated,Applied');
process.exitCode = 1;
return;
}
const fromTracker = args.includes('--tracker');
const discardExpired = args.includes('--discard-expired');
const applyDiscards = args.includes('--apply');
const positional = urlPositionals(args);
if (discardExpired && !fromTracker) {
console.error('--discard-expired requires --tracker.');
process.exitCode = 1;
return;
}
if (positional.length === 0 && !fromTracker && !hasFlag(args, '--file')) {
console.error(USAGE);
process.exitCode = 1;
return;
}
let urls;
const labels = new Map();
let trackerPath = null;
let trackerText = null;
let reportsRoot = null;
if (fromTracker) {
const root = getCareerOpsRoot();
trackerPath = resolveTrackerPath(root);
reportsRoot = join(root, 'reports');
if (!existsSync(trackerPath)) {
console.error(`No tracker at ${trackerPath}. Evaluate a role first.`);
process.exitCode = 1;
return;
}
trackerText = readFileSync(trackerPath, 'utf-8');
const statuses = statusGiven ? statusValue.split(',').map((s) => s.trim()).filter(Boolean) : DEFAULT_TRACKER_STATUSES;
const { targets, noUrl } = trackerLivenessTargets(trackerText, { statuses, reportsRoot });
console.log(`Tracker rows with status ${statuses.join(' or ')}: ${targets.length} with a posting URL${noUrl.length ? `, ${noUrl.length} without (${noUrl.map((n) => `#${n}`).join(' ')})` : ''}`);
for (const t of targets) labels.set(t.url, `#${t.num} ${t.company} · ${t.role} (${t.status})`);
urls = targets.map((t) => t.url);
} else if (hasFlag(args, '--file')) {
const text = await readFile(flagValue(args, '--file'), 'utf-8');
urls = text.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#'));
} else {
urls = positional;
}
const notes = [
noFallback ? null : 'headed fallback on challenge',
throttleBaseMs ? `throttle ~${throttleBaseMs / 1000}-${(throttleBaseMs * 2) / 1000}s` : null,
record ? null : 'no-record',
].filter(Boolean);
console.log(`Checking ${urls.length} URL(s)...${notes.length ? ` (${notes.join(', ')})` : ''}\n`);
// Lazy browser: the API rung resolves ATS postings with no browser at all, so we
// only launch Playwright if a URL actually needs the fallback.
let browser = null, page = null, headed = null;
async function ensureBrowser() {
if (browser) return;
const { chromium } = await import('playwright');
const launched = await chromium.launch(browserLaunchOptions({ headless: true }));
try {
page = await newLivenessPage(launched);
headed = noFallback ? null : createHeadedPageProvider(chromium);
} catch (err) {
// Never leave a launched process behind when setup fails half way.
await launched.close().catch(() => {});
throw err;
}
browser = launched;
}
let active = 0, expired = 0, uncertain = 0, viaApi = 0;
const verdicts = [];
// --throttle spaces repeat browser visits to the same host; the API rung is
// cheap and not WAF-rate-limited, so it never waits.
const spacer = createHostSpacer(throttleBaseMs);
// An Ashby board answers for every posting under its org: fetch each board
// once per run, not once per posting.
const boardCache = new Map();
// Sequential — project rule: never Playwright in parallel
for (let i = 0; i < urls.length; i++) {
const url = urls[i];
let result, reason, code;
// Rung 1: zero-token ATS API check. A conclusive active/expired wins; otherwise fall through.
const api = await checkLivenessViaApi(url, { boardCache });
if (api) {
({ result, reason, code } = api);
viaApi++;
} else {
// Rung 2: Playwright — handles non-ATS pages and inconclusive API results.
// A browser that will not launch (missing binaries after a Playwright
// update, no display) must not end the batch: every URL after it would
// lose its verdict, including API-checkable ATS URLs that never needed
// the browser. Report the URLs that needed it as uncertain with the
// install hint and keep going; the exit code still reflects that they
// were not confirmed.
try {
await ensureBrowser();
const wait = spacer.waitBefore(url);
if (wait) await sleep(wait);
const getHeadedPage = headed ? () => headed.get() : undefined;
try {
({ result, reason, code } = await checkUrlLivenessWithFallback(page, url, { getHeadedPage, robotsGate: robotsAllows }));
} finally {
// A failed visit counts too: that is when a WAF is most likely involved.
spacer.visited(url);
}
} catch (err) {
result = 'uncertain';
code = 'browser_unavailable';
const fix = err?.code === 'ERR_MODULE_NOT_FOUND' ? 'npm install' : 'npx playwright install chromium';
reason = `browser unavailable (${String(err?.message ?? err).split('\n')[0]}); run \`${fix}\` and re-run`;
}
}
const icon = { active: '✅', expired: '❌', uncertain: '⚠️' }[result];
console.log(`${icon} ${result.padEnd(10)} ${api ? '(api) ' : ' '}${labels.has(url) ? `${labels.get(url)} ` : ''}${url}`);
if (result !== 'active') console.log(` ${reason}`);
if (result === 'active') active++;
else if (result === 'expired') expired++;
else uncertain++;
verdicts.push({ url, result, code });
}
if (headed) await headed.close();
if (browser) await browser.close();
console.log(`\nResults: ${active} active ${expired} expired ${uncertain} uncertain (${viaApi} via API, no browser)`);
const quietBlocks = takeSuppressedBlockCount();
if (quietBlocks) console.log(`(${quietBlocks} tracker/analytics requests skipped: their host had no usable address, e.g. a DNS ad-blocker; not counted against any posting)`);
// Close the loop: the verdict is the expensive part of this run and it used
// to die with the process. `--no-record` leaves no trace, so the write is
// skipped entirely rather than computed and thrown away.
if (record) {
const recorded = await recordExpiredVerdicts(verdicts);
if (recorded > 0) console.log(`Recorded ${recorded} expired posting(s) in ${SCAN_HISTORY_PATH}`);
}
if (discardExpired) {
// Only closure EVIDENCE closes a row. An `insufficient_content` verdict
// means the page could not be read, whether it arrives as `uncertain` or,
// from an older verdict, as `expired`, and this path writes Discarded to
// the tracker — see isClosureEvidence().
const { closed: expiredUrls, unreadable } = discardableExpiredUrls(verdicts);
if (unreadable.length > 0) {
console.log(`\nDiscard expired: ${unreadable.length} page(s) could not be read and were left as they are `
+ '(an unreadable page is unconfirmed, not closed):');
for (const url of unreadable) console.log(` · ${url}`);
}
const planned = planExpiredDiscards(trackerText, expiredUrls, {
statuses: DEFAULT_DISCARD_STATUSES,
reportsRoot,
});
const note = postingClosedNote(localToday());
if (planned.length === 0) {
console.log('\nDiscard expired: no Evaluated rows matched an expired posting.');
} else {
const mode = applyDiscards ? 'apply' : 'dry-run';
console.log(`\nDiscard expired (${mode}): ${planned.length} Evaluated row(s) → Discarded`);
const results = discardExpiredRows(planned, { apply: applyDiscards, note, trackerPath, reportsRoot });
for (const r of results) {
if (r.skipped) {
console.log(` · #${r.num} ${r.company} · ${r.role} — skipped, ${r.skipped}`);
continue;
}
if (r.error) {
console.log(` ❌ #${r.num} ${r.company} · ${r.role} — ${r.error}`);
process.exitCode = 1;
continue;
}
const verb = r.dryRun ? 'would discard' : (r.changed ? 'discarded' : 'already Discarded');
console.log(` ${r.dryRun ? '·' : '✅'} #${r.num} ${r.company} · ${r.role} — ${verb} (${note})`);
}
if (!applyDiscards) {
console.log('Re-run with --apply to write. Uncertain and Applied rows are left alone.');
}
}
}
if (expired > 0 || uncertain > 0) process.exitCode = 1;
}
// Only run main() when invoked directly (`node check-liveness.mjs`), not when
// imported — the arg parsing above would otherwise read an importing test's argv.
if (isMainModule(import.meta.url)) {
main().catch(err => {
console.error('Fatal:', err.message);
process.exitCode = 1;
});
}