discover-new-companies.mjs
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* discover-new-companies.mjs — propose scanner-discovered companies for discovery
*
* The missing hop between the scanners and discover-ats.mjs. See modes/discover.md.
*
* ── The gap this closes ────────────────────────────────────────────────────
* The zero-token scanners (scan.mjs, scan-ats-full.mjs) record every company
* they touch in data/scan-history.tsv, including companies that are NOT in
* portals.yml — the Gupy provider in particular sweeps that whole platform by
* keyword, so it constantly surfaces employers nobody configured.
*
* Nothing ever fed those names back. They stayed in the history file, and each
* run rediscovered them from scratch. discover-ats.mjs can resolve a company to
* a scannable Greenhouse/Ashby/Lever/Workday board, but it takes a hand-written
* list as input and does not read scan-history.tsv.
*
* This script is that missing hop:
*
* scan-history.tsv → [this script] → companies.yml → discover-ats.mjs → portals.yml
*
* It only ever WRITES a candidate list. portals.yml is user-layer and is
* touched exclusively by `discover-ats.mjs --write`, after you review the
* preview. Zero LLM tokens; the only I/O is reading two local files.
*
* company-history.mjs also reads scan-history.tsv, but it is read-only and
* renders per-company evidence cards; it never emits a discover-ats input, so
* the two are complementary rather than overlapping.
*
* Issue #4181 — github.com/career-ops-hq/career-ops
*/
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import * as yaml from 'js-yaml';
import { loadYaml } from './lib/yaml-document.mjs';
import { getCareerOpsRoot } from './path-resolver.mjs';
import { buildCompanyCanonicalizer, findBlacklistEntry, loadBlacklist } from './scan.mjs';
import { parseScanHistoryLine, scanHistoryLineHasColumn } from './lib/scan-history-columns.mjs';
import { atsFromUrl } from './coverage-check.mjs';
import { flagValue, hasFlag, validateFlags } from './lib/cli-flags.mjs';
import { localToday } from './lib/local-today.mjs';
const DATA_ROOT = getCareerOpsRoot();
const HISTORY_PATH = process.env.CAREER_OPS_SCAN_HISTORY || join(DATA_ROOT, 'data', 'scan-history.tsv');
const PORTALS_PATH = process.env.CAREER_OPS_PORTALS || join(DATA_ROOT, 'portals.yml');
const BLACKLIST_PATH = process.env.CAREER_OPS_BLACKLIST || join(DATA_ROOT, 'data', 'blacklist.md');
const USAGE = `Usage: node discover-new-companies.mjs [options]
Reads data/scan-history.tsv, subtracts every company already tracked in
portals.yml or on data/blacklist.md, and emits the remainder as a
discover-ats.mjs input file.
Options:
--since <n> Only companies first seen in the last N days (default: 30)
--min-rows <n> Only companies with >= N history rows (default: 2)
--added-only Only companies that produced at least one 'added' row
--limit <n> Cap the output list (default: 40)
--out <file> Write YAML here (default: stdout)
--summary Human-readable table instead of YAML
--json JSON envelope instead of YAML
--help This message
Typical loop:
node discover-new-companies.mjs --out /tmp/new.yml
node discover-ats.mjs --in /tmp/new.yml --summary # preview, writes nothing
node discover-ats.mjs --in /tmp/new.yml --write # only after you review
`;
const args = process.argv.slice(2);
const KNOWN_FLAGS = ['--since','--min-rows','--limit','--added-only','--out','--summary','--json','--help','-h'];
const VALUE_FLAGS = ['--since','--min-rows','--limit','--out'];
validateFlags(args, KNOWN_FLAGS, USAGE, { valueFlags: VALUE_FLAGS, requireOperand: true });
// A flag given twice (in either --flag value or --flag=value form) is an error;
// compare the part before '=' so --since 7 and --since=7 count as the same flag.
const seenFlags = new Set();
for (const arg of args) {
if (typeof arg !== 'string' || !arg.startsWith('-')) continue;
const flag = arg.split('=')[0];
if (!KNOWN_FLAGS.includes(flag)) continue;
if (seenFlags.has(flag)) {
console.error(`ERROR: ${flag} may only be specified once`);
process.exit(1);
}
seenFlags.add(flag);
}
const has = (f) => hasFlag(args, f);
const val = (f, d = null) => flagValue(args, f) ?? d;
if (has('--help')) {
console.log(USAGE);
process.exit(0);
}
const SINCE_DAYS = Number(val('--since', '30'));
const MIN_ROWS = Number(val('--min-rows', '2'));
const LIMIT = Number(val('--limit', '40'));
const ADDED_ONLY = has('--added-only');
const OUT_FILE = val('--out');
for (const [flag, v] of [['--since', SINCE_DAYS], ['--min-rows', MIN_ROWS], ['--limit', LIMIT]]) {
if (!Number.isFinite(v) || v < 0) {
console.error(`ERROR: ${flag} must be a non-negative number; got ${JSON.stringify(val(flag))}`);
process.exit(1);
}
}
if (!existsSync(HISTORY_PATH)) {
console.error(`ERROR: ${HISTORY_PATH} not found — run a scan first.`);
process.exit(1);
}
// ── Already-tracked set ─────────────────────────────────────────────────────
// Match on the canonicalized name so alias drift ("Stone" vs "Pagar.me (Stone)")
// does not resurface a company that is already configured. Disabled entries
// count as tracked: the user turned them off deliberately, and re-proposing
// them every run would be noise.
//
// The boards entries read count too. The reverse sweep (scan-ats-full.mjs)
// names a company by its board slug ("injective-labs"), which canonicalizes to
// nothing an entry is called, so a name-only check re-proposed boards that
// portals.yml already scans.
const portalsRaw = existsSync(PORTALS_PATH)
? (loadYaml(readFileSync(PORTALS_PATH, 'utf-8')) || {})
: {};
const canon = buildCompanyCanonicalizer(portalsRaw.company_aliases);
const tracked = new Set();
const trackedBoards = new Set();
for (const entry of portalsRaw.tracked_companies || []) {
if (entry && typeof entry.name === 'string') tracked.add(canon(entry.name));
for (const field of ['careers_url', 'api']) {
const board = typeof entry?.[field] === 'string' ? atsFromUrl(entry[field]) : null;
if (board?.slug) trackedBoards.add(`${board.ats}:${board.slug}`);
}
}
const boardOf = (url) => {
const board = atsFromUrl(url);
return board.slug ? `${board.ats}:${board.slug}` : null;
};
// ── Do-not-apply list ───────────────────────────────────────────────────────
// A blacklisted employer is never worth resolving to a board: scan.mjs would
// drop every posting from it anyway. Matched per row with the scanner's own
// findBlacklistEntry (company name, or an opt-in domain-scope host suffix), and
// one matching row excludes the whole company — a name-only row and a
// domain-matched row for the same employer must not split into "keep" and "skip".
const blacklist = loadBlacklist(BLACKLIST_PATH);
const blacklisted = new Set();
// ── Walk the history ────────────────────────────────────────────────────────
// Legacy headerless scan-history.tsv files exist and are never rewritten (see
// appendToScanHistory in scan.mjs), so detect the header instead of assuming
// row 0 is one — the same guard parseScanHistory (detect-reposts.mjs) uses.
// An unconditional skip would silently drop the first company in a headerless
// file. Columns are read by name through lib/scan-history-columns.mjs, whose
// order is fixed whatever header (if any) the file carries.
const rows = readFileSync(HISTORY_PATH, 'utf-8').split('\n');
const hasHeader = /^\s*url\s*\t/i.test(rows[0] ?? '');
const cutoffMs = SINCE_DAYS > 0 ? Date.now() - SINCE_DAYS * 86_400_000 : null;
/** @type {Map<string, {name:string, rows:number, added:number, lastSeen:string, titles:Set<string>, hosts:Set<string>}>} */
const found = new Map();
for (let i = hasHeader ? 1 : 0; i < rows.length; i++) {
if (!scanHistoryLineHasColumn(rows[i], 'status')) continue;
const row = parseScanHistoryLine(rows[i]);
// A title-filtered row is evidence the company posted something the user
// does not want, not a reason to track it (#2695).
if (row.status.trim() === 'skipped_title') continue;
const rawName = row.company.trim();
if (!rawName) continue;
const key = canon(rawName);
if (!key || tracked.has(key) || blacklisted.has(key)) continue;
// One row on a tracked board settles the whole company, as one blacklisted
// row does below. Checked before the --since window: a board is tracked
// however long ago a scanner last saw it.
if (trackedBoards.has(boardOf(row.url))) {
tracked.add(key);
found.delete(key);
continue;
}
const seen = row.first_seen.trim();
if (cutoffMs != null) {
const t = Date.parse(seen);
// Undated rows pass — same "don't penalize missing data" rule the scanners use.
if (Number.isFinite(t) && t < cutoffMs) continue;
}
if (findBlacklistEntry(blacklist, rawName, row.url)) {
blacklisted.add(key);
found.delete(key);
continue;
}
if (!found.has(key)) {
found.set(key, { name: rawName, rows: 0, added: 0, lastSeen: seen, titles: new Set(), hosts: new Set() });
}
const rec = found.get(key);
rec.rows++;
if (row.status.trim() === 'added') rec.added++;
if (seen > rec.lastSeen) rec.lastSeen = seen;
const title = row.title.trim();
if (title && rec.titles.size < 3) rec.titles.add(title);
try { rec.hosts.add(new URL(row.url).hostname); } catch { /* malformed URL — skip host hint */ }
}
// ── Rank and cut ────────────────────────────────────────────────────────────
// Companies that actually produced pipeline entries rank above ones that only
// ever got filtered out — those are the boards most worth resolving.
let candidates = [...found.values()]
.filter((c) => c.rows >= MIN_ROWS)
.filter((c) => (ADDED_ONLY ? c.added > 0 : true))
.sort((a, b) => b.added - a.added || b.rows - a.rows || a.name.localeCompare(b.name));
const totalBeforeLimit = candidates.length;
candidates = candidates.slice(0, LIMIT);
// ── Emit ────────────────────────────────────────────────────────────────────
// One branch per format, and no process.exit(): stdout to a pipe is written
// asynchronously, and exiting right after console.log cut off everything past
// the pipe's 64 KiB buffer.
if (has('--summary')) {
console.log('='.repeat(78));
console.log(' New companies seen by scanners but absent from portals.yml');
console.log(` candidates: ${totalBeforeLimit} | shown: ${candidates.length} | window: ${SINCE_DAYS}d | blacklisted skipped: ${blacklisted.size}`);
console.log('='.repeat(78));
for (const c of candidates) {
console.log(` ${c.name}`);
console.log(` rows: ${c.rows} | added: ${c.added} | last: ${c.lastSeen} | ${[...c.hosts][0] || 'n/a'}`);
if (c.titles.size) console.log(` e.g. ${[...c.titles][0]}`);
}
if (!candidates.length) console.log(' (nothing new — every scanned company is already tracked)');
console.log('\nNext: node discover-new-companies.mjs --out /tmp/new.yml');
console.log(' node discover-ats.mjs --in /tmp/new.yml --summary');
} else if (has('--json')) {
console.log(JSON.stringify({
metadata: { candidates: totalBeforeLimit, emitted: candidates.length, blacklistedSkipped: blacklisted.size, sinceDays: SINCE_DAYS, minRows: MIN_ROWS, addedOnly: ADDED_ONLY },
companies: candidates.map((c) => ({
name: c.name, rows: c.rows, added: c.added, lastSeen: c.lastSeen,
hosts: [...c.hosts], sampleTitles: [...c.titles],
})),
}, null, 2));
} else {
const doc = yaml.dump({ companies: candidates.map((c) => ({ name: c.name })) });
const banner = `# Generated by discover-new-companies.mjs — ${localToday()}\n`
+ `# ${totalBeforeLimit} candidate(s) found, ${candidates.length} emitted (window ${SINCE_DAYS}d, min-rows ${MIN_ROWS}); ${blacklisted.size} blacklisted company/companies skipped.\n`
+ `# Feed to: node discover-ats.mjs --in <this file> --summary\n`;
if (OUT_FILE) {
writeFileSync(OUT_FILE, banner + doc);
console.error(`Wrote ${candidates.length} company/companies to ${OUT_FILE}`);
} else {
process.stdout.write(banner + doc);
}
}