intake.mjs
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
// intake.mjs — deterministic half of the `intake` mode (#1723).
//
// Scans the documents/ intake folder (master CV, LinkedIn export, diplomas,
// reference letters), extracts text from each source locally, and
// fingerprints every source so re-runs surface only genuinely new material.
//
// Division of labor (mirrors modes/add.md + add-entry.mjs):
// - this script: enumeration, text extraction, idempotency bookkeeping —
// everything deterministic. It NEVER writes cv.md / config/profile.yml /
// modes/_profile.md; per repo convention those are agent-edited only,
// after explicit user confirmation (see modes/intake.md).
// - modes/intake.md: semantic mapping (CV → experience/skills, LinkedIn →
// certifications, …), conflict display, the confirm gate, and the
// source-annotated writes.
//
// PDF extraction ladder (zero new package.json deps, mirrors the
// generate-latex.mjs engine ladder): born-digital PDFs — the dominant case
// for CVs / LinkedIn "Save to PDF" exports / transcripts — carry a text
// layer that `pdftotext -layout` (Poppler) extracts directly. No extractor
// on PATH degrades to an install hint, never a crash. Scanned/image-only
// PDFs and .docx are out of scope for v1 (the summary tells the user to
// convert them).
//
// Snapshots (#5779): `--commit` also keeps the extracted text of each recorded
// source in data/intake-snapshots/, so `--diff` can show a `changed` source as
// the lines that moved instead of the whole document again. A snapshot is
// trusted only while its text still hashes to the ledger entry; anything else
// (an install from before snapshots, a deleted ledger, a crash between the two
// writes) falls back to the full text, labelled as such.
//
// Usage:
// node intake.mjs # scan + extract, JSON to stdout
// node intake.mjs --summary # human-readable table instead of JSON
// node intake.mjs --text <path> # full extracted text of one source
// node intake.mjs --diff <path> # what changed in one source since it was
// # last committed (unified line diff)
// node intake.mjs --commit <path …>
// # record the confirmed sources as ingested
// # (run only after the user confirmed)
// node intake.mjs --commit --all # record every source with new material —
// # only when the user merged all of them
// node intake.mjs --self-test # pure-function self-test, no filesystem
// node intake.mjs --help # usage, then exit 0. An unknown flag exits 1.
import { createHash } from 'crypto';
import { execFileSync } from 'child_process';
import {
existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync,
} from 'fs';
import { dirname, extname, join, relative, resolve, sep } from 'path';
import { fileURLToPath } from 'url';
import { isMainModule } from './lib/is-main-module.mjs';
import { flagValue, hasFlag, validateFlags } from './lib/cli-flags.mjs';
import { getCareerOpsRoot } from './path-resolver.mjs';
import { writeFileAtomic, acquireTrackerLock } from './tracker-utils.mjs';
import { isNestedCheckout } from './lib/mjs-files.mjs';
const ROOT = dirname(fileURLToPath(import.meta.url));
const DATA_ROOT = getCareerOpsRoot();
const DOCS_DIR = process.env.CAREER_OPS_DOCUMENTS_DIR || join(DATA_ROOT, 'documents');
const STATE_FILE = process.env.CAREER_OPS_INTAKE_STATE || join(DATA_ROOT, 'data', 'intake-state.json');
// Beside the ledger rather than under DATA_ROOT, so an overridden state file
// (CAREER_OPS_INTAKE_STATE, which the tests use) takes its snapshots with it.
// With no override this is {DATA_ROOT}/data/intake-snapshots/.
const SNAPSHOT_DIR = join(dirname(STATE_FILE), 'intake-snapshots');
// The four intake folders from the issue spec. Files directly under
// documents/ are picked up too — the folders are guidance, not a gate.
export const INTAKE_FOLDERS = ['cv', 'linkedin', 'diplomas', 'references'];
// Extraction ladder for PDFs, in preference order. v1 has a single rung;
// an OCR rung for scanned PDFs is an explicit later opt-in (see #1723
// thread), NOT a silent fallback — OCR output is too lossy to mix in
// unannounced.
const PDF_EXTRACTORS = [
{
name: 'pdftotext',
probeArgs: ['-v'],
// -layout preserves column layout, which keeps two-column CVs from
// bleeding into scrambled text. `-` sends the text to stdout.
extract: (path) => execFileSync(
'pdftotext', ['-layout', path, '-'],
{ stdio: ['ignore', 'pipe', 'pipe'], timeout: 30_000, maxBuffer: 16 * 1024 * 1024 },
).toString('utf-8'),
},
];
const PDF_INSTALL_HINT =
'No PDF text extractor found. Optional: install poppler for PDF intake '
+ '(brew install poppler / apt install poppler-utils) — .md/.txt/.tex '
+ 'sources work without it.';
/** Classify a source file by extension. Pure. */
export function classifySource(relPath) {
const ext = extname(relPath).toLowerCase();
if (['.md', '.txt', '.tex'].includes(ext)) return { kind: 'direct' };
if (ext === '.pdf') return { kind: 'pdf' };
if (['.png', '.jpg', '.jpeg', '.webp', '.gif', '.tiff'].includes(ext)) {
return { kind: 'unsupported', reason: 'image — convert to a text-layer PDF or .md/.txt first' };
}
if (['.docx', '.doc', '.odt', '.rtf'].includes(ext)) {
return { kind: 'unsupported', reason: `${ext} — export to PDF or .md/.txt first` };
}
return { kind: 'unsupported', reason: `unrecognized extension ${ext || '(none)'}` };
}
/**
* Walk the ladder and return the first extractor whose binary answers a
* version probe, or null. `probe` is injectable for tests.
*/
export function detectPdfExtractor(probe = defaultProbe) {
for (const candidate of PDF_EXTRACTORS) {
if (probe(candidate)) return candidate;
}
return null;
}
function defaultProbe(candidate) {
try {
// Bounded like extract() below: a wedged binary must not hang every run.
execFileSync(candidate.name, candidate.probeArgs, { stdio: 'pipe', timeout: 5_000 });
return true;
} catch (err) {
return probeRan(err);
}
}
/**
* Given whatever execFileSync threw from a version probe, did the binary
* actually run? Pure and exported so the self-test can cover every error shape
* without needing a real binary on PATH.
*
* A nonzero exit is NOT proof of absence. Poppler's pdftotext exits 0 on `-v`,
* but Xpdf's — the Glyph & Cog build shipped by xpdfreader.com and by
* MSYS2/mingw64 — prints its version and exits 99. Reading that as "not
* installed" silently skipped every PDF on a machine where extraction worked
* perfectly, and the scan summary blamed a missing extractor.
*
* What separates present-but-grumpy from absent is whether the process ran at
* all: execFileSync sets `status` to the exit code when it did, and leaves it
* null with code ENOENT / EACCES / ETIMEDOUT when the binary is missing,
* unrunnable, or wedged. A wedged binary stays excluded on purpose — extract()
* would hit the same timeout on every source.
*/
export function probeRan(err) {
return typeof err?.status === 'number';
}
export function sha256(text) {
return createHash('sha256').update(text).digest('hex');
}
/**
* Annotate extracted sources against the recorded intake state. Pure.
* A source is `new` (never ingested), `changed` (ingested before but the
* extracted text differs), or `ingested` (hash matches the record — the
* agent must not re-propose it; that is what makes re-runs idempotent).
*/
export function computeDelta(state, sources) {
const recorded = new Map(Object.entries(state.ingested || {}));
return sources.map((s) => {
if (!s.hash) return { ...s, status: s.status || 'error' };
const prev = recorded.get(s.path);
if (!prev) return { ...s, status: 'new' };
if (prev.hash !== s.hash) return { ...s, status: 'changed' };
return { ...s, status: 'ingested' };
});
}
/**
* Raised when the ingest ledger exists and cannot be trusted.
*
* Its own class so the CLI boundary can report it as a one-line message the way
* every other expected failure in this file does, while a genuine bug still
* arrives as a stack trace.
*/
class StateError extends Error {
constructor(message) { super(message); this.name = 'StateError'; }
}
function loadState() {
// Absent is a first run — an empty ledger is correct.
if (!existsSync(STATE_FILE)) return { ingested: {} };
let parsed;
try {
parsed = JSON.parse(readFileSync(STATE_FILE, 'utf-8'));
} catch (err) {
// Existing and unreadable is NOT a first run. Returning an empty ledger
// makes every already-ingested document look new, so intake re-proposes
// additions the user has already reviewed — and the review is the whole
// human-in-the-loop step this mode exists for. Saying so costs one message;
// hiding it costs the user's trust in the proposals.
throw new StateError(
`${STATE_FILE} is not valid JSON (${String(err.message).split('\n')[0]}) — `
+ 'fix or delete it; deleting means already-ingested documents are proposed again.',
);
}
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new StateError(`${STATE_FILE} does not contain a JSON object — fix or delete it.`);
}
// Absent is an empty ledger and `{}` is right. Present but not an object is
// the same silent reset one level down — it was replaced with `{}` and then
// written back by commitState, so every reviewed document looked new again.
if (parsed.ingested === undefined || parsed.ingested === null) parsed.ingested = {};
else if (typeof parsed.ingested !== 'object' || Array.isArray(parsed.ingested)) {
throw new StateError(`${STATE_FILE} has an "ingested" that is not an object — fix or delete it.`);
}
return parsed;
}
function listSourceFiles() {
if (!existsSync(DOCS_DIR)) return { files: [], unreadable: [] };
const out = [];
// Directories that could not be listed (permissions, a dead mount behind a
// link). They are skipped rather than fatal — one locked folder must not
// abort the scan of everything else — but they are reported, because a
// directory that silently vanishes is a source the user dropped in and never
// hears about again.
const unreadable = new Set();
const readDir = (dir) => {
try {
return readdirSync(dir, { withFileTypes: true });
} catch {
unreadable.add(dir);
return [];
}
};
// Directories already walked, by real path. Symlinks are followed (see
// below), so without this a link back up the tree — documents/cv/loop ->
// documents/ — re-enters it until the path length gives out, reporting
// one CV a dozen times over. Two links to the same folder collapse too.
const walked = new Set();
// Every directory reachable without following a link, by real path. Dirent's
// isDirectory() is false for a symlink, so this pass naturally stays inside
// the real tree.
//
// It exists because the walk keeps whichever alias of a folder it reaches
// first, and that path becomes the key in intake-state.json. With two ways
// into one folder — documents/cv plus a symlink documents/current -> cv — the
// key would otherwise depend on readdirSync order, which is filesystem
// dependent: the same documents/ could produce `cv/master.md` on one machine
// and `current/master.md` on another, and an already ingested source would
// resurface as new. Knowing the real directories up front lets a link step
// aside for the path the user actually created.
const realDirs = new Set();
const claimRealDirs = (dir) => {
let real;
try { real = realpathSync(dir); } catch { return; }
if (realDirs.has(real)) return;
realDirs.add(real);
for (const entry of readDir(dir)) {
if (entry.name.startsWith('.')) continue;
if (entry.isDirectory() && !isNestedCheckout(join(dir, entry.name))) claimRealDirs(join(dir, entry.name));
}
};
claimRealDirs(DOCS_DIR);
const walk = (dir) => {
let real;
try { real = realpathSync(dir); } catch { return; }
if (walked.has(real)) return;
walked.add(real);
// Sorted so the traversal itself doesn't vary with readdirSync order.
const entries = readDir(dir)
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
for (const entry of entries) {
if (entry.name.startsWith('.') || entry.name === 'README.md' && dir === DOCS_DIR) continue;
const abs = join(dir, entry.name);
// Follow symlinks (a symlinked master CV is a natural setup) —
// Dirent.isFile()/isDirectory() are both false for them, which would
// silently drop the source. Broken links are skipped.
let isDir = entry.isDirectory();
let isFile = entry.isFile();
if (entry.isSymbolicLink()) {
try {
const st = statSync(abs);
isDir = st.isDirectory();
isFile = st.isFile();
} catch { continue; }
}
if (isDir) {
// A link onto a directory that is already in the tree under its own
// name adds nothing but a second alias — skip it wherever it sits, so
// documents/a/link -> documents/z cannot claim `z` before the walk
// reaches it. Links pointing outside documents/ have no real in-tree
// name, so they are still followed.
if (entry.isSymbolicLink()) {
let target;
try { target = realpathSync(abs); } catch { continue; }
if (realDirs.has(target)) continue;
}
// A repository someone cloned into documents/ is not a document they
// dropped in: every source file in it would be offered as intake
// material for the profile (#3762).
if (isNestedCheckout(abs)) continue;
walk(abs);
} else if (isFile) out.push(abs);
}
};
walk(DOCS_DIR);
return { files: out.sort(), unreadable: [...unreadable].sort() };
}
/** documents/-relative path with forward slashes — the key used everywhere. */
function toRelPath(abs) {
return relative(DOCS_DIR, abs).split(sep).join('/');
}
/**
* Snapshot filename for a documents/-relative source path. Pure.
*
* A hash, not a slug: slugs collide (`cv/a-b.md` and `cv/a/b.md`; `cv/CV.md`
* and `cv/cv.md` on a case-insensitive disk), and a source filename can carry
* the user's name, which then stays out of data/intake-snapshots/ listings.
*/
export function snapshotFileName(relPath) {
return `${sha256(relPath)}.txt`;
}
/** Split text into lines for diffing: CRLF folded, no phantom last line. Pure. */
function toLines(text) {
const lines = text.replace(/\r\n/g, '\n').split('\n');
if (lines.length && lines[lines.length - 1] === '') lines.pop();
return lines;
}
// Above this many DP cells (lines changed in the old text × in the new) the
// LCS table would cost hundreds of MB; the middle is reported as removed then
// added instead. Still a correct diff, just not a minimal one. A CV never gets
// near it; a several-hundred-page export might.
const LCS_CELL_LIMIT = 25_000_000;
/**
* Line diff of two texts as a list of `{ op, line, a, b }`, where op is ' '
* (kept), '-' (only in the old text) or '+' (only in the new), and a / b are
* 1-based line numbers in the old / new text. Pure; longest-common-subsequence
* over the lines left after trimming the shared prefix and suffix. No package
* dependency: package.json carries no diff library and this is ~40 lines.
*/
export function diffLines(oldText, newText) {
const a = toLines(oldText);
const b = toLines(newText);
let start = 0;
while (start < a.length && start < b.length && a[start] === b[start]) start += 1;
let endA = a.length;
let endB = b.length;
while (endA > start && endB > start && a[endA - 1] === b[endB - 1]) { endA -= 1; endB -= 1; }
const ops = [];
for (let i = 0; i < start; i += 1) ops.push({ op: ' ', line: a[i], a: i + 1, b: i + 1 });
const n = endA - start;
const m = endB - start;
if (n > 0 && m > 0 && (n + 1) * (m + 1) <= LCS_CELL_LIMIT) {
// lcs[i * (m + 1) + j] = LCS length of a[start+i..endA) and b[start+j..endB).
const w = m + 1;
const lcs = new Uint32Array((n + 1) * w);
for (let i = n - 1; i >= 0; i -= 1) {
for (let j = m - 1; j >= 0; j -= 1) {
lcs[i * w + j] = a[start + i] === b[start + j]
? lcs[(i + 1) * w + j + 1] + 1
: Math.max(lcs[(i + 1) * w + j], lcs[i * w + j + 1]);
}
}
let i = 0;
let j = 0;
while (i < n && j < m) {
if (a[start + i] === b[start + j]) {
ops.push({ op: ' ', line: a[start + i], a: start + i + 1, b: start + j + 1 });
i += 1; j += 1;
} else if (lcs[(i + 1) * w + j] >= lcs[i * w + j + 1]) {
ops.push({ op: '-', line: a[start + i], a: start + i + 1, b: start + j });
i += 1;
} else {
ops.push({ op: '+', line: b[start + j], a: start + i, b: start + j + 1 });
j += 1;
}
}
for (; i < n; i += 1) ops.push({ op: '-', line: a[start + i], a: start + i + 1, b: start + m });
for (; j < m; j += 1) ops.push({ op: '+', line: b[start + j], a: start + n, b: start + j + 1 });
} else {
for (let i = start; i < endA; i += 1) ops.push({ op: '-', line: a[i], a: i + 1, b: start });
for (let j = start; j < endB; j += 1) ops.push({ op: '+', line: b[j], a: endA, b: j + 1 });
}
for (let k = 0; k < a.length - endA; k += 1) {
ops.push({ op: ' ', line: a[endA + k], a: endA + k + 1, b: endB + k + 1 });
}
return ops;
}
const HEADING_RE = /^#{1,6}\s+\S/;
/**
* Format diffLines() output as unified-diff hunks (`@@ -a,b +c,d @@`), with
* `context` unchanged lines around each change. Pure; returns '' when nothing
* changed.
*
* Each hunk header carries the nearest Markdown heading above the hunk, the
* way `git diff` names the enclosing function, so a changed bullet arrives
* labelled with the `### Company` block it sits in. That is the cheap answer
* to "line-based or section-aware" (#5779): line-based, with the section shown.
*/
export function formatHunks(ops, context = 2) {
const changed = ops.map((o, idx) => (o.op === ' ' ? -1 : idx)).filter((idx) => idx !== -1);
if (!changed.length) return '';
// Group change indices whose context windows touch into one hunk.
const groups = [];
for (const idx of changed) {
const last = groups[groups.length - 1];
if (last && idx - last[1] <= context * 2 + 1) last[1] = idx;
else groups.push([idx, idx]);
}
const out = [];
for (const [first, lastIdx] of groups) {
const from = Math.max(0, first - context);
const to = Math.min(ops.length - 1, lastIdx + context);
const slice = ops.slice(from, to + 1);
const oldCount = slice.filter((o) => o.op !== '+').length;
const newCount = slice.filter((o) => o.op !== '-').length;
// Unified-diff convention: a zero-length side names the line before it.
const oldStart = oldCount ? slice.find((o) => o.op !== '+').a : slice[0].a;
const newStart = newCount ? slice.find((o) => o.op !== '-').b : slice[0].b;
let heading = '';
for (let k = from - 1; k >= 0; k -= 1) {
if (ops[k].op !== '-' && HEADING_RE.test(ops[k].line)) { heading = ops[k].line.trim(); break; }
}
out.push(`@@ -${oldStart},${oldCount} +${newStart},${newCount} @@${heading ? ` ${heading}` : ''}`);
for (const o of slice) out.push(`${o.op}${o.line}`);
}
return out.join('\n') + '\n';
}
function extractAll({ withText = false } = {}) {
const extractor = detectPdfExtractor();
const { files, unreadable } = listSourceFiles();
const sources = files.map((abs) => {
// Normalise to forward slashes: this path is the key in intake-state.json
// and is echoed back to `--text`, so it must not vary by platform. On
// Windows relative() yields `cv\master.md`, which would both break the
// folder split below and make a state file unportable across machines.
const path = toRelPath(abs);
const cls = classifySource(path);
const base = { path, folder: path.includes('/') ? path.split('/')[0] : '(root)' };
if (cls.kind === 'unsupported') {
return { ...base, status: 'skipped', reason: cls.reason };
}
try {
let text;
if (cls.kind === 'direct') {
text = readFileSync(abs, 'utf-8');
base.extractor = 'direct';
} else {
if (!extractor) return { ...base, status: 'skipped', reason: PDF_INSTALL_HINT };
text = extractor.extract(abs);
base.extractor = extractor.name;
}
if (!text.trim()) {
return {
...base,
status: 'skipped',
reason: 'no text extracted — likely a scanned/image-only PDF; convert or re-export with a text layer',
};
}
// `text` only on the --commit path, which writes it as the snapshot:
// the scan JSON must keep printing a preview, not every whole CV.
return {
...base, chars: text.length, hash: sha256(text), preview: text.slice(0, 400),
...(withText ? { text } : {}),
};
} catch (err) {
return { ...base, status: 'error', reason: String(err.message || err).split('\n')[0] };
}
});
for (const dir of unreadable) {
const path = dir === DOCS_DIR ? '(root)' : `${toRelPath(dir)}/`;
sources.push({
path,
folder: path.includes('/') ? path.split('/')[0] : '(root)',
status: 'skipped',
reason: 'directory could not be listed (permissions?) — nothing under it was scanned',
});
}
return {
documentsDir: DOCS_DIR,
pdfExtractor: extractor ? extractor.name : null,
...(extractor ? {} : { pdfHint: PDF_INSTALL_HINT }),
sources: computeDelta(loadState(), sources),
};
}
function ensureScaffold() {
for (const folder of INTAKE_FOLDERS) mkdirSync(join(DOCS_DIR, folder), { recursive: true });
}
// Sentinel for `--commit --all`. The safe case ("record what the user
// confirmed") and the destructive one ("record everything") must not be
// spelled the same way: `only.length &&` made an empty list mean "all", so
// `--commit --summary` — main() filters flags out of the path list — recorded
// sources the user never saw and buried them forever (#1843 review).
const COMMIT_ALL = Symbol('commit-all');
// Record sources as ingested. `only` is the list of paths the user actually
// confirmed for merge, or COMMIT_ALL. A blanket commit after a per-item
// confirmation would mark declined sources as ingested and silently bury them
// on every future run, so an empty list records nothing.
async function commitState(result, only = []) {
mkdirSync(dirname(STATE_FILE), { recursive: true });
const lock = await acquireTrackerLock(`${STATE_FILE}.lock`, { tracker: STATE_FILE });
try {
const all = only === COMMIT_ALL;
const state = loadState();
state.ingested = state.ingested || {};
const now = new Date().toISOString();
let count = 0;
for (const s of computeDelta(state, result.sources.filter(source => source.status !== 'ingested'))) {
if (!s.hash || s.status === 'ingested') continue;
if (!all && !only.includes(s.path)) continue;
// Snapshot first, in the same branch that records the source, so a
// committer that records nothing (a stale scan, see
// tests/intake-concurrent-commit.test.mjs) writes no snapshot either. If
// the ledger write below then fails, the snapshot no longer hashes to
// the ledger entry and --diff falls back to the full text.
if (typeof s.text === 'string') {
mkdirSync(SNAPSHOT_DIR, { recursive: true });
writeFileAtomic(join(SNAPSHOT_DIR, snapshotFileName(s.path)), s.text);
}
state.ingested[s.path] = { hash: s.hash, ingestedAt: now };
count += 1;
}
// Atomic for the same reason as hired-share's: a truncating write that is
// interrupted leaves unparseable JSON, which is the condition the loader
// above now refuses to silently paper over.
writeFileAtomic(STATE_FILE, JSON.stringify(state, null, 2) + '\n');
return count;
} finally {
lock.release();
}
}
function printSummary(result) {
console.log(`documents/: ${result.documentsDir}`);
console.log(`PDF extractor: ${result.pdfExtractor || 'none — ' + PDF_INSTALL_HINT}`);
if (!result.sources.length) {
console.log('No sources found. Drop files into documents/cv, linkedin/, diplomas/, references/.');
return;
}
const w = Math.max(...result.sources.map((s) => s.path.length), 6);
console.log(`\n${'source'.padEnd(w)} status detail`);
for (const s of result.sources) {
const detail = s.status === 'skipped' || s.status === 'error'
? s.reason
: `${s.chars} chars via ${s.extractor}`;
console.log(`${s.path.padEnd(w)} ${s.status.padEnd(8)} ${detail}`);
}
const actionable = result.sources.filter((s) => s.status === 'new' || s.status === 'changed').length;
console.log(`\n${actionable} source(s) with new material.`);
}
function runSelfTest() {
let passed = 0;
let failed = 0;
const eq = (label, actual, expected) => {
if (JSON.stringify(actual) === JSON.stringify(expected)) { passed += 1; return; }
failed += 1;
console.log(` FAIL: ${label}\n actual: ${JSON.stringify(actual)}\n expected: ${JSON.stringify(expected)}`);
};
eq('md is direct', classifySource('cv/master.md'), { kind: 'direct' });
eq('tex is direct', classifySource('cv/master.tex'), { kind: 'direct' });
eq('pdf is pdf', classifySource('linkedin/Profile.PDF'), { kind: 'pdf' });
eq('docx unsupported', classifySource('cv/old.docx').kind, 'unsupported');
eq('image unsupported', classifySource('diplomas/scan.jpg').kind, 'unsupported');
eq('ladder picks first probing rung',
detectPdfExtractor(() => true)?.name, 'pdftotext');
eq('ladder degrades to null', detectPdfExtractor(() => false), null);
// A version probe that exits nonzero is not proof the binary is absent:
// Xpdf's pdftotext prints its version and exits 99. Only a process that never
// ran counts as missing.
eq('exit 99 (Xpdf -v) counts as present', probeRan({ status: 99 }), true);
eq('exit 1 counts as present', probeRan({ status: 1 }), true);
eq('exit 0 counts as present', probeRan({ status: 0 }), true);
eq('ENOENT counts as absent', probeRan({ code: 'ENOENT', status: null }), false);
eq('EACCES counts as absent', probeRan({ code: 'EACCES', status: null }), false);
eq('timeout counts as absent',
probeRan({ code: 'ETIMEDOUT', status: null, signal: 'SIGTERM' }), false);
eq('missing error object counts as absent', probeRan(undefined), false);
const state = { ingested: { 'cv/master.md': { hash: sha256('old') } } };
const delta = computeDelta(state, [
{ path: 'cv/master.md', hash: sha256('old') },
{ path: 'cv/master2.md', hash: sha256('x') },
{ path: 'linkedin/p.pdf', hash: sha256('new') },
{ path: 'diplomas/scan.jpg', status: 'skipped' },
]);
eq('unchanged source is ingested', delta[0].status, 'ingested');
eq('never-seen source is new', delta[1].status, 'new');
eq('never-seen pdf is new', delta[2].status, 'new');
eq('skipped source keeps its status', delta[3].status, 'skipped');
const delta2 = computeDelta(state, [{ path: 'cv/master.md', hash: sha256('edited') }]);
eq('re-extracted source with different text is changed', delta2[0].status, 'changed');
// Line diff + hunks (#5779).
const ops = diffLines('a\nb\nc\n', 'a\nB\nc\nd\n');
eq('diffLines marks a replaced line as - then +',
ops.map((o) => `${o.op}${o.line}`), [' a', '-b', '+B', ' c', '+d']);
eq('diffLines of equal texts has no changes',
diffLines('x\r\ny\n', 'x\ny').every((o) => o.op === ' '), true);
eq('formatHunks is empty when nothing changed', formatHunks(diffLines('x\n', 'x\n')), '');
eq('formatHunks names the heading above the hunk',
formatHunks(diffLines('### Acme\n\n- one\n- two\n- three\n- four\n', '### Acme\n\n- one\n- two\n- three\n- 4\n'), 1),
'@@ -5,2 +5,2 @@ ### Acme\n - three\n-- four\n+- 4\n');
eq('formatHunks counts a pure insertion from the line before it',
formatHunks(diffLines('a\n', 'a\nb\n'), 0), '@@ -1,0 +2,1 @@\n+b\n');
eq('snapshot names are a hash of the path', /^[0-9a-f]{64}\.txt$/.test(snapshotFileName('cv/Jane Doe.md')), true);
eq('snapshot names keep case-variant paths apart',
snapshotFileName('cv/CV.md') !== snapshotFileName('cv/cv.md'), true);
console.log(`\nself-test: ${passed} passed, ${failed} failed`);
process.exit(failed > 0 ? 1 : 0);
}
/**
* Resolve, contain and extract one source named on the command line. Shared by
* --text and --diff so both refuse the same paths the same way. Exits 1 with a
* one-line message on any expected failure.
*/
function readSourceText(target, flag) {
if (!target) { console.error(`Usage: node intake.mjs ${flag} <path relative to documents/>`); process.exit(1); }
const abs = resolve(DOCS_DIR, target);
// Scan output only ever emits paths inside documents/ — refuse anything
// that resolves outside it.
if (!abs.startsWith(resolve(DOCS_DIR) + sep)) {
console.error(`Path escapes documents/: ${target}`);
process.exit(1);
}
if (!existsSync(abs)) { console.error(`Not found: ${target}`); process.exit(1); }
const cls = classifySource(target);
let text;
// existsSync() only proves the name resolves — a directory called `cv.md`,
// a file the user can't read, or a malformed PDF all throw from here. Fail
// the way extractAll() does (first line of the message, nonzero) instead of
// dumping a stack trace at someone who mistyped a path.
try {
if (cls.kind === 'direct') text = readFileSync(abs, 'utf-8');
else if (cls.kind === 'pdf') {
const extractor = detectPdfExtractor();
if (!extractor) { console.error(PDF_INSTALL_HINT); process.exit(1); }
text = extractor.extract(abs);
} else { console.error(`Unsupported source: ${cls.reason}`); process.exit(1); }
} catch (err) {
console.error(`Could not read ${target}: ${String(err.message || err).split('\n')[0]}`);
process.exit(1);
}
if (!text.trim()) {
console.error('No text extracted — likely a scanned/image-only PDF; convert or re-export with a text layer.');
process.exit(1);
}
// The ledger key: documents/-relative with forward slashes, whatever
// spelling (`./cv/master.md`, `cv\\master.md`) the caller used.
return { path: toRelPath(abs), text };
}
/**
* The --diff output for one extracted source. Every outcome is an answer, not
* an error, so all of them exit 0; the first line says which one it is.
*
* - unchanged since the last commit → one line saying so;
* - a snapshot whose text still hashes to the ledger entry → a unified diff;
* - anything else (never committed, committed before snapshots existed,
* snapshot missing or out of step with the ledger) → a one-line notice,
* then the full text, so the agent reads it exactly as it would --text.
* The next --commit writes the snapshot, so this happens once per source.
*/
function diffSource({ path, text }) {
const entry = loadState().ingested[path];
if (entry && entry.hash === sha256(text)) {
return `intake: ${path} is unchanged since it was last committed (${entry.ingestedAt || 'date unknown'}).\n`;
}
let snapshot = null;
let reason;
if (!entry) reason = 'it has never been committed';
else {
try {
snapshot = readFileSync(join(SNAPSHOT_DIR, snapshotFileName(path)), 'utf-8');
} catch {
reason = 'it was committed before intake kept snapshots';
}
if (snapshot !== null && sha256(snapshot) !== entry.hash) {
snapshot = null;
reason = 'the snapshot does not match the recorded fingerprint';
}
}
if (snapshot === null) {
const body = text.endsWith('\n') ? text : `${text}\n`;
return `intake: no snapshot to diff ${path} against (${reason}); full text follows.\n${body}`;
}
// The ledger hashes raw text, but toLines() folds CRLF and the final newline,
// so a source that changed only in those has a new hash and no hunks.
const hunks = formatHunks(diffLines(snapshot, text));
if (!hunks) {
return `intake: ${path} differs from its snapshot only in line endings or the final newline; nothing to propose.\n`;
}
return `--- ${path} (committed ${entry.ingestedAt || 'date unknown'})\n+++ ${path} (current)\n` + hunks;
}
const isMain = isMainModule(import.meta.url);
// A function, not a bare `if (isMain)` block, so the success paths can `return`
// instead of calling process.exit(0). `--text` writes a whole CV to stdout and
// is meant to be piped; process.exit() is synchronous and can truncate a pipe
// write that hasn't drained. Falling off the end leaves exitCode at 0.
const INTAKE_USAGE = [
'Usage: node intake.mjs [--summary]',
' node intake.mjs --text <path>',
' node intake.mjs --diff <path>',
' node intake.mjs --commit <path> [<path> ...]',
' node intake.mjs --commit --all',
' node intake.mjs --self-test',
].join('\n');
const INTAKE_KNOWN_FLAGS = ['--summary', '--text', '--diff', '--commit', '--all', '--self-test', '--help', '-h'];
async function main() {
const args = process.argv.slice(2);
// Unknown flags used to be ignored, so `intake --txt path` scanned the
// whole folder and exited 0. --commit is not a value flag: `--commit --all`
// and `--commit --summary` are real. --text=path has to be read below or
// declaring --text as a value flag would scan again. --diff is the same.
validateFlags(args, INTAKE_KNOWN_FLAGS, INTAKE_USAGE, { valueFlags: ['--text', '--diff'], requireOperand: true });
if (args.includes('--self-test')) runSelfTest();
ensureScaffold();
const textTarget = flagValue(args, '--text');
if (hasFlag(args, '--text')) {
process.stdout.write(readSourceText(textTarget, '--text').text);
return;
}
if (hasFlag(args, '--diff')) {
process.stdout.write(diffSource(readSourceText(flagValue(args, '--diff'), '--diff')));
return;
}
const commitIdx = args.indexOf('--commit');
// Full text only when committing: that is where it becomes the snapshot.
const result = extractAll({ withText: commitIdx !== -1 });
if (commitIdx !== -1) {
const only = args.slice(commitIdx + 1).filter((a) => !a.startsWith('--'));
const all = args.includes('--all');
// Refuse before writing: no paths and no --all is a mistake, not consent.
if (!all && only.length === 0) {
console.error(
'Nothing to commit. Pass the confirmed sources — node intake.mjs --commit <path> [<path> …] — '
+ 'or node intake.mjs --commit --all when every source with new material was merged. '
+ 'Nothing was recorded.',
);
process.exit(1);
}
const count = await commitState(result, all ? COMMIT_ALL : only);
console.log(`Recorded ${count} source(s) as ingested → ${STATE_FILE}`);
return;
}
if (args.includes('--summary')) printSummary(result);
else console.log(JSON.stringify(result, null, 2));
}
if (isMain) {
// Only StateError. Every other expected failure in main() already prints one
// line and exits 1; a ledger this function cannot trust is the one that
// reaches here as a throw, because it is raised deep inside extractAll().
// Anything else is a bug in this script and keeps its stack trace.
try {
await main();
} catch (err) {
if (!(err instanceof StateError)) throw err;
console.error(err.message);
process.exit(1);
}
}