lib/dir-lock.mjs
Rama main · réplica de lectura del repositorio original.
// lib/dir-lock.mjs — the one cross-process directory lock.
//
// Every core lock (data/pipeline.md, the applications tracker, follow-ups.md,
// portal-health.tsv) is this protocol with a different path, owner stamp and
// timeout error. Each used to carry its own acquire/recover/release body, and
// the copies drifted apart one fix at a time: #2777 (Windows EPERM) was learned
// by one copy at a time, #2506 and #2835 (lockstep wakeups) reached three of
// them late, #3895 found "a fourth copy of that bound", and #5358 found copies
// that deleted a peer's lock on a failed owner write, crashed or deadlocked on
// one, or skipped the identity recheck and the stamp read-back pipeline-lock
// had added after measuring the race. So the whole protocol lives here, and
// the callers only pick what differs. tests/lock-rm-contention.test.mjs fails
// if an acquire loop of its own appears anywhere else.
//
// web/src/lib/core/dir-lock.mjs is a byte-for-byte copy of this file, because
// the web bundle cannot import the repo-root core (web/next.config.mjs pins the
// Turbopack root to web/). web/tests/lib/dir-lock.test.mjs fails if the two
// differ by a single byte, so edit this file and copy it across, never the
// other way round.
//
// Protocol:
// - the lock is a directory ("<path>.lock"); a mkdir is atomic.
// - the holder records owner.json — pid, a unique token, started_at — so
// both stale-reclaim and release can verify who actually owns the lock
// before deleting anything.
// - staleness is judged by owner-PID liveness first, falling back to
// directory age only when the metadata is missing or unreadable. An old
// lock whose owner is still running is NOT stale, and an ownerless
// directory gets a fixed grace period (OWNERLESS_GRACE_MS) before age
// alone can condemn it, so a directory created microseconds ago is never
// reclaimable no matter how aggressive the caller's staleMs.
// - stale reclamation is serialized behind a second atomic guard directory
// ("<path>.lock.recover"). Without it, reclamation is itself a TOCTOU
// race: two callers that both judge the same lock stale can have the
// second one's rmSync delete the first one's freshly created lock, after
// which both believe they hold it — reintroducing the very race the lock
// exists to close, just gated behind a crash + contention window.
//
// Timing is caller-supplied so tests can exercise contention in milliseconds
// instead of waiting out a multi-second constant.
import { mkdirSync, rmSync, statSync, existsSync, writeFileSync, readFileSync } from 'fs';
import { join, dirname } from 'path';
import { randomUUID } from 'crypto';
// Env-overridable like the timing knobs each caller reads (staleMs, timeoutMs,
// retryMs) for the same reason: a test that backdates a lock by a small,
// fixed amount to stay just past a tiny staleMs needs the OTHER side of that
// window — the floor itself — to have slack too, or the real cost of spawning
// the child process (Node startup, the module graph, retries) can eat into the
// 1s default before the child ever looks. On a slower CI runner that reads
// as the lock "aging out" prematurely and the test asserting the opposite of
// what it measured (#4537) — read once at module load, which is fine since
// every caller is a fresh process.
//
// Only a finite, positive override is honoured. `|| 1_000` alone would accept
// a NEGATIVE value (a truthy number, so it survives the `||`) and reduce the
// floor below 1,000ms — the opposite of every legitimate use, which widens
// it. It would also accept `Infinity`, which stops ownerless locks from ever
// aging out and can hang acquisition on a genuinely abandoned lock. Both are
// clearly misconfiguration, not a caller asking for "no floor"; unlike
// acquireDirLock's `maxWaitMs`, this constant has no documented
// Infinity-means-unbounded convention to preserve.
const configuredOwnerlessGraceMs = Number(process.env.CAREER_OPS_OWNERLESS_GRACE_MS);
export const OWNERLESS_GRACE_MS = Number.isFinite(configuredOwnerlessGraceMs) && configuredOwnerlessGraceMs > 0
? configuredOwnerlessGraceMs
: 1_000;
// Ceiling on progress-extended waiting (see createLockWaitPolicy). A multiple
// of timeoutMs rather than a constant, so a caller that tunes one tunes both.
const DEFAULT_MAX_WAIT_FACTOR = 10;
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// The same pause for tryAcquireDirLockSync, which has no event-loop turn to
// give back. It only ever waits out an inconclusive stamp read-back, below.
const sleepSync = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
// How long acquisition keeps re-reading its own stamp when the read fails
// with something other than ENOENT: a handful of short tries, enough to ride
// out a transient Windows sharing violation without stretching acquisition.
const READBACK_RETRIES = 5;
const READBACK_RETRY_MS = 10;
/**
* Owner metadata for a lock directory, plus whether its ABSENCE is a fact.
*
* `inspected` is the half that matters for safety, and it is the same
* distinction #2984 drew for `statSync` one line further down — applied to the
* read that happens first. ENOENT is a FACT: there is genuinely no stamp, which
* happens by construction for a lock between its mkdir and its owner.json
* write, and for the recover guard, which never carries one. The age rule is
* the right judge for both.
*
* Any other code is not a fact. On Windows a live holder's stamp is momentarily
* unreadable while the directory is churning under concurrent waiters, and
* collapsing that into the same `null` as "no stamp" lets the age rule condemn
* a lock that is very much alive. Absence we established, versus absence we
* merely failed to observe.
*
* ENOTDIR is the same fact for a path that is not a directory at all: there is
* no stamp because there is nowhere for one to be. A lock path holding a plain
* FILE is never a live lock under this protocol. It is a leftover, typically
* from a caller that locked with an O_EXCL file at the same name before moving
* here (the web's assistant chat store did, #5358). Reading it as "could not
* look" would pin it LIVE for ever and wedge every acquirer; reading it as
* ownerless hands it to the age rule, which reclaims it once it is old enough.
*/
export function readLockOwner(lockDir) {
let raw;
try {
raw = readFileSync(join(lockDir, 'owner.json'), 'utf-8');
} catch (err) {
return { inspected: err?.code === 'ENOENT' || err?.code === 'ENOTDIR', owner: null };
}
try {
return { inspected: true, owner: JSON.parse(raw) };
} catch {
// A torn or corrupt stamp is readable but meaningless. Fall through to the
// age rule rather than pinning the lock as unrecoverable for ever.
return { inspected: true, owner: null };
}
}
// stat, or null when the path is gone or unreadable. Callers that are deciding
// whether to DELETE something need "I could not establish this" and "it is not
// there" to arrive as the same cautious answer.
function statOrNull(dir) {
try {
return statSync(dir);
} catch {
return null;
}
}
// A stat taken to compare identities: the stats, GONE when the directory does
// not exist (ENOENT is a fact), or null when it could not be looked at — no
// evidence either way, the same distinction lockRecoveryVerdict draws.
const GONE = Symbol('gone');
function statForIdentity(dir) {
try {
return statSync(dir);
} catch (err) {
return err?.code === 'ENOENT' ? GONE : null;
}
}
// Whether `now` is the directory `created` was taken from: true, false, or
// null when either look failed and the question cannot be answered.
function sameAsCreated(created, now) {
if (now === GONE) return false;
if (created === GONE || created === null || now === null) return null;
return sameLockDirectory(created, now);
}
// Identity of a directory, so a lock that was removed and recreated by another
// process is never mistaken for the one this caller created.
export function sameLockDirectory(left, right) {
return left.dev === right.dev && left.ino === right.ino
&& (left.ino !== 0 || left.birthtimeMs === right.birthtimeMs);
}
// mkdir's "someone else already has this" answer is NOT portable. POSIX gives
// EEXIST; Windows gives EEXIST *sometimes* and EPERM/EACCES when the target is
// mid-flight — being created, or being removed, by another process at that
// instant. That is not an error condition here, it is the contention this loop
// exists to handle, and it is precisely what a burst of concurrent writers
// manufactures.
//
// Treating it as fatal is how an item gets LOST. Measured on windows-latest
// (#2777, run 31745798742): one of 30 concurrent `agent-inbox add` processes
// died with `EPERM: operation not permitted, mkdir '…agent-inbox.md.lock.recover'`
// and its line was never appended. The two budget increases before this one
// (#2506 jitter, #2825's 30s) both moved the symptom without touching this,
// because a starving writer and a writer killed by EPERM produce the same
// `kept=29 of 30` and only the second is a crash.
//
// A genuine permissions problem still surfaces: it simply stops being an
// instant throw and becomes a timeout that names the last error it saw, which
// is the correct trade when the alternative is silent data loss.
export function isMkdirContention(err) {
return err?.code === 'EEXIST' || err?.code === 'EPERM' || err?.code === 'EACCES';
}
// rm's Windows answer under contention mirrors mkdir's. Removing a directory
// another process is inside of — mid-mkdir, mid-scan (antivirus, indexer), or
// mid-rm — fails with EPERM/EACCES/EBUSY/ENOTEMPTY, and `force: true` only
// silences ENOENT. Measured on windows-latest (#2777, run 32044401225): two of
// 30 concurrent `agent-inbox add` processes died on
// `EPERM … agent-inbox.md.lock.recover` thrown by a bare rmSync of the recover
// guard, and their items were never appended — the same loss the mkdir half of
// this handling removed, resurfacing one call later.
export function isRmContention(err) {
return err?.code === 'EPERM' || err?.code === 'EACCES' || err?.code === 'EBUSY' || err?.code === 'ENOTEMPTY';
}
// Best-effort removal of a lock artifact (the lock dir or the recover guard).
// Contention is swallowed and reported via the return value; anything else
// (EROFS/ENOSPC-class breakage) still throws. Never fatal on contention
// because every lock artifact ages out: an abandoned guard or lock is
// reclaimed by lockRecoveryVerdict on a later attempt, so the worst case of
// leaving one behind is one extra retry for some caller. Killing the writer
// loses its item; waiting does not.
export function rmLockArtifactSync(dir) {
try {
rmSync(dir, { recursive: true, force: true });
return true;
} catch (err) {
if (isRmContention(err)) return false;
throw err;
}
}
function processIsAlive(pid) {
if (!Number.isInteger(pid) || pid <= 0) return false;
try {
process.kill(pid, 0);
return true;
} catch (err) {
return err?.code === 'EPERM'; // exists, just not signalable by this user
}
}
// Conservative: a lock whose recorded owner is still running is never stale,
// however old it is. Age is the fallback only when there's no readable owner.
//
// That fallback needs a floor. Two directories are ownerless by construction,
// not by accident: a lock between its mkdir and its owner.json write, and the
// recover guard, which never carries owner.json at all. Judging those on
// `age > staleMs` alone lets a caller with an aggressive staleMs delete a
// directory created microseconds ago — either stealing a winner's lock inside
// its acquisition window, or evicting a live guard and putting two callers
// inside the decide-then-delete window the guard exists to serialize.
// OWNERLESS_GRACE_MS is a lower bound on that patience, never a cap: a larger
// caller staleMs still wins, and a genuinely abandoned directory still ages
// out, so a crash while holding the guard cannot disable recovery for good.
//
// The answer is a VERDICT, not a boolean, because "recoverable" was two
// different answers wearing one hat and only one of them licenses a delete:
//
// STALE — a directory is THERE and its holder is gone. Deleting it IS the
// recovery, and it is the only case that should reach rmSync.
// VANISHED — there was nothing there when we looked. Also "not blocked", but
// emphatically NOT a licence to delete: the caller acts on this
// verdict microseconds later, and by then the path may be owned by
// a process that legitimately created it in between. Deleting on
// this answer is how an acquirer destroys a live lock it never
// observed (see the caller).
// LIVE — someone holds it, or we could not establish otherwise. Wait.
export const RECOVER_STALE = 'stale';
export const RECOVER_VANISHED = 'vanished';
export const RECOVER_LIVE = 'live';
// Tokens of the handles this module has handed out and not yet released.
//
// A live pid proves its holder is alive only when the holder is another
// process. For THIS process's pid it proves nothing: a long-lived process (the
// web server) can be left holding a directory no handle owns — a release whose
// rm was contended on Windows (swallowed by design, see createRelease), or a
// stamp this caller wrote but could not confirm. The pid rule would then keep
// that lock LIVE until the server restarts, answering "busy" to every request
// that needs it (#5390 review). So our own pid counts as live only for a token
// we are actually holding; any other stamp of ours goes to the age rule below.
//
// The age rule, not an immediate reclaim, because this set is per module
// instance: a second copy of this module in the same process (a bundler chunk,
// a worker thread, a dev-server reload) holds tokens this one cannot see. Its
// lock is reclaimable only once it is older than staleMs, which no critical
// section in this repo approaches.
const activeTokens = new Set();
export function lockRecoveryVerdict(lockDir, staleMs) {
const { inspected, owner } = readLockOwner(lockDir);
// Unreadable is not ownerless. A stamp this process could not read proves
// nothing about whether its holder is alive, and falling through to the age
// rule on that basis is how a LIVE lock gets condemned — the same reasoning
// #2984 applied to the stat below, one step earlier in the same function.
if (!inspected) return RECOVER_LIVE;
if (owner?.pid) {
if (!processIsAlive(owner.pid)) return RECOVER_STALE;
if (owner.pid !== process.pid || activeTokens.has(owner.token)) return RECOVER_LIVE;
// Our pid, but no handle of ours: fall through to the age rule.
}
try {
return Date.now() - statSync(lockDir).mtimeMs > Math.max(staleMs, OWNERLESS_GRACE_MS)
? RECOVER_STALE
: RECOVER_LIVE;
} catch (err) {
// Any stat failure other than ENOENT — Windows EPERM/EBUSY while the
// directory is mid-flight — is "could not look", and answering
// "recoverable" to that hands the caller an rmSync of a LIVE lock created
// microseconds ago: its winner then dies with ENOENT writing owner.json
// (#2777, third face — measured after the rm-contention fix exposed it).
return err?.code === 'ENOENT' ? RECOVER_VANISHED : RECOVER_LIVE;
}
}
/**
* The waiting half of the protocol.
*
* Both rules here were bought with measured failures in `pipeline-lock`, and
* for a long time neither reached the other lock modules, which slept a FIXED
* `retryMs` and timed out on a plain elapsed check:
*
* - jitter (#2506). A fixed retry wakes every waiter at the same instant to
* re-race, which is the coupon-collector problem: serving N waiters takes
* about N·H(N) rounds, and the loser's write is lost. Measured at 20 runs
* per arm, a fixed retry lost an item in 12 of 20 against 2 of 20 jittered.
* - the progress rule (#2835). An expired deadline only means "give up" when
* the lock has NOT changed hands since we last looked. Without it a caller
* waiting on a healthy, briskly handed-round lock is killed for being
* unlucky rather than for anything being stuck.
*
* Returned as closures over one caller's state because both rules are
* stateful: the backoff needs the ceiling, and the progress rule needs the
* previous fingerprint and a re-armable window.
*
* @param {string} lockDir - The lock directory this caller is waiting on.
* @param {{timeoutMs: number, retryMs: number, deadline: number, hardDeadline?: number}} timing
* `hardDeadline` is OPTIONAL: omit it and the policy applies its own default
* ceiling, DEFAULT_MAX_WAIT_FACTOR x timeoutMs from now. Pass one only when
* the caller has a real knob of its own (acquireDirLock's `maxWaitMs`).
* @returns {{backoffMs: () => number, holderStillWedged: () => boolean, noteWaiting: () => void, ceilingReached: () => boolean}}
*/
export function createLockWaitPolicy(lockDir, { timeoutMs, retryMs, deadline, hardDeadline }) {
let perHolderDeadline = deadline;
let lastFingerprint;
// The ceiling is the policy's, not each caller's (#3895). Three lock modules
// used to write `Date.now() + timeoutMs * 10` at their own call sites, so the
// bound the clamp below applies existed in four places. Copies of an
// invariant stay correct only until one is edited, and a wrong ceiling fails
// silently — it changes retry timing, which nothing asserts and nobody
// reports. Deriving it here leaves one copy for them all.
const ceiling = hardDeadline ?? Date.now() + timeoutMs * DEFAULT_MAX_WAIT_FACTOR;
// A ceiling that is not a number is not a ceiling, and it fails INVISIBLY:
// Math.min(x, NaN) is NaN, Math.max(0, NaN) is NaN, and setTimeout(NaN) fires
// immediately — so a caller that got this wrong would spin hot through the
// retry loop rather than raise anything. Infinity is a deliberate, legible
// "no ceiling" (acquireDirLock passes it for maxWaitMs: Infinity) and is
// kept; NaN and non-numbers are mistakes and are refused where they are made.
if (typeof ceiling !== 'number' || Number.isNaN(ceiling)) {
throw new TypeError(
`createLockWaitPolicy: hardDeadline must be a number or omitted, got ${String(hardDeadline)}`,
);
}
// Jittered backoff, never sleeping past EITHER deadline the loop can exit on.
//
// ceiling: an uncapped sleep can cross it and let the NEXT mkdir succeed,
// returning a lock after the documented absolute limit — an overshoot of up
// to 1.5x retryMs. Waking exactly at the ceiling means the check at the top
// of the loop is what decides, rather than whichever of the two happened to
// be later.
//
// perHolderDeadline: the loop also exits when holderStillWedged() flips true,
// which cannot happen until this deadline passes. Sleeping past it means the
// caller's own timeoutMs is only observed at whatever moment the sleep
// happens to end. With retryMs above timeoutMs the overshoot is the whole
// retry: a 150ms timeout waited five seconds. Waking at the nearer deadline
// puts the decision back in the check at the top of the loop.
//
// Clamping OUTSIDE this function is not equivalent. A caller clamping to its
// own fixed `startedAt + timeoutMs` goes to zero once that passes, while
// holderStillWedged() RE-ARMS perHolderDeadline every time the lock changes
// hands. The loop then legitimately continues to the ceiling while every
// sleep is 0, which is a busy-wait at the setTimeout floor and the thundering
// herd the jitter above exists to prevent.
const backoffMs = () => Math.max(0, Math.min(
retryMs * (0.5 + Math.random()),
ceiling - Date.now(),
perHolderDeadline - Date.now(),
));
// The per-holder deadline, evaluated the SAME way everywhere: an expired
// deadline only means "give up" when the lock has not changed hands since we
// last looked. Otherwise the window is re-armed and the caller waits again.
//
// A helper rather than inline code because three separate paths can time a
// caller out — the main retry, a non-EEXIST guard refusal, and an ENOENT
// owner-write retry — and the progress rule holding on one while the other
// two throw directly is the same bug in two more places: a healthy lock
// being handed round briskly could still kill a caller through them.
const holderStillWedged = () => {
if (Date.now() <= perHolderDeadline) return false;
const fingerprint = lockFingerprint(lockDir);
// Only a lock we can SEE, unchanged across a full window, is evidence that
// waiting longer is futile. A fingerprint that moved means the lock is
// being handed round. A null one means it was free or unobservable at the
// instant we looked — no evidence either way, and not to be mistaken for a
// wedged holder.
if (fingerprint !== null && fingerprint === lastFingerprint) return true;
lastFingerprint = fingerprint;
perHolderDeadline = Date.now() + timeoutMs;
return false;
};
// Sampled lazily on the first failed acquisition, not at construction, so the
// very first window is measured against the state the caller actually started
// waiting on. Without it the first expiry compares against `undefined`,
// always re-arms, and every caller gets one free window it did not ask for.
const noteWaiting = () => {
if (lastFingerprint === undefined) lastFingerprint = lockFingerprint(lockDir);
};
// The absolute ceiling, exposed because holderStillWedged() RE-ARMS on
// progress: a lock being handed round briskly resets the window every time,
// so the progress rule alone never terminates. acquireDirLock checks this at
// the top of its loop; the lock modules that once had their own loops had no
// ceiling at all before they adopted the progress rule, and adopting it
// without this would trade a premature timeout for an unbounded wait.
const ceilingReached = () => Date.now() > ceiling;
return { backoffMs, holderStillWedged, noteWaiting, ceilingReached };
}
// A cheap identity for "which holder currently has this lock". Used only to
// tell a lock that is CHANGING HANDS (the system is making progress; this
// caller is merely unlucky) from one that is WEDGED (a single holder is not
// letting go, which is what the timeout exists to escape).
function lockFingerprint(lockDir) {
try {
const st = statSync(lockDir);
const { owner } = readLockOwner(lockDir);
// ino is 0 on some Windows volumes, hence birthtime as the tiebreaker —
// the same pairing sameLockDirectory() already relies on.
return `${owner?.token ?? ''}|${st.ino}|${st.birthtimeMs}`;
} catch {
return null; // free or unobservable — no evidence either way
}
}
/**
* Error thrown when a lock stays busy and the caller supplied no `timeoutError`.
* Callers with an established error type of their own pass a factory instead.
*/
export class DirLockTimeoutError extends Error {
constructor(lockDir, expiredMs) {
super(`lock timeout: ${lockDir} held > ${expiredMs}ms`);
this.name = 'DirLockTimeoutError';
this.code = 'LOCK_TIMEOUT';
this.lockDir = lockDir;
}
}
// What one pass of the protocol can end in. The waiting caller
// (acquireDirLock) and the non-waiting one (tryAcquireDirLockSync) differ only
// in what they do with these, so the pass itself is written once.
// STAMPED — this pass created the lock and wrote its stamp; confirmStamp()
// decides whether it is still ours.
// RECLAIMED — a stale lock was removed under the recover guard: try again now.
// LOST_RACE — the lock this pass created was taken from it before it could
// be handed out: try again now, unless the holder is wedged.
// BUSY — someone else holds it, or the guard is mid-flight: back off.
const PASS_STAMPED = 'stamped';
const PASS_RECLAIMED = 'reclaimed';
const PASS_LOST_RACE = 'lost-race';
const PASS_BUSY = 'busy';
// The non-waiting caller's bound on immediate retries. A reclaim and a lost race
// each earn one more pass; anything still contended after that is a holder, and
// the honest answer is "busy".
const SYNC_MAX_PASSES = 3;
/**
* One acquisition attempt: mkdir, and on contention a guarded stale-reclaim;
* on success, the owner stamp. Never sleeps. Returns a PASS_* outcome; on
* PASS_STAMPED the caller runs confirmStamp() before handing out a lock.
*
* `state` carries what persists across passes for one caller: its token, the
* attempt count, whether it reclaimed a stale lock, the last mkdir error it read
* as contention, the identity of the directory this pass created, and
* `onContention`, the waiting caller's hook for sampling the lock it started
* waiting on.
*/
function attemptDirLock(lockDir, state, options) {
const { staleMs } = options;
const recoverGuardDir = `${lockDir}.recover`;
state.attempts++;
try {
mkdirSync(lockDir);
} catch (err) {
// Not just EEXIST: Windows reports a directory that is mid-create or
// mid-remove by another process as EPERM/EACCES. That is contention, not
// failure — treating it as fatal kills a concurrent writer and loses its
// write (#2777, measured on windows-latest).
if (!isMkdirContention(err)) throw err;
state.lastContentionError = err;
state.onContention();
// Serialize stale-reclaim behind a second atomic guard so only one
// caller can be inside the decide-then-delete window at a time.
let hasRecoverGuard = false;
try {
mkdirSync(recoverGuardDir);
hasRecoverGuard = true;
} catch (guardErr) {
if (!isMkdirContention(guardErr)) throw guardErr;
state.lastContentionError = guardErr;
// An EPERM/EACCES here says the guard directory is mid-flight, not that
// it is sitting there abandoned, so the age check below would be
// reasoning about a directory it cannot even stat reliably. Back off to
// the retry loop instead of judging it.
if (guardErr.code !== 'EEXIST') return PASS_BUSY;
// A process killed between taking the guard and cleaning it up would
// otherwise disable stale recovery forever. The guard normally lives
// for milliseconds, so an old one is judged by the same age rule.
// STALE only. A guard that was already gone when we looked needs no
// eviction, and evicting on that answer would delete the guard a
// different caller has just legitimately taken — putting two callers
// inside the decide-then-delete window this guard exists to serialize.
if (lockRecoveryVerdict(recoverGuardDir, staleMs) === RECOVER_STALE) {
rmLockArtifactSync(recoverGuardDir);
}
}
if (hasRecoverGuard) {
try {
options.onRecoverGuard?.();
} catch { /* an observational hook must never change acquisition */ }
try {
// Only STALE reaches the rm. VANISHED means the lock was absent when
// we looked, and the gap between that observation and this line is
// long enough for another acquirer to have won the mkdir and be
// partway through writing its owner.json. Deleting on that answer
// destroys the fresh, live lock: measured on Windows, the winner's
// mkdir landed 472 us after the vanished verdict and the rm 68 us
// after that, killing it with `ENOENT ... open '<path>.lock/owner.json'`
// and losing its queued item. The rm is the lucky outcome — had the
// stamp landed first, both callers would have held the lock with no
// error at all.
//
// So VANISHED falls through to the normal backoff and retries
// acquisition. It costs one backoff in a rare case; deleting a live
// lock costs an item.
//
// STALE is not enough on its own either: it says the directory we
// READ was abandoned, and the rm acts on whatever is at the path a
// moment later. Reaching the verdict costs a readFileSync and a
// process.kill, which is ample room to be descheduled, and in that
// gap the stale lock can be reclaimed by someone else and replaced by
// a live one. Deleting then destroys a lock this caller never judged.
//
// Not hypothetical. Logging the directory's inode either side of the
// verdict, the ONE stale reclaim in a 2,400-acquisition run came back
// idBefore=5910974513639709 ownerBefore=39392
// idAfter =6192449490350378 ownerAfter =27256 alive=true
// — a different directory, owned by a live process, deleted anyway.
//
// So the identity of the judged directory is carried to the rm and
// rechecked. A path that changed underneath us is left alone; we go
// back and compete for it normally.
const judged = statOrNull(lockDir);
if (judged && lockRecoveryVerdict(lockDir, staleMs) === RECOVER_STALE
&& sameLockDirectory(judged, statOrNull(lockDir) ?? {})) {
if (rmLockArtifactSync(lockDir)) {
state.staleRecovered = true;
return PASS_RECLAIMED; // retry acquisition immediately, still holding the guard's decision
}
// rm hit contention: another process is touching the stale lock at
// this instant, so fall through to the normal backoff instead of
// treating the collision as fatal.
}
} finally {
rmLockArtifactSync(recoverGuardDir);
}
}
return PASS_BUSY;
}
// The directory we just won, by identity. confirmStamp() checks it
// alongside the stamp, and falls back on it alone when the stamp cannot be
// read at all.
state.created = statForIdentity(lockDir);
// Acquired. Record ownership; an owner-less lock would block every future
// acquirer until the age-out, so clean up if the stamp can't be written.
// Its own try, NOT the mkdir's: a failure here is about a directory this
// caller created, and the mkdir catch above would read EPERM/EACCES from
// the write as contention and wait on our own ownerless lock (#5358).
try {
writeFileSync(join(lockDir, 'owner.json'), JSON.stringify({
pid: process.pid,
token: state.token,
started_at: new Date().toISOString(),
...options.owner,
}, null, 2), { flag: 'wx' });
} catch (ownerErr) {
// ENOENT writing owner.json means the just-won lock directory is gone:
// another caller (mis)judged it reclaimable and deleted it. That is a
// lost race, not a failure — re-enter the loop and compete again.
// Dying here loses the caller's queued write (#2777, third face). And
// deleting here would be worse: the path may already hold a NEWER
// caller's live lock, which an rm would free underneath it (#5358).
//
// EEXIST is the same race seen from the other side. The directory we
// created was empty, so a stamp already in it means our directory was
// replaced by another caller's before we wrote. The exclusive write is
// what keeps us from overwriting that holder's stamp: a plain write would
// put our token in its lock, our read-back would then accept it, and two
// processes would enter the critical section (#5389 review).
if (ownerErr?.code === 'ENOENT' || ownerErr?.code === 'EEXIST') return PASS_LOST_RACE;
// Best-effort: a contended rm here must not mask ownerErr, and an
// orphaned owner-less lock ages out via lockRecoveryVerdict anyway.
//
// The try/catch is what makes that sentence true for EVERY failure, not
// just a contended one. rmLockArtifactSync deliberately rethrows the
// non-contention class (EROFS/ENOSPC), so without this an unwritable
// filesystem would propagate from the CLEANUP and the `throw ownerErr`
// below would never run — the caller would be told the lock could not be
// removed, and never learn why the owner stamp could not be written.
// A leftover lock ages out; a swallowed cause is recovered by nothing.
try {
rmLockArtifactSync(lockDir);
} catch {
/* cleanup must never outrank the reason we are here */
}
throw ownerErr;
}
return PASS_STAMPED;
}
/**
* Read our own stamp back before handing the caller a lock handle. A generator
* so the retry schedule is written once for both entry points: it yields each
* pause and returns whether the lock is still ours. acquireDirLock awaits the
* pauses (settleAsync); tryAcquireDirLockSync blocks on them (settleSync).
*
* The identity recheck in attemptDirLock stops this caller from deleting a
* directory it did not judge, but it cannot stop another caller from doing so
* to US, and that failure has no error attached to it: the delete lands after
* our owner.json write succeeds, so nothing in the acquisition path notices and
* two processes enter the critical section believing they hold it.
*
* The ENOENT branch of the owner write recovers the case where the delete
* lands BEFORE the stamp. This is the same recovery for the case where it lands
* just after. A stamp that is missing, or that carries someone else's token,
* means we no longer hold what we took, so we compete again rather than
* proceed.
*
* A read that FAILED (not ENOENT) is neither answer. On Windows a live
* holder's stamp is momentarily unreadable while waiters churn the
* directory (see readLockOwner), and treating that as "lost" sends this
* caller back to wait on its own lock: the mkdir meets EEXIST, the verdict
* finds our own live pid and answers LIVE, and the caller times out while
* the orphan it leaves blocks every writer until this process exits. So an
* inconclusive read is retried briefly, and if it never becomes readable
* the directory's identity decides: a replacement lock is a new directory.
* Identity is the fallback, not the rule, because a filesystem can reuse
* an inode number for a directory recreated at once, which only the stamp
* can tell apart.
*/
function* confirmStamp(lockDir, state) {
let readBack = readLockOwner(lockDir);
for (let retry = 0; !readBack.inspected && retry < READBACK_RETRIES; retry++) {
yield READBACK_RETRY_MS;
readBack = readLockOwner(lockDir);
}
// A readable stamp with our token must also sit in the directory we
// created. The exclusive write already refuses another holder's stamp, so
// this is the second guard on the same race, not the only one. A stat that
// merely failed is no evidence against us (the Windows churn again), so it
// only blocks us when the identity is known to differ or the directory is
// known to be gone.
const identity = sameAsCreated(state.created, statForIdentity(lockDir));
return readBack.inspected
? readBack.owner?.token === state.token && identity !== false
: identity === true;
}
async function settleAsync(steps) {
let step = steps.next();
while (!step.done) {
await sleep(step.value);
step = steps.next();
}
return step.value;
}
function settleSync(steps) {
let step = steps.next();
while (!step.done) {
sleepSync(step.value);
step = steps.next();
}
return step.value;
}
function newAttemptState(onContention = () => {}) {
return { token: randomUUID(), attempts: 0, staleRecovered: false, lastContentionError: null, created: null, onContention };
}
function lockHandle(lockDir, state, startedAt, options) {
activeTokens.add(state.token);
return {
lockDir,
token: state.token,
attempts: state.attempts,
waitMs: Date.now() - startedAt,
staleRecovered: state.staleRecovered,
release: createRelease(lockDir, state.token, options.removeLock),
};
}
/**
* Blocks until the directory lock at `lockDir` is held, then returns a handle
* whose release() frees it.
*
* @param {string} lockDir - The lock directory itself (not the file it guards).
* @param {object} options
* @param {number} options.timeoutMs - Max time a SINGLE holder may block this caller. Waiting is extended while the lock keeps changing hands (see createLockWaitPolicy).
* @param {number} options.retryMs - Mean delay between acquisition attempts (jittered).
* @param {number} options.staleMs - Age threshold for a lock with no readable owner, floored at OWNERLESS_GRACE_MS.
* @param {number} [options.maxWaitMs] - Absolute ceiling on waiting, however much progress the lock makes. Omitted: the policy's default, DEFAULT_MAX_WAIT_FACTOR x timeoutMs.
* @param {object} [options.owner] - Extra owner.json fields, recorded for diagnostics.
* @param {boolean} [options.createParent=false] - mkdir -p the lock's parent first.
* @param {(info: {lockDir: string, expiredMs: number, owner: object, lastMkdirError: string|null, diagnosis: string}) => Error} [options.timeoutError]
* Builds the error thrown when the wait gives up. `expiredMs` is the bound that
* actually ran out; `diagnosis` is a ready-made " — owner=…" suffix.
* @param {() => void} [options.onRecoverGuard] - Called each time this caller takes the recover guard (test ordering hook).
* @param {(lockDir: string) => void} [options.removeLock] - Release's rm, injectable for fault tests.
* @returns {Promise<{lockDir: string, token: string, attempts: number, waitMs: number, staleRecovered: boolean, release: () => void}>}
*/
export async function acquireDirLock(lockDir, options) {
const { timeoutMs, retryMs, maxWaitMs } = options;
const startedAt = Date.now();
// Deliberately NOT floored at timeoutMs. A caller that asks for a ceiling
// below one per-holder window means it, and quietly raising it would make
// maxWaitMs a no-op in precisely the range where it was stated most
// explicitly. The ceiling wins; timeoutMs just stops mattering.
const hardDeadline = maxWaitMs === undefined ? undefined : startedAt + maxWaitMs;
const ceilingMs = maxWaitMs ?? timeoutMs * DEFAULT_MAX_WAIT_FACTOR;
const { backoffMs, holderStillWedged, noteWaiting, ceilingReached } = createLockWaitPolicy(lockDir, {
timeoutMs, retryMs, deadline: startedAt + timeoutMs, hardDeadline,
});
// state.lastContentionError is the last mkdir error this caller treated as
// contention. On POSIX it is always EEXIST and says nothing; on Windows an
// EPERM/EACCES that persists to the deadline is the difference between
// "crowded" and "this process cannot create directories here at all", and
// without it a real permissions problem would present as a plain,
// unexplained timeout.
const state = newAttemptState(noteWaiting);
// Built at the throw site so the diagnosis reflects the moment it gave up.
// A timeout on a critical section that is a single sub-millisecond append
// has more than one explanation, and the owner record separates them:
// - no owner, or an owner whose PID is dead: real contention, and the
// structural answer is a fair queue rather than a bigger budget.
// - an owner reported ALIVE after tens of seconds: the directory outlived
// its holder, because release()'s rm can fail on Windows while a handle
// is open and is swallowed by design.
// Diagnostic only: it never changes whether the error is thrown.
// `expiredMs` is the bound that actually ran out, so the message names the
// limit that was applied. Reporting timeoutMs when the CEILING expired states
// a number that was never in force.
const buildTimeoutError = (expiredMs = timeoutMs) => {
const info = { lockDir, expiredMs, owner: null, lastMkdirError: null, diagnosis: '' };
try {
const { inspected, owner } = readLockOwner(lockDir);
info.owner = owner
? { pid: owner.pid, alive: processIsAlive(owner.pid), started_at: owner.started_at, heldMs: Date.parse(owner.started_at) ? Date.now() - Date.parse(owner.started_at) : null }
// The note separates the two cases the diagnosis could not otherwise
// tell apart, because they point at different bugs: a stamp that is
// absent is an acquisition still in flight, while one that could not be
// read is the condition lockRecoveryVerdict refuses to condemn.
: {
pid: null,
alive: null,
note: !inspected ? 'owner.json exists but could not be read'
: existsSync(lockDir) ? 'lock exists with no owner.json'
: 'lock vanished before it could be read',
};
info.diagnosis = ` — owner=${JSON.stringify(info.owner)}`;
const lastContentionError = state.lastContentionError;
if (lastContentionError && lastContentionError.code !== 'EEXIST') {
info.lastMkdirError = lastContentionError.code;
info.diagnosis += `, last mkdir error=${lastContentionError.code} on ${lastContentionError.path ?? '?'}`;
}
} catch { /* diagnosis must never mask the timeout it describes */ }
return options.timeoutError ? options.timeoutError(info) : new DirLockTimeoutError(lockDir, expiredMs);
};
// A fresh install may not have data/ yet — plugins.mjs's cmdRun calls
// appendToPipeline with no directory pre-creation, and appendPortalHealth()
// creates data/ only after its lock is taken — so create it here rather
// than letting mkdirSync(lockDir) throw a raw ENOENT.
if (options.createParent) mkdirSync(dirname(lockDir), { recursive: true });
for (;;) {
// The ceiling is tested FIRST, on its own, on every pass. Nesting it inside
// the per-holder deadline made it conditional on a branch that may never
// run: a caller whose maxWaitMs is below timeoutMs never reaches the inner
// check, a re-arm pushes the next look a whole window away, and the reclaim
// fast-path continues straight past both. A bound that holds only when
// another bound happens to fire is not a bound.
if (ceilingReached()) throw buildTimeoutError(ceilingMs);
const outcome = attemptDirLock(lockDir, state, options);
if (outcome === PASS_STAMPED) {
if (await settleAsync(confirmStamp(lockDir, state))) return lockHandle(lockDir, state, startedAt, options);
// Not ours any more: a lost race, judged like the others below.
} else if (outcome === PASS_RECLAIMED) {
continue;
}
if (holderStillWedged()) throw buildTimeoutError();
if (outcome !== PASS_BUSY) continue;
// Jitter, because a FIXED retry makes every waiter wake at the same
// instant and re-race, and whoever loses is picked at random rather than
// queued. With N waiters that is the coupon-collector problem: serving
// all of them takes about N·H(N) rounds, so 30 concurrent adds need ~120
// and the default budget only affords 8000/80 = 100. The starving writer
// then times out and its item is LOST — the exact failure #2777 removed
// from the silent path, reappearing as a loud one under contention.
// Spreading wake-ups over [0.5x, 1.5x) breaks the herd so waiters stop
// colliding on every round. It does not make the lock fair; it makes
// unfairness cost a retry instead of an item.
//
// Measured, 20 runs per arm alternated on the same machine with the
// budget forced to 220ms (2.75 rounds for 30 writers, short by
// construction so the effect is visible without waiting out 8s):
// with jitter 2 of 20 runs lose an item
// without jitter 12 of 20
// So this is a ~6x reduction, NOT a cure: a starving writer is still
// possible, and the structural fix is a fair queue rather than a retry
// lottery. Left as a lottery because fairness needs an ordered wait and
// that is a different lock; recorded here so nobody reads the jitter as
// "the race is gone".
await sleep(backoffMs());
}
}
/**
* Takes the directory lock at `lockDir` if it can be had now, without waiting:
* returns the same handle acquireDirLock does, or null when another holder has
* it. For a caller whose answer to contention is "busy, retry" rather than a
* queue — a synchronous request handler returning 409, say.
*
* Not a weaker lock. It is the same pass acquireDirLock makes, so a stale lock
* (a crashed holder, an ownerless leftover past its age-out) is reclaimed in
* line and the caller proceeds; only a LIVE holder answers null. The one wait
* it shares is confirmStamp's: an unreadable read-back of its own stamp
* blocks for at most READBACK_RETRIES x READBACK_RETRY_MS (50ms) before the
* directory's identity decides. A readable stamp, the normal case, never waits.
*
* @param {string} lockDir - The lock directory itself (not the file it guards).
* @param {object} options
* @param {number} options.staleMs - Age threshold for a lock with no readable owner, floored at OWNERLESS_GRACE_MS.
* @param {object} [options.owner] - Extra owner.json fields, recorded for diagnostics.
* @param {boolean} [options.createParent=false] - mkdir -p the lock's parent first.
* @param {() => void} [options.onRecoverGuard] - Called each time this caller takes the recover guard (test ordering hook).
* @param {(lockDir: string) => void} [options.removeLock] - Release's rm, injectable for fault tests.
* @returns {{lockDir: string, token: string, attempts: number, waitMs: number, staleRecovered: boolean, release: () => void} | null}
*/
export function tryAcquireDirLockSync(lockDir, options) {
const startedAt = Date.now();
const state = newAttemptState();
if (options.createParent) mkdirSync(dirname(lockDir), { recursive: true });
for (let pass = 0; pass < SYNC_MAX_PASSES; pass++) {
const outcome = attemptDirLock(lockDir, state, options);
if (outcome === PASS_STAMPED && settleSync(confirmStamp(lockDir, state))) {
return lockHandle(lockDir, state, startedAt, options);
}
if (outcome === PASS_BUSY) return null;
}
return null;
}
/**
* The handle's release(). Idempotent, and retryable: a release that throws can
* be called again, and it remembers what it had already verified.
*
* It removes the lock only when it can show the directory is still the one this
* caller stamped: a stat on either side of reading the stamp, compared by
* identity, so a lock that was reclaimed and recreated in between is left to
* its new owner. Our operation can outlive staleMs, and another process can
* then legitimately reclaim the lock — deleting it here would free someone
* else's critical section.
*
* Throws when ownership cannot be established either way (the directory is
* there but its stamp is missing or unreadable) and on a removal failure that
* is not Windows contention. Contention is swallowed: ownership was verified,
* so a contended rm (EPERM/EBUSY while another process stats the directory)
* must not fail a caller whose work already succeeded; the orphan ages out via
* lockRecoveryVerdict.
*/
function createRelease(lockDir, token, removeLock = (path) => rmSync(path, { recursive: true, force: true })) {
let released = false;
let verifiedDir = null;
// Released, whether or not the rm below succeeded: the handle no longer
// holds the lock, so its token stops counting as live (see activeTokens).
const finish = () => {
released = true;
activeTokens.delete(token);
};
return function release() {
if (released) return;
if (verifiedDir) {
// A previous call verified ownership and then failed to remove. Its own
// partial cleanup may have taken owner.json with it, so a missing stamp
// is no longer evidence against us; the directory's identity is.
let currentDir;
try {
currentDir = statSync(lockDir);
} catch (err) {
if (err?.code === 'ENOENT') {
finish();
return;
}
throw err;
}
if (!sameLockDirectory(verifiedDir, currentDir)) {
finish();
return;
}
const { owner } = readLockOwner(lockDir);
if (owner && owner.token !== token) {
finish();
return;
}
if (!owner && existsSync(join(lockDir, 'owner.json'))) {
throw new Error(`Cannot verify lock ownership at ${lockDir}`);
}
} else {
let beforeRead;
try {
beforeRead = statSync(lockDir);
} catch (err) {
if (err?.code === 'ENOENT') {
finish();
return;
}
throw err;
}
const { owner } = readLockOwner(lockDir);
if (owner?.token !== token) {
if (owner) {
finish(); // reclaimed by someone else — leave it alone
return;
}
throw new Error(`Cannot verify lock ownership at ${lockDir}`);
}
const afterRead = statSync(lockDir);
if (!sameLockDirectory(beforeRead, afterRead)) {
finish(); // swapped underneath us
return;
}
verifiedDir = afterRead;
}
try {
removeLock(lockDir);
} catch (rmErr) {
if (!isRmContention(rmErr)) throw rmErr;
}
finish();
};
}
/**
* Release a handle from a `finally`, where a throw would replace the caller's
* own outcome: a failed release must neither mask the error that is unwinding
* nor turn a completed write into a reported failure. A warning is the honest
* answer, and the handle is then abandoned: nothing will retry this release.
*
* So its token stops counting as live here, even though release() itself
* keeps it (a handle whose release threw may be retried, and until then it
* still holds the lock). An abandoned token left in activeTokens would pin the
* leftover LIVE until this process exits — a chat answering 409, a store
* timing out, until the web server restarts. Dropped, the leftover goes to the
* age rule like any other stamp of ours with no handle behind it.
*/
export function releaseOrWarn(lock) {
try {
lock.release();
} catch (err) {
activeTokens.delete(lock.token);
console.error(`Warning: lock cleanup failed at ${lock.lockDir}: ${err.message}`);
}
}