tests/test_caveman_stats.js
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
// Tests for /caveman-stats — direct script invocation and via mode tracker.
// Run: node tests/test_caveman_stats.js
const fs = require('fs');
const path = require('path');
const os = require('os');
const assert = require('assert');
const { execFileSync, spawnSync } = require('child_process');
const ROOT = path.resolve(__dirname, '..');
const STATS = path.join(ROOT, 'src', 'hooks', 'caveman-stats.js');
const TRACKER = path.join(ROOT, 'src', 'hooks', 'caveman-mode-tracker.js');
const SESSION_END_SESSION_ID = 'session-end-stats';
const INVALID_HOOK_JSON = '{not-json';
let passed = 0;
let failed = 0;
function test(name, fn) {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'caveman-stats-test-'));
try {
fn(tmp);
passed++;
console.log(` ✓ ${name}`);
} catch (e) {
failed++;
console.error(` ✗ ${name}\n ${e.message}`);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
}
function makeSession(dir, lines) {
const projDir = path.join(dir, '.claude', 'projects', 'p');
fs.mkdirSync(projDir, { recursive: true });
const sessFile = path.join(projDir, 's.jsonl');
fs.writeFileSync(sessFile, lines.map(l => JSON.stringify(l)).join('\n'));
return sessFile;
}
function assertSavingsUnknown(out) {
assert.match(out, /savings:? unknown/i);
assert.doesNotMatch(out, /Est\. (?:without|tokens saved|saved|output reduction|rule overhead|net)|\bSaved \d|\d[\d,.]*%|\$[\d.]+/i);
}
console.log('caveman-stats tests\n');
test('Gemini commands cannot read or mutate unrelated Claude history (#403)', (tmp) => {
makeSession(tmp, [{ type: 'assistant', message: { usage: { output_tokens: 987654, cache_read_input_tokens: 12345 } } }]);
const claudeDir = path.join(tmp, '.claude');
const history = path.join(claudeDir, '.caveman-history.jsonl');
const suffix = path.join(claudeDir, '.caveman-statusline-suffix');
fs.writeFileSync(history, 'existing Claude history\n');
fs.writeFileSync(suffix, 'existing Claude suffix');
for (const args of [[], ['--all'], ['--share'], ['--host', 'gemini']]) {
const out = execFileSync(process.execPath, [STATS, ...args], {
encoding: 'utf8', env: { ...process.env, GEMINI_CLI: '1', CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(out, /\/stats model/);
assert.match(out, /\/stats session/);
assertSavingsUnknown(out);
assert.doesNotMatch(out, /987[,.]?654|12[,.]?345/);
assert.strictEqual(fs.readFileSync(history, 'utf8'), 'existing Claude history\n');
assert.strictEqual(fs.readFileSync(suffix, 'utf8'), 'existing Claude suffix');
}
});
test('explicit Claude hook ownership works when launched below a Gemini shell', (tmp) => {
const sess = makeSession(tmp, [{ type: 'assistant', message: { usage: { output_tokens: 431, cache_read_input_tokens: 0 } } }]);
const out = execFileSync(process.execPath, [STATS, '--host', 'claude', '--session-file', sess], {
encoding: 'utf8', env: { ...process.env, GEMINI_CLI: '1', CLAUDE_CONFIG_DIR: path.join(tmp, '.claude') },
});
assert.match(out, /Output tokens:\s+431/);
assertSavingsUnknown(out);
});
test('full-mode output and historical estimates never establish measured savings (#991/#789)', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 1000, cache_read_input_tokens: 2400 } } },
]);
const claudeDir = path.join(tmp, '.claude');
const historyPath = path.join(claudeDir, '.caveman-history.jsonl');
const historical = JSON.stringify({ ts: 1, session_id: 'old', output_tokens: 1000,
est_saved_tokens: 1857, est_saved_usd: 0.027855, turns: 1 }) + '\n';
fs.writeFileSync(historyPath, historical);
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const suffixPath = path.join(claudeDir, '.caveman-statusline-suffix');
for (const args of [['--session-file', sess], ['--session-file', sess, '--share'], ['--all']]) {
fs.writeFileSync(suffixPath, '⛏ 1.9k');
const out = execFileSync(process.execPath, [STATS, ...args], {
encoding: 'utf8', env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
assert.strictEqual(fs.readFileSync(suffixPath, 'utf8'), '');
}
const history = fs.readFileSync(historyPath, 'utf8');
assert.ok(history.startsWith(historical), 'original history bytes must be preserved');
for (const line of history.slice(historical.length).trim().split('\n')) {
const row = JSON.parse(line);
assert.strictEqual(row.output_tokens, 1000);
assert.strictEqual(row.cache_read_input_tokens, 2400);
assert.ok(!Object.hasOwn(row, 'est_saved_tokens'));
assert.ok(!Object.hasOwn(row, 'est_saved_usd'));
}
});
test('reads --session-file directly and sums output tokens', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100, cache_read_input_tokens: 200 } } },
{ type: 'user', message: { content: 'hi' } },
{ type: 'assistant', message: { usage: { output_tokens: 50, cache_read_input_tokens: 50 } } },
]);
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: path.join(tmp, '.claude') },
});
assert.match(out, /Turns:\s+2/);
assert.match(out, /Output tokens:\s+150/);
assert.match(out, /Cache-read tokens:\s+250/);
});
test('counts a multi-block API response once, not once per JSONL line', (tmp) => {
// Claude Code writes one assistant line per content block (text + each
// tool_use) of the same API response — same message.id + requestId, same
// usage repeated. Only one line per response may count.
const usage = { output_tokens: 487, cache_read_input_tokens: 1000 };
const sess = makeSession(tmp, [
{ type: 'assistant', requestId: 'req_1', message: { id: 'msg_a', usage } },
{ type: 'assistant', requestId: 'req_1', message: { id: 'msg_a', usage } },
{ type: 'assistant', requestId: 'req_1', message: { id: 'msg_a', usage } },
{ type: 'user', message: { content: 'tool result' } },
{ type: 'assistant', requestId: 'req_2', message: { id: 'msg_b', usage: { output_tokens: 13, cache_read_input_tokens: 500 } } },
]);
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: path.join(tmp, '.claude') },
});
assert.match(out, /Turns:\s+2/);
assert.match(out, /Output tokens:\s+500\b/);
assert.match(out, /Cache-read tokens:\s+1,?\.?500\b/);
});
test('same message.id under different requestIds counts per response (retry path)', (tmp) => {
// A retried request re-sends the same message.id under a new requestId —
// those are distinct billed responses and must both count.
const sess = makeSession(tmp, [
{ type: 'assistant', requestId: 'req_1', message: { id: 'msg_a', usage: { output_tokens: 100 } } },
{ type: 'assistant', requestId: 'req_2', message: { id: 'msg_a', usage: { output_tokens: 100 } } },
]);
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: path.join(tmp, '.claude') },
});
assert.match(out, /Turns:\s+2/);
assert.match(out, /Output tokens:\s+200\b/);
});
test('entries without message.id keep per-line counting (no dedupe key)', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
]);
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: path.join(tmp, '.claude') },
});
assert.match(out, /Turns:\s+2/);
assert.match(out, /Output tokens:\s+200\b/);
});
test('caveman mode (legacy "full" flag) reports observed output with savings unknown', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
assert.match(out, /Output tokens:\s+350/);
assert.match(out, /Mode: caveman\b/);
});
test('other modes (legacy "ultra" flag) also leave savings unknown', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'ultra');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(out, /Mode: ultracave\b/);
assertSavingsUnknown(out);
});
test('reports no-session when no .jsonl exists', (tmp) => {
fs.mkdirSync(path.join(tmp, '.claude', 'projects'), { recursive: true });
let err = null;
try {
execFileSync(process.execPath, [STATS], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: path.join(tmp, '.claude') },
});
} catch (e) { err = e; }
assert.ok(err, 'should exit non-zero');
assert.match(err.stderr, /no Claude Code session found/);
});
// Without --session-file, the current project's transcript beats a newer one
// from another project. Claude Code's folder name replaces every
// non-alphanumeric character with '-', so a dot in the path must not defeat it.
test('manual run prefers the current project transcript over a newer one elsewhere (#563)', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
const cwd = path.join(tmp, 'my.repo');
fs.mkdirSync(cwd, { recursive: true });
const slug = fs.realpathSync(cwd).replace(/[^A-Za-z0-9]/g, '-');
const write = (dir, name, outputTokens, mtimeSec) => {
fs.mkdirSync(dir, { recursive: true });
const file = path.join(dir, name);
fs.writeFileSync(file, JSON.stringify({ type: 'assistant', message: { usage: { output_tokens: outputTokens } } }));
fs.utimesSync(file, mtimeSec, mtimeSec);
};
const now = Math.floor(Date.now() / 1000);
write(path.join(claudeDir, 'projects', slug), 'a.jsonl', 111, now - 3600);
write(path.join(claudeDir, 'projects', '-elsewhere'), 'b.jsonl', 999, now);
const out = execFileSync(process.execPath, [STATS, '--host', 'claude'], {
encoding: 'utf8', cwd, env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(out, /a\.jsonl/);
assert.doesNotMatch(out, /b\.jsonl/);
assert.match(out, /Output tokens:\s+111/);
});
test('mode tracker delivers /caveman-stats via additionalContext', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [TRACKER], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, HOME: tmp },
input: JSON.stringify({ prompt: '/caveman-stats', transcript_path: sess }),
});
const parsed = JSON.parse(out);
assert.strictEqual(parsed.hookSpecificOutput.hookEventName, 'UserPromptSubmit');
assert.match(parsed.hookSpecificOutput.additionalContext, /Caveman Stats/);
assert.match(parsed.hookSpecificOutput.additionalContext, /Output tokens:\s+100/);
});
test('mode tracker preserves caveman flag when /caveman-stats fires', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 50 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
execFileSync(process.execPath, [TRACKER], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, HOME: tmp },
input: JSON.stringify({ prompt: '/caveman-stats', transcript_path: sess }),
});
// The flag must still say 'full' — the stats command must not change mode.
assert.strictEqual(fs.readFileSync(path.join(claudeDir, '.caveman-active'), 'utf8'), 'full');
});
test('known models do not imply monetary savings', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-20250514', usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
assert.match(out, /Output tokens:\s+350/);
});
test('unknown models retain output counts without a savings estimate', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'some-future-model-xyz', usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
// A model identifier is not a measured comparison.
assertSavingsUnknown(out);
assert.doesNotMatch(out, /Est\. saved \(USD\)/);
});
test('parseSession retains the observed model without inventing pricing', (tmp) => {
const { parseSession } = require(STATS);
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
]);
assert.strictEqual(parseSession(sess).model, 'claude-sonnet-4-7');
});
test('formatStats handles empty session gracefully', () => {
const { formatStats } = require(path.join(ROOT, 'src', 'hooks', 'caveman-stats.js'));
const out = formatStats({ outputTokens: 0, cacheReadTokens: 0, turns: 0, mode: 'full', model: null });
assert.match(out, /No conversation yet/);
});
test('--share prints single-line tweetable summary', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess, '--share'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.strictEqual(out.split('\n').filter(Boolean).length, 1);
assert.match(out, /^🪨 1 turn, 350 output tokens this session; savings unknown — caveman\.sh$/m);
assertSavingsUnknown(out);
});
test('--share reports observed usage in lite mode', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 200 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'lite');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess, '--share'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(out, /^🪨 1 turn, 200 output tokens this session; savings unknown — caveman\.sh$/m);
});
test('appends to lifetime history on each run', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
const histPath = path.join(claudeDir, '.caveman-history.jsonl');
assert.ok(fs.existsSync(histPath), 'history file should be created');
const lines = fs.readFileSync(histPath, 'utf8').split('\n').filter(Boolean);
assert.strictEqual(lines.length, 1);
const entry = JSON.parse(lines[0]);
assert.strictEqual(entry.session_id, 's');
assert.strictEqual(entry.output_tokens, 350);
assert.strictEqual(entry.turns, 1);
assert.ok(!Object.hasOwn(entry, 'est_saved_tokens'));
assert.ok(!Object.hasOwn(entry, 'est_saved_usd'));
assert.deepStrictEqual(entry.output_tokens_by_mode, { caveman: 350 });
assert.strictEqual(entry.mode, 'caveman');
assert.strictEqual(entry.model, 'claude-sonnet-4-7');
});
// SessionEnd runs `caveman-stats.js --record` with the hook payload on stdin.
function runRecord(claudeDir, input, extraEnv = {}) {
return spawnSync(process.execPath, [STATS, '--record'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, ...extraEnv },
input,
});
}
test('--record appends a SessionEnd snapshot silently, with this session\'s own mode', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350, cache_read_input_tokens: 40 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(path.join(claudeDir, '.caveman-sessions'), { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-sessions', `${SESSION_END_SESSION_ID}.mode`), 'ultracave');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'caveman'); // another window's last write
const r = runRecord(claudeDir, JSON.stringify({ session_id: SESSION_END_SESSION_ID, transcript_path: sess, reason: 'other' }));
assert.strictEqual(r.status, 0, r.stderr);
assert.strictEqual(r.stdout, '');
assert.strictEqual(r.stderr, '');
const lines = fs.readFileSync(path.join(claudeDir, '.caveman-history.jsonl'), 'utf8').split('\n').filter(Boolean);
assert.strictEqual(lines.length, 1);
const entry = JSON.parse(lines[0]);
assert.strictEqual(entry.session_id, SESSION_END_SESSION_ID);
assert.strictEqual(entry.output_tokens, 350);
assert.strictEqual(entry.cache_read_input_tokens, 40);
assert.strictEqual(entry.turns, 1);
assert.strictEqual(entry.mode, 'ultracave');
for (const key of Object.keys(entry)) assert.doesNotMatch(key, /saved|savings|usd/i, `no savings field: ${key}`);
// The lifetime view now counts the session without anyone typing /caveman-stats.
const all = execFileSync(process.execPath, [STATS, '--all'], {
encoding: 'utf8', env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(all, /Sessions:\s+1\b/);
assert.match(all, /Output tokens:\s+350/);
});
test('--record ignores malformed hook stdin without guessing a session', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
const r = runRecord(claudeDir, INVALID_HOOK_JSON);
assert.strictEqual(r.status, 0, r.stderr);
assert.strictEqual(r.stdout + r.stderr, '');
assert.strictEqual(fs.existsSync(path.join(claudeDir, '.caveman-history.jsonl')), false);
assert.ok(fs.existsSync(sess), 'fixture session should exist but not be auto-selected');
});
test('--record never falls back to the newest transcript when the payload names none', (tmp) => {
makeSession(tmp, [{ type: 'assistant', message: { usage: { output_tokens: 350 } } }]);
const claudeDir = path.join(tmp, '.claude');
const r = runRecord(claudeDir, JSON.stringify({ session_id: SESSION_END_SESSION_ID, reason: 'other' }));
assert.strictEqual(r.status, 0, r.stderr);
assert.strictEqual(r.stdout + r.stderr, '');
assert.strictEqual(fs.existsSync(path.join(claudeDir, '.caveman-history.jsonl')), false);
});
test('--record stays silent with exit 0 on an unreadable or empty transcript', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
const empty = path.join(tmp, 'empty.jsonl');
fs.writeFileSync(empty, '');
for (const transcript of [path.join(tmp, 'missing.jsonl'), empty]) {
const r = runRecord(claudeDir, JSON.stringify({ session_id: SESSION_END_SESSION_ID, transcript_path: transcript }));
assert.strictEqual(r.status, 0, r.stderr);
assert.strictEqual(r.stdout + r.stderr, '');
}
assert.strictEqual(fs.existsSync(path.join(claudeDir, '.caveman-history.jsonl')), false);
});
test('--record runs as a Claude hook even below a Gemini shell', (tmp) => {
const sess = makeSession(tmp, [{ type: 'assistant', message: { usage: { output_tokens: 12 } } }]);
const claudeDir = path.join(tmp, '.claude');
const r = runRecord(claudeDir, JSON.stringify({ session_id: SESSION_END_SESSION_ID, transcript_path: sess }), { GEMINI_CLI: '1' });
assert.strictEqual(r.status, 0, r.stderr);
assert.strictEqual(r.stdout, '');
const entry = JSON.parse(fs.readFileSync(path.join(claudeDir, '.caveman-history.jsonl'), 'utf8').trim());
assert.strictEqual(entry.output_tokens, 12);
});
test('--all aggregates latest entry per session', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
const histPath = path.join(claudeDir, '.caveman-history.jsonl');
// Two sessions, second one has two snapshots — only latest counts.
fs.writeFileSync(histPath, [
{ ts: 1000, session_id: 'a', mode: 'full', output_tokens: 100, est_saved_tokens: 185, est_saved_usd: 0.0028 },
{ ts: 2000, session_id: 'b', mode: 'full', output_tokens: 50, est_saved_tokens: 92, est_saved_usd: 0.0014 },
{ ts: 3000, session_id: 'b', mode: 'full', output_tokens: 200, est_saved_tokens: 371, est_saved_usd: 0.0056 },
].map(o => JSON.stringify(o)).join('\n') + '\n');
const out = execFileSync(process.execPath, [STATS, '--all'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(out, /Sessions:\s+2/);
assertSavingsUnknown(out);
assert.match(out, /Output tokens:\s+300/);
});
test('--since filters by time window', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
const histPath = path.join(claudeDir, '.caveman-history.jsonl');
const now = Date.now();
const twoDaysAgo = now - 2 * 86_400_000;
const tenMinAgo = now - 10 * 60_000;
fs.writeFileSync(histPath, [
{ ts: twoDaysAgo, session_id: 'old', mode: 'full', output_tokens: 100, est_saved_tokens: 185, est_saved_usd: 0.003 },
{ ts: tenMinAgo, session_id: 'new', mode: 'full', output_tokens: 50, est_saved_tokens: 92, est_saved_usd: 0.001 },
].map(o => JSON.stringify(o)).join('\n') + '\n');
const out = execFileSync(process.execPath, [STATS, '--since', '1d'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
// Only the recent session is counted.
assert.match(out, /Sessions:\s+1/);
assertSavingsUnknown(out);
assert.match(out, /Output tokens:\s+50/);
assert.match(out, /\(last 1d\)/);
});
test('--since rejects malformed durations', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
let err = null;
try {
execFileSync(process.execPath, [STATS, '--since', 'sometime'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
} catch (e) { err = e; }
assert.ok(err, 'should exit non-zero');
assert.match(err.stderr, /--since takes Nh or Nd/);
});
test('--all reports empty when no history', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
const out = execFileSync(process.execPath, [STATS, '--all'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(out, /No sessions logged yet/);
assert.match(out, /recorded when (?:it|they) ends?/);
});
test('reports original and current memory file bytes without provider-savings claims', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
// Make a fake compressed/original pair: original is 800 bytes, compressed 200 bytes.
fs.writeFileSync(path.join(claudeDir, 'CLAUDE.original.md'), 'x'.repeat(800));
fs.writeFileSync(path.join(claudeDir, 'CLAUDE.md'), 'y'.repeat(200));
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
]);
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(out, /Memory file sizes:\s+1 pair, 800 original bytes → 200 current bytes \(600 fewer bytes\)/);
assert.doesNotMatch(out, /tokens saved|per session start/);
});
test('omits memory line when no compressed pairs exist', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
]);
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.doesNotMatch(out, /Memory file sizes/);
});
test('skips pairs where compressed is not actually smaller', (tmp) => {
const { findCompressedPairs } = require(path.join(ROOT, 'src', 'hooks', 'caveman-stats.js'));
fs.writeFileSync(path.join(tmp, 'foo.original.md'), 'small');
fs.writeFileSync(path.join(tmp, 'foo.md'), 'this is actually larger somehow');
const pairs = findCompressedPairs([tmp]);
assert.strictEqual(pairs.length, 0);
});
test('retires the old numeric statusline suffix after a stats run', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 1500 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
const suffixPath = path.join(claudeDir, '.caveman-statusline-suffix');
assert.ok(fs.existsSync(suffixPath));
const suffix = fs.readFileSync(suffixPath, 'utf8');
assert.strictEqual(suffix, '');
});
test('compressed pair comparison counts bytes rather than assuming a tokenizer', (tmp) => {
const { findCompressedPairs, summarizeCompressed } = require(STATS);
fs.writeFileSync(path.join(tmp, 'MEMORY.original.md'), '文'.repeat(100));
fs.writeFileSync(path.join(tmp, 'MEMORY.md'), '文'.repeat(20));
assert.deepStrictEqual(summarizeCompressed(findCompressedPairs([tmp])), {
count: 1, totalOriginal: 300, totalCompressed: 60, bytesReduced: 240,
});
});
// The statusline ships as two scripts with one contract: caveman-statusline.sh
// for POSIX hosts and caveman-statusline.ps1 for Windows. These tests used to
// bail out on win32, which left the .ps1 — the script Windows users actually
// run — with no coverage at all, including its control-byte stripping. Run
// whichever script the host would run instead.
const STATUSLINE = process.platform === 'win32'
? { command: 'powershell', args: ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File',
path.join(ROOT, 'src', 'hooks', 'caveman-statusline.ps1')] }
: { command: 'bash', args: [path.join(ROOT, 'src', 'hooks', 'caveman-statusline.sh')] };
function runStatusline(env) {
return execFileSync(STATUSLINE.command, STATUSLINE.args, { encoding: 'utf8', env });
}
test('statusline ignores legacy savings before stats runs even with old opt-in', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
fs.writeFileSync(path.join(claudeDir, '.caveman-statusline-suffix'), '⛏ 2.8k');
const out = runStatusline({ ...process.env, CLAUDE_CONFIG_DIR: claudeDir, CAVEMAN_STATUSLINE_SAVINGS: '1' });
assert.match(out, /\[CAVEMAN\]/);
assert.doesNotMatch(out, /⛏|2\.8k/);
assert.strictEqual(fs.readFileSync(path.join(claudeDir, '.caveman-statusline-suffix'), 'utf8'), '⛏ 2.8k');
});
test('statusline ignores legacy savings before stats runs by default', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
fs.writeFileSync(path.join(claudeDir, '.caveman-statusline-suffix'), '⛏ 2.8k');
const env = { ...process.env, CLAUDE_CONFIG_DIR: claudeDir };
delete env.CAVEMAN_STATUSLINE_SAVINGS;
const out = runStatusline(env);
assert.match(out, /\[CAVEMAN\]/);
assert.doesNotMatch(out, /⛏|2\.8k/);
assert.strictEqual(fs.readFileSync(path.join(claudeDir, '.caveman-statusline-suffix'), 'utf8'), '⛏ 2.8k');
});
test('statusline omits savings when CAVEMAN_STATUSLINE_SAVINGS=0', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
fs.writeFileSync(path.join(claudeDir, '.caveman-statusline-suffix'), '⛏ 2.8k');
const out = runStatusline({ ...process.env, CLAUDE_CONFIG_DIR: claudeDir, CAVEMAN_STATUSLINE_SAVINGS: '0' });
assert.match(out, /\[CAVEMAN\]/);
assert.doesNotMatch(out, /⛏/);
});
test('statusline omits savings when suffix file is missing (fresh install)', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
// No suffix file written — simulates the moment after install but before
// /caveman-stats has run. Default-on must NOT fabricate a number.
const env = { ...process.env, CLAUDE_CONFIG_DIR: claudeDir };
delete env.CAVEMAN_STATUSLINE_SAVINGS;
const out = runStatusline(env);
assert.match(out, /\[CAVEMAN\]/);
assert.doesNotMatch(out, /⛏/);
});
test('statusline never renders content from a legacy suffix', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
// Plant a malicious suffix with ANSI escape (control byte \x1b).
fs.writeFileSync(path.join(claudeDir, '.caveman-statusline-suffix'), '\x1b[31mEVIL');
const out = runStatusline({ ...process.env, CLAUDE_CONFIG_DIR: claudeDir, CAVEMAN_STATUSLINE_SAVINGS: '1' });
// The retired suffix must not be read at all.
assert.doesNotMatch(out, /\x1b\[31m|EVIL/);
});
test('PowerShell keeps session badges and ignores old savings before stats runs', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
seedSessions(claudeDir, { session: 'ultracave' });
const suffixPath = path.join(claudeDir, '.caveman-statusline-suffix');
fs.writeFileSync(suffixPath, '⛏ 1.9k');
const command = process.platform === 'win32' ? 'powershell' : 'pwsh';
for (const value of [undefined, '1', '0']) {
const env = { ...process.env, CLAUDE_CONFIG_DIR: claudeDir };
if (value === undefined) delete env.CAVEMAN_STATUSLINE_SAVINGS;
else env.CAVEMAN_STATUSLINE_SAVINGS = value;
let out;
try {
out = execFileSync(command, ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File',
path.join(ROOT, 'src', 'hooks', 'caveman-statusline.ps1')], {
encoding: 'utf8', input: '{"session_id":"session"}', env,
});
} catch (e) {
if (e.code === 'ENOENT' && process.platform !== 'win32') {
console.log(' PowerShell unavailable on this host; native statusline tests still run.');
return;
}
throw e;
}
assert.match(out, /\[ULTRACAVE\]/);
assert.doesNotMatch(out, /⛏|1\.9k/);
}
assert.strictEqual(fs.readFileSync(suffixPath, 'utf8'), '⛏ 1.9k');
});
// ── statusline: per-session badge ──────────────────────────────────────────
//
// Claude Code pipes session JSON (including session_id) to the statusline
// command. These drive the script the same way, so the badge is proven to
// reflect the window it belongs to rather than a machine-wide flag.
function statusline(claudeDir, stdin) {
return execFileSync('bash', [path.join(ROOT, 'src', 'hooks', 'caveman-statusline.sh')], {
encoding: 'utf8',
input: stdin === undefined ? '' : stdin,
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, CAVEMAN_STATUSLINE_SAVINGS: '0' },
});
}
function seedSessions(claudeDir, modes) {
const dir = path.join(claudeDir, '.caveman-sessions');
fs.mkdirSync(dir, { recursive: true });
for (const [sid, mode] of Object.entries(modes)) {
fs.writeFileSync(path.join(dir, `${sid}.mode`), mode);
}
}
test('statusline.sh renders the session mode, not the shared flag', (tmp) => {
if (process.platform === 'win32') return;
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
seedSessions(claudeDir, { sessA: 'ultracave', sessB: 'megacave' });
assert.match(statusline(claudeDir, '{"session_id":"sessA"}'), /\[ULTRACAVE\]/);
assert.match(statusline(claudeDir, '{"session_id":"sessB"}'), /\[MEGACAVE\]/);
});
test('statusline.sh badge for every mode id and every legacy level name', (tmp) => {
if (process.platform === 'win32') return;
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
const badges = {
caveman: '[CAVEMAN]', ultracave: '[ULTRACAVE]', megacave: '[MEGACAVE]',
commit: '[CAVEMAN:COMMIT]', review: '[CAVEMAN:REVIEW]', compress: '[CAVEMAN:COMPRESS]',
lite: '[CAVEMAN]', full: '[CAVEMAN]', ultra: '[ULTRACAVE]', wenyan: '[MEGACAVE]',
'wenyan-lite': '[MEGACAVE]', 'wenyan-full': '[MEGACAVE]', 'wenyan-ultra': '[MEGACAVE]',
};
for (const [mode, badge] of Object.entries(badges)) {
seedSessions(claudeDir, { s: mode });
assert.strictEqual(statusline(claudeDir, '{"session_id":"s"}'), '\x1b[38;5;172m' + badge + '\x1b[0m', mode);
}
for (const mode of ['off', 'manual', 'bogus', '']) {
seedSessions(claudeDir, { s: mode });
assert.strictEqual(statusline(claudeDir, '{"session_id":"s"}'), '', `${JSON.stringify(mode)} must render nothing`);
}
});
test('statusline.sh renders nothing for a durable off session', (tmp) => {
if (process.platform === 'win32') return;
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
// Another window is still on 'full' — this one must stay silent anyway.
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
seedSessions(claudeDir, { sessOff: 'off' });
const out = statusline(claudeDir, '{"session_id":"sessOff"}');
assert.strictEqual(out, '', `expected empty badge, got ${JSON.stringify(out)}`);
assert.doesNotMatch(out, /OFF/, 'must never render [CAVEMAN:OFF]');
});
test('statusline.sh falls back to the legacy flag without a usable session id', (tmp) => {
if (process.platform === 'win32') return;
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'wenyan');
seedSessions(claudeDir, { sessA: 'ultracave' });
for (const stdin of [
'{"session_id":"unknown-session"}', // valid id, no state file yet
'{"model":{"id":"x"}}', // payload without session_id
'{"session_id":"../../etc/passwd"}', // traversal attempt
'not json at all',
'', // no payload
]) {
assert.match(statusline(claudeDir, stdin), /\[MEGACAVE\]/, `stdin: ${stdin}`);
}
});
test('statusline.sh never reads a session file outside the sessions dir', (tmp) => {
if (process.platform === 'win32') return;
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
// Plant a file the traversal would reach if the id were interpolated raw.
fs.mkdirSync(path.join(claudeDir, '.caveman-sessions'), { recursive: true });
fs.writeFileSync(path.join(claudeDir, 'escaped.mode'), 'ultra');
const out = statusline(claudeDir, '{"session_id":"../escaped"}');
assert.match(out, /\[CAVEMAN\]/, 'must fall back to the legacy full badge');
assert.doesNotMatch(out, /ULTRA/, 'traversal must not reach the planted file');
});
test('statusline.sh tolerates multiline and whitespace-padded JSON', (tmp) => {
if (process.platform === 'win32') return;
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
seedSessions(claudeDir, { sessA: 'ultracave' });
const pretty = '{\n "model": { "id": "x" },\n "session_id" : "sessA"\n}\n';
assert.match(statusline(claudeDir, pretty), /\[ULTRACAVE\]/);
});
test('appendFlag is symlink-safe (refuses symlinked target)', (tmp) => {
// Creating a symlink on Windows needs Developer Mode or admin (#115), so the
// fixture cannot be built here. The Windows guard is the ReparsePoint check in
// caveman-statusline.ps1 and the O_NOFOLLOW path in caveman-config.js; both are
// exercised by tests/test_symlink_flag.js on POSIX. Accepted platform gap.
if (process.platform === 'win32') return;
const { appendFlag } = require(path.join(ROOT, 'src', 'hooks', 'caveman-config.js'));
const target = path.join(tmp, 'real-target');
fs.writeFileSync(target, 'do-not-clobber\n');
const linkPath = path.join(tmp, 'history.jsonl');
fs.symlinkSync(target, linkPath);
appendFlag(linkPath, JSON.stringify({ ts: 1, session_id: 'x' }));
// Original target must be untouched.
assert.strictEqual(fs.readFileSync(target, 'utf8'), 'do-not-clobber\n');
});
test('mode tracker forwards --share to stats script', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [TRACKER], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, HOME: tmp },
input: JSON.stringify({ prompt: '/caveman-stats --share', transcript_path: sess }),
});
const parsed = JSON.parse(out);
assert.strictEqual(parsed.hookSpecificOutput.hookEventName, 'UserPromptSubmit');
assert.match(parsed.hookSpecificOutput.additionalContext, /🪨 1 turn, 350 output tokens/);
assertSavingsUnknown(parsed.hookSpecificOutput.additionalContext);
});
// ── No counterfactual from transcript usage ────────────────────────────
test('history aggregation excludes all legacy estimated-savings fields', (tmp) => {
const { aggregateHistory } = require(STATS);
const historyPath = path.join(tmp, 'history.jsonl');
const original = [
{ ts: 1, session_id: 'one', output_tokens: 350, est_saved_tokens: 650, est_saved_usd: 0.01, turns: 2 },
{ ts: 2, session_id: 'two', output_tokens: 50, est_saved_tokens: 0, est_saved_usd: 0 },
].map(row => JSON.stringify(row)).join('\n') + '\n';
fs.writeFileSync(historyPath, original);
assert.deepStrictEqual(aggregateHistory(historyPath, null), { sessions: 2, outputTokens: 400, outputAvailability: 'complete' });
assert.strictEqual(fs.readFileSync(historyPath, 'utf8'), original);
});
test('session view never infers output or budget reductions from usage', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
// Observed output does not establish either kind of reduction.
assertSavingsUnknown(out);
assert.ok(!/budget|of your usage|of tracked usage/i.test(out),
'must not relabel output reduction as a usage/budget share');
// API pricing cannot establish a missing baseline.
assertSavingsUnknown(out);
// The missing comparison is explicit.
assert.match(out, /no measured comparison for this session/);
assert.ok(!/weekly limit|5-hour limit/i.test(out), 'must not fabricate Anthropic quota sizes');
});
test('--all ignores historical reduction percentages and dollars', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
const history = [
{ ts: Date.now(), session_id: 'a', output_tokens: 350, est_saved_tokens: 650, est_saved_usd: 0.01 },
{ ts: Date.now(), session_id: 'b', output_tokens: 650, est_saved_tokens: 350, est_saved_usd: 0.005 },
];
fs.writeFileSync(
path.join(claudeDir, '.caveman-history.jsonl'),
history.map(h => JSON.stringify(h)).join('\n') + '\n',
);
const out = execFileSync(process.execPath, [STATS, '--all'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
assert.ok(!/budget|of your usage|of tracked usage/i.test(out),
'must not relabel output reduction as a usage/budget share');
});
test('--all treats legacy zero estimates as unknown savings', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(
path.join(claudeDir, '.caveman-history.jsonl'),
JSON.stringify({ ts: Date.now(), session_id: 'a', output_tokens: 350, est_saved_tokens: 0, est_saved_usd: 0 }) + '\n',
);
const out = execFileSync(process.execPath, [STATS, '--all'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
});
// ── Mid-session mode-change attribution (#601) ─────────────────────────────
// Tokens must be attributed to the mode active WHEN each message happened,
// via the .caveman-mode-log.jsonl transition log — never to whatever mode the
// flag holds at stats time (which inflated savings after a late activation,
// and zeroed them after a late deactivation).
test('attributes tokens to the mode active when each message happened (#601)', (tmp) => {
const now = Date.now();
const iso = (minAgo) => new Date(now - minAgo * 60_000).toISOString();
// 300 verbose tokens BEFORE caveman was activated, 350 after.
const sess = makeSession(tmp, [
{ type: 'assistant', timestamp: iso(60), message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 300 } } },
{ type: 'assistant', timestamp: iso(10), message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-mode-log.jsonl'),
JSON.stringify({ ts: now - 30 * 60_000, mode: 'full', prev: null }) + '\n');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
// The old whole-session-at-current-mode math would claim 1,207 (inflated).
assertSavingsUnknown(out);
assert.doesNotMatch(out, /1,207/);
assert.match(out, /Mode changed mid-session/);
assert.match(out, /caveman off:\s+300 tokens/);
// A pre-three-skill 'full' log row is attributed to caveman.
assert.match(out, /caveman:\s+350 tokens/);
assertSavingsUnknown(out);
// Lifetime snapshots preserve the observed mode attribution.
const hist = fs.readFileSync(path.join(claudeDir, '.caveman-history.jsonl'), 'utf8')
.split('\n').filter(Boolean).map(l => JSON.parse(l));
assert.ok(!Object.hasOwn(hist[hist.length - 1], 'est_saved_tokens'));
assert.deepStrictEqual(hist[hist.length - 1].output_tokens_by_mode, { none: 300, caveman: 350 });
});
test('retains mode attribution after caveman is turned off mid-session (#601)', (tmp) => {
const now = Date.now();
const iso = (minAgo) => new Date(now - minAgo * 60_000).toISOString();
const sess = makeSession(tmp, [
{ type: 'assistant', timestamp: iso(60), message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
{ type: 'assistant', timestamp: iso(10), message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 200 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-mode-log.jsonl'),
JSON.stringify({ ts: now - 90 * 60_000, mode: 'full', prev: null }) + '\n' +
JSON.stringify({ ts: now - 30 * 60_000, mode: null, prev: 'full' }) + '\n');
// No .caveman-active flag — caveman is off at stats time. The old behavior
// printed "Caveman not active this session." and logged zero savings.
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.doesNotMatch(out, /Caveman not active this session/);
assert.match(out, /caveman:\s+350 tokens/);
assertSavingsUnknown(out);
});
test('mixed and unattributed output stays observed usage in reports, shares and history', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', timestamp: new Date(1000).toISOString(), message: { usage: { output_tokens: 1000 } } },
{ type: 'assistant', timestamp: new Date(3000).toISOString(), message: { usage: { output_tokens: 200 } } },
{ type: 'assistant', message: { usage: { output_tokens: 50 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'ultra');
fs.writeFileSync(path.join(claudeDir, '.caveman-mode-log.jsonl'), [
{ ts: 0, mode: 'full', prev: null },
{ ts: 2000, mode: 'ultra', prev: 'full' },
].map(row => JSON.stringify(row)).join('\n') + '\n');
for (const share of [false, true]) {
const args = [STATS, '--session-file', sess, ...(share ? ['--share'] : [])];
const out = execFileSync(process.execPath, args, {
encoding: 'utf8', env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
if (share) {
assert.match(out, /3 turns, 1,250 output tokens/);
} else {
assert.match(out, /Output tokens:\s+1,250/);
assert.match(out, /caveman:\s+1,000 tokens/);
assert.match(out, /ultracave:\s+200 tokens/);
assert.match(out, /unattributed:\s+50 tokens \(mode unknown\)/);
}
}
const rows = fs.readFileSync(path.join(claudeDir, '.caveman-history.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
for (const row of rows) {
assert.strictEqual(row.output_tokens, 1250);
assert.deepStrictEqual(row.output_tokens_by_mode, { caveman: 1000, ultracave: 200 });
assert.strictEqual(row.unattributed_output_tokens, 50);
assert.strictEqual(row.mode_attribution, 'log');
assert.ok(!Object.hasOwn(row, 'est_saved_tokens'));
assert.ok(!Object.hasOwn(row, 'est_saved_usd'));
}
});
test('mode tracker logs timestamped transitions, deduping unchanged modes (#601)', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const run = (prompt) => execFileSync(process.execPath, [TRACKER], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, HOME: tmp },
input: JSON.stringify({ prompt }),
});
const logPath = path.join(claudeDir, '.caveman-mode-log.jsonl');
const rows = () => fs.readFileSync(logPath, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
run('/caveman ultra');
run('/ultracave'); // same mode under its own command — must not append a duplicate row
assert.strictEqual(rows().length, 1);
assert.strictEqual(rows()[0].mode, 'ultracave');
assert.strictEqual(rows()[0].prev, 'caveman', 'legacy "full" flag logs as caveman');
assert.ok(Number.isFinite(rows()[0].ts));
run('/caveman off'); // deactivation is a transition too
assert.strictEqual(rows().length, 2);
assert.strictEqual(rows()[1].mode, null);
assert.strictEqual(rows()[1].prev, 'ultracave');
});
test('excludes tokens that predate a mid-session flag write with no log (#601)', (tmp) => {
const now = Date.now();
const sess = makeSession(tmp, [
{ type: 'assistant', timestamp: new Date(now - 60 * 60_000).toISOString(), message: { model: 'claude-sonnet-4-7', usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
// Flag written NOW (after the message), no transition log: the mode during
// the message is unknown; its token count is still observed.
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
assert.match(out, /unattributed:\s+350 tokens/);
assert.match(out, /mode unknown/);
assert.doesNotMatch(out, /Est\. without caveman/);
});
// ── No net or overhead claims without a measured comparison ────────────
test('longer output does not establish positive net savings', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 1500 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
});
test('short output does not establish negative net savings (#145)', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
assert.match(out, /Output tokens:\s+100/);
assert.doesNotMatch(out, /cost more|consider turning it off/);
});
test('legacy overhead override cannot create a comparison', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 1500 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, CAVEMAN_RULE_OVERHEAD_TOKENS: '500' },
});
assertSavingsUnknown(out);
});
test('legacy overhead settings cannot manufacture a net result', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 1234 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
for (const overhead of ['500', '0', '-100', 'garbage']) {
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8', env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, CAVEMAN_RULE_OVERHEAD_TOKENS: overhead },
});
assertSavingsUnknown(out);
assert.match(out, /Output tokens:\s+1,234/);
}
});
test('unattributed output remains counted with unknown mode and savings', (tmp) => {
const now = Date.now();
const sess = makeSession(tmp, [
{ type: 'assistant', timestamp: new Date(now - 60 * 60_000).toISOString(), message: { usage: { output_tokens: 350 } } },
]);
const claudeDir = path.join(tmp, '.claude');
// Flag written now, no transition log → mode during the message is unknown.
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(out, /unattributed:\s+350 tokens/);
assert.doesNotMatch(out, /Est\. net:/); // no attributed savings basis → no net claim
});
test('ultracave mode also has no measured net result', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'ultra');
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assert.match(out, /Mode: ultracave\b/);
assertSavingsUnknown(out);
assert.doesNotMatch(out, /Est\. net:/);
});
test('lifetime view ignores legacy estimates even with turn counts', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
const histPath = path.join(claudeDir, '.caveman-history.jsonl');
fs.writeFileSync(histPath, [
{ ts: 1000, session_id: 'a', mode: 'full', output_tokens: 1500, est_saved_tokens: 2786, est_saved_usd: 0, turns: 1 },
{ ts: 2000, session_id: 'b', mode: 'full', output_tokens: 100, est_saved_tokens: 186, est_saved_usd: 0, turns: 1 },
].map(o => JSON.stringify(o)).join('\n') + '\n');
const out = execFileSync(process.execPath, [STATS, '--all'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
assert.match(out, /Output tokens:\s+1,600/);
});
test('lifetime view retains actual counts from legacy rows without turns', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-history.jsonl'),
JSON.stringify({ ts: 1000, session_id: 'a', mode: 'full', output_tokens: 350, est_saved_tokens: 650, est_saved_usd: 0 }) + '\n');
const out = execFileSync(process.execPath, [STATS, '--all'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
assert.match(out, /Output tokens:\s+350/);
});
test('number formatting is pinned to en-US even under a dot-grouping locale', (tmp) => {
// Regression for the locale pin: toLocaleString() alone inherits the host OS
// locale (de-DE renders 1234 as "1.234"), which would break machine-readable
// output and any test oracle. fmt() forces 'en-US' grouping everywhere.
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 1234, cache_read_input_tokens: 5678 } } },
]);
const out = execFileSync(process.execPath, [STATS, '--session-file', sess], {
encoding: 'utf8',
env: {
...process.env,
CLAUDE_CONFIG_DIR: path.join(tmp, '.claude'),
LANG: 'de-DE.UTF-8',
LC_ALL: 'de-DE.UTF-8',
},
});
// en-US grouping: commas, never the de-DE dot separator.
assert.match(out, /Output tokens:\s+1,234/);
assert.match(out, /Cache-read tokens:\s+5,678/);
assert.doesNotMatch(out, /Output tokens:\s+1\.234/);
});
test('mixed legacy history preserves output totals without net claims', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
fs.writeFileSync(path.join(claudeDir, '.caveman-history.jsonl'), [
// Legacy row: no turns field, but its recorded output remains usable.
{ ts: 1000, session_id: 'legacy', mode: 'full', output_tokens: 350, est_saved_tokens: 650, est_saved_usd: 0 },
{ ts: 2000, session_id: 'new', mode: 'full', output_tokens: 1500, est_saved_tokens: 2786, est_saved_usd: 0, turns: 1 },
].map(o => JSON.stringify(o)).join('\n') + '\n');
const out = execFileSync(process.execPath, [STATS, '--all'], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir },
});
assertSavingsUnknown(out);
assert.match(out, /Output tokens:\s+1,850/);
});
// #789 — the two caveman-stats docs describe the delivery mechanism the hook
// actually uses. This is drift, not prose: SKILL.md is loaded into the model's
// context when /caveman-stats fires, so "the hook returns decision: block, the
// model does not need to do anything" tells the model to stay silent at the
// exact moment additionalContext is asking it to print the block. The hook has
// emitted no `decision` since #618; `grep decision src/hooks/*.js` finds none.
//
// Ground truth comes from running the real hook rather than from a second
// hardcoded string, so this case cannot pass a doc that agrees with a contract
// the code has moved off.
test('caveman-stats docs describe the delivery mechanism the hook actually uses', (tmp) => {
const sess = makeSession(tmp, [
{ type: 'assistant', message: { usage: { output_tokens: 100 } } },
]);
const claudeDir = path.join(tmp, '.claude');
fs.writeFileSync(path.join(claudeDir, '.caveman-active'), 'full');
const out = execFileSync(process.execPath, [TRACKER], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, HOME: tmp },
input: JSON.stringify({ prompt: '/caveman-stats', transcript_path: sess }),
});
const parsed = JSON.parse(out);
assert.strictEqual(parsed.decision, undefined,
'ground truth: the hook does not block, so no doc may say it does');
assert.ok(parsed.hookSpecificOutput && parsed.hookSpecificOutput.additionalContext,
'ground truth: the hook delivers through additionalContext');
for (const rel of ['skills/caveman-stats/SKILL.md', 'skills/caveman-stats/README.md']) {
const doc = fs.readFileSync(path.join(ROOT, rel), 'utf8');
assert.ok(!/decision:\s*"?block/i.test(doc) && !/blocked-decision/i.test(doc),
`${rel} still describes the retired decision:"block" delivery`);
assert.ok(!/does not need to do anything/i.test(doc),
`${rel} still tells the model to do nothing, while the hook asks it to print the block`);
assert.match(doc, /additionalContext/,
`${rel} must name the additionalContext delivery the hook actually uses`);
}
});
// The same #789 report also flagged the `hooks/…` paths in these docs. They are
// not wrong — the installer copies HOOK_FILES into $CLAUDE_CONFIG_DIR/hooks/,
// so that IS the installed layout — but they leave a repo reader with no path
// that exists here. Both spellings have to be reachable, so pin that the doc
// names the repo source and that the file is really there.
test('caveman-stats docs point a repo reader at a path that exists', () => {
const doc = fs.readFileSync(path.join(ROOT, 'skills/caveman-stats/SKILL.md'), 'utf8');
const referenced = [...doc.matchAll(/`(src\/hooks\/[A-Za-z0-9._-]+)`/g)].map(m => m[1]);
assert.ok(referenced.includes('src/hooks/caveman-stats.js'),
'SKILL.md must name the repo source of the stats script');
for (const rel of referenced) {
assert.ok(fs.existsSync(path.join(ROOT, rel)), `SKILL.md references a missing path: ${rel}`);
}
});
// Third site of the same #789 root cause: when the stats script cannot run, the
// hook told the user to `node hooks/caveman-stats.js`. That relative path is
// only real for a standalone install rooted at $CLAUDE_CONFIG_DIR; a plugin
// user has no `hooks/` directory to run it from, and neither does anyone whose
// cwd is not the config dir. The hook already knows the resolved path.
test('stats fallback message names the script path that actually exists', (tmp) => {
const claudeDir = path.join(tmp, '.claude');
fs.mkdirSync(claudeDir, { recursive: true });
// No transcript_path and an empty config dir: the stats child exits non-zero,
// which is the branch that produces the fallback message.
const out = execFileSync(process.execPath, [TRACKER], {
encoding: 'utf8',
env: { ...process.env, CLAUDE_CONFIG_DIR: claudeDir, HOME: tmp },
input: JSON.stringify({ prompt: '/caveman-stats' }),
stdio: ['pipe', 'pipe', 'pipe'], // the failing child's stderr is expected noise
});
const ctx = JSON.parse(out).hookSpecificOutput.additionalContext;
assert.match(ctx, /could not run stats script/);
const suggested = /Try manually: node (.+)$/m.exec(ctx);
assert.ok(suggested, `fallback must suggest a command: ${ctx}`);
assert.ok(fs.existsSync(suggested[1].trim()),
`fallback suggests a path that does not exist: ${suggested[1].trim()}`);
});
console.log(`\n${passed} passed, ${failed} failed`);
process.exit(failed ? 1 : 0);