test/devin-connect-finish-callsite.test.js
Rama master · réplica de lectura del repositorio original.
// The finish EVENT, not the pure functions behind it.
//
// Handoff item 5: normalizeConnectUsage and the finish-reason resolution both had
// unit tests, but nothing asserted that the terminal `finish` event actually calls
// them. A bug could be reinstated at the call site — yielding the raw upstream
// usage, or the bare enum map — and the whole suite would stay green, because every
// existing test either drives the pure function directly or ignores the finish
// payload. These tests drive a real streamChat() over a mock transport and assert
// on what the event carries, which is the only thing the four protocol handlers
// ever see.
import { describe, it, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { EventEmitter } from 'node:events';
import { streamChat, __setRequestImpl, normalizeConnectUsage } from '../src/devin-connect.js';
import { writeStringField, writeVarintField, writeMessageField } from '../src/proto.js';
import { wrapEnvelope, endOfStreamEnvelope } from '../src/connect.js';
const TOKEN = 'devin-session-token$test.jwt.sig';
// Metadata sub-message (#7) tags, per the calibrated layout: #2 fresh prompt
// tokens, #3 completion tokens, #4 cache_write, #5 cache_read.
function metaFrame({ prompt, completion, cacheRead, cacheWrite }) {
const parts = [];
if (prompt != null) parts.push(writeVarintField(2, prompt));
if (completion != null) parts.push(writeVarintField(3, completion));
if (cacheWrite != null) parts.push(writeVarintField(4, cacheWrite));
if (cacheRead != null) parts.push(writeVarintField(5, cacheRead));
return writeMessageField(7, Buffer.concat(parts));
}
function mockTransport(framePayloads) {
return (_opts, cb) => {
const req = new EventEmitter();
req.setTimeout = () => req;
req.write = () => {};
req.destroy = () => {};
req.end = () => {
const res = new EventEmitter();
res.statusCode = 200;
setImmediate(() => {
for (const p of framePayloads) res.emit('data', wrapEnvelope(p, { compress: false }));
res.emit('data', endOfStreamEnvelope());
res.emit('end');
});
cb(res);
};
return req;
};
}
async function finishEventFrom(framePayloads, { completion, env } = {}) {
__setRequestImpl(mockTransport(framePayloads));
let finish = null;
for await (const ev of streamChat({
messages: [{ role: 'user', content: 'hi' }],
model: 'claude-sonnet-4.6',
token: TOKEN,
...(completion ? { completion } : {}),
env: { DEVIN_CONNECT_TOKEN: TOKEN, ...(env || {}) },
})) {
if (ev.type === 'finish') finish = ev;
}
return finish;
}
afterEach(() => __setRequestImpl(null));
describe('the finish event calls normalizeConnectUsage (call-site guard)', () => {
it('yields NORMALIZED usage, never the raw upstream counters', async () => {
// The live-reproduced bug this guards: on a cache-hit turn the upstream sends a
// tiny fresh-input figure (3) alongside a large cache_read (1765). Passing that
// through raw emitted cached_tokens > prompt_tokens — a subset field larger than
// its superset — and under-reported total_tokens by ~91%. Every billing relay in
// front of this proxy meters from exactly these numbers.
const finish = await finishEventFrom([
Buffer.concat([
writeStringField(3, 'ok'),
writeVarintField(5, 2),
metaFrame({ prompt: 3, completion: 155, cacheRead: 1765, cacheWrite: 40 }),
]),
]);
assert.ok(finish, 'the stream must reach its terminal finish event');
// Compare against the production normalizer applied to the decoded upstream
// shape — not a re-derivation of the arithmetic here, which would pass even if
// production degraded (a mirror test is not a test).
assert.deepEqual(finish.usage, normalizeConnectUsage({
prompt: 3, completion: 155, cache_read_tokens: 1765, cache_write_tokens: 40,
}));
// And the invariant that made it a billing bug, asserted directly.
const cached = finish.usage.prompt_tokens_details.cached_tokens;
assert.ok(cached <= finish.usage.prompt_tokens,
`cached_tokens (${cached}) must be a SUBSET of prompt_tokens (${finish.usage.prompt_tokens})`);
assert.equal(finish.usage.prompt_tokens, 3 + 1765 + 40,
'prompt_tokens must include cache_read AND cache_write (O2: the full input count)');
});
it('yields null usage when the upstream sent no metadata', async () => {
const finish = await finishEventFrom([
Buffer.concat([writeStringField(3, 'ok'), writeVarintField(5, 2)]),
]);
assert.equal(finish.usage, null, 'a missing usage block passes through as null');
});
});
describe('the finish event resolves truncation from usage (call-site guard)', () => {
it('reports "length" when completion_tokens hits the cap the caller requested', async () => {
// The enum's clean-stop values (2/4) say nothing about truncation; the
// cap-equality check is the second line of defense, and it has to be wired
// into the event, not just unit-tested.
const finish = await finishEventFrom([
Buffer.concat([
writeStringField(3, 'truncated here'),
writeVarintField(5, 2),
metaFrame({ prompt: 100, completion: 64 }),
]),
], { completion: { maxTokens: 64 } });
assert.equal(finish.reason, 'length');
});
it('reports "stop" for a complete answer under the cap', async () => {
const finish = await finishEventFrom([
Buffer.concat([
writeStringField(3, 'HI'),
writeVarintField(5, 4),
metaFrame({ prompt: 100, completion: 2 }),
]),
], { completion: { maxTokens: 300 } });
assert.equal(finish.reason, 'stop',
'the paid-tier shape that used to report every complete answer as truncated');
});
it('reports "length" for the descriptor-pinned MAX_TOKENS enum (3) on its own', async () => {
// 3 = MAX_TOKENS in the descriptor set, so the enum alone is now a
// truncation signal; the call site must surface the mapped reason rather
// than the raw enum or a default 'stop'.
const finish = await finishEventFrom([
Buffer.concat([
writeStringField(3, 'complete answer'),
writeVarintField(5, 3),
metaFrame({ prompt: 100, completion: 20 }),
]),
], { completion: { maxTokens: 4096 } });
assert.equal(finish.reason, 'length');
});
it('honours a calibrated DEVIN_CONNECT_STOP_REASON_MAP at the call site', async () => {
// An operator who captured the real integers must be able to restore 'length'
// for 3 — and that override has to survive the trip through the event.
const finish = await finishEventFrom([
Buffer.concat([
writeStringField(3, 'cut'),
writeVarintField(5, 3),
metaFrame({ prompt: 100, completion: 20 }),
]),
], { env: { DEVIN_CONNECT_STOP_REASON_MAP: '3=length' } });
assert.equal(finish.reason, 'length');
});
});