test/chat-p2-attribution.test.js
Rama master · réplica de lectura del repositorio original.
// P2-6 (evidence/ReviewChatCore.md §P2-6): the DEVIN_CONNECT usage-attribution
// residue must name the SERVED model on every surface, not just the
// client-visible `model` field.
//
// The echoed name was already unified on `connectDisplayModel`
// (mapped ? requested : the name that actually rides the wire). The residue was
// the dashboard rows:
//
// stream recordTokenUsage(_sr.usage, reqModelName) ← requested
// non-stream recordTokenUsage(body.usage, body.model||req) ← served
// both finalizeConnectAccount({ model: reqModelName })← requested
//
// so on a degrade (WINDSURFAPI_STRICT_MODEL=0) or a router hop
// (DEVIN_CONNECT_ASSIGN_MODEL=1) the same conversation's streamed and
// non-streamed turns landed in DIFFERENT model buckets, and neither matched the
// request row finalizeConnectAccount writes for both.
//
// These run the real handler with the connect network stubbed and read the
// dashboard state the way the UI does (getStats().tokensByModel / modelCounts).
import { afterEach, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { handleChatCompletions, __setConnectDeps, __resetConnectDeps } from '../src/handlers/chat.js';
import { addAccountByKey, removeAccount, getAccountInternal } from '../src/auth.js';
import { resetStats, getStats } from '../src/dashboard/stats.js';
const FREE_SELECTOR = 'swe-1-6-slow';
const UNMAPPED_PAID = 'claude-opus-4.9';
const MAPPED_ALIAS = 'claude-sonnet-4.6';
const ASSIGNED = 'claude-opus-4-8-medium';
const SAVED = {};
const ENV_KEYS = ['DEVIN_CONNECT', 'WINDSURFAPI_STRICT_MODEL', 'DEVIN_CONNECT_TOKEN', 'WINDSURF_API_KEY', 'DEVIN_CONNECT_ASSIGN_MODEL'];
for (const k of ENV_KEYS) SAVED[k] = process.env[k];
const seeded = [];
function seedAccount() {
const a = addAccountByKey('sk-p2-6-' + Math.random().toString(36).slice(2, 12), 'p2-6');
const acct = getAccountInternal(a.id);
acct.status = 'active';
acct.tier = 'pro';
seeded.push(a.id);
return acct;
}
function fakeRes() {
const listeners = new Map();
return {
body: '', writableEnded: false,
write(chunk) { this.body += String(chunk); return true; },
end(chunk) { if (chunk) this.write(chunk); this.writableEnded = true; for (const cb of listeners.get('close') || []) cb(); },
on(event, cb) { if (!listeners.has(event)) listeners.set(event, []); listeners.get(event).push(cb); return this; },
};
}
const USAGE = { prompt_tokens: 12, completion_tokens: 7, total_tokens: 19 };
/** A stream stub that mirrors the real adapter: it stamps the display name on
* every chunk and returns the terminal usage. */
function stubStream() {
const seen = { wire: null, display: null };
__setConnectDeps({
streamChatCompletion: async (params, send, meta) => {
seen.wire = params.model;
seen.display = meta.displayModel;
send({
id: 's', object: 'chat.completion.chunk', created: 0, model: meta.displayModel,
choices: [{ index: 0, delta: { content: 'ok' }, finish_reason: null }],
});
send({
id: 's', object: 'chat.completion.chunk', created: 0, model: meta.displayModel,
choices: [{ index: 0, delta: {}, finish_reason: 'stop' }],
});
return { usage: USAGE };
},
});
return seen;
}
/** Non-stream stub, same display-name precedence as the real adapter. */
function stubNonStream() {
const seen = { wire: null, display: null };
__setConnectDeps({
toChatCompletion: async (params, meta) => {
seen.wire = params.model;
seen.display = meta.displayModel;
return {
status: 200,
body: {
id: 'c', object: 'chat.completion', created: 0,
model: meta.displayModel || params.model,
choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
usage: USAGE,
},
};
},
});
return seen;
}
async function runStream(model) {
const res = await handleChatCompletions(
{ model, stream: true, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: 'p2-6-caller' },
);
assert.equal(res.status, 200, `stream precondition: handler returned ${res.status}`);
const fake = fakeRes();
await res.handler(fake);
return fake;
}
async function runNonStream(model) {
return handleChatCompletions(
{ model, stream: false, messages: [{ role: 'user', content: 'hi' }] },
{ callerKey: 'p2-6-caller' },
);
}
function buckets() {
const stats = getStats();
return {
usage: Object.keys(stats.tokensByModel || {}),
requests: Object.keys(stats.modelCounts || {}),
};
}
beforeEach(() => {
process.env.DEVIN_CONNECT = '1';
process.env.DEVIN_CONNECT_TOKEN = 'test-token-shape-only';
delete process.env.WINDSURF_API_KEY;
delete process.env.DEVIN_CONNECT_ASSIGN_MODEL;
resetStats();
seedAccount();
});
afterEach(() => {
while (seeded.length) removeAccount(seeded.pop());
__resetConnectDeps();
resetStats();
for (const k of ENV_KEYS) {
if (SAVED[k] === undefined) delete process.env[k];
else process.env[k] = SAVED[k];
}
});
describe('P2-6: usage attribution follows the served model', () => {
it('stream: a degraded turn files usage AND the request row under the served selector', async () => {
process.env.WINDSURFAPI_STRICT_MODEL = '0';
const seen = stubStream();
await runStream(UNMAPPED_PAID);
assert.equal(seen.wire, FREE_SELECTOR, 'precondition: the free selector ran upstream');
assert.equal(seen.display, FREE_SELECTOR, 'precondition: the served name is what the client is told');
const b = buckets();
assert.deepEqual(b.usage, [FREE_SELECTOR],
`usage must be filed under the served selector, got [${b.usage}]`);
assert.equal(getStats().tokensByModel[FREE_SELECTOR].requests_with_usage, 1);
assert.equal(getStats().tokensByModel[FREE_SELECTOR].total, 19, 'the usage numbers land intact');
assert.deepEqual(b.requests, [FREE_SELECTOR],
`the request row must name what ran, got [${b.requests}]`);
});
it('non-stream: the same degrade files the same buckets (anchor for the stream arm)', async () => {
process.env.WINDSURFAPI_STRICT_MODEL = '0';
const seen = stubNonStream();
const res = await runNonStream(UNMAPPED_PAID);
assert.equal(res.status, 200);
assert.equal(seen.wire, FREE_SELECTOR);
const b = buckets();
assert.deepEqual(b.usage, [FREE_SELECTOR]);
assert.deepEqual(b.requests, [FREE_SELECTOR]);
});
it('stream and non-stream land in ONE bucket for the same model (the split this fixes)', async () => {
process.env.WINDSURFAPI_STRICT_MODEL = '0';
stubStream();
await runStream(UNMAPPED_PAID);
const streamBuckets = buckets();
resetStats();
stubNonStream();
await runNonStream(UNMAPPED_PAID);
const nonStreamBuckets = buckets();
assert.deepEqual(streamBuckets, nonStreamBuckets,
'the same request must not be split across buckets by the transport it used');
assert.deepEqual(streamBuckets.usage, [FREE_SELECTOR]);
assert.deepEqual(streamBuckets.requests, [FREE_SELECTOR]);
});
it('a mapped alias keeps the requested name on BOTH transports (documented alias behaviour)', async () => {
process.env.WINDSURFAPI_STRICT_MODEL = '0';
const seen = stubStream();
await runStream(MAPPED_ALIAS);
assert.notEqual(seen.wire, MAPPED_ALIAS, 'precondition: the wire selector differs from the alias');
assert.equal(seen.display, MAPPED_ALIAS, 'precondition: the alias is echoed');
assert.deepEqual(buckets().usage, [MAPPED_ALIAS],
'a mapped alias is not a degrade — attribution follows the echoed name');
resetStats();
stubNonStream();
await runNonStream(MAPPED_ALIAS);
assert.deepEqual(buckets().usage, [MAPPED_ALIAS]);
assert.deepEqual(buckets().requests, [MAPPED_ALIAS]);
});
it('stream: a router hop files usage under the ASSIGNED uid, not the router name', async () => {
process.env.WINDSURFAPI_STRICT_MODEL = '0';
process.env.DEVIN_CONNECT_ASSIGN_MODEL = '1';
const seen = stubStream();
__setConnectDeps({ assignModel: async () => ({ model_uid: ASSIGNED }) });
await runStream('adaptive');
assert.equal(seen.wire, ASSIGNED, 'precondition: the router hop put the assigned uid on the wire');
const b = buckets();
assert.deepEqual(b.usage, [ASSIGNED],
`usage must be filed under the model the router assigned, got [${b.usage}]`);
assert.deepEqual(b.requests, [ASSIGNED]);
});
// The Responses route runs this same chat handler underneath, so it must
// inherit the attribution — the report lists it as the third surface, and its
// client-visible name was already deferred to the served name. Driven through
// the real handler (no stubbed chatHandler) to prove the inheritance.
it('Responses non-stream: a degraded turn files usage under the served selector', async () => {
process.env.WINDSURFAPI_STRICT_MODEL = '0';
stubNonStream();
const { handleResponses } = await import('../src/handlers/responses.js');
const res = await handleResponses({
model: UNMAPPED_PAID,
input: [{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] }],
}, { context: { callerKey: 'p2-6-responses' } });
assert.equal(res.status, 200);
assert.equal(res.body.model, FREE_SELECTOR, 'precondition: the served name is what the client is told');
const b = buckets();
assert.deepEqual(b.usage, [FREE_SELECTOR], `usage buckets were [${b.usage}]`);
assert.deepEqual(b.requests, [FREE_SELECTOR]);
});
it('Responses stream: the same degrade files the same bucket', async () => {
process.env.WINDSURFAPI_STRICT_MODEL = '0';
stubStream();
const { handleResponses } = await import('../src/handlers/responses.js');
const res = await handleResponses({
model: UNMAPPED_PAID,
stream: true,
input: [{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] }],
}, { context: { callerKey: 'p2-6-responses' } });
assert.equal(res.status, 200);
await res.handler(fakeRes());
const b = buckets();
assert.deepEqual(b.usage, [FREE_SELECTOR], `usage buckets were [${b.usage}]`);
assert.deepEqual(b.requests, [FREE_SELECTOR]);
});
});