test/connect-image-capability-gate.test.js
Rama master · réplica de lectura del repositorio original.
// C3-01 (evidence/HistoryGapAudit/gaps.md): /v1/models advertises
// `supports_images` from the live DEVIN_CONNECT catalog (GetCliModelConfigs
// field #5), but no request path ever read the bit — images were forwarded to a
// model the same server had just told the client was text-only.
//
// Measurement (2026-10-25, production account, evidence/Wave4-Images.md):
// 240 catalog rows — 224 supports_images=true, 0 false, 16 silent. No live model
// exercises the false arm, so there is no capture of a false model rejecting an
// image. The gate therefore STRIPS image blocks and counts them
// (`images_dropped_capability`) instead of refusing the request with a 400: a
// hard failure derived from a bit whose false arm has never been observed is
// the same class of bug as the Codex `parallel_tool_calls` 400 that broke a
// working client (chat.js:3623-3635). Only an EXPLICIT false gates; a missing
// bit stays unknown and forwards unchanged, exactly like `disabled` (#4).
//
// Red-first: these tests were written before the gate existed and failed on the
// unmodified handler (images still in params.messages, counter undefined).
import { after, before, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { handleChatCompletions, __setConnectDeps, __resetConnectDeps } from '../src/handlers/chat.js';
import { handleResponses } from '../src/handlers/responses.js';
import { handleMessages } from '../src/handlers/messages.js';
import { setLiveCatalogSelectors, clearLiveCatalogSelectors } from '../src/devin-connect-models.js';
import { getConnectMetrics } from '../src/devin-connect-metrics.js';
import { log } from '../src/config.js';
const TEXT_ONLY = 'w4-text-only-model';
const VISION = 'w4-vision-model';
const SILENT = 'w4-silent-model';
// 1x1 transparent PNG.
const DATA_URL = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
const B64 = DATA_URL.split(',')[1];
const ENV_KEYS = ['DEVIN_CONNECT', 'DEVIN_CONNECT_TOKEN', 'WINDSURF_API_KEY', 'WINDSURFAPI_STRICT_MODEL'];
const SAVED_ENV = {};
for (const key of ENV_KEYS) SAVED_ENV[key] = process.env[key];
const upstream = { hits: 0, lastParams: null };
function fakeRes() {
const listeners = new Map();
return {
body: '',
writableEnded: false,
write(chunk) { this.body += String(chunk); return true; },
end(chunk) {
if (chunk) this.write(chunk);
this.writableEnded = true;
for (const cb of listeners.get('close') || []) cb();
},
on(event, cb) {
if (!listeners.has(event)) listeners.set(event, []);
listeners.get(event).push(cb);
return this;
},
removeListener() {},
};
}
function chatBody(model) {
return {
id: 'chatcmpl-w4', object: 'chat.completion', created: 0, model,
choices: [{ index: 0, message: { role: 'assistant', content: 'OK' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
}
function ask(body, context = { callerKey: '' }) {
return handleChatCompletions(body, context);
}
async function captureLogs(fn) {
const lines = [];
const origInfo = log.info;
const origWarn = log.warn;
log.info = (...args) => { lines.push(args.join(' ')); };
log.warn = (...args) => { lines.push(args.join(' ')); };
try {
const value = await fn();
return { value, lines };
} finally {
log.info = origInfo;
log.warn = origWarn;
}
}
function imageDrops() { return getConnectMetrics().images_dropped_capability; }
/** Every image block that would ride the connect wire. */
function wireImages(messages) {
const out = [];
for (const msg of messages || []) {
if (!Array.isArray(msg?.content)) continue;
for (const part of msg.content) {
if (part?.type === 'image_url' || part?.type === 'image' || part?.type === 'input_image') out.push(part);
}
}
return out;
}
before(() => {
process.env.DEVIN_CONNECT = '1';
process.env.DEVIN_CONNECT_TOKEN = 'env-token-shape-only';
delete process.env.WINDSURF_API_KEY;
delete process.env.WINDSURFAPI_STRICT_MODEL;
// Only these three selectors exist in the authoritative catalog for the run:
// false, true, and silent (bit absent) — the three-state matrix.
setLiveCatalogSelectors([
{ selector: TEXT_ONLY, provider: 'test', supportsImages: false },
{ selector: VISION, provider: 'test', supportsImages: true },
{ selector: SILENT, provider: 'test' },
]);
__setConnectDeps({
toChatCompletion: async (params) => {
upstream.hits++;
upstream.lastParams = params;
return { status: 200, body: chatBody(params.model) };
},
streamChatCompletion: async (params, send) => {
upstream.hits++;
upstream.lastParams = params;
send({ id: 'x', object: 'chat.completion.chunk', created: 0, model: params.model, choices: [{ index: 0, delta: { role: 'assistant', content: 'OK' } }] });
send({ id: 'x', object: 'chat.completion.chunk', created: 0, model: params.model, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] });
return { usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } };
},
});
});
beforeEach(() => {
upstream.hits = 0;
upstream.lastParams = null;
});
after(() => {
__resetConnectDeps();
clearLiveCatalogSelectors();
for (const key of ENV_KEYS) {
if (SAVED_ENV[key] === undefined) delete process.env[key];
else process.env[key] = SAVED_ENV[key];
}
});
describe('connect image capability gate — supports_images:false', () => {
it('strips the image, preserves the text turn, counts the drop and warns', async () => {
const before = imageDrops();
const messages = [{
role: 'user',
content: [
{ type: 'text', text: 'what is in this image?' },
{ type: 'image_url', image_url: { url: DATA_URL } },
],
}];
const { value, lines } = await captureLogs(() => ask({ model: TEXT_ONLY, stream: false, messages }));
assert.equal(value.status, 200, JSON.stringify(value.body));
assert.equal(upstream.hits, 1);
assert.equal(wireImages(upstream.lastParams.messages).length, 0, 'no image may ride the wire to a false model');
assert.deepEqual(upstream.lastParams.messages, [{
role: 'user',
content: [{ type: 'text', text: 'what is in this image?' }],
}], 'the text part must survive verbatim');
assert.equal(imageDrops() - before, 1, 'one dropped image is counted');
const warn = lines.find((l) => l.includes('images_dropped_capability'));
assert.ok(warn, `a warn naming the counter must be emitted; got:\n${lines.join('\n')}`);
assert.ok(warn.includes(TEXT_ONLY), 'the warn names the resolved selector');
assert.ok(warn.includes('supports_images=false'), 'the warn names the capability');
});
it('strips images from replayed history and tool results too, counting every block', async () => {
const before = imageDrops();
const messages = [
{ role: 'user', content: [
{ type: 'text', text: 'first' },
{ type: 'image_url', image_url: { url: DATA_URL } },
] },
{ role: 'assistant', content: 'looked at it' },
{ role: 'user', content: [
{ type: 'text', text: 'and this one?' },
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: B64 } },
{ type: 'image_url', image_url: { url: DATA_URL } },
] },
{ role: 'tool', tool_call_id: 'call_1', content: [
{ type: 'text', text: 'tool output' },
{ type: 'image_url', image_url: { url: DATA_URL } },
] },
];
const { value } = await captureLogs(() => ask({ model: TEXT_ONLY, stream: false, messages }));
assert.equal(value.status, 200, JSON.stringify(value.body));
assert.equal(wireImages(upstream.lastParams.messages).length, 0);
assert.equal(imageDrops() - before, 4, 'every block counts: user history, two in the current turn, and the tool result');
assert.equal(upstream.lastParams.messages[0].content[0].text, 'first');
assert.equal(upstream.lastParams.messages[3].content[0].text, 'tool output');
});
it('strips on the streaming path before the wire adapter runs', async () => {
const before = imageDrops();
const result = await ask({
model: TEXT_ONLY,
stream: true,
messages: [{ role: 'user', content: [
{ type: 'text', text: 'hi' },
{ type: 'image_url', image_url: { url: DATA_URL } },
] }],
});
assert.equal(result.status, 200);
const res = fakeRes();
await result.handler(res);
assert.equal(upstream.hits, 1);
assert.equal(wireImages(upstream.lastParams.messages).length, 0);
assert.equal(imageDrops() - before, 1);
});
it('leaves an image-free request byte-identical even for a false model', async () => {
const before = imageDrops();
const messages = [{ role: 'user', content: 'plain text only' }];
const { value, lines } = await captureLogs(() => ask({ model: TEXT_ONLY, stream: false, messages }));
assert.equal(value.status, 200);
assert.deepEqual(upstream.lastParams.messages, messages);
assert.equal(imageDrops() - before, 0);
assert.equal(lines.some((l) => l.includes('images_dropped_capability')), false, 'no warn when nothing was dropped');
});
// P2-3 (Kimi wave review): the answerability guard runs BEFORE stripping and
// counts a non-text part as content. An image-only last turn therefore passed
// it, was stripped to `content: []`, and the handler returned 200 while the
// builder encoded an empty user frame — which consecutive user turns can merge
// away, leaving the model to answer a stale earlier prompt. Nothing was left
// to answer, so the honest answer is a clear 400.
it('refuses an image-only last turn with a clear 400 instead of an empty user frame', async () => {
const before = imageDrops();
const messages = [{
role: 'user',
content: [{ type: 'image_url', image_url: { url: DATA_URL } }],
}];
const { value, lines } = await captureLogs(() => ask({ model: TEXT_ONLY, stream: false, messages }));
assert.equal(value.status, 400, JSON.stringify(value.body));
assert.equal(upstream.hits, 0, 'a turn with no answerable content must not reach the upstream');
assert.equal(value.body.error.type, 'invalid_request_error');
assert.equal(value.body.error.param, 'messages');
assert.match(value.body.error.message, /supports_images/,
'the error must name the capability that caused the strip');
assert.match(value.body.error.message, /image/i);
assert.equal(imageDrops() - before, 1, 'the block was still dropped and is counted');
assert.ok(lines.some((l) => /image-only|no answerable/i.test(l)),
`a warn must name the refusal; got:\n${lines.join('\n')}`);
});
// Control for the refusal: a conversation that ends on a TOOL RESULT is still
// answerable even if an earlier user turn was image-only and got emptied by
// the strip. The 400 is about the terminal turn, not about any empty turn.
it('keeps a tool-result continuation answerable when an earlier user image is stripped', async () => {
const messages = [
{ role: 'user', content: [{ type: 'image_url', image_url: { url: DATA_URL } }] },
{ role: 'assistant', content: null, tool_calls: [
{ id: 'call_1', type: 'function', function: { name: 'f', arguments: '{}' } },
] },
{ role: 'tool', tool_call_id: 'call_1', content: 'tool output' },
];
const { value } = await captureLogs(() => ask({ model: TEXT_ONLY, stream: false, messages }));
assert.equal(value.status, 200, JSON.stringify(value.body));
assert.equal(upstream.hits, 1, 'a tool-result-terminated request is answerable');
assert.equal(wireImages(upstream.lastParams.messages).length, 0);
const toolTurn = upstream.lastParams.messages.find((m) => m.role === 'tool');
assert.equal(toolTurn?.content, 'tool output', 'the tool result must survive');
});
});
describe('connect image capability gate — true and unknown are untouched', () => {
const messages = [{
role: 'user',
content: [
{ type: 'text', text: 'what is in this image?' },
{ type: 'image_url', image_url: { url: DATA_URL } },
],
}];
it('forwards images unchanged to a model that declares supports_images=true', async () => {
const before = imageDrops();
const result = await ask({ model: VISION, stream: false, messages });
assert.equal(result.status, 200, JSON.stringify(result.body));
assert.deepEqual(upstream.lastParams.messages, messages);
assert.equal(imageDrops() - before, 0);
});
it('forwards images unchanged when the catalog is silent (unknown is not a no)', async () => {
const before = imageDrops();
const result = await ask({ model: SILENT, stream: false, messages });
assert.equal(result.status, 200, JSON.stringify(result.body));
assert.deepEqual(upstream.lastParams.messages, messages);
assert.equal(imageDrops() - before, 0);
});
});
describe('connect image capability gate is shared by every surface', () => {
it('/v1/responses input_image is stripped for a false model', async () => {
const before = imageDrops();
const result = await handleResponses({
model: TEXT_ONLY,
stream: false,
input: [{
role: 'user',
content: [
{ type: 'input_text', text: 'what is this?' },
{ type: 'input_image', image_url: DATA_URL },
],
}],
});
assert.equal(result.status, 200, JSON.stringify(result.body));
assert.equal(upstream.hits, 1);
assert.equal(wireImages(upstream.lastParams.messages).length, 0);
assert.equal(imageDrops() - before, 1);
});
it('/v1/messages Anthropic image block is stripped for a false model', async () => {
const before = imageDrops();
const result = await handleMessages({
model: TEXT_ONLY,
max_tokens: 16,
messages: [{
role: 'user',
content: [
{ type: 'text', text: 'what is this?' },
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: B64 } },
],
}],
});
assert.equal(result.status, 200, JSON.stringify(result.body));
assert.equal(upstream.hits, 1);
assert.equal(wireImages(upstream.lastParams.messages).length, 0);
assert.equal(imageDrops() - before, 1);
});
});