scripts/cline-compat-verify.mjs
Rama master · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* Cline OpenAI-Compatible contract verifier.
*
* Cline's "OpenAI Compatible" provider is Vercel `@ai-sdk/openai-compatible`
* (includeUsage:true). It talks standard `/v1/chat/completions` with streaming,
* native `tools`/`tool_calls` deltas, `stream_options.include_usage`, and an
* optional `/v1/models` listing. This script asserts a gateway satisfies every
* request shape Cline sends so we can hand a partner a green self-check.
*
* Two modes:
* - OFFLINE (default): spins up an in-process mock that returns the exact
* OpenAI-compatible shapes, then runs the assertion suite against it. This
* proves the ASSERTIONS themselves are correct (a green offline run means the
* contract checks are sound), with zero upstream billing.
* - REAL (`--real` or CLINE_VERIFY_REAL=1): runs the identical suite against a
* live gateway (BASE_URL + API_KEY). A green real run means the gateway
* honors the Cline contract end to end.
*
* Zero-dep: built-in `node:http` + global `fetch` only.
*
* Env / flags:
* --real | CLINE_VERIFY_REAL=1 hit a live gateway instead of the mock
* BASE_URL gateway base, no trailing slash (default http://127.0.0.1:3003)
* API_KEY downstream auth (default 'test'; required in real mode)
* MODEL model id to exercise (default 'swe-1-6-slow', a free selector)
* VERIFY_TIMEOUT_MS per-request timeout (default 120000)
*
* Exit code 0 = all checks passed, 1 = one or more failed, 2 = bad usage.
*/
import { createServer } from 'node:http';
const REAL = process.argv.includes('--real') || process.env.CLINE_VERIFY_REAL === '1';
const API_KEY = process.env.API_KEY || 'test';
const MODEL = process.env.MODEL || 'swe-1-6-slow';
const TIMEOUT_MS = Number(process.env.VERIFY_TIMEOUT_MS || 120000);
// ── result plumbing ────────────────────────────────────────────────
const results = [];
function record(name, ok, detail) {
results.push({ name, ok, detail });
const tag = ok ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';
console.log(` ${tag} ${name}${detail ? ` — ${detail}` : ''}`);
}
async function check(name, fn) {
try {
const detail = await fn();
record(name, true, detail || '');
} catch (err) {
record(name, false, String(err?.message || err));
}
}
function assert(cond, msg) {
if (!cond) throw new Error(msg);
}
// ── HTTP client ────────────────────────────────────────────────────
async function post(base, path, body, { stream = false } = {}) {
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
try {
const res = await fetch(base + path, {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${API_KEY}`,
},
body: JSON.stringify(body),
signal: ctrl.signal,
});
if (stream) return res; // caller reads the SSE body
const text = await res.text();
let json = null;
try { json = JSON.parse(text); } catch {}
return { status: res.status, json, text };
} finally {
clearTimeout(timer);
}
}
async function get(base, path) {
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
try {
const res = await fetch(base + path, {
headers: { authorization: `Bearer ${API_KEY}` },
signal: ctrl.signal,
});
const text = await res.text();
let json = null;
try { json = JSON.parse(text); } catch {}
return { status: res.status, json, text };
} finally {
clearTimeout(timer);
}
}
// Parse an SSE `/v1/chat/completions` stream into ordered `data:` payloads.
// Returns { events: [obj...], sawDone: bool }. `[DONE]` is recorded, not parsed.
async function readSSE(res) {
assert(res.ok, `stream HTTP ${res.status}`);
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buf = '';
const events = [];
let sawDone = false;
for (;;) {
const { value, done } = await reader.read();
if (done) break;
buf += decoder.decode(value, { stream: true });
let idx;
while ((idx = buf.indexOf('\n')) >= 0) {
const line = buf.slice(0, idx).trimEnd();
buf = buf.slice(idx + 1);
if (!line.startsWith('data:')) continue;
const payload = line.slice(5).trim();
if (payload === '[DONE]') { sawDone = true; continue; }
try { events.push(JSON.parse(payload)); } catch {}
}
}
return { events, sawDone };
}
// ── the Cline contract suite ───────────────────────────────────────
const WEATHER_TOOL = {
type: 'function',
function: {
name: 'get_weather',
description: 'Get the current weather for a city.',
parameters: {
type: 'object',
properties: { city: { type: 'string' } },
required: ['city'],
additionalProperties: false,
},
},
};
async function runSuite(base) {
// 1. Non-stream chat: OpenAI envelope with choices[0].message.content.
await check('non-stream /v1/chat/completions returns OpenAI envelope', async () => {
const { status, json } = await post(base, '/v1/chat/completions', {
model: MODEL,
messages: [{ role: 'user', content: 'Say hello in one word.' }],
stream: false,
});
assert(status === 200, `HTTP ${status}`);
assert(json?.object === 'chat.completion', `object=${json?.object}`);
assert(Array.isArray(json.choices) && json.choices.length > 0, 'no choices');
assert(typeof json.choices[0].message?.content === 'string', 'no message.content');
assert(json.choices[0].finish_reason != null, 'no finish_reason');
return `finish_reason=${json.choices[0].finish_reason}`;
});
// 2. Streaming chat: role-first delta chain + [DONE].
await check('streaming /v1/chat/completions emits deltas + [DONE]', async () => {
const res = await post(base, '/v1/chat/completions', {
model: MODEL,
messages: [{ role: 'user', content: 'Count: one two three.' }],
stream: true,
}, { stream: true });
const { events, sawDone } = await readSSE(res);
assert(events.length > 0, 'no SSE events');
assert(events.every(e => e.object === 'chat.completion.chunk'), 'non-chunk object in stream');
const text = events.map(e => e.choices?.[0]?.delta?.content || '').join('');
assert(text.length > 0, 'empty streamed content');
assert(sawDone, 'missing [DONE] terminator');
return `${events.length} chunks, ${text.length}c`;
});
// 3. stream_options.include_usage → a usage-bearing tail frame.
await check('stream_options.include_usage yields a usage tail frame', async () => {
const res = await post(base, '/v1/chat/completions', {
model: MODEL,
messages: [{ role: 'user', content: 'Hi.' }],
stream: true,
stream_options: { include_usage: true },
}, { stream: true });
const { events } = await readSSE(res);
const usageFrame = events.find(e => e.usage && typeof e.usage.total_tokens === 'number');
assert(usageFrame, 'no frame carried usage.total_tokens');
assert(typeof usageFrame.usage.prompt_tokens === 'number', 'usage.prompt_tokens missing');
assert(typeof usageFrame.usage.completion_tokens === 'number', 'usage.completion_tokens missing');
return `usage total=${usageFrame.usage.total_tokens}`;
});
// 4. Native tools → streamed tool_calls fragments that reassemble.
await check('tools → streamed tool_calls (name + id + reassembled arguments)', async () => {
const res = await post(base, '/v1/chat/completions', {
model: MODEL,
messages: [{ role: 'user', content: 'What is the weather in Paris? Use the tool.' }],
tools: [WEATHER_TOOL],
tool_choice: 'auto',
stream: true,
}, { stream: true });
const { events } = await readSSE(res);
// Collect tool_call deltas by index and reassemble arguments.
const byIndex = new Map();
let finishReason = null;
for (const e of events) {
const choice = e.choices?.[0];
const tcs = choice?.delta?.tool_calls;
if (choice?.finish_reason) finishReason = choice.finish_reason;
if (!Array.isArray(tcs)) continue;
for (const tc of tcs) {
const i = tc.index ?? 0;
const acc = byIndex.get(i) || { id: '', name: '', args: '' };
if (tc.id) acc.id = tc.id;
if (tc.function?.name) acc.name = tc.function.name;
if (tc.function?.arguments) acc.args += tc.function.arguments;
byIndex.set(i, acc);
}
}
assert(byIndex.size > 0, 'no tool_calls in stream');
const call = byIndex.get(0);
assert(call.id, 'tool_call missing id');
assert(call.name === 'get_weather', `tool name=${call.name}`);
const parsed = JSON.parse(call.args); // must be valid JSON once reassembled
assert(typeof parsed === 'object' && parsed !== null, 'arguments not a JSON object');
assert(finishReason === 'tool_calls', `finish_reason=${finishReason}`);
return `${call.name}(${call.args})`;
});
// 4b. Parameterless tool → arguments must still be PARSEABLE JSON (the
// vercel/ai#6687 case: Claude sends "" for a no-arg tool, which
// @ai-sdk/openai-compatible silently drops. The Cline compat layer normalizes
// it to "{}"). We can't force the model to pick this tool, so we only assert
// the invariant WHEN a tool_call is produced: whatever arguments arrive must
// JSON.parse. A gateway with the compat layer active can never emit "".
await check('parameterless tool → reassembled arguments are parseable (never "")', async () => {
const res = await post(base, '/v1/chat/completions', {
model: MODEL,
messages: [{ role: 'user', content: 'List the files in the current directory using the tool.' }],
tools: [{ type: 'function', function: {
name: 'list_files',
description: 'List files in the current directory. Takes no arguments.',
parameters: { type: 'object', properties: {}, additionalProperties: false },
} }],
tool_choice: 'auto',
stream: true,
}, { stream: true });
const { events } = await readSSE(res);
let args = '';
let sawToolCall = false;
for (const e of events) {
for (const tc of (e.choices?.[0]?.delta?.tool_calls || [])) {
sawToolCall = true;
if (tc.function?.arguments) args += tc.function.arguments;
}
}
if (!sawToolCall) return 'model chose not to call the tool (invariant not exercised)';
// The critical assertion: arguments are NEVER the empty string, and always parse.
assert(args !== '', 'parameterless tool emitted empty-string arguments (@ai-sdk would drop it)');
JSON.parse(args);
return `arguments=${args}`;
});
// 5. reasoning_effort passes through without a 4xx/5xx.
await check('reasoning_effort passthrough does not error', async () => {
const { status, json } = await post(base, '/v1/chat/completions', {
model: MODEL,
messages: [{ role: 'user', content: 'Briefly: 2+2?' }],
reasoning_effort: 'low',
stream: false,
});
assert(status === 200, `HTTP ${status} (${json?.error?.message || 'no body'})`);
assert(json?.choices?.[0]?.message, 'no message on reasoning_effort response');
return 'accepted reasoning_effort=low';
});
// 6. /v1/models returns an OpenAI-shaped catalog.
await check('/v1/models returns a catalog', async () => {
const { status, json } = await get(base, '/v1/models');
assert(status === 200, `HTTP ${status}`);
assert(json?.object === 'list', `object=${json?.object}`);
assert(Array.isArray(json.data) && json.data.length > 0, 'empty model list');
assert(json.data.every(m => typeof m.id === 'string'), 'a model entry lacks id');
return `${json.data.length} models`;
});
// 7. Multi-turn history with a role:tool turn must not 500 (Cline's tool loop).
await check('multi-turn history incl. role:tool does not 500', async () => {
const { status, json } = await post(base, '/v1/chat/completions', {
model: MODEL,
messages: [
{ role: 'user', content: 'What is the weather in Paris?' },
{
role: 'assistant',
content: null,
tool_calls: [{
id: 'call_1', type: 'function',
function: { name: 'get_weather', arguments: '{"city":"Paris"}' },
}],
},
{ role: 'tool', tool_call_id: 'call_1', content: '{"tempC":21,"sky":"clear"}' },
],
stream: false,
});
assert(status < 500, `HTTP ${status} (${json?.error?.message || 'no body'})`);
assert(status === 200, `HTTP ${status} (expected 200 for a valid tool-result turn)`);
return `follow-up turn accepted (HTTP ${status})`;
});
}
// ── offline mock: a minimal OpenAI-compatible gateway ───────────────
// Returns exactly the shapes the suite asserts, so a green offline run means
// the assertions are self-consistent (not that any real gateway passes).
function sseChunk(obj) { return `data: ${JSON.stringify(obj)}\n\n`; }
function startMock() {
const server = createServer((req, res) => {
const chunks = [];
req.on('data', d => chunks.push(d));
req.on('end', () => {
const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString()) : {};
const url = req.url;
if (req.method === 'GET' && url === '/v1/models') {
res.writeHead(200, { 'content-type': 'application/json' });
res.end(JSON.stringify({
object: 'list',
data: [
{ id: MODEL, object: 'model', owned_by: 'windsurfapi' },
{ id: 'claude-opus-4-8', object: 'model', owned_by: 'windsurfapi' },
],
}));
return;
}
if (req.method === 'POST' && url === '/v1/chat/completions') {
const wantsTools = Array.isArray(body.tools) && body.tools.length > 0;
const wantsUsage = body.stream_options?.include_usage === true;
if (!body.stream) {
// Non-stream envelope.
res.writeHead(200, { 'content-type': 'application/json' });
res.end(JSON.stringify({
id: 'chatcmpl-mock', object: 'chat.completion', created: 1,
model: body.model,
choices: [{
index: 0,
message: { role: 'assistant', content: 'hello' },
finish_reason: 'stop',
}],
usage: { prompt_tokens: 5, completion_tokens: 1, total_tokens: 6 },
}));
return;
}
// Streaming.
res.writeHead(200, {
'content-type': 'text/event-stream',
'cache-control': 'no-cache',
connection: 'keep-alive',
});
const toolName = body.tools?.[0]?.function?.name;
if (wantsTools && toolName === 'list_files') {
// Parameterless tool. A compat-active gateway normalizes the empty
// arguments Claude would send to "{}" — never emits "". The mock
// reflects that post-compat shape (a single "{}" arguments fragment).
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [{ index: 0, delta: { role: 'assistant', tool_calls: [{ index: 0, id: 'call_lf', type: 'function', function: { name: 'list_files', arguments: '{}' } }] }, finish_reason: null }] }));
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] }));
} else if (wantsTools) {
// Split tool_call across frames to exercise reassembly.
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [{ index: 0, delta: { role: 'assistant', tool_calls: [{ index: 0, id: 'call_mock', type: 'function', function: { name: 'get_weather', arguments: '' } }] }, finish_reason: null }] }));
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, function: { arguments: '{"city":' } }] }, finish_reason: null }] }));
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [{ index: 0, delta: { tool_calls: [{ index: 0, function: { arguments: '"Paris"}' } }] }, finish_reason: null }] }));
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] }));
} else {
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [{ index: 0, delta: { role: 'assistant', content: 'one ' }, finish_reason: null }] }));
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [{ index: 0, delta: { content: 'two three' }, finish_reason: null }] }));
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }));
}
if (wantsUsage) {
res.write(sseChunk({ object: 'chat.completion.chunk', choices: [], usage: { prompt_tokens: 3, completion_tokens: 4, total_tokens: 7 } }));
}
res.write('data: [DONE]\n\n');
res.end();
return;
}
res.writeHead(404, { 'content-type': 'application/json' });
res.end(JSON.stringify({ error: { message: 'not found' } }));
});
});
return new Promise(resolve => {
server.listen(0, '127.0.0.1', () => {
const { port } = server.address();
resolve({ server, base: `http://127.0.0.1:${port}` });
});
});
}
// ── main ───────────────────────────────────────────────────────────
async function main() {
let base;
let mock = null;
if (REAL) {
base = (process.env.BASE_URL || 'http://127.0.0.1:3003').replace(/\/+$/, '');
console.log(`Cline compat verify — REAL mode against ${base} (model=${MODEL})\n`);
} else {
mock = await startMock();
base = mock.base;
console.log(`Cline compat verify — OFFLINE mock mode (proves the contract assertions)\n`);
}
try {
await runSuite(base);
} finally {
if (mock) mock.server.close();
}
const failed = results.filter(r => !r.ok);
console.log(`\n${results.length - failed.length}/${results.length} checks passed`);
if (failed.length) {
console.log(`\x1b[31m${failed.length} FAILED:\x1b[0m ${failed.map(f => f.name).join('; ')}`);
process.exit(1);
}
console.log('\x1b[32mAll Cline contract checks passed.\x1b[0m');
}
main().catch(err => {
console.error('verifier crashed:', err);
process.exit(1);
});