scripts/devin-connect-smoke.mjs
Rama master · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* DEVIN_CONNECT smoke — end-to-end check of the direct cloud GetChatMessage
* adapter through the public OpenAI/Anthropic surfaces.
*
* Stages:
* 0. Entitlement preflight (zero-billable): GetUserStatus + GetCliModelConfigs
* via the direct catalog probe. Prints the account tier and the selectors
* it can name. No chat turn, so no allowance is spent.
* 0c. Recovery-chain preflight (zero-billable): liveness probe agrees with
* preflight; encrypted credential store roundtrips with no plaintext on
* disk; auto-relogin/liveness-sweep config is reported (and the half-armed
* "relogin on, no key" case is flagged).
* 1. /v1/chat/completions non-stream on the free selector (swe-1-6-slow).
* 2. /v1/chat/completions streaming — SSE deltas + [DONE].
* 3. Multi-turn — prior assistant turn is honored.
* 4. /v1/messages (Anthropic dialect) over the same connect backend.
* 5. Error classification — a paid selector on a free account must surface a
* clean 402/401 (MODEL_BLOCKED/UNAUTHORIZED), not an opaque 500.
*
* Paid-account verification (#15): set CONNECT_SMOKE_PAID_MODEL=claude-opus-4.8
* (or any paid selector). On a paid account stage 5 flips to expect a 200 with
* real content — the one-command confirmation that paid routing works.
*
* Env:
* BASE_URL server base (default http://127.0.0.1:3003)
* API_KEY downstream auth (default 'test')
* CONNECT_SMOKE_TOKEN devin session token for the direct preflight;
* falls back to accounts.json[0] then DEVIN_* env
* CONNECT_SMOKE_FREE_MODEL free selector (default swe-1-6-slow)
* CONNECT_SMOKE_PAID_MODEL paid selector to probe (default claude-opus-4.8)
* CONNECT_SMOKE_REAL_CALLS=0 skip billable chat stages (preflight only)
* CONNECT_SMOKE_TIMEOUT_MS per-request timeout (default 120000)
*/
import { readFileSync, mkdtempSync, rmSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { fetchCatalog, fetchUserStatus, checkSessionLiveness } from '../src/devin-connect-catalog.js';
import { storeCredential, getCredential } from '../src/devin-connect-credentials.js';
import { resolveConnectSelector } from '../src/devin-connect-models.js';
const baseUrl = (process.env.BASE_URL || process.env.WINDSURFAPI_BASE_URL || 'http://127.0.0.1:3003').replace(/\/+$/, '');
const apiKey = process.env.API_KEY || process.env.WINDSURFAPI_API_KEY || 'test';
const freeModel = process.env.CONNECT_SMOKE_FREE_MODEL || 'swe-1-6-slow';
const paidModel = process.env.CONNECT_SMOKE_PAID_MODEL || 'claude-opus-4.8';
const realCalls = process.env.CONNECT_SMOKE_REAL_CALLS !== '0';
const requestTimeoutMs = Math.max(5_000, Number(process.env.CONNECT_SMOKE_TIMEOUT_MS || 120_000));
function resolveToken() {
if (process.env.CONNECT_SMOKE_TOKEN) return process.env.CONNECT_SMOKE_TOKEN;
for (const k of ['DEVIN_CONNECT_TOKEN', 'DEVIN_SESSION_TOKEN', 'WINDSURF_SESSION_TOKEN']) {
if (process.env[k]) return process.env[k];
}
// Last resort: first persisted account (local dev convenience).
try {
const accounts = JSON.parse(readFileSync(new URL('../accounts.json', import.meta.url), 'utf8'));
const a = accounts.find((x) => String(x.apiKey || '').startsWith('devin-session-token$'));
if (a) return a.apiKey;
} catch { /* none */ }
return '';
}
function compact(text, max = 600) {
const s = String(text ?? '').replace(/\s+/g, ' ').trim();
return s.length > max ? `${s.slice(0, max)}...<+${s.length - max}>` : s;
}
async function postJson(path, payload) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), requestTimeoutMs);
try {
const res = await fetch(`${baseUrl}${path}`, {
method: 'POST',
signal: controller.signal,
headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },
body: JSON.stringify(payload),
});
const text = await res.text();
let body = null;
try { body = text ? JSON.parse(text) : null; } catch { /* keep text */ }
return { status: res.status, body, text };
} finally {
clearTimeout(timer);
}
}
async function postSSE(path, payload) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), requestTimeoutMs);
const dataLines = [];
try {
const res = await fetch(`${baseUrl}${path}`, {
method: 'POST',
signal: controller.signal,
headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json', accept: 'text/event-stream' },
body: JSON.stringify(payload),
});
if (!res.ok || !res.body) {
const text = await res.text().catch(() => '');
return { status: res.status, dataLines, error: text };
}
const decoder = new TextDecoder();
let buf = '';
for await (const chunk of res.body) {
buf += decoder.decode(chunk, { stream: true });
let nl;
while ((nl = buf.indexOf('\n')) >= 0) {
const line = buf.slice(0, nl).trimEnd();
buf = buf.slice(nl + 1);
if (line.startsWith('data:')) dataLines.push(line.slice(5).trim());
}
}
return { status: res.status, dataLines };
} finally {
clearTimeout(timer);
}
}
const results = [];
function record(stage, ok, detail) {
results.push({ stage, ok, ...detail });
const tag = ok ? 'PASS' : 'FAIL';
console.log(`[${tag}] ${stage}${detail?.note ? ` — ${detail.note}` : ''}`);
}
// ─── Stage 0: entitlement preflight (zero-billable) ─────────────────────────
const token = resolveToken();
let isPaid = false;
let liveCatalog = null;
if (!token) {
record('preflight', false, { note: 'no devin session token (set CONNECT_SMOKE_TOKEN or persist an account)' });
} else {
try {
const status = await fetchUserStatus({ token });
isPaid = status.isPaid;
const catalog = await fetchCatalog({ token });
liveCatalog = catalog;
const byProvider = catalog.reduce((acc, m) => { (acc[m.provider] ||= []).push(m.selector); return acc; }, {});
record('preflight', true, {
note: `tier=${status.plan} paid=${status.isPaid} models=${catalog.length}`,
plan: status.plan,
isPaid: status.isPaid,
providers: Object.fromEntries(Object.entries(byProvider).map(([p, s]) => [p, s.length])),
});
} catch (err) {
record('preflight', false, { note: `${err.code || 'ERR'}: ${err.message}` });
}
}
// ─── Stage 0b: catalog drift diff (zero-billable) ───────────────────────────
// Compare the LIVE catalog against the committed snapshot + the resolver. New
// or renamed selectors that the SELECTOR_MAP can't resolve are surfaced here so
// the map + fixture get refreshed before a client silently degrades to free.
if (liveCatalog) {
try {
const snap = JSON.parse(readFileSync(new URL('../test/fixtures/devin-catalog-snapshot.json', import.meta.url), 'utf8'));
const snapSelectors = new Set(snap.models.map((m) => m.selector));
const liveSelectors = new Set(liveCatalog.map((m) => m.selector));
const added = [...liveSelectors].filter((s) => !snapSelectors.has(s));
const removed = [...snapSelectors].filter((s) => !liveSelectors.has(s));
// Resolvability: every live selector AND its advertised alias must map
// without degrading to the free default.
const unresolved = [];
for (const m of liveCatalog) {
for (const name of [m.selector, m.alias].filter(Boolean)) {
const { mapped } = resolveConnectSelector(name);
if (!mapped) unresolved.push(name);
}
}
const drift = added.length || removed.length || unresolved.length;
record('catalog-drift', !drift, {
note: drift
? `DRIFT: +${added.length} -${removed.length} unresolved=${unresolved.length} — refresh snapshot + SELECTOR_MAP`
: `in sync with snapshot (${liveCatalog.length} models, all resolvable)`,
added, removed, unresolved,
});
} catch (err) {
record('catalog-drift', false, { note: `diff failed: ${err.message}` });
}
}
// ─── Stage 0c: recovery-chain preflight (zero-billable) ─────────────────────
// Exercises the dead-session_id recovery wiring WITHOUT spending allowance:
// - checkSessionLiveness on the live token (the same GetUserStatus probe the
// scheduled liveness sweep uses) must agree with the preflight.
// - the encrypted credential store roundtrips under an ephemeral key.
// - the auto-relogin + liveness-probe env flags are reported so an operator
// can see at a glance whether hands-off recovery is actually armed.
if (token) {
try {
const live = await checkSessionLiveness({ token });
record('liveness', live.alive === true, {
note: live.alive ? `session alive (plan=${live.plan || '?'})` : `session DEAD (${live.code})`,
code: live.code,
});
} catch (err) {
record('liveness', false, { note: `probe threw: ${err.message}` });
}
}
try {
// Ephemeral key in a temp store so we never touch the real accounts.creds.json.
const tmp = mkdtempSync(join(tmpdir(), 'connect-smoke-cred-'));
const prevKey = process.env.DEVIN_CONNECT_CRED_KEY;
const prevFile = process.env.DEVIN_CONNECT_CRED_FILE;
process.env.DEVIN_CONNECT_CRED_KEY = 'smoke-ephemeral-key-' + Math.random().toString(36).slice(2);
process.env.DEVIN_CONNECT_CRED_FILE = join(tmp, 'creds.json');
storeCredential('smoke@example.com', 's3cret-pw');
const ok = getCredential('smoke@example.com') === 's3cret-pw';
const onDisk = readFileSync(process.env.DEVIN_CONNECT_CRED_FILE, 'utf8');
const leaks = onDisk.includes('s3cret-pw');
record('cred-store', ok && !leaks, {
note: ok && !leaks ? 'roundtrip OK, plaintext absent from disk' : `roundtrip=${ok} leaked=${leaks}`,
});
rmSync(tmp, { recursive: true, force: true });
if (prevKey === undefined) delete process.env.DEVIN_CONNECT_CRED_KEY; else process.env.DEVIN_CONNECT_CRED_KEY = prevKey;
if (prevFile === undefined) delete process.env.DEVIN_CONNECT_CRED_FILE; else process.env.DEVIN_CONNECT_CRED_FILE = prevFile;
} catch (err) {
record('cred-store', false, { note: `store failed: ${err.message}` });
}
{
const autoRelogin = process.env.DEVIN_CONNECT_AUTO_RELOGIN === '1';
const livenessSweep = process.env.DEVIN_CONNECT_LIVENESS_PROBE === '1';
const credKeySet = !!process.env.DEVIN_CONNECT_CRED_KEY;
const armed = autoRelogin && credKeySet;
// Informational, not a hard fail: recovery is opt-in. We only flag the
// half-configured case (relogin on, no key) since that silently can't work.
record('recovery-config', !(autoRelogin && !credKeySet), {
note: armed
? 'hands-off recovery ARMED (auto-relogin + cred key + ' + (livenessSweep ? 'liveness sweep on)' : 'sweep OFF)')
: `auto-relogin=${autoRelogin} cred-key=${credKeySet} liveness-sweep=${livenessSweep}`,
});
}
// ─── Stage 0d: LIVE re-login round-trip (opt-in, zero-billable) ──────────────
// The recovery-config stage above only REPORTS whether hands-off recovery is
// armed. This stage actually FIRES it end-to-end against the real windsurfLogin:
// plant a dead token, run reLoginAccount, confirm a fresh live session token
// swaps in. Opt-in (SMOKE_RELOGIN_LIVE=1 + real email/password + a cred key) and
// billing-free (login + GetUserStatus only). Uses a temp cred store + an
// ephemeral pool account so the real accounts.json is never touched.
if (process.env.SMOKE_RELOGIN_LIVE === '1') {
const email = process.env.SMOKE_RELOGIN_EMAIL || '';
const password = process.env.SMOKE_RELOGIN_PASSWORD || '';
if (!email || !password || !process.env.DEVIN_CONNECT_CRED_KEY) {
record('relogin-live', false, { note: 'armed but missing SMOKE_RELOGIN_EMAIL/PASSWORD or DEVIN_CONNECT_CRED_KEY' });
} else {
const tmp = mkdtempSync(join(tmpdir(), 'connect-smoke-relogin-'));
const prevFile = process.env.DEVIN_CONNECT_CRED_FILE;
const prevFlag = process.env.DEVIN_CONNECT_AUTO_RELOGIN;
process.env.DEVIN_CONNECT_CRED_FILE = join(tmp, 'creds.json');
process.env.DEVIN_CONNECT_AUTO_RELOGIN = '1';
let acct;
try {
const { addAccountByKey, removeAccount, reLoginAccount } = await import('../src/auth.js');
storeCredential(email, password);
acct = addAccountByKey('devin-session-token$DEAD-SMOKE', email);
acct.email = email; acct.method = 'email';
const fresh = await reLoginAccount(acct.id, { force: true });
const ok = typeof fresh === 'string' && fresh.startsWith('devin-session-token$') && fresh !== 'devin-session-token$DEAD-SMOKE';
let live = false;
if (ok) {
const st = await fetchUserStatus({ token: fresh }).catch(() => null);
live = !!st;
}
record('relogin-live', ok && live, {
note: ok && live ? 'dead token revived via real login + fresh token is live'
: `relogin=${ok} freshTokenLive=${live}`,
});
if (acct) removeAccount(acct.id);
} catch (err) {
record('relogin-live', false, { note: `threw: ${err.message}` });
if (acct) { try { (await import('../src/auth.js')).removeAccount(acct.id); } catch {} }
} finally {
rmSync(tmp, { recursive: true, force: true });
if (prevFile === undefined) delete process.env.DEVIN_CONNECT_CRED_FILE; else process.env.DEVIN_CONNECT_CRED_FILE = prevFile;
if (prevFlag === undefined) delete process.env.DEVIN_CONNECT_AUTO_RELOGIN; else process.env.DEVIN_CONNECT_AUTO_RELOGIN = prevFlag;
}
}
}
// ─── Billable chat stages ───────────────────────────────────────────────────
if (!realCalls) {
console.log('\n[skip] CONNECT_SMOKE_REAL_CALLS=0 — chat stages skipped (preflight only).');
} else {
// Stage 1: non-stream
const r1 = await postJson('/v1/chat/completions', {
model: freeModel, stream: false,
messages: [{ role: 'user', content: 'reply with exactly: CONNECT_OK' }],
});
const c1 = r1.body?.choices?.[0]?.message?.content;
record('non-stream', r1.status === 200 && /CONNECT_OK/i.test(c1 || ''), {
note: `status=${r1.status} content="${compact(c1, 80)}"`,
usage: r1.body?.usage,
});
// Stage 2: streaming
const r2 = await postSSE('/v1/chat/completions', {
model: freeModel, stream: true,
messages: [{ role: 'user', content: 'count: one two three' }],
});
const deltas = r2.dataLines.filter((l) => l && l !== '[DONE]');
const sawDone = r2.dataLines.includes('[DONE]');
record('stream', r2.status === 200 && deltas.length > 0 && sawDone, {
note: `status=${r2.status} deltas=${deltas.length} done=${sawDone}`,
});
// Stage 3: multi-turn
const r3 = await postJson('/v1/chat/completions', {
model: freeModel, stream: false,
messages: [
{ role: 'user', content: 'My favorite color is teal. Remember it.' },
{ role: 'assistant', content: 'Got it, teal.' },
{ role: 'user', content: 'What is my favorite color? One word.' },
],
});
const c3 = r3.body?.choices?.[0]?.message?.content;
record('multi-turn', r3.status === 200 && /teal/i.test(c3 || ''), {
note: `status=${r3.status} content="${compact(c3, 80)}"`,
});
// Stage 4: Anthropic /v1/messages over the same backend
const r4 = await postJson('/v1/messages', {
model: freeModel, max_tokens: 64, stream: false,
messages: [{ role: 'user', content: 'reply with exactly: MSG_OK' }],
});
const c4 = Array.isArray(r4.body?.content) ? r4.body.content.map((b) => b.text).join('') : '';
record('anthropic-messages', r4.status === 200 && /MSG_OK/i.test(c4 || ''), {
note: `status=${r4.status} content="${compact(c4, 80)}"`,
});
// Stage 5: paid-model behavior. Free account → clean 402/401, not an opaque
// 5xx. Paid account → 200 with content (the #15 verification).
const r5 = await postJson('/v1/chat/completions', {
model: paidModel, stream: false,
messages: [{ role: 'user', content: 'reply with exactly: PAID_OK' }],
});
if (isPaid) {
const c5 = r5.body?.choices?.[0]?.message?.content;
record('paid-model', r5.status === 200 && /PAID_OK/i.test(c5 || ''), {
note: `[PAID ACCOUNT] ${paidModel} status=${r5.status} content="${compact(c5, 80)}"`,
usage: r5.body?.usage,
});
} else {
// A free account asking for a paid selector is an ENTITLEMENT wall, which
// must surface as 402 MODEL_BLOCKED — not 401 "dead session tokens". The
// latter was a live-fire bug (#42): upstream returns a bare permission_denied
// indistinguishable from a retired token, and the old code ran a re-login +
// full-pool failover storm before lying with "all accounts exhausted". A 401
// here now means that regression is back. 403 still tolerated (hard auth).
const code = r5.body?.error?.code;
const correct = r5.status === 402 && code === 'MODEL_BLOCKED';
const tolerated = r5.status === 403;
record('paid-model', correct || tolerated, {
note: `[FREE ACCOUNT] ${paidModel} → status=${r5.status} code=${code || '-'} (expected 402/MODEL_BLOCKED${r5.status === 401 ? ' — 401 = #42 dead-token-misclassification REGRESSED' : ''}) error="${compact(r5.body?.error?.message || r5.text, 100)}"`,
});
}
// Stage 6: tool/function calling (text-emulation). Works on the free swe-1.6
// model — tool defs are injected into the prompt and <tool_call> markup is
// parsed back into OpenAI tool_calls. A weather query with a tool defined
// should come back as a tool_calls turn, not a prose answer.
const weatherTool = [{
type: 'function',
function: {
name: 'get_weather',
description: 'Get the current weather for a city.',
parameters: {
type: 'object',
properties: { city: { type: 'string', description: 'City name' } },
required: ['city'],
},
},
}];
const r6 = await postJson('/v1/chat/completions', {
model: freeModel, stream: false, tools: weatherTool, tool_choice: 'auto',
messages: [{ role: 'user', content: 'What is the weather in Tokyo right now? Use the tool.' }],
});
const m6 = r6.body?.choices?.[0]?.message;
const tc6 = m6?.tool_calls?.[0];
const calledWeather = tc6?.function?.name === 'get_weather';
record('tool-call-nonstream', r6.status === 200 && calledWeather, {
note: `status=${r6.status} finish=${r6.body?.choices?.[0]?.finish_reason} tool=${tc6?.function?.name || 'none'} args=${compact(tc6?.function?.arguments || '', 60)}`,
});
// Stage 7: tool calling over SSE — the parser must surface a tool_calls delta.
const r7 = await postSSE('/v1/chat/completions', {
model: freeModel, stream: true, tools: weatherTool, tool_choice: 'auto',
messages: [{ role: 'user', content: 'Check the weather in Berlin. Use the tool.' }],
});
const toolDelta = r7.dataLines
.filter((l) => l && l !== '[DONE]')
.map((l) => { try { return JSON.parse(l); } catch { return null; } })
.find((o) => o?.choices?.[0]?.delta?.tool_calls);
const streamToolName = toolDelta?.choices?.[0]?.delta?.tool_calls?.[0]?.function?.name;
record('tool-call-stream', r7.status === 200 && streamToolName === 'get_weather', {
note: `status=${r7.status} streamedTool=${streamToolName || 'none'}`,
});
}
// ─── Summary ────────────────────────────────────────────────────────────────
const failed = results.filter((r) => !r.ok);
console.log(`\n${'─'.repeat(60)}`);
console.log(JSON.stringify({ ok: failed.length === 0, isPaid, stages: results }, null, 2));
process.exit(failed.length === 0 ? 0 : 1);