scripts/devin-connect-calibrate.mjs
Rama master · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* devin-connect-calibrate.mjs — ONE command to discover every unknown wire tag.
*
* WHY: five tasks (#15/#28 paid selectors, #29 vision image tag, #46 cache/billing
* tokens, #47 actual_model_uid, #49 native tool_calls) are all blocked on the SAME
* thing — a wire TAG that a free capture physically never emits. Their decoders are
* already shipped behind env flags (ready-to-fire); the only missing input is the
* integer tag, which appears the instant a paid token / tool-using / cached / router
* turn is captured. Until now that capture had to be wired up by hand each time.
*
* This harness collapses all of it into `npm run calibrate:devin`. Point it at ANY
* token, it fires one completion with DEVIN_CONNECT_DEBUG_META on, aggregates EVERY
* top-level + metadata tag the stream emitted, diffs them against the known free
* baseline, classifies each NEW tag by wire shape into the likely field bucket, and
* writes the calibrated `DEVIN_CONNECT_*` env lines to disk. A status table shows,
* per target, whether it's calibrated, still pending, and which task it unblocks.
*
* BILLABLE? One tiny completion per run. OFF by default (CALIBRATE_REAL=1 to fire);
* without it, the offline self-test proves collection + classification + persistence.
*
* Usage:
* CALIBRATE_REAL=1 CONNECT_SMOKE_TOKEN=<token> node scripts/devin-connect-calibrate.mjs
* Env:
* CALIBRATE_REAL=1 actually fire (default: offline self-test)
* CALIBRATE_MODEL=<alias> model to probe (default: swe-1.6-slow free default)
* CALIBRATE_PROMPT=<text> prompt (default a tool-encouraging ask, to coax tool_calls)
* CALIBRATE_OUT=<path> env output file (default ../.devin-connect-calibrated.env)
* CALIBRATE_TIMEOUT_MS per-completion timeout (default 90000)
*/
import { readFileSync, writeFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
const REAL = process.env.CALIBRATE_REAL === '1';
const TIMEOUT_MS = Number(process.env.CALIBRATE_TIMEOUT_MS || 90000);
// Default MUST be a resolved proto #21 selector (dashed), not the dotted alias:
// streamChat passes `model` RAW (alias resolution lives in the chat.js handler,
// not the transport), and the upstream returns "an internal error occurred
// (trace ID: ...)" for an UNRECOGNIZED selector. The dotted "swe-1.6-slow"
// looked like a dead-token/backend-fault for hours until this was isolated on
// free account <redacted> (dashed swe-1-6-slow → alive, dotted → error).
// The same trap bites CALIBRATE_MODEL: an operator naturally passes the
// client-facing alias ("claude-sonnet-4.6") and gets the identical, deeply
// misleading UPSTREAM_INTERNAL — indistinguishable from a dead token or a walled
// account, which is exactly the state a paid calibration run is trying to escape.
// Resolve the alias to its real selector here so either form works, and say so
// when the two differ. (Cost two failed paid runs on 2026-07-27 before the
// dotted/dashed distinction was spotted.)
const RAW_MODEL = process.env.CALIBRATE_MODEL || 'swe-1-6-slow';
const DEFAULT_MODEL = await (async () => {
try {
const { resolveConnectSelector } = await import('../src/devin-connect-models.js');
const { selector, mapped } = resolveConnectSelector(RAW_MODEL);
if (mapped && selector && selector !== RAW_MODEL) {
console.log(`[calibrate] resolved alias ${RAW_MODEL} -> selector ${selector}`);
return selector;
}
return RAW_MODEL;
} catch {
return RAW_MODEL; // resolution is a convenience, never a hard dependency
}
})();
const DEFAULT_PROMPT = process.env.CALIBRATE_PROMPT
|| 'Reply with exactly: PONG';
// ─── Known FREE baseline (re-dumped live 2026-06-30, free acct <redacted>,
// selector swe-1-6-slow) ──
// A plain free TEXT turn (no tools) emits, per-frame: #1 bot-id, #2 content
// sub-message (present EVERY frame), #7 a second sub-message (present EVERY
// frame), #17 uuid. On terminal frames it adds #3 final-text, #4/#5 finish/stop
// enum, #9 reasoning delta, #28 a trailing standalone sub-message. #11 seen on
// some turns. ALL of these are normal free output — the previous baseline
// ({1,3,4,5,9,11,17}) was missing #2/#7/#28, so a clean free turn falsely
// surfaced them as tool_calls/billing candidates (every dump "discovered" them).
// Meta sub-message: #6 constant (=6) on free; #2/#3 varints appear on free text
// turns too (NOT paid-only billing — confirmed: present with no tools, no paid
// selector), so they are baseline, not credit_cost candidates.
export const FREE_BASELINE = {
top: new Set([1, 2, 3, 4, 5, 7, 9, 11, 17, 28]),
meta: new Set([2, 3, 6]),
};
// ─── Targets: the five blocked unknowns, each with the env var that arms its
// already-shipped decoder, the wire shape it appears as, and the task it unblocks.
export const TARGETS = [
{ key: 'actual_model_uid', env: 'DEVIN_CONNECT_ACTUAL_MODEL_TAG', scope: 'top', shape: 'string', task: '#47', note: 'concrete model behind a router turn' },
{ key: 'tool_calls', env: 'DEVIN_CONNECT_TOOL_CALL_TAGS', scope: 'top', shape: 'message', task: '#49', note: 'repeated ChatToolCall (delta_tool_calls)' },
{ key: 'billing', env: 'DEVIN_CONNECT_BILLING_TAGS', scope: 'meta', shape: 'varint', task: '#46', note: 'credit_cost / committed_*_cost' },
{ key: 'cache_tokens', env: 'DEVIN_CONNECT_BILLING_TAGS', scope: 'meta', shape: 'varint', task: '#46', note: 'cache_read_tokens / cache_write_tokens' },
{ key: 'image_tag', env: 'DEVIN_CONNECT_IMAGE_TAG', scope: 'request', shape: 'message', task: '#29', note: 'vision images field — needs the dedicated image-calibrate sweep' },
];
export function resolveToken(env = process.env) {
if (env.CONNECT_SMOKE_TOKEN) return env.CONNECT_SMOKE_TOKEN.trim();
for (const k of ['DEVIN_CONNECT_TOKEN', 'DEVIN_SESSION_TOKEN', 'WINDSURF_SESSION_TOKEN']) {
if (env[k]) return env[k].trim();
}
try {
const accountsUrl = env.CALIBRATE_ACCOUNTS_FILE
? new URL(`file://${env.CALIBRATE_ACCOUNTS_FILE.replace(/\\/g, '/')}`)
: new URL('../accounts.json', import.meta.url);
const accounts = JSON.parse(readFileSync(accountsUrl, 'utf8'));
const first = accounts.find((a) => a.apiKey);
if (first) return first.apiKey;
} catch { /* none */ }
return '';
}
export function resolveOutPath(env = process.env) {
return env.CALIBRATE_OUT || fileURLToPath(new URL('../.devin-connect-calibrated.env', import.meta.url));
}
/**
* Classify one observed top-level/meta tag (not in the baseline) into the target
* bucket its wire shape best fits. Pure + exported for the self-test.
* - meta varint → billing / cache_tokens (#46)
* - top numeric → billing (#46) — the top-level cost coordinates (#14/#22/#26/#27)
* - top string → actual_model_uid (#47)
* - top message → tool_calls (#49)
*/
export function classifyTag({ scope, tag, kind, preview, topTag, path }) {
const loc = path || (topTag != null ? `${topTag}.${tag}` : tag);
if (scope === 'meta' && kind === 'varint') {
return { bucket: 'billing/cache', targets: ['billing', 'cache_tokens'], task: '#46',
detail: `meta varint #${tag}=${preview} — credit_cost / cache token candidate` };
}
if (scope === 'top' && kind === 'string') {
return { bucket: 'actual_model_uid', targets: ['actual_model_uid'], task: '#47',
detail: `top string #${tag}="${preview}" — actual_model_uid candidate` };
}
if (scope === 'top' && kind === 'message') {
return { bucket: 'tool_calls', targets: ['tool_calls'], task: '#49',
detail: `top sub-message #${tag} (${preview}) — delta_tool_calls (tool_calls) candidate` };
}
// A top-level NUMERIC field — varint, or a fixed64 double / fixed32 float, the
// shapes a cost field rides (committed_acu_cost #22 is a descriptor-verified
// double: docs/DEVIN-CONNECT-CUTOVER.md §7). The four .proto reimplementations
// put the top-level cost coordinates at #14/#22/#26/#27, and the shipped
// decoder reads them via the `^N` form of DEVIN_CONNECT_BILLING_TAGS — so this
// is a REAL billing candidate, unlike a #28 inner varint (the decoder cannot
// reach those; they stay informational). The tag itself stays un-pinned until
// an operator maps it; only the bucket is decided here.
if (scope === 'top' && (kind === 'varint' || kind === 'fixed64' || kind === 'fixed32')) {
return { bucket: 'billing/cache', targets: ['billing'], task: '#46',
detail: `top numeric #${tag}=${preview} (${kind}) — top-level cost candidate (credit_cost / committed_acu_cost / quota / overage); pin via ^${tag}` };
}
// Inner fields of a top-level sub-message (e.g. the recurring #28 trailer). A
// varint inner field is the strongest billing/usage/stop-metadata signal — this
// is where credit_cost / committed_*_cost / cache tokens most likely live when
// they don't ride the #7 meta block. Strings inside are model-id / stop-reason.
if (scope === 'sub' && kind === 'varint') {
// Informational only (targets: []): the shipped billing decoder reads the #7
// meta block, so a #28 inner varint must NOT auto-fill DEVIN_CONNECT_BILLING_TAGS.
// It's surfaced for the operator to inspect, with a dedicated env hint below.
return { bucket: 'sub-billing', targets: [], task: '#46',
detail: `sub #${loc} varint=${preview} — billing/usage/stop-metadata candidate` };
}
if (scope === 'sub' && kind === 'string') {
return { bucket: 'sub-metadata', targets: [], task: '#46/#47',
detail: `sub #${loc} string="${preview}" — model-id / stop-reason candidate` };
}
if (scope === 'sub') {
return { bucket: 'sub-nested', targets: [], task: '#46',
detail: `sub #${loc} (${kind} ${preview}) — nested sub-message candidate` };
}
return { bucket: 'unknown', targets: [], task: '?', detail: `#${tag} (${kind}) — unrecognized shape` };
}
/**
* Aggregate the per-frame dumps emitted over a stream into a stable tag inventory.
* Each dump entry value tells us the wire kind: number→varint, "<msg Nb>"→message,
* a fixed64/fixed32 dump entry keeps its kind (double/float preview + raw hex),
* else→string. Returns { top:{tag:{kind,preview}}, meta:{tag:{kind,preview}} }.
*/
export function aggregateDumps(frameDumps, metaDumps, subDumps) {
const classify = (v) => {
if (typeof v === 'number') return { kind: 'varint', preview: v };
if (typeof v === 'string' && /^<msg \d+b>$/.test(v)) return { kind: 'message', preview: v };
// Fixed-width dump entries from decodeFrame's fixedWidthDumpEntry():
// { kind: 'fixed64'|'fixed32', preview: <number>, raw: <hex> }. Preserve the
// kind/raw — letting the object fall through to the string branch turned it
// into "[object Object]" and sent a top-level double (#22 committed_acu_cost)
// into the top-string / actual_model_uid bucket.
if (v && typeof v === 'object' && (v.kind === 'fixed64' || v.kind === 'fixed32')) {
return { kind: v.kind, preview: v.preview, raw: v.raw };
}
return { kind: 'string', preview: String(v).slice(0, 48) };
};
const top = {}, meta = {}, sub = {};
for (const d of frameDumps || []) for (const [tag, v] of Object.entries(d)) top[tag] = classify(v);
for (const d of metaDumps || []) for (const [tag, v] of Object.entries(d)) meta[tag] = classify(v);
// sub: { <topTag>: { <innerTag>: {kind, preview, fields?} } } — decodeSubMessage
// emits the {kind, preview} shape (with a nested `fields` map when an inner field
// is itself a decodable message, e.g. #28.2). Merge as-is (later frames win,
// matching top/meta's last-writer semantics); nested `fields` ride along.
for (const d of subDumps || []) for (const [topTag, inner] of Object.entries(d)) {
sub[topTag] = { ...(sub[topTag] || {}), ...inner };
}
return { top, meta, sub };
}
/**
* Given an aggregated inventory, return the NEW tags (not in baseline) classified
* into target buckets. Returns { candidates:[{scope,tag,kind,preview,...classify}] }.
*/
export function findCandidates(inventory, baseline = FREE_BASELINE) {
const candidates = [];
for (const [tag, info] of Object.entries(inventory.top)) {
if (baseline.top.has(Number(tag))) continue;
candidates.push({ scope: 'top', tag: Number(tag), ...info, ...classifyTag({ scope: 'top', tag: Number(tag), kind: info.kind, preview: info.preview }) });
}
for (const [tag, info] of Object.entries(inventory.meta)) {
if (baseline.meta.has(Number(tag))) continue;
candidates.push({ scope: 'meta', tag: Number(tag), ...info, ...classifyTag({ scope: 'meta', tag: Number(tag), kind: info.kind, preview: info.preview }) });
}
// Inner fields of top-level sub-messages (e.g. #28). Every inner field is a
// candidate — there is no free baseline for sub-message internals (a free
// trailer's inner structure was never decoded before this harness existed), so
// all are surfaced for operator inspection. Keyed by both the outer top tag and
// the inner tag so the report reads "sub #28.3 varint=…".
// Walk the sub tree recursively. `path` is the dotted tag chain from the top-level
// tag down (e.g. "28.2.3" for the counter nested inside #28's Response Statistics
// message). Nested `.fields` maps come from decodeSubMessage recursing into inner
// messages; a leaf's own tag is the last path segment.
const walkSub = (topTag, node, path) => {
for (const [tag, info] of Object.entries(node)) {
const p = [...path, Number(tag)];
const { fields, ...leaf } = info;
candidates.push({ scope: 'sub', topTag, tag: Number(tag), path: p.join('.'), ...leaf,
...classifyTag({ scope: 'sub', topTag, tag: Number(tag), path: p.join('.'), kind: leaf.kind, preview: leaf.preview }) });
if (fields) walkSub(topTag, fields, p); // descend into the nested message
}
};
for (const [topTag, inner] of Object.entries(inventory.sub || {})) {
walkSub(Number(topTag), inner, [Number(topTag)]);
}
return { candidates };
}
/**
* Run a calibration probe. `deps.streamChat` injectable for the self-test. It does
* NOT decode with calibrated tags — it only collects raw dumps, so it works on a
* token where the tags are still unknown. Returns the full report.
*/
export async function runCalibration({ token, model = DEFAULT_MODEL, prompt = DEFAULT_PROMPT, tools = null, env = process.env, real = false, deps = {} } = {}) {
const frameDumps = [];
const metaDumps = [];
const subDumps = [];
let modelAlive = false;
let error = null;
if (real) {
const { streamChat = (await import('../src/devin-connect.js')).streamChat } = deps;
const probeEnv = { ...env, DEVIN_CONNECT_DEBUG_META: '1' };
// Capture the dumps directly off decodeFrame via a stream consumer. streamChat
// logs them; we instead read the structured events it yields plus a frame hook.
const ac = new AbortController();
const timer = setTimeout(() => ac.abort(), TIMEOUT_MS);
try {
const chatArgs = { token, model, messages: [{ role: 'user', content: prompt }], env: probeEnv, signal: ac.signal };
if (tools) chatArgs.tools = tools;
for await (const ev of streamChat(chatArgs)) {
if (ev.type === 'frame-dump' && ev.frameDump) frameDumps.push(ev.frameDump);
if (ev.type === 'frame-dump' && ev.metaDump) metaDumps.push(ev.metaDump);
if (ev.type === 'frame-dump' && ev.subDump) subDumps.push(ev.subDump);
if (ev.type === 'delta' || ev.type === 'finish') modelAlive = true;
}
} catch (e) {
error = { code: e.code || 'ERR', message: String(e.message || '').slice(0, 80) };
} finally {
clearTimeout(timer);
}
} else {
// self-test path provides dumps directly via deps
if (deps.frameDumps) frameDumps.push(...deps.frameDumps);
if (deps.metaDumps) metaDumps.push(...deps.metaDumps);
if (deps.subDumps) subDumps.push(...deps.subDumps);
modelAlive = deps.modelAlive !== false;
}
const inventory = aggregateDumps(frameDumps, metaDumps, subDumps);
const { candidates } = findCandidates(inventory);
// Build the env lines for any discovered candidates.
const envLines = [];
const byTarget = {};
for (const c of candidates) for (const t of c.targets) (byTarget[t] ||= []).push(c);
if (byTarget.actual_model_uid?.length) envLines.push(`DEVIN_CONNECT_ACTUAL_MODEL_TAG=${byTarget.actual_model_uid[0].tag}`);
if (byTarget.tool_calls?.length) envLines.push(`# tool_calls outer candidate at tag ${byTarget.tool_calls[0].tag} — confirm subfields then set:\n# DEVIN_CONNECT_TOOL_CALL_TAGS="outer=${byTarget.tool_calls[0].tag},id=?,name=?,arguments_json=?"`);
// Split the billing hints by scope: a meta varint is pinned by its bare tag,
// while a top-level numeric (e.g. #22 committed_acu_cost, a double) needs the
// decoder's `^N` form. Same candidates, same convention — the scope picks the
// env syntax, so the written .env can't point the operator at the wrong block.
// Meta candidates target both billing AND cache_tokens, so UNION the two lists
// (not concat) or every meta tag is emitted twice ([14,15,14,15]).
const billingCands = [...new Set([...(byTarget.billing || []), ...(byTarget.cache_tokens || [])])];
const metaBilling = billingCands.filter((c) => c.scope === 'meta');
const topBilling = billingCands.filter((c) => c.scope === 'top');
if (metaBilling.length) envLines.push(`# meta varint candidates at tags [${metaBilling.map((c) => c.tag).join(',')}] — map to credit_cost/cache_*; then set DEVIN_CONNECT_BILLING_TAGS / cache via DEVIN_CONNECT_BILLING_TAGS`);
// The example stays `<field>=^N`: with several top candidates the first tag's
// key is unknown, and naming a specific key (#22 committed_acu_cost) next to a
// different tag (#14 credit_cost) would wire the wrong field.
if (topBilling.length) envLines.push(`# top-level numeric candidates at tags [${topBilling.map((c) => c.tag).join(',')}] — map to credit_cost / committed_acu_cost / quota / overage; pin via the ^N form, e.g. DEVIN_CONNECT_BILLING_TAGS=<field>=^${topBilling[0].tag}`);
// Sub-message inner varints (e.g. the #28 trailer): informational — the shipped
// billing decoder reads the #7 meta block, so these are NOT auto-wired. Surface
// them so the operator can decide whether #28 carries the billing/usage fields.
const subVarints = candidates.filter((c) => c.scope === 'sub' && c.kind === 'varint');
if (subVarints.length) {
// Group by the parent path (everything but the leaf tag) so nested counters
// read as "#28.2 inner varints: {3=…, 4=…}" — the Response Statistics message.
const byParent = {};
for (const c of subVarints) {
const segs = String(c.path || `${c.topTag}.${c.tag}`).split('.');
const parent = segs.slice(0, -1).join('.');
(byParent[parent] ||= []).push(`${segs[segs.length - 1]}=${c.preview}`);
}
for (const [parent, fields] of Object.entries(byParent)) {
envLines.push(`# sub-message #${parent} inner varints: {${fields.join(', ')}} — inspect for credit_cost/cache/stop-metadata (NOT auto-wired; #7-meta drives billing decode today)`);
}
}
return { modelAlive, error, inventory, candidates, envLines };
}
/** Status table: for each target, calibrated? pending? which task. */
export function statusTable(report, env = process.env) {
const found = new Set(report.candidates.flatMap((c) => c.targets));
return TARGETS.map((t) => {
const already = env[t.env] && String(env[t.env]).trim().length > 0;
const discovered = found.has(t.key);
const state = already ? 'CALIBRATED (env set)' : discovered ? 'CANDIDATE FOUND' : 'pending';
return { target: t.key, task: t.task, env: t.env, state, note: t.note };
});
}
// ─── Offline self-test: proves collection + classify + persistence with fake
// dumps that mimic a paid/tool/cached/router capture — no token, no network.
async function selfTest() {
const assert = (c, m) => { if (!c) { console.error(`[SELFTEST FAIL] ${m}`); process.exitCode = 1; } };
// classifyTag buckets
assert(classifyTag({ scope: 'meta', tag: 10, kind: 'varint', preview: 42 }).bucket === 'billing/cache', 'meta varint → billing/cache');
assert(classifyTag({ scope: 'top', tag: 8, kind: 'string', preview: 'claude-opus' }).bucket === 'actual_model_uid', 'top string → actual_model_uid');
assert(classifyTag({ scope: 'top', tag: 12, kind: 'message', preview: '<msg 40b>' }).bucket === 'tool_calls', 'top message → tool_calls');
assert(classifyTag({ scope: 'sub', topTag: 28, tag: 3, kind: 'varint', preview: 42 }).bucket === 'sub-billing', 'sub varint → sub-billing');
assert(classifyTag({ scope: 'sub', topTag: 28, tag: 3, kind: 'varint', preview: 42 }).targets.length === 0, 'sub varint NOT auto-wired');
assert(classifyTag({ scope: 'sub', topTag: 28, tag: 1, kind: 'string', preview: 'stop' }).bucket === 'sub-metadata', 'sub string → sub-metadata');
// Top-level numeric (#22 committed_acu_cost is a descriptor-verified double,
// docs/DEVIN-CONNECT-CUTOVER.md §7). A fixed64 dump entry used to be mangled to
// "[object Object]" and land in actual_model_uid — emitting ACTUAL_MODEL_TAG=22.
assert(classifyTag({ scope: 'top', tag: 22, kind: 'fixed64', preview: 0.00067 }).bucket === 'billing/cache', 'top fixed64 → billing/cache');
assert(classifyTag({ scope: 'top', tag: 26, kind: 'varint', preview: 1200 }).bucket === 'billing/cache', 'top varint → billing/cache');
// aggregate + findCandidates against the free baseline
const frameDumps = [
{ 1: 'bot-x', 9: 'thinking', 17: 'uuid' }, // all baseline → no candidates
{ 1: 'bot-x', 3: 'PONG', 4: 2, 8: 'claude-opus-4-8', 12: '<msg 47b>' }, // #8 actual_model, #12 tool_calls
];
const metaDumps = [{ 6: 6, 14: 1500, 15: 200 }]; // #14/#15 new varints → billing/cache
// #28 trailer — the recurring "Response Statistics" container captured on PAID-1
// 2026-07-03: #28.1 is the string label, #28.2 is a NESTED message whose inner
// varints (#28.2.3, #28.2.4) are the real usage/billing counters, one level down.
const subDumps = [{ 28: {
1: { kind: 'string', preview: 'Response Statistics' },
2: { kind: 'message', preview: '<msg 40b>', fields: {
3: { kind: 'varint', preview: 1200 },
4: { kind: 'varint', preview: 34 },
} },
} }];
const report = await runCalibration({ real: false, deps: { frameDumps, metaDumps, subDumps } });
const buckets = report.candidates.map((c) => c.bucket).sort();
assert(report.candidates.some((c) => c.scope === 'top' && c.tag === 8 && c.bucket === 'actual_model_uid'), 'found actual_model_uid #8');
assert(report.candidates.some((c) => c.scope === 'top' && c.tag === 12 && c.bucket === 'tool_calls'), 'found tool_calls #12');
assert(report.candidates.filter((c) => c.scope === 'meta' && c.bucket === 'billing/cache').length === 2, 'found 2 billing/cache meta varints');
assert(!report.candidates.some((c) => c.tag === 6), 'baseline meta #6 not flagged');
assert(!report.candidates.some((c) => c.scope === 'top' && [1, 3, 4, 9, 17].includes(c.tag)), 'baseline top tags not flagged');
// sub-message inner fields surfaced (the #28 decode — the previous session's blocker).
// The recursion must reach the NESTED counters at #28.2.3 / #28.2.4 (one level down).
assert(report.candidates.filter((c) => c.scope === 'sub' && c.kind === 'varint').length === 2, 'found 2 nested sub varints');
assert(report.candidates.some((c) => c.path === '28.2.3' && c.kind === 'varint' && c.preview === 1200), 'reached nested #28.2.3');
assert(report.candidates.some((c) => c.path === '28.2.4' && c.kind === 'varint' && c.preview === 34), 'reached nested #28.2.4');
assert(report.candidates.some((c) => c.path === '28.1' && c.bucket === 'sub-metadata'), 'found sub #28.1 label string');
// env line generation
assert(report.envLines.some((l) => l === 'DEVIN_CONNECT_ACTUAL_MODEL_TAG=8'), 'emits actual_model env line');
assert(report.envLines.some((l) => /outer=12/.test(l)), 'emits tool_call outer candidate');
assert(report.envLines.some((l) => /14,15/.test(l)), 'emits billing meta candidates');
assert(report.envLines.some((l) => /^# meta varint candidates at tags \[14,15\] —/.test(l)), 'meta hint lists each tag once (no billing/cache_tokens duplication)');
assert(report.envLines.some((l) => /sub-message #28\.2 inner varints/.test(l) && /3=1200/.test(l) && /4=34/.test(l)), 'emits nested sub #28.2 informational env hint');
// Top-level numeric tag #22 in the exact dump shape decodeFrame emits for a
// double ({kind:'fixed64', preview:<number>, raw:<hex>}): it must classify as a
// billing candidate and NEVER emit the model-uid env line.
const acuRaw = Buffer.alloc(8); acuRaw.writeDoubleLE(0.0006735000060871243, 0);
const acuFrame = { 1: 'bot-x', 22: { kind: 'fixed64', preview: 0.0006735000060871243, raw: acuRaw.toString('hex') } };
assert(aggregateDumps([acuFrame], [], []).top[22].kind === 'fixed64', 'fixed64 dump entry keeps its kind');
const acuReport = await runCalibration({ real: false, deps: { frameDumps: [acuFrame], metaDumps: [] } });
assert(acuReport.candidates.some((c) => c.scope === 'top' && c.tag === 22 && c.bucket === 'billing/cache' && c.task === '#46'), 'top numeric #22 → billing/cache (#46)');
assert(!acuReport.envLines.some((l) => /ACTUAL_MODEL_TAG=22/.test(l)), 'top numeric #22 never emits the actual_model env line');
assert(acuReport.envLines.some((l) => /\^22/.test(l)), 'top numeric #22 emits the ^N top-level pin hint');
// Multi-candidate top frame: the hint's example must stay generic — pairing a
// specific key with the FIRST tag (#14 = credit_cost) would wire credit as ACU.
const acuRaw2 = Buffer.alloc(8); acuRaw2.writeDoubleLE(0.0006735000060871243, 0);
const raw14 = Buffer.alloc(8); raw14.writeDoubleLE(1.25, 0);
const multiFrame = {
1: 'bot-x',
14: { kind: 'fixed64', preview: 1.25, raw: raw14.toString('hex') },
22: { kind: 'fixed64', preview: 0.0006735000060871243, raw: acuRaw2.toString('hex') },
};
const multiReport = await runCalibration({ real: false, deps: { frameDumps: [multiFrame], metaDumps: [] } });
const multiLine = multiReport.envLines.find((l) => /top-level numeric candidates/.test(l));
assert(multiLine && !/committed_acu_cost=\^14/.test(multiLine), 'multi-candidate hint never pairs a key with the wrong tag');
assert(multiLine && /<field>=\^14/.test(multiLine), 'multi-candidate hint shows the generic ^N form');
// status table reflects discoveries + already-set env
const tbl = statusTable(report, {});
assert(tbl.find((r) => r.target === 'actual_model_uid').state === 'CANDIDATE FOUND', 'status: actual_model candidate');
const tbl2 = statusTable({ candidates: [] }, { DEVIN_CONNECT_ACTUAL_MODEL_TAG: '8' });
assert(tbl2.find((r) => r.target === 'actual_model_uid').state.startsWith('CALIBRATED'), 'status: env-set → CALIBRATED');
// a pure-free capture yields zero candidates (proves no false positives)
const freeOnly = await runCalibration({ real: false, deps: { frameDumps: [{ 1: 'b', 3: 'PONG', 4: 2, 9: 't', 17: 'u' }], metaDumps: [{ 6: 6 }] } });
assert(freeOnly.candidates.length === 0, 'pure-free capture → no candidates');
if (process.exitCode) console.error('\n[SELFTEST] FAILED — do not trust the harness until fixed.');
else console.log('[SELFTEST] OK — collect + classify + env-gen + status wiring verified (no token, no network, no billing).');
}
// ─── Main ────────────────────────────────────────────────────────────────────
async function main() {
const model = DEFAULT_MODEL;
console.log(`[calibrate] model=${model} real=${REAL}`);
if (!REAL) {
console.log('CALIBRATE_REAL is not 1 — running offline self-test only (no token, no network).');
console.log('To calibrate against a real (ideally paid/tool-using/router) token:');
console.log(' CALIBRATE_REAL=1 CONNECT_SMOKE_TOKEN=<token> node scripts/devin-connect-calibrate.mjs\n');
await selfTest();
process.exit(process.exitCode || 0);
}
const token = resolveToken();
if (!token) {
console.error('CALIBRATE_REAL=1 but no token — set CONNECT_SMOKE_TOKEN or persist an account.');
process.exit(2);
}
console.log(`[calibrate] firing one DEBUG_META probe (timeout ${TIMEOUT_MS}ms)\n`);
const report = await runCalibration({ token, model, real: true });
if (report.error) console.log(` probe error: ${report.error.code} ${report.error.message}`);
console.log(` model alive: ${report.modelAlive}`);
console.log(` top tags seen: ${Object.keys(report.inventory.top).join(', ') || '(none)'}`);
console.log(` meta tags seen: ${Object.keys(report.inventory.meta).join(', ') || '(none)'}`);
console.log(`\n─── NEW candidate tags (not in free baseline) ───`);
if (!report.candidates.length) {
console.log(' none — this token/turn emitted only known free fields.');
console.log(' (paid selectors, a tool-using turn, a router model, or a cached turn are needed to surface the rest.)');
} else {
for (const c of report.candidates) console.log(` ${c.detail} [${c.task}]`);
}
console.log(`\n─── Calibration status ───`);
for (const r of statusTable(report)) {
console.log(` ${r.target.padEnd(18)} ${r.task.padEnd(4)} ${r.state.padEnd(22)} ${r.env}`);
}
if (report.envLines.length) {
const outPath = resolveOutPath();
const banner = `# DEVIN_CONNECT calibration — ${new Date().toISOString()}\n# Review each candidate before pinning in production.\n`;
writeFileSync(outPath, banner + report.envLines.join('\n') + '\n', 'utf8');
console.log(`\nWrote candidate env lines → ${outPath}`);
console.log('Review, confirm subfields where noted, then copy the DEVIN_CONNECT_* lines into prod env + memory.');
}
process.exit(0);
}
const isEntry = import.meta.main
?? (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1].replace(/\\/g, '/')}`).href);
if (isEntry) await main();