scripts/verify-model-compatibility-live.mjs
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env node
/**
* Explicit live check for one reviewed Chickpea model-catalog lane. The script
* never prints credentials or model output and refuses all egress except the
* selected provider host. Subscription checks reuse an already-authorized
* Node settings database; API-key checks read the provider key from env.
*/
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { readFile } from 'node:fs/promises';
import { resolveModel } from '@flue/runtime/internal';
import { withStatelessModelAccess } from '../src/config/installation-model-access.ts';
import { resolveRuntimeModel } from '../src/config/runtime-model.ts';
import { resolveOpenAiAuthMethod } from '../src/config/openai-auth.ts';
import { registeredPiProvider } from '../src/config/pi-provider-registry.ts';
import { bootstrapRuntimeProviders } from '../src/runtime-bootstrap.ts';
import { SqliteSettingsStore } from '../src/config/settings-store.ts';
import { activateModelCatalog } from '../src/model-catalog/catalog.ts';
import { parseModelCatalogBytes } from '../src/model-catalog/schema.ts';
import { clearOpenAiSubscriptionTransport } from '../src/openai-subscription/transport.ts';
const REASONING_LEVELS = ['off', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'];
const args = parseArgs(process.argv.slice(2));
if (args.has('--help')) {
console.log(
'Usage: npm run verify:model-compatibility:live -- --live ' +
'--lane <subscription|openai-api-key|anthropic-api-key> --model <id> ' +
`[--reasoning <${REASONING_LEVELS.join('|')}>] [--state-db <path>] [--catalog-file <path>]`,
);
console.log('--reasoning sends that thinking level the way an Agent turn does. Without it the request carries no level, and the model profile decides whether that sends the effort none or omits it.');
console.log('Consumes provider or subscription quota and never prints model output or credentials.');
process.exit(0);
}
assert.equal(args.has('--live'), true, 'refusing model traffic without the explicit --live flag');
const lane = String(args.get('--lane') ?? '');
assert.ok(
lane === 'subscription' || lane === 'openai-api-key' || lane === 'anthropic-api-key',
'--lane must be subscription, openai-api-key, or anthropic-api-key',
);
const modelId = String(args.get('--model') ?? '');
assert.match(modelId, /^[a-z0-9][a-z0-9._-]{0,127}$/, '--model must be a safe model id');
const reasoning = args.get('--reasoning');
assert.ok(
reasoning === undefined || REASONING_LEVELS.includes(reasoning),
`--reasoning must be one of ${REASONING_LEVELS.join(', ')}`,
);
const provider = lane === 'anthropic-api-key' ? 'anthropic' : 'openai';
const canonicalModel = `${provider}/${modelId}`;
const expectedHost = lane === 'subscription' ? 'chatgpt.com' : `api.${provider}.com`;
const catalogFile = args.get('--catalog-file');
const hostedCatalog = typeof catalogFile === 'string'
? await readHostedCatalog(catalogFile)
: undefined;
const statePath = lane === 'subscription' ? String(args.get('--state-db') ?? '') : ':memory:';
if (lane === 'subscription') {
assert.ok(statePath, '--state-db is required for a subscription check');
} else {
const keyName = provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY';
assert.ok(process.env[keyName], `${keyName} is required`);
if (provider === 'openai') {
assert.equal(
process.env.OPENAI_BASE_URL ?? '',
'',
'OPENAI_BASE_URL must be unset for the live compatibility gate',
);
}
if (provider === 'anthropic') {
assert.equal(
process.env.ANTHROPIC_BASE_URL ?? '',
'',
'ANTHROPIC_BASE_URL must be unset for the live compatibility gate',
);
}
}
const nativeFetch = globalThis.fetch;
const destinations = [];
const sentReasoningEfforts = [];
globalThis.fetch = async (input, init) => {
const url = new URL(
typeof input === 'string'
? input
: input instanceof URL
? input.href
: input.url,
);
destinations.push(url.hostname);
if (url.hostname !== expectedHost && !(lane === 'subscription' && url.hostname === 'auth.openai.com')) {
throw new Error(`Live compatibility gate blocked unexpected host ${url.hostname}.`);
}
if (url.hostname === expectedHost && typeof init?.body === 'string') {
sentReasoningEfforts.push(JSON.parse(init.body).reasoning?.effort ?? null);
}
return nativeFetch(input, init);
};
bootstrapRuntimeProviders();
const settings = new SqliteSettingsStore(statePath);
try {
if (provider === 'openai') {
const selected = await resolveOpenAiAuthMethod(settings);
assert.equal(
selected,
lane === 'subscription' ? 'subscription' : 'api_key',
`persisted OpenAI authentication method must be ${lane === 'subscription' ? 'subscription' : 'api_key'}`,
);
}
const route = await resolveRuntimeModel('live_model_compatibility', canonicalModel, {
settings,
...(hostedCatalog
? {
loadCatalog: async () => {
const activation = activateModelCatalog(hostedCatalog);
return {
status: activation.status === 'restart_required'
? 'restart_required'
: 'activated',
revision: activation.snapshot.revision,
};
},
}
: {}),
});
const model = resolveModel(route.model);
const piProvider = registeredPiProvider(model.provider);
assert.ok(piProvider, `Pi provider ${model.provider} must be registered`);
const marker = 'CHICKPEA_MODEL_COMPATIBILITY_OK';
// An API-key lane sends only inside model access, which reads the key from env.
const result = await withStatelessModelAccess(route.model, { env: undefined, settings, runId: 'live-model-compatibility', purpose: 'reply' }, () =>
piProvider.streamSimple(
model,
{
messages: [{
role: 'user',
content: `Return exactly ${marker} and nothing else.`,
timestamp: Date.now(),
}],
},
{
...(lane === 'subscription' ? { apiKey: 'chickpea-boundary-managed' } : {}),
...(reasoning === undefined ? {} : { reasoning }),
// Reasoning tokens count against the output limit.
maxTokens: reasoning === undefined || reasoning === 'off' ? 64 : 4_096,
},
).result());
const output = result.content
.filter((part) => part.type === 'text')
.map((part) => part.text)
.join('');
assert.match(output, new RegExp(marker));
assert.equal(result.stopReason === 'error', false);
assert.ok(destinations.includes(expectedHost), `model traffic must reach ${expectedHost}`);
assert.equal(
destinations.every((host) =>
host === expectedHost || (lane === 'subscription' && host === 'auth.openai.com')
),
true,
);
console.log(JSON.stringify({
ok: true,
lane,
canonicalModel,
runtimeModel: route.model,
provider: result.provider,
destination: expectedHost,
reasoning: reasoning ?? null,
...(provider === 'openai' ? { sentReasoningEfforts } : {}),
outputVerified: true,
}));
} finally {
clearOpenAiSubscriptionTransport();
globalThis.fetch = nativeFetch;
settings.close();
}
function parseArgs(values) {
const result = new Map();
for (let index = 0; index < values.length; index += 1) {
const value = values[index];
if (!value?.startsWith('--')) continue;
const next = values[index + 1];
if (next && !next.startsWith('--')) {
result.set(value, next);
index += 1;
} else {
result.set(value, true);
}
}
return result;
}
async function readHostedCatalog(file) {
const bytes = new Uint8Array(await readFile(file));
return {
document: parseModelCatalogBytes(bytes),
sha256: createHash('sha256').update(bytes).digest('hex'),
};
}