test/dashboard-models-connect-parity.test.js
Rama master · réplica de lectura del repositorio original.
// Dashboard /models and /v1/models must answer the SAME question about the Connect
// namespace (#234's last acceptance criterion).
//
// They did not. With DEVIN_CONNECT=1 on a free-only pool, measured before the fix:
//
// /v1/models 1 row (swe-1-6-slow)
// dashboard /models 163 rows, ZERO overlap — every model the account cannot call,
// and the one it CAN was absent entirely
//
// The dashboard resolved through the Cascade namespace (filterModelKeysByCloudCatalog,
// which early-returns unfiltered when devinConnect is on — correct as a namespace
// boundary) while /v1/models resolved through the Connect one. Two views deriving one
// rule from two namespaces: this repo's most frequent defect shape.
//
// The fix is one exported predicate (buildConnectReachability) that both call, so these
// tests are written to fail if a future change lands on one side only.
//
// Two properties beyond parity, both load-bearing:
//
// 1. rows are ANNOTATED, not dropped. The panel renders the allow/deny chips from this
// list alone, so narrowing it would make an existing allow-list entry invisible and
// therefore unremovable — the operator cannot click a chip that is not rendered.
// 2. the free-reachable selector is SYNTHESIZED. `swe-1-6-slow` is in neither the frozen
// snapshot nor the live catalog nor MODELS, so no Cascade-derived list contains it —
// yet it is the only selector every account can serve.
import { afterEach, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { config } from '../src/config.js';
import {
__resetModelCatalogState,
__setModelCatalogDeps,
__waitForModelCatalogSync,
addAccountByKey,
configureBindHost,
getAccountInternal,
removeAccount,
} from '../src/auth.js';
import { handleDashboardApi } from '../src/dashboard/api.js';
import { handleModels, buildConnectReachability } from '../src/handlers/models.js';
import { setLiveCatalogSelectors, clearLiveCatalogSelectors, resolveConnectSelector, __testing as connectModelsTesting } from '../src/devin-connect-models.js';
import { mergeCloudModels } from '../src/models.js';
import { _resetRuntimeConfigForTests } from '../src/runtime-config.js';
// Written literally rather than imported from FREE_REACHABLE_SELECTORS: importing the set
// the implementation reads would make the assertion pass whatever that set contains — the
// "guard satisfied by its own source" antipattern this repo has been bitten by.
const CONNECT_FREE_SELECTOR = 'swe-1-6-slow';
// A paid Cascade-namespace model. Free accounts must see it listed-but-unreachable.
const PAID_KEY = 'claude-4-sonnet';
const ORIGINAL_ALLOW_NO_AUTH = process.env.DASHBOARD_ALLOW_NO_AUTH;
const ORIGINAL_DEVIN_CONNECT = process.env.DEVIN_CONNECT;
const ORIGINAL_DEVIN_CONNECT_TOKEN = process.env.DEVIN_CONNECT_TOKEN;
const ORIGINAL_WINDSURF_API_KEY = process.env.WINDSURF_API_KEY;
const ORIGINAL_API_KEY = config.apiKey;
const ORIGINAL_DASHBOARD_PASSWORD = config.dashboardPassword;
const created = [];
let liveCatalogDirty = false;
function fakeRes() {
return {
statusCode: 0,
body: '',
writeHead(status) { this.statusCode = status; },
end(chunk) { this.body += chunk ? String(chunk) : ''; },
json() { return this.body ? JSON.parse(this.body) : null; },
};
}
const localReq = (path) => ({
url: `/dashboard/api${path}`,
headers: {},
socket: { remoteAddress: '127.0.0.1' },
});
function seed(tier, apiKey = null) {
const a = addAccountByKey(
apiKey || `devin-session-token$xk-parity-${Math.random().toString(36).slice(2)}`,
`parity-${tier}`,
);
created.push(a.id);
const acct = getAccountInternal(a.id);
acct.tier = tier;
acct.status = 'active';
return acct;
}
async function dashboardModels() {
const res = fakeRes();
await handleDashboardApi('GET', '/models', {}, localReq('/models'), res);
// Guard the measurement itself: a 401 yields zero rows, and "every row agrees" is
// vacuously true over zero rows. The first draft of this probe reported success off a
// 401 for exactly that reason.
assert.equal(res.statusCode, 200, `dashboard /models returned ${res.statusCode}`);
const rows = res.json()?.models || [];
assert.ok(rows.length > 0, 'dashboard /models returned no rows — nothing was measured');
return rows;
}
function v1Ids() {
const data = handleModels(process.env)?.data || [];
assert.ok(data.length > 0, '/v1/models returned no rows — nothing to compare against');
return new Set(data.map((m) => m.id));
}
beforeEach(() => {
process.env.DEVIN_CONNECT = '1';
delete process.env.DEVIN_CONNECT_TOKEN;
delete process.env.WINDSURF_API_KEY;
process.env.DASHBOARD_ALLOW_NO_AUTH = '1';
// The suite must not inherit the operator's real env/file-backed dashboard
// credentials. A configured API key or runtime dashboard-password hash makes
// the deliberately open-local fixture return 401 before /models is measured.
_resetRuntimeConfigForTests();
config.apiKey = '';
config.dashboardPassword = '';
configureBindHost('127.0.0.1');
// These are namespace/view tests. Keep them deterministic and offline: account
// additions must not launch real catalog RPCs with fixture tokens.
__setModelCatalogDeps({
disableConnectSync: true,
getCascadeModelConfigs: async () => ({ configs: [] }),
});
});
afterEach(async () => {
// Only ids this file created. NEVER map removeAccount over the account list — a cleanup
// written that way once deleted a user's real accounts, unrecoverably.
for (const id of created.splice(0)) { try { removeAccount(id); } catch {} }
await __waitForModelCatalogSync();
__resetModelCatalogState();
__setModelCatalogDeps(null);
if (liveCatalogDirty) {
clearLiveCatalogSelectors();
liveCatalogDirty = false;
}
_resetRuntimeConfigForTests();
if (ORIGINAL_DEVIN_CONNECT === undefined) delete process.env.DEVIN_CONNECT;
else process.env.DEVIN_CONNECT = ORIGINAL_DEVIN_CONNECT;
if (ORIGINAL_DEVIN_CONNECT_TOKEN === undefined) delete process.env.DEVIN_CONNECT_TOKEN;
else process.env.DEVIN_CONNECT_TOKEN = ORIGINAL_DEVIN_CONNECT_TOKEN;
if (ORIGINAL_WINDSURF_API_KEY === undefined) delete process.env.WINDSURF_API_KEY;
else process.env.WINDSURF_API_KEY = ORIGINAL_WINDSURF_API_KEY;
if (ORIGINAL_ALLOW_NO_AUTH === undefined) delete process.env.DASHBOARD_ALLOW_NO_AUTH;
else process.env.DASHBOARD_ALLOW_NO_AUTH = ORIGINAL_ALLOW_NO_AUTH;
config.apiKey = ORIGINAL_API_KEY;
config.dashboardPassword = ORIGINAL_DASHBOARD_PASSWORD;
configureBindHost('0.0.0.0');
});
describe('dashboard /models agrees with /v1/models on the Connect namespace (#234)', () => {
it('every row\'s reachable flag matches whether /v1/models lists it', async () => {
seed('free');
const ids = v1Ids();
const rows = await dashboardModels();
const disagree = rows.filter((r) => r.reachable !== ids.has(r.id));
assert.deepEqual(
disagree.map((r) => `${r.id}:dash=${r.reachable},v1=${ids.has(r.id)}`),
[],
'a row disagrees with /v1/models. Before the fix EVERY row disagreed: the dashboard '
+ 'listed 163 unreachable models and omitted the one reachable selector',
);
// Pin that the comparison had teeth in BOTH directions — an all-true or all-false
// answer would satisfy the equality above while measuring nothing.
assert.ok(rows.some((r) => r.reachable), 'no row was reachable — the filter is stuck closed');
assert.ok(rows.some((r) => !r.reachable), 'every row was reachable — the filter is stuck open');
});
it('lists paid models as unreachable rather than dropping them', async () => {
seed('free');
const rows = await dashboardModels();
const paid = rows.find((r) => r.id === PAID_KEY);
assert.ok(paid, `${PAID_KEY} must still be LISTED — the allow/deny panel renders only `
+ 'this list, so dropping a row makes an existing allow-list entry unclickable');
assert.equal(paid.reachable, false, `${PAID_KEY} is not callable by a free account`);
});
it('synthesizes the free-reachable selector, which no Cascade-derived list contains', async () => {
seed('free');
const rows = await dashboardModels();
const free = rows.find((r) => r.id === CONNECT_FREE_SELECTOR);
assert.ok(free, `${CONNECT_FREE_SELECTOR} absent. It is in neither the snapshot, the `
+ 'live catalog, nor MODELS, so it has to be synthesized — and it is the only '
+ 'selector every account can serve');
assert.equal(free.reachable, true);
});
it('reports the resolved selector so "unreachable" and "unmapped" are distinguishable', async () => {
seed('free');
const rows = await dashboardModels();
const free = rows.find((r) => r.id === CONNECT_FREE_SELECTOR);
assert.equal(free.connectSelector, CONNECT_FREE_SELECTOR);
// Every row carries the key, present-but-null included: a missing key would make the
// UI unable to tell "resolves but you lack entitlement" from "maps to nothing here".
for (const r of rows) {
assert.ok('connectSelector' in r, `${r.id} is missing connectSelector`);
}
});
it('includes live-only selectors that /v1/models synthesizes', async () => {
seed('pro');
const liveOnly = 'grok-4-5-medium-dashboard-parity';
const aliasBacked = 'claude-opus-4-6';
setLiveCatalogSelectors([
{ selector: ` ${liveOnly} `, provider: 'xai', label: 'Grok live only' },
{ selector: aliasBacked, provider: 'anthropic', label: 'Claude Opus 4.8 Medium' },
]);
liveCatalogDirty = true;
const ids = v1Ids();
assert.ok(ids.has(liveOnly), 'precondition: /v1/models synthesized the live-only selector');
const rows = await dashboardModels();
const row = rows.find((r) => r.id === liveOnly);
assert.ok(row, 'Dashboard omitted a selector that /v1/models advertises');
assert.equal(row.reachable, true);
assert.equal(row.connectSelector, liveOnly);
assert.ok('freeNow' in row,
'the live-only Dashboard producer must carry the pricing field too');
assert.equal(row.freeNow, null,
'a live-only selector without a rate-table entry has unknown pricing');
assert.equal(row.freeReason, null);
assert.equal(row.quotaState, 'unknown');
assert.equal(rows.filter((r) => r.reachable && r.connectSelector === aliasBacked).length, 1,
'Dashboard must not append a canonical row when a visible alias already represents it');
});
it('keeps a restricted live-only row visible but unreachable to a free pool', async () => {
seed('free');
const paidLiveOnly = 'gpt-5-6-sol-max-dashboard-parity';
setLiveCatalogSelectors([
{ selector: paidLiveOnly, provider: 'openai', label: 'Paid live only' },
]);
liveCatalogDirty = true;
const rows = await dashboardModels();
const row = rows.find((candidate) => candidate.id === paidLiveOnly);
assert.ok(row, 'the Dashboard must retain a live-only row so operators can inspect it');
assert.equal(row.reachable, false,
'a free-only pool must annotate the paid live-only row as unreachable');
assert.equal(row.connectSelector, paidLiveOnly);
assert.ok('freeNow' in row,
'an unreachable live-only row must still carry the pricing field');
assert.equal(row.freeNow, null,
'reachability does not turn absent pricing data into a billable claim');
assert.equal(row.quotaState, 'unknown');
});
it('keeps snapshot rows reachable for an env token beside a restricted live pool', async () => {
seed('free');
setLiveCatalogSelectors([
{ selector: 'claude-opus-4-8-medium', provider: 'anthropic' },
]);
liveCatalogDirty = true;
process.env.DEVIN_CONNECT_TOKEN = 'dashboard-parity-env-token';
const rows = await dashboardModels();
const snapshot = rows.find((row) => row.id === 'gpt-5.5');
assert.ok(snapshot, 'precondition: the snapshot-backed public row must remain listed');
assert.equal(snapshot.reachable, true,
'an env token is outside per-account live restrictions and retains snapshot fallback');
});
it('fails open on an empty pool even when a restricted live catalog is present', async () => {
setLiveCatalogSelectors([
{ selector: 'claude-opus-4-8-medium', provider: 'anthropic' },
]);
liveCatalogDirty = true;
const rows = await dashboardModels();
const snapshot = rows.find((row) => row.id === 'gpt-5.5');
assert.ok(snapshot, 'precondition: the snapshot-backed row must remain listed');
assert.equal(snapshot.reachable, true,
'without accounts there is no per-account authority to narrow discovery');
});
it('drops snapshot reachability when the final no-LKG contributor is removed', async () => {
__setModelCatalogDeps({
getCascadeModelConfigs: async () => ({ configs: [] }),
scheduleCatalogRetry: () => () => {},
fetchConnectCatalog: async ({ token }) => (
token === 'devin-session-token$xk-dashboard-live-fixture'
? [{ selector: 'claude-opus-4-8-medium', provider: 'anthropic' }]
: []
),
});
seed('pro', 'devin-session-token$xk-dashboard-live-fixture');
await __waitForModelCatalogSync();
const incomplete = seed('pro', 'devin-session-token$xk-dashboard-incomplete-fixture');
await __waitForModelCatalogSync();
const before = await dashboardModels();
assert.equal(before.find((row) => row.id === 'gpt-5.5')?.reachable, true,
'a contributor without LKG rows keeps the controlled snapshot fallback active');
removeAccount(incomplete.id);
const after = await dashboardModels();
assert.equal(after.find((row) => row.id === 'gpt-5.5')?.reachable, false,
'removing the final incomplete contributor makes the remaining live union authoritative');
});
it('widens when a paid account joins — the flag is computed, not baked in', async () => {
seed('free');
const before = await dashboardModels();
const reachableBefore = before.filter((r) => r.reachable).length;
seed('pro');
const after = await dashboardModels();
const reachableAfter = after.filter((r) => r.reachable).length;
assert.ok(
reachableAfter > reachableBefore,
`reachable count did not grow when a pro account joined (${reachableBefore} -> `
+ `${reachableAfter}). A hardcoded answer would pass every assertion above`,
);
});
// #235: the panel must say whether a model COSTS QUOTA, and must not guess when it cannot
// tell. The reporter had a pro account, set GLM-5-2 believing it was free on that plan, and
// burned the whole weekly allowance. The rate-table wiring that answers this shipped in
// v3.9.8 but had one production caller, inside the drought decision — the data existed and
// no surface showed it.
//
// The load-bearing case is UNKNOWN. `isConnectSelectorCurrentlyFree` returns a plain
// `false` both when a model costs money and when no account has a rate table at all, so
// reporting its result directly would render "we have no idea" as "billable" — and the
// mirror of that mistake, rendering unknown as free, is the reporter's exact loss.
describe('#235: whether a model costs quota', () => {
it('reports null (unknown), not false, when no account has a rate table', async () => {
// A seeded account has no credits.rateTable, so the pool-wide answer is "no data".
seed('free');
const rows = await dashboardModels();
const withSelector = rows.filter((r) => r.connectSelector);
assert.ok(withSelector.length > 0, 'precondition: some row must resolve to a selector');
// BOTH directions. A first draft asserted only "nothing is wrongly false" and a
// mutation that flattened unknown to TRUE survived it — the dangerous direction, since
// "free" on an unpriced model is precisely the reporter's loss. Assert the exact value.
const nonFree = withSelector.filter((r) => r.connectSelector !== CONNECT_FREE_SELECTOR);
assert.ok(nonFree.length > 0, 'precondition: need a resolving non-free selector');
const wrong = nonFree.filter((r) => r.freeNow !== null);
assert.deepEqual(wrong.map((r) => `${r.id}=${r.freeNow}`), [],
'with no rate table anywhere, a resolving non-free selector must read null (unknown). '
+ '`false` presents a guess as a fact; `true` is the exact claim that burned the '
+ 'reporter\'s weekly allowance. isConnectSelectorCurrentlyFree returns plain false '
+ 'for both "billable" and "no data", so the route must consult the pool-wide '
+ 'free-set to tell them apart');
});
it('reports the static-fallback selector as free only while there is no live data', async () => {
seed('free');
const rows = await dashboardModels();
const free = rows.find((r) => r.id === CONNECT_FREE_SELECTOR);
assert.equal(free.freeNow, true,
'with no live cost tier and no rate table, the documented static fallback is the '
+ 'only free source — the cold-start case it exists for');
assert.equal(free.freeReason, 'fallback_whitelist');
});
it('carries the full pricing/quota shape on every row so the UI never infers it', async () => {
seed('free');
const rows = await dashboardModels();
for (const r of rows) {
for (const key of ['costTier', 'promoEndsAt', 'freeNow', 'freeReason', 'quotaState']) {
assert.ok(key in r, `${r.id} is missing ${key}`);
}
assert.ok(
r.freeNow === true || r.freeNow === false || r.freeNow === null,
`${r.id} has a non-tri-state freeNow: ${JSON.stringify(r.freeNow)}`,
);
assert.ok(
r.quotaState === null || ['free', 'blocked_daily', 'blocked_weekly', 'not_in_catalog', 'unknown'].includes(r.quotaState),
`${r.id} has an unknown quotaState: ${JSON.stringify(r.quotaState)}`,
);
}
});
it('reports false — not unknown — once a rate table says the selector is billable', async () => {
// Without this, "always null" would satisfy every assertion above.
const acct = seed('free');
const rows0 = await dashboardModels();
const target = rows0.find((r) => r.connectSelector && r.connectSelector !== CONNECT_FREE_SELECTOR);
assert.ok(target, 'precondition: need a resolving non-free selector to price');
acct.credits = { rateTable: { [target.connectSelector]: 4 } };
const rows1 = await dashboardModels();
const priced = rows1.find((r) => r.id === target.id);
assert.equal(priced.freeNow, false,
`${target.id} resolves to ${target.connectSelector} which the rate table prices at 4, `
+ 'so it must read as billable rather than unknown');
acct.credits = { rateTable: { [target.connectSelector]: 0 } };
const rows2 = await dashboardModels();
const free = rows2.find((r) => r.id === target.id);
assert.equal(free.freeNow, true,
'rate === 0 must read as free — otherwise the field is stuck and tells the operator '
+ 'nothing that changes');
assert.equal(free.freeReason, 'rate_zero');
});
});
// The live Pro account (2026-10-22) is the fixture: its catalog marks exactly three
// selectors MODEL_COST_TIER_FREE with a promo ending 2026-11-01, its rate table has
// zero zero-rate entries, its daily bucket is at 0 %, and the selector the static
// whitelist called free (`swe-1-6-slow`) is absent from the catalog and 502s.
// Before the fix the panel showed the inverse of all of that.
describe('rows carry upstream cost tier, promo and account quota state', () => {
const PROMO_END = '2099-12-31T23:59:59.999999999Z';
const CONNECT_ROWS = [
{ selector: 'swe-2-medium', provider: 'windsurf', label: 'SWE-2 Medium' },
{ selector: 'swe-2-high', provider: 'windsurf', label: 'SWE-2 High' },
{ selector: 'swe-2-max', provider: 'windsurf', label: 'SWE-2 Max' },
{ selector: 'swe-2-medium-lightning', provider: 'windsurf', label: 'SWE-2 Medium Lightning' },
];
const COST_CONFIGS = [
{ modelUid: 'swe-2-medium', creditMultiplier: 6, modelCostTier: 'MODEL_COST_TIER_FREE', promoStatus: { isActive: true, endDate: PROMO_END } },
{ modelUid: 'swe-2-high', creditMultiplier: 9, modelCostTier: 'MODEL_COST_TIER_FREE', promoStatus: { isActive: true, endDate: PROMO_END } },
{ modelUid: 'swe-2-max', creditMultiplier: 12, modelCostTier: 'MODEL_COST_TIER_FREE', promoStatus: { isActive: true, endDate: PROMO_END } },
{
modelUid: 'swe-2-medium-lightning',
creditMultiplier: 12,
modelCostTier: 'MODEL_COST_TIER_HIGH',
promoStatus: { isActive: true, endDate: '2027-01-01T07:59:59.999999999Z', label: '83% off' },
},
];
async function seedLiveProAccount() {
__setModelCatalogDeps({
getCascadeModelConfigs: async () => ({ configs: [] }),
scheduleCatalogRetry: () => () => {},
fetchConnectCatalog: async () => CONNECT_ROWS,
});
const acct = seed('pro');
acct.credits = { dailyPercent: 0, weeklyPercent: 47 };
await __waitForModelCatalogSync();
// The Cascade cost-tier map is account-scoped in models.js; mergeCloudModels
// (the legacy entry point) records it for the default catalog key, which the
// pool-wide lookup unions. Bypasses the shrink-confirmation guard so the
// fixture stays the five rows the upstream actually sent.
mergeCloudModels(COST_CONFIGS);
return acct;
}
it('marks the three cost-tier-FREE selectors free, with the promo end', async () => {
await seedLiveProAccount();
const rows = await dashboardModels();
for (const id of ['swe-2-medium', 'swe-2-high', 'swe-2-max']) {
const row = rows.find((r) => r.id === id);
assert.ok(row, `${id} missing from the panel`);
assert.equal(row.reachable, true, `${id} is in the live catalog and the tier is paid`);
assert.equal(row.costTier, 'MODEL_COST_TIER_FREE', id);
assert.equal(row.freeNow, true, id);
assert.equal(row.freeReason, 'cost_tier_free', id);
assert.equal(row.promoEndsAt, '2099-12-31T23:59:59.999Z', id);
assert.equal(row.quotaState, 'free', id);
}
});
it('marks a discounted paid selector blocked_daily instead of a bare QUOTA chip', async () => {
await seedLiveProAccount();
const rows = await dashboardModels();
const row = rows.find((r) => r.id === 'swe-2-medium-lightning');
assert.ok(row);
assert.equal(row.costTier, 'MODEL_COST_TIER_HIGH');
assert.equal(row.freeNow, false);
assert.equal(row.freeReason, null);
assert.equal(row.promoEndsAt, '2027-01-01T07:59:59.999Z',
'a discount is not free, but the operator still needs to see when it ends');
assert.equal(row.quotaState, 'blocked_daily',
'the account is daily-dry, which is why this selector cannot run right now');
});
it('marks the absent phantom not_in_catalog instead of FREE', async () => {
await seedLiveProAccount();
const rows = await dashboardModels();
const phantom = rows.find((r) => r.id === CONNECT_FREE_SELECTOR);
assert.ok(phantom, 'the row stays visible so an allow-list entry remains removable');
assert.equal(phantom.reachable, false,
'the account live catalog omits it, and it 502s with quota exhausted when called');
assert.equal(phantom.costTier, null);
assert.equal(phantom.freeNow, false,
'live pricing data exists and does not name it free — the static floor is not a claim');
assert.equal(phantom.freeReason, null);
assert.equal(phantom.quotaState, 'not_in_catalog');
});
});
it('treats every model as reachable when the Connect backend is off', async () => {
delete process.env.DEVIN_CONNECT;
_resetRuntimeConfigForTests();
seed('free');
const rows = await dashboardModels();
assert.ok(
rows.every((r) => r.reachable),
'on a Cascade deployment the Connect namespace does not apply, so nothing may be '
+ 'reported unreachable — reporting otherwise would grey out the whole panel',
);
assert.ok(
rows.every((r) => r.connectSelector === null),
'connectSelector must be null off the Connect transport, not a stale resolution',
);
});
});
describe('read-only discovery does not consume the one-time degrade warning', () => {
it('discovery is silent; the next real request warns exactly once', () => {
const name = 'zzscale-parity-discovery-ghost';
const warns = [];
const originalWarn = console.warn;
connectModelsTesting.degradeWarned.clear();
console.warn = (...args) => { warns.push(args.join(' ')); };
try {
const reach = buildConnectReachability({ ...process.env, DEVIN_CONNECT: '1' });
reach(name);
assert.equal(warns.filter((l) => l.includes(name)).length, 0,
'catalog discovery is not a paid request and must not spend the warning');
resolveConnectSelector(name);
assert.equal(warns.filter((l) => l.includes(name)).length, 1,
'the first real request after discovery must still warn');
} finally {
console.warn = originalWarn;
connectModelsTesting.degradeWarned.clear();
}
});
});