test/backend-router.test.js

Rama master · réplica de lectura del repositorio original.

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { selectBackend, usesCascadeFlow, BACKEND } from '../src/backend-router.js';
import { MODELS } from '../src/models.js';

// Behaviour-preserving decision table. These cases mirror the inline routing
// logic in handlers/chat.js (isSpecialAgentModelInfo → special-agent, then
// useCascade = !!(modelUid || enumValue), else legacy). If any of these change
// the router has diverged from the legacy behaviour P1 promised to preserve.

describe('backend-router selectBackend — behaviour parity with legacy', () => {
  it('special_agent backend + default mode → devin-print', () => {
    const sel = selectBackend({ modelInfo: { backend: 'special_agent' }, env: {} });
    assert.equal(sel.backend, BACKEND.DEVIN_PRINT);
    assert.equal(sel.flow, 'special_agent');
  });

  it('special_agent backend + DEVIN_CLI_MODE=acp → devin-acp', () => {
    const sel = selectBackend({ modelInfo: { backend: 'special_agent' }, env: { DEVIN_CLI_MODE: 'acp' } });
    assert.equal(sel.backend, BACKEND.DEVIN_ACP);
    assert.equal(sel.flow, 'special_agent');
  });

  it('special_agent wins even when a modelUid is also present', () => {
    const sel = selectBackend({ modelInfo: { backend: 'special_agent', modelUid: 'MODEL_X' }, env: {} });
    assert.equal(sel.flow, 'special_agent');
  });

  it('modelUid present → cascade', () => {
    const sel = selectBackend({ modelInfo: { modelUid: 'MODEL_CLAUDE_4_SONNET' }, env: {} });
    assert.equal(sel.backend, BACKEND.CASCADE);
    assert.equal(sel.flow, 'cascade');
    assert.equal(sel.reason, 'modelUid');
    assert.ok(usesCascadeFlow(sel));
  });

  it('gpt-5.6 stays on cascade unless Devin CLI special-agent is enabled', () => {
    const sel = selectBackend({
      modelInfo: { name: 'gpt-5.6-luna', modelUid: 'gpt-5-6-luna-medium' },
      env: {},
    });
    assert.equal(sel.flow, 'cascade');
  });

  it('gpt-5.6-luna → ACP when DEVIN_CLI_ENABLED=1 and DEVIN_CLI_MODE=acp', () => {
    const sel = selectBackend({
      modelInfo: { name: 'gpt-5.6-luna', modelUid: 'gpt-5-6-luna-medium' },
      env: { DEVIN_CLI_ENABLED: '1', DEVIN_CLI_MODE: 'acp' },
    });
    assert.equal(sel.backend, BACKEND.DEVIN_ACP);
    assert.equal(sel.flow, 'special_agent');
    assert.equal(sel.reason, 'devin_local_only');
  });

  it('gpt-5.6-luna → print when CLI is on but mode is not acp', () => {
    const sel = selectBackend({
      modelInfo: { name: 'gpt-5.6-luna', modelUid: 'gpt-5-6-luna-medium' },
      env: { DEVIN_CLI_ENABLED: '1' },
    });
    assert.equal(sel.backend, BACKEND.DEVIN_PRINT);
    assert.equal(sel.reason, 'devin_local_only');
  });

  it('does not treat gpt-5.60 as a Devin Local gpt-5.6 SKU', () => {
    const env = { DEVIN_CLI_ENABLED: '1', DEVIN_CLI_MODE: 'acp' };
    // No uid: hijack would be special_agent / devin_local_only.
    const noUid = selectBackend({ modelInfo: { name: 'gpt-5.60' }, env });
    assert.equal(noUid.flow, 'legacy');
    assert.notEqual(noUid.reason, 'devin_local_only');
    // With uid the fall-through is cascade, still not the local-only branch.
    const withUid = selectBackend({
      modelInfo: { name: 'gpt-5.60', modelUid: 'gpt-5-60' },
      env,
    });
    assert.equal(withUid.flow, 'cascade');
    assert.equal(withUid.reason, 'modelUid');
    assert.notEqual(withUid.reason, 'devin_local_only');
  });

  it('every catalogued gpt-6 SKU → Devin Local when the CLI is enabled (DEVIN_CONNECT=0)', () => {
    const keys = Object.keys(MODELS).filter((k) => k.startsWith('gpt-6'));
    assert.ok(keys.length >= 20, `expected the gpt-6 family in MODELS, found ${keys.length}`);
    for (const key of keys) {
      const sel = selectBackend({
        modelInfo: MODELS[key],
        env: { DEVIN_CONNECT: '0', DEVIN_CLI_ENABLED: '1', DEVIN_CLI_MODE: 'acp' },
      });
      assert.equal(sel.backend, BACKEND.DEVIN_ACP, key);
      assert.equal(sel.flow, 'special_agent', key);
      assert.equal(sel.reason, 'devin_local_only', key);
    }
  });

  it('gpt-6-luna-max → print when the CLI is on but mode is not acp', () => {
    const sel = selectBackend({
      modelInfo: MODELS['gpt-6-luna-max'],
      env: { DEVIN_CONNECT: '0', DEVIN_CLI_ENABLED: '1' },
    });
    assert.equal(sel.backend, BACKEND.DEVIN_PRINT);
    assert.equal(sel.reason, 'devin_local_only');
  });

  it('gpt-6-luna-max → devin-connect when DEVIN_CONNECT=1 (connect wins)', () => {
    const sel = selectBackend({
      modelInfo: MODELS['gpt-6-luna-max'],
      env: { DEVIN_CONNECT: '1', DEVIN_CLI_ENABLED: '1', DEVIN_CLI_MODE: 'acp' },
    });
    assert.equal(sel.backend, BACKEND.DEVIN_CONNECT);
    assert.equal(sel.flow, 'devin_connect');
    assert.equal(sel.reason, 'devin_connect');
  });

  it('gpt-6 lookalikes are not hijacked into the local-only branch', () => {
    const env = { DEVIN_CONNECT: '0', DEVIN_CLI_ENABLED: '1', DEVIN_CLI_MODE: 'acp' };
    for (const name of ['gpt-6', 'gpt-60-luna', 'gpt-6-lunar', 'gpt-6.0-luna']) {
      const sel = selectBackend({ modelInfo: { name, modelUid: name }, env });
      assert.equal(sel.reason, 'modelUid', name);
      assert.equal(sel.flow, 'cascade', name);
    }
  });

  it('gpt-6-luna-max stays on cascade when no Devin backend is configured', () => {
    // DEVIN_CONNECT=0 with the CLI off leaves no Devin transport, so the
    // documented operator fix is to enable one (DEVIN_CONNECT=1 or
    // DEVIN_CLI_ENABLED=1). Pinned so this fall-through stays deliberate.
    const sel = selectBackend({
      modelInfo: MODELS['gpt-6-luna-max'],
      env: { DEVIN_CONNECT: '0' },
    });
    assert.equal(sel.flow, 'cascade');
    assert.equal(sel.reason, 'modelUid');
  });

  it('swe-2 catalog entries use backend=special_agent (not the gpt-5.6 local-only branch)', () => {
    const sel = selectBackend({
      modelInfo: { name: 'swe-2-medium', modelUid: 'swe-2-medium', backend: 'special_agent' },
      env: { DEVIN_CLI_ENABLED: '1', DEVIN_CLI_MODE: 'acp' },
    });
    assert.equal(sel.backend, BACKEND.DEVIN_ACP);
    assert.equal(sel.flow, 'special_agent');
    assert.equal(sel.reason, 'modelInfo.backend=special_agent');
  });

  it('DEVIN_CONNECT=1 still wins over swe-2 special_agent', () => {
    const sel = selectBackend({
      modelInfo: { name: 'swe-2-medium', modelUid: 'swe-2-medium', backend: 'special_agent' },
      env: { DEVIN_CONNECT: '1', DEVIN_CLI_ENABLED: '1', DEVIN_CLI_MODE: 'acp' },
    });
    assert.equal(sel.backend, BACKEND.DEVIN_CONNECT);
    assert.equal(sel.reason, 'devin_connect');
  });

  it('enumValue > 0 (no uid) → cascade', () => {
    const sel = selectBackend({ modelInfo: { enumValue: 166 }, env: {} });
    assert.equal(sel.backend, BACKEND.CASCADE);
    assert.equal(sel.reason, 'enumValue');
    assert.ok(usesCascadeFlow(sel));
  });

  it('no uid and no enum → legacy', () => {
    const sel = selectBackend({ modelInfo: { enumValue: 0 }, env: {} });
    assert.equal(sel.backend, BACKEND.LEGACY);
    assert.equal(sel.flow, 'legacy');
    assert.equal(usesCascadeFlow(sel), false);
  });

  it('null modelInfo → legacy (defensive)', () => {
    const sel = selectBackend({ modelInfo: null, env: {} });
    assert.equal(sel.backend, BACKEND.LEGACY);
    assert.equal(sel.flow, 'legacy');
  });

  it('uses process.env by default (no env arg)', () => {
    // Smoke: must not throw and must return a known backend.
    const sel = selectBackend({ modelInfo: { modelUid: 'X' } });
    assert.ok(Object.values(BACKEND).includes(sel.backend));
  });

  it('DEVIN_REST constant exists but is never auto-selected in P1', () => {
    // P1 must not route anything to devin-rest yet.
    const cases = [
      { backend: 'special_agent' },
      { modelUid: 'X' },
      { enumValue: 5 },
      { enumValue: 0 },
      null,
    ];
    for (const modelInfo of cases) {
      const sel = selectBackend({ modelInfo, env: {} });
      assert.notEqual(sel.backend, BACKEND.DEVIN_REST);
    }
  });
});

// DEVIN_ONLY: Cascade retired — every request is forced onto Devin regardless
// of the model. This is the "Devin is the only core" kill-switch.
describe('backend-router selectBackend — DEVIN_ONLY (Cascade retired)', () => {
  it('forces a cascade model (claude) onto Devin when DEVIN_ONLY=1', () => {
    const sel = selectBackend({
      modelInfo: { modelUid: 'MODEL_CLAUDE_4_5_SONNET', enumValue: 200 },
      env: { DEVIN_ONLY: '1' },
    });
    assert.equal(sel.flow, 'special_agent');
    assert.equal(sel.reason, 'devin_only');
    assert.equal(sel.backend, BACKEND.DEVIN_PRINT); // default sub-mode
    assert.ok(!usesCascadeFlow(sel), 'no longer a cascade flow');
  });

  it('honours DEVIN_CLI_MODE=acp under DEVIN_ONLY', () => {
    const sel = selectBackend({
      modelInfo: { modelUid: 'MODEL_GPT_5' },
      env: { DEVIN_ONLY: '1', DEVIN_CLI_MODE: 'acp' },
    });
    assert.equal(sel.flow, 'special_agent');
    assert.equal(sel.backend, BACKEND.DEVIN_ACP);
  });

  it('forces even a legacy (no uid, no enum) model onto Devin', () => {
    const sel = selectBackend({ modelInfo: { enumValue: 0 }, env: { DEVIN_ONLY: '1' } });
    assert.equal(sel.flow, 'special_agent');
    assert.equal(sel.reason, 'devin_only');
  });

  it('forces null modelInfo onto Devin (defensive)', () => {
    const sel = selectBackend({ modelInfo: null, env: { DEVIN_ONLY: '1' } });
    assert.equal(sel.flow, 'special_agent');
    assert.equal(sel.reason, 'devin_only');
  });

  it('DEVIN_ONLY wins over a special_agent model too (same flow, devin_only reason)', () => {
    const sel = selectBackend({ modelInfo: { backend: 'special_agent' }, env: { DEVIN_ONLY: '1' } });
    assert.equal(sel.flow, 'special_agent');
    assert.equal(sel.reason, 'devin_only');
  });

  it('DEVIN_ONLY=0 / unset leaves legacy routing intact (cascade still selected)', () => {
    const off = selectBackend({ modelInfo: { modelUid: 'MODEL_CLAUDE_4_5_SONNET' }, env: { DEVIN_ONLY: '0' } });
    assert.equal(off.flow, 'cascade');
    const unset = selectBackend({ modelInfo: { modelUid: 'MODEL_CLAUDE_4_5_SONNET' }, env: {} });
    assert.equal(unset.flow, 'cascade');
  });

  it('only the exact value "1" enables DEVIN_ONLY (truthy-string guard)', () => {
    for (const v of ['true', 'yes', '2', 'on', ' ']) {
      const sel = selectBackend({ modelInfo: { modelUid: 'MODEL_X' }, env: { DEVIN_ONLY: v } });
      assert.equal(sel.flow, 'cascade', `DEVIN_ONLY=${JSON.stringify(v)} must NOT enable`);
    }
    // surrounding whitespace around "1" is tolerated
    const padded = selectBackend({ modelInfo: { modelUid: 'MODEL_X' }, env: { DEVIN_ONLY: ' 1 ' } });
    assert.equal(padded.flow, 'special_agent');
  });
});

// DEVIN_CONNECT: pure-HTTP cloud egress (no local CLI). Highest precedence —
// retires both Cascade and the Devin CLI.
describe('backend-router selectBackend — DEVIN_CONNECT (pure-HTTP egress)', () => {
  it('routes a cascade model to DEVIN_CONNECT when DEVIN_CONNECT=1', () => {
    const sel = selectBackend({
      modelInfo: { modelUid: 'MODEL_CLAUDE_4_5_SONNET' },
      env: { DEVIN_CONNECT: '1' },
    });
    assert.equal(sel.backend, BACKEND.DEVIN_CONNECT);
    assert.equal(sel.flow, 'devin_connect');
    assert.equal(sel.reason, 'devin_connect');
  });

  it('routes a legacy (no-uid-no-enum) request to DEVIN_CONNECT too', () => {
    const sel = selectBackend({ modelInfo: { enumValue: 0 }, env: { DEVIN_CONNECT: '1' } });
    assert.equal(sel.backend, BACKEND.DEVIN_CONNECT);
  });

  it('wins over DEVIN_ONLY when both are set', () => {
    const sel = selectBackend({
      modelInfo: { backend: 'special_agent' },
      env: { DEVIN_CONNECT: '1', DEVIN_ONLY: '1' },
    });
    assert.equal(sel.backend, BACKEND.DEVIN_CONNECT);
    assert.equal(sel.flow, 'devin_connect');
  });

  it('wins over a special_agent model', () => {
    const sel = selectBackend({ modelInfo: { backend: 'special_agent' }, env: { DEVIN_CONNECT: '1' } });
    assert.equal(sel.backend, BACKEND.DEVIN_CONNECT);
  });

  it('DEVIN_CONNECT=0 / unset leaves routing intact', () => {
    const off = selectBackend({ modelInfo: { modelUid: 'MODEL_X' }, env: { DEVIN_CONNECT: '0' } });
    assert.equal(off.flow, 'cascade');
    const viaOnly = selectBackend({ modelInfo: { modelUid: 'MODEL_X' }, env: { DEVIN_ONLY: '1' } });
    assert.equal(viaOnly.flow, 'special_agent');
  });

  it('only the exact value "1" enables DEVIN_CONNECT (truthy-string guard)', () => {
    for (const v of ['true', 'yes', '2', 'on', ' ']) {
      const sel = selectBackend({ modelInfo: { modelUid: 'MODEL_X' }, env: { DEVIN_CONNECT: v } });
      assert.notEqual(sel.flow, 'devin_connect', `DEVIN_CONNECT=${JSON.stringify(v)} must NOT enable`);
    }
    const padded = selectBackend({ modelInfo: { modelUid: 'MODEL_X' }, env: { DEVIN_CONNECT: ' 1 ' } });
    assert.equal(padded.flow, 'devin_connect');
  });
});

// handlers/chat.js calls selectBackend TWICE per request — once early at
// chat.js:2151 (`selectBackend({ modelInfo }).flow === 'devin_connect'`, to
// short-circuit into the DEVIN_CONNECT egress BEFORE the Cascade "unsupported
// model" gate) and once as the main decision at chat.js:2779. Between those two
// calls `modelInfo` can be REASSIGNED (chat.js:2764, the model-blocked →
// default-model fallback). If the two calls could disagree, a request could be
// early-routed to DEVIN_CONNECT and then main-routed elsewhere (or vice versa),
// splitting the pipeline. These tests pin the invariants that keep the two
// calls in lockstep so a future edit to selectBackend can't silently fork them.
describe('backend-router selectBackend — double-call consistency (chat.js:2151 vs :2779)', () => {
  it('is a pure function: repeated calls with identical input are byte-identical', () => {
    const cases = [
      { modelInfo: { modelUid: 'MODEL_CLAUDE_4_5_SONNET' }, env: {} },
      { modelInfo: { backend: 'special_agent' }, env: { DEVIN_CLI_MODE: 'acp' } },
      { modelInfo: { enumValue: 0 }, env: { DEVIN_ONLY: '1' } },
      { modelInfo: { modelUid: 'X' }, env: { DEVIN_CONNECT: '1' } },
      { modelInfo: null, env: {} },
    ];
    for (const args of cases) {
      const a = selectBackend(args);
      const b = selectBackend(args);
      assert.deepEqual(a, b, `selectBackend must be deterministic for ${JSON.stringify(args)}`);
    }
  });

  it('devin_connect decision is env-only — invariant to any modelInfo swap at chat.js:2764', () => {
    // This is THE invariant protecting the early short-circuit: the 2151 check
    // only looks at .flow==='devin_connect', which selectBackend derives purely
    // from env (devinConnectEnabled). So the model-blocked fallback reassigning
    // modelInfo between 2151 and 2779 CANNOT flip the devin_connect routing.
    const env = { DEVIN_CONNECT: '1' };
    const modelInfos = [
      { modelUid: 'MODEL_CLAUDE_4_5_SONNET' },   // original requested model
      { modelUid: 'MODEL_DEFAULT_FALLBACK' },    // reassigned at 2764
      { backend: 'special_agent' },
      { enumValue: 0 },
      null,
    ];
    for (const modelInfo of modelInfos) {
      assert.equal(
        selectBackend({ modelInfo, env }).flow, 'devin_connect',
        `devin_connect must hold regardless of modelInfo=${JSON.stringify(modelInfo)}`
      );
    }
    // And when DEVIN_CONNECT is off, NO modelInfo can early-route to it.
    for (const modelInfo of modelInfos) {
      assert.notEqual(selectBackend({ modelInfo, env: {} }).flow, 'devin_connect');
    }
  });

  it('same modelInfo at both call sites → identical backend/flow/reason', () => {
    // Simulate chat.js: modelInfo resolved once, used at 2151 and again at 2779
    // WITHOUT the blocked-fallback reassignment (the common path). Both call
    // sites must land on the same backend.
    for (const env of [{}, { DEVIN_ONLY: '1' }, { DEVIN_CONNECT: '1' }]) {
      const modelInfo = { modelUid: 'MODEL_CLAUDE_4_5_SONNET', enumValue: 200 };
      const early = selectBackend({ modelInfo, env });
      const main = selectBackend({ modelInfo, env });
      assert.equal(early.backend, main.backend);
      assert.equal(early.flow, main.flow);
      assert.equal(early.reason, main.reason);
    }
  });
});