test/devin-acp-probe.test.js

Rama master · réplica de lectura del repositorio original.

import { afterEach, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { probeDevinCliAvailable, __resetDevinAcpProbeCache } from '../src/devin-acp.js';
import { handleChatCompletions } from '../src/handlers/chat.js';

// AP1: zero-billable proactive availability probe.
//
// The probe answers "is the Devin CLI runnable on this box?" by spawning
// `devin --version` ONLY — it never runs initialize/authenticate/session.new/
// session/prompt, so it cannot consume account quota or upstream tokens. These
// tests prove: availability detection (runnable / ENOENT / non-zero exit),
// caching + path-keyed invalidation, the DEVIN_ACP_PROBE=0 opt-out, that the
// probe NEVER speaks ACP (zero-billable), and that the handler fails fast with
// 503 backend_unavailable WITHOUT checking out an account. All against fakes /
// node itself — no real Devin CLI, no account, no network.

const ENV_KEYS = [
  'DEVIN_CLI_PATH',
  'DEVIN_ACP_PROBE',
  'DEVIN_ACP_PROBE_ARGS_JSON',
  'DEVIN_ACP_PROBE_TTL_MS',
  'DEVIN_ACP_PROBE_TIMEOUT_MS',
  'WINDSURFAPI_SPECIAL_AGENT_BACKEND',
  'DEVIN_CLI_ENABLED',
  'DEVIN_CLI_MODE',
  'DEVIN_CLI_USE_ACCOUNT_POOL',
];
const originalEnv = Object.fromEntries(ENV_KEYS.map(k => [k, process.env[k]]));
const tmpDirs = [];

beforeEach(() => {
  __resetDevinAcpProbeCache();
});

afterEach(() => {
  for (const k of ENV_KEYS) {
    if (originalEnv[k] === undefined) delete process.env[k];
    else process.env[k] = originalEnv[k];
  }
  __resetDevinAcpProbeCache();
  while (tmpDirs.length) {
    try { rmSync(tmpDirs.pop(), { recursive: true, force: true }); } catch { /* ignore */ }
  }
});

function mkScript(source) {
  const dir = mkdtempSync(join(tmpdir(), 'windsurfapi-acp-probe-'));
  tmpDirs.push(dir);
  const script = join(dir, 'fake-probe.mjs');
  writeFileSync(script, source, 'utf8');
  return script;
}

describe('probeDevinCliAvailable — zero-billable availability detection', () => {
  it('reports available when the binary runs and exits 0 (node --version)', async () => {
    process.env.DEVIN_CLI_PATH = process.execPath; // node, `node --version` → exit 0
    const r = await probeDevinCliAvailable();
    assert.equal(r.available, true);
    assert.equal(r.reason, 'ok');
    assert.equal(r.exitCode, 0);
    assert.equal(r.cached, false);
  });

  it('reports unavailable with reason "not_found" when the binary is missing (ENOENT)', async () => {
    process.env.DEVIN_CLI_PATH = join(tmpdir(), 'no-such-devin-binary-probe-xyz');
    const r = await probeDevinCliAvailable();
    assert.equal(r.available, false);
    assert.equal(r.reason, 'not_found');
  });

  it('stays available on a non-zero exit (binary present, just does not grok --version)', async () => {
    // A fake that exits non-zero regardless of args: presence is proven by the
    // fact it ran to exit at all, so we must NOT false-negative a working CLI.
    const script = mkScript(`process.exit(3);`);
    process.env.DEVIN_CLI_PATH = process.execPath;
    process.env.DEVIN_ACP_PROBE_ARGS_JSON = JSON.stringify([script]);
    const r = await probeDevinCliAvailable();
    assert.equal(r.available, true);
    assert.equal(r.reason, 'present_nonzero_exit');
    assert.equal(r.exitCode, 3);
  });

  it('honours DEVIN_ACP_PROBE=0 — skips the probe and assumes available (no spawn)', async () => {
    // Point at a missing binary: if the probe actually ran it would report
    // not_found. skipped:true proves it short-circuited without spawning.
    process.env.DEVIN_CLI_PATH = join(tmpdir(), 'no-such-devin-binary-skip');
    process.env.DEVIN_ACP_PROBE = '0';
    const r = await probeDevinCliAvailable();
    assert.equal(r.available, true);
    assert.equal(r.skipped, true);
    assert.equal(r.reason, 'probe_disabled');
  });

  it('caches the verdict within TTL and re-probes when the binary path changes', async () => {
    process.env.DEVIN_CLI_PATH = process.execPath;
    process.env.DEVIN_ACP_PROBE_TTL_MS = '60000';
    const first = await probeDevinCliAvailable();
    assert.equal(first.cached, false);
    const second = await probeDevinCliAvailable();
    assert.equal(second.cached, true);
    assert.equal(second.available, true);

    // Switching to a different (missing) path must invalidate the cache —
    // a stale "available" verdict for the old binary must not leak.
    process.env.DEVIN_CLI_PATH = join(tmpdir(), 'different-missing-binary');
    const third = await probeDevinCliAvailable();
    assert.equal(third.cached, false);
    assert.equal(third.available, false);
    assert.equal(third.reason, 'not_found');
  });

  it('force:true bypasses a fresh cache entry', async () => {
    process.env.DEVIN_CLI_PATH = process.execPath;
    await probeDevinCliAvailable();
    const forced = await probeDevinCliAvailable({ force: true });
    assert.equal(forced.cached, false);
  });

  it('reports unavailable (probe_timeout) for a binary that hangs past the timeout', async () => {
    // A fake that never exits. The probe must give up and treat it unavailable
    // rather than block routing forever.
    const script = mkScript(`setInterval(() => {}, 1000);`);
    process.env.DEVIN_CLI_PATH = process.execPath;
    process.env.DEVIN_ACP_PROBE_ARGS_JSON = JSON.stringify([script]);
    process.env.DEVIN_ACP_PROBE_TIMEOUT_MS = '300';
    const r = await probeDevinCliAvailable();
    assert.equal(r.available, false);
    assert.equal(r.reason, 'probe_timeout');
  });

  it('is zero-billable: the probe never speaks ACP (no initialize/authenticate/session/prompt)', async () => {
    // The fake records every JSON-RPC method it receives on stdin to a marker
    // file. The probe spawns with stdio ['ignore','ignore','ignore'] and writes
    // NOTHING, so the recorded method list must be empty — proving the probe
    // opens no session and sends no prompt (cannot consume quota/tokens).
    const dir = mkdtempSync(join(tmpdir(), 'windsurfapi-acp-probe-bill-'));
    tmpDirs.push(dir);
    const marker = join(dir, 'methods.json');
    const script = join(dir, 'fake-probe-record.mjs');
    writeFileSync(script, `
import readline from 'node:readline';
import { writeFileSync } from 'node:fs';
const methods = [];
writeFileSync(${JSON.stringify(marker)}, JSON.stringify(methods));
const rl = readline.createInterface({ input: process.stdin });
rl.on('line', line => {
  try { const m = JSON.parse(line); if (m.method) methods.push(m.method); } catch {}
  writeFileSync(${JSON.stringify(marker)}, JSON.stringify(methods));
});
// Exit promptly so the probe sees a clean close; no stdin should ever arrive.
setTimeout(() => process.exit(0), 50);
`, 'utf8');
    process.env.DEVIN_CLI_PATH = process.execPath;
    process.env.DEVIN_ACP_PROBE_ARGS_JSON = JSON.stringify([script]);
    process.env.DEVIN_ACP_PROBE_TIMEOUT_MS = '2000';
    const r = await probeDevinCliAvailable();
    assert.equal(r.available, true);
    assert.ok(existsSync(marker), 'fake probe target should have run');
    const recorded = JSON.parse(readFileSync(marker, 'utf8'));
    assert.deepEqual(recorded, [], 'probe must not send any JSON-RPC method (no session/prompt)');
  });
});

describe('handler — proactive probe gates ACP routing before checkout', () => {
  it('returns 503 backend_unavailable WITHOUT checking out an account when the probe fails', async () => {
    process.env.WINDSURFAPI_SPECIAL_AGENT_BACKEND = 'devin-cli';
    process.env.DEVIN_CLI_MODE = 'acp';

    let checkoutCalls = 0;
    let runnerCalls = 0;
    let reportedFailure = 0;
    const result = await handleChatCompletions({
      model: 'swe-1.6',
      messages: [{ role: 'user', content: 'hi' }],
    }, {
      specialAgent: {
        probeDevinCliAvailable: async () => ({ available: false, reason: 'not_found' }),
        checkoutAccount: () => { checkoutCalls++; return { id: 'a', apiKey: 'k' }; },
        runDevinAcp: async () => { runnerCalls++; return { text: 'x' }; },
        reportError: () => { reportedFailure++; },
      },
    });

    assert.equal(result.status, 503);
    assert.equal(result.body.error.type, 'backend_unavailable');
    assert.equal(result.body.error.probe_reason, 'not_found');
    assert.equal(checkoutCalls, 0, 'must not reserve an account when the CLI is unavailable');
    assert.equal(runnerCalls, 0, 'must not spawn the runner when the CLI is unavailable');
    assert.equal(reportedFailure, 0, 'pool health must be untouched — this is an env fault, not an account fault');
  });

  it('proceeds to checkout + runner when the probe reports available', async () => {
    process.env.WINDSURFAPI_SPECIAL_AGENT_BACKEND = 'devin-cli';
    process.env.DEVIN_CLI_MODE = 'acp';

    let checkoutCalls = 0;
    const result = await handleChatCompletions({
      model: 'swe-1.6',
      messages: [{ role: 'user', content: 'hi' }],
    }, {
      specialAgent: {
        probeDevinCliAvailable: async () => ({ available: true, reason: 'ok' }),
        checkoutAccount: () => { checkoutCalls++; return { id: 'a', apiKey: 'k', apiServerUrl: '' }; },
        runDevinAcp: async () => ({ text: 'ACP_OK' }),
        releaseAccount: () => {},
      },
    });

    assert.equal(result.status, 200);
    assert.equal(result.body.choices[0].message.content, 'ACP_OK');
    assert.equal(checkoutCalls, 1);
  });

  it('does NOT probe in print mode (print keeps its own post-spawn ENOENT handling)', async () => {
    process.env.WINDSURFAPI_SPECIAL_AGENT_BACKEND = 'devin-cli';
    process.env.DEVIN_CLI_MODE = 'print';
    process.env.DEVIN_CLI_USE_ACCOUNT_POOL = '0';

    let probeCalls = 0;
    const result = await handleChatCompletions({
      model: 'swe-1.6',
      messages: [{ role: 'user', content: 'hi' }],
    }, {
      specialAgent: {
        probeDevinCliAvailable: async () => { probeCalls++; return { available: false }; },
        runDevinPrint: async () => ({ text: 'PRINT_OK' }),
      },
    });

    assert.equal(probeCalls, 0, 'print mode must not invoke the ACP probe');
    assert.equal(result.status, 200);
    assert.equal(result.body.choices[0].message.content, 'PRINT_OK');
  });
});

describe('reportRunFailure — 503 backend_unavailable does not burn pool health (gap-c)', () => {
  // reportRunFailure is private; exercise it through the handler by injecting a
  // runner that throws the relevant error and spying on the pool-feedback deps.
  async function runWithFailure(err, extraDeps = {}) {
    process.env.WINDSURFAPI_SPECIAL_AGENT_BACKEND = 'devin-cli';
    process.env.DEVIN_CLI_MODE = 'acp';
    process.env.DEVIN_ACP_PROBE = '0'; // skip the gate so the runner actually runs
    const calls = { generic: 0, rateLimited: 0, internal: 0, ban: 0 };
    const result = await handleChatCompletions({
      model: 'swe-1.6',
      messages: [{ role: 'user', content: 'hi' }],
    }, {
      specialAgent: {
        checkoutAccount: () => ({ id: 'a', apiKey: 'k', apiServerUrl: '' }),
        releaseAccount: () => {},
        refundReservation: () => {},
        reportError: () => { calls.generic++; },
        markRateLimited: () => { calls.rateLimited++; },
        reportInternalError: () => { calls.internal++; },
        reportBanSignal: () => { calls.ban++; },
        runDevinAcp: async () => { throw err; },
        ...extraDeps,
      },
    });
    return { result, calls };
  }

  it('a 503 backend_unavailable from the runner is NOT counted against the account', async () => {
    const err = Object.assign(new Error('Devin CLI not found: devin'), {
      status: 503, type: 'backend_unavailable',
    });
    const { result, calls } = await runWithFailure(err);
    assert.equal(result.status, 503);
    assert.equal(result.body.error.type, 'backend_unavailable');
    assert.equal(calls.generic, 0, 'env fault must not feed the generic-error streak');
    assert.equal(calls.rateLimited, 0);
    assert.equal(calls.internal, 0);
    assert.equal(calls.ban, 0);
  });

  it('a 499 request_aborted is NOT counted against the account', async () => {
    const err = Object.assign(new Error('Request aborted'), {
      status: 499, type: 'request_aborted',
    });
    const { calls } = await runWithFailure(err);
    assert.equal(calls.generic, 0, 'client abort must not feed the error streak');
  });

  it('a genuine 502 backend_error STILL feeds the generic-error streak (sanity)', async () => {
    const err = Object.assign(new Error('something broke'), {
      status: 502, type: 'backend_error',
    });
    const { calls } = await runWithFailure(err);
    assert.equal(calls.generic, 1, 'a real backend fault must still report to the pool');
  });
});