test/cascade-completion-controls.test.js

Rama master · réplica de lectura del repositorio original.

// V-14 (KH-005) — the Cascade flow silently dropped every completion control.
//
// MEASURED on the audited baseline: `temperature`, `top_p`, `top_k` and
// `max_tokens` were placed only into the Devin Connect `completionOverrides`;
// neither Cascade entry point received them, and the wire builder hardcoded
// `max_output_tokens = 32768`. A caller asking for a tiny cap therefore got a
// much longer answer that still reported `finish_reason: "stop"` — a lie in the
// one field an auto-continuing client trusts.
//
// The contract this file pins:
//   * the caller's cap reaches the wire (CascadePlannerConfig field 6), clamped
//     to the value the real IDE is observed to send;
//   * the cap is enforced locally as `finish_reason: "length"`, so an upstream
//     that ignores the request cannot turn it into a clean stop;
//   * temperature / top_p / top_k are REFUSED on Cascade with a 400 that names
//     the parameter, because the wire has no field to carry them;
//   * the Devin Connect backend still honours all three (control).

import { afterEach, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import {
  handleChatCompletions,
  __resetConnectDeps,
  __setConnectDeps,
} from '../src/handlers/chat.js';
import {
  buildSendCascadeMessageRequest,
  normalizeCascadeMaxOutputTokens,
  CASCADE_MAX_OUTPUT_TOKENS_CEILING,
} from '../src/windsurf.js';
import { parseFields, getField } from '../src/proto.js';
import { addAccountByKey, getApiKey, removeAccount } from '../src/auth.js';

// The response cache would replay the previous test's capped answer (its key does
// not carry max_tokens) and mask the cap under test.
process.env.RESPONSE_CACHE_ENABLED = '0';

const CALLER = 'api:v14:user:u1';
const CASCADE_MODEL = 'gemini-2.5-flash';
const CONNECT_MODEL = 'swe-1-6-slow';
const created = [];
let savedDevinConnect = null;

function seedAccount(label) {
  const a = addAccountByKey(`v14-${label}-${Math.random().toString(36).slice(2)}`, label);
  created.push(a.id);
  return a;
}

beforeEach(() => { savedDevinConnect = process.env.DEVIN_CONNECT; });

afterEach(() => {
  __resetConnectDeps();
  if (savedDevinConnect === undefined) delete process.env.DEVIN_CONNECT;
  else process.env.DEVIN_CONNECT = savedDevinConnect;
  while (created.length) {
    try { removeAccount(created.pop()); } catch {}
  }
});

/** Read max_output_tokens (field 6) out of the planner config (request field 5). */
function wireMaxOutputTokens(sendProto) {
  const request = parseFields(sendProto);
  const config = getField(request, 5, 2);
  assert.ok(config && config.value && config.value.length, 'the request must carry cascade_config (field 5)');
  // CascadeConfig.planner_config is field 1; max_output_tokens is field 6 of it.
  const plannerWrap = getField(parseFields(config.value), 1, 2);
  assert.ok(plannerWrap && plannerWrap.value, 'cascade_config must carry the planner config (field 1)');
  const field = getField(parseFields(plannerWrap.value), 6, 0);
  return field ? Number(field.value) : null;
}

function cascadeDeps(text, capture) {
  class FakeClient {
    async cascadeChat(_messages, _modelEnum, _modelUid, opts = {}) {
      if (capture) capture.completion = opts.completion;
      if (opts.onChunk) await opts.onChunk({ text, thinking: '', isError: false });
      // The non-stream path reads the returned array; the stream path uses onChunk.
      const chunks = opts.onChunk ? [] : [{ text, thinking: '', isError: false }];
      chunks.toolCalls = [];
      chunks.totalYielded = text.length;
      chunks.totalThinking = 0;
      return chunks;
    }
  }
  return {
    waitForAccount(tried) { return tried.length === 0 ? getApiKey(tried) : null; },
    ensureLs: async () => {},
    getLsFor: () => ({ port: 17991, csrfToken: 'csrf', generation: 1 }),
    WindsurfClient: FakeClient,
    callerKey: CALLER,
  };
}

function fakeRes() {
  const listeners = new Map();
  return {
    body: '',
    writableEnded: false,
    write(chunk) { this.body += String(chunk); return true; },
    end(chunk) {
      if (chunk) this.write(chunk);
      this.writableEnded = true;
      for (const cb of listeners.get('close') || []) cb();
    },
    on(event, cb) { if (!listeners.has(event)) listeners.set(event, []); listeners.get(event).push(cb); return this; },
    once(event, cb) { this.on(event, cb); return this; },
    removeListener() {},
    writeHead() { return this; },
  };
}

function streamFrames(res) {
  return res.body.split('\n\n').filter(Boolean).map((frame) => {
    const line = frame.split('\n').find((l) => l.startsWith('data: '));
    if (!line) return null;
    try { return JSON.parse(line.slice(6)); } catch { return null; }
  }).filter(Boolean);
}

function body(extra = {}) {
  return {
    model: CASCADE_MODEL,
    stream: false,
    messages: [{ role: 'user', content: 'count to a thousand' }],
    ...extra,
  };
}

describe('V-14: the caller output cap reaches the Cascade wire', () => {
  it('writes max_tokens into CascadePlannerConfig field 6', () => {
    const withCap = buildSendCascadeMessageRequest('key', 'cascade-1', 'hi', 0, 'MODEL', 'sess', { maxOutputTokens: 64 });
    assert.equal(wireMaxOutputTokens(withCap), 64);
    const withoutCap = buildSendCascadeMessageRequest('key', 'cascade-1', 'hi', 0, 'MODEL', 'sess', {});
    assert.equal(wireMaxOutputTokens(withoutCap), 32768, 'an unset cap keeps the historical default byte-identically');
  });

  it('clamps to the IDE ceiling and ignores unusable values', () => {
    assert.equal(normalizeCascadeMaxOutputTokens(64), 64);
    assert.equal(normalizeCascadeMaxOutputTokens(1e9), CASCADE_MAX_OUTPUT_TOKENS_CEILING);
    assert.equal(normalizeCascadeMaxOutputTokens(0), CASCADE_MAX_OUTPUT_TOKENS_CEILING);
    assert.equal(normalizeCascadeMaxOutputTokens(-5), CASCADE_MAX_OUTPUT_TOKENS_CEILING);
    assert.equal(normalizeCascadeMaxOutputTokens('nope'), CASCADE_MAX_OUTPUT_TOKENS_CEILING);
    assert.equal(normalizeCascadeOutputFloor(), 1);
  });
});

function normalizeCascadeOutputFloor() {
  // A fractional cap must still ask for at least one token, never zero.
  return normalizeCascadeMaxOutputTokens(0.4);
}

describe('V-14: controls the Cascade wire cannot carry are refused, not dropped', () => {
  it('temperature on Cascade returns 400 param=temperature', async () => {
    process.env.DEVIN_CONNECT = '0';
    seedAccount('cascade-temp');
    const r = await handleChatCompletions(body({ temperature: 0.3 }), cascadeDeps('x'));
    assert.equal(r.status, 400, JSON.stringify(r.body));
    assert.equal(r.body.error.param, 'temperature');
    assert.equal(r.body.error.type, 'invalid_request_error');
    assert.match(r.body.error.message, /temperature/);
  });

  it('top_p and top_k are refused the same way', async () => {
    process.env.DEVIN_CONNECT = '0';
    seedAccount('cascade-topp');
    const topP = await handleChatCompletions(body({ top_p: 0.9 }), cascadeDeps('x'));
    assert.equal(topP.status, 400);
    assert.equal(topP.body.error.param, 'top_p');
    const topK = await handleChatCompletions(body({ top_k: 40 }), cascadeDeps('x'));
    assert.equal(topK.status, 400);
    assert.equal(topK.body.error.param, 'top_k');
  });

  it('control: the Devin Connect backend still honours them', async () => {
    process.env.DEVIN_CONNECT = '1';
    seedAccount('connect-temp');
    let seen = null;
    __setConnectDeps({
      toChatCompletion: async (params) => {
        seen = params.completion || null;
        return {
          status: 200,
          body: {
            id: 'x', object: 'chat.completion', created: 1, model: CONNECT_MODEL,
            choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
          },
        };
      },
    });
    const r = await handleChatCompletions(
      body({ model: CONNECT_MODEL, temperature: 0.3, top_p: 0.9, top_k: 40 }),
      { callerKey: CALLER },
    );
    assert.equal(r.status, 200, JSON.stringify(r.body));
    assert.deepEqual(seen, { temperature: 0.3, topP: 0.9, topK: 40 }, 'the connect adapter must still receive them');
  });
});

describe('V-14: the cap is enforced locally and reported as length', () => {
  const LONG = 'A'.repeat(400); // 400 chars ≈ 100 tokens

  it('non-stream: a small max_tokens truncates the answer and reports length', async () => {
    process.env.DEVIN_CONNECT = '0';
    seedAccount('cascade-cap');
    const capture = {};
    const r = await handleChatCompletions(body({ max_tokens: 10 }), cascadeDeps(LONG, capture));
    assert.equal(r.status, 200, JSON.stringify(r.body));
    assert.deepEqual(capture.completion, { maxTokens: 10 }, 'the cap must reach the producer');
    const choice = r.body.choices[0];
    assert.equal(choice.message.content.length, 40, '10 tokens × 4 chars/token');
    assert.equal(choice.finish_reason, 'length', 'a cut answer is never reported as a clean stop');
  });

  it('non-stream control: no cap means the whole answer and finish_reason stop', async () => {
    process.env.DEVIN_CONNECT = '0';
    seedAccount('cascade-nocap');
    const r = await handleChatCompletions(body({}), cascadeDeps(LONG));
    assert.equal(r.status, 200);
    assert.equal(r.body.choices[0].message.content.length, 400);
    assert.equal(r.body.choices[0].finish_reason, 'stop');
  });

  it('stream: the cap stops the content channel and the terminal says length', async () => {
    process.env.DEVIN_CONNECT = '0';
    seedAccount('cascade-stream-cap');
    const r = await handleChatCompletions(body({ stream: true, max_tokens: 10 }), cascadeDeps(LONG));
    assert.equal(r.status, 200);
    const res = fakeRes();
    await r.handler(res);
    const chunks = streamFrames(res);
    const content = chunks.flatMap((c) => c.choices || []).map((c) => c.delta?.content || '').join('');
    assert.equal(content.length, 40);
    assert.equal(chunks.some((c) => c.choices?.[0]?.finish_reason === 'length'), true,
      `expected a length terminal, got ${JSON.stringify(chunks.map((c) => c.choices?.[0]?.finish_reason ?? c.error?.code))}`);
    assert.equal(chunks.some((c) => c.choices?.[0]?.finish_reason === 'stop'), false);
    assert.ok(res.body.includes('data: [DONE]'), 'a length terminal is still a well-formed stream end');
  });

  it('stream control: no cap streams the whole answer and stops', async () => {
    process.env.DEVIN_CONNECT = '0';
    seedAccount('cascade-stream-nocap');
    const r = await handleChatCompletions(body({ stream: true }), cascadeDeps(LONG));
    const res = fakeRes();
    await r.handler(res);
    const chunks = streamFrames(res);
    const content = chunks.flatMap((c) => c.choices || []).map((c) => c.delta?.content || '').join('');
    assert.equal(content.length, 400);
    assert.equal(chunks.some((c) => c.choices?.[0]?.finish_reason === 'stop'), true);
  });
});