evidence/ChatToolGate-probe.mjs

Rama master · réplica de lectura del repositorio original.

// F3 (evidence/DataAuditTools/findings.md): does /v1/chat/completions refuse
// hosted / server-executed tool types, or run the model with the tool gone?
//
// Real handleChatCompletions, DEVIN_CONNECT route, single env token (no pool),
// connect deps stubbed so a model call is observable and no bytes leave the box.
// Run with: node --import ./test/setup-env.mjs evidence/ChatToolGate-probe.mjs

import { handleChatCompletions, __setConnectDeps, __resetConnectDeps } from '../src/handlers/chat.js';
import { log } from '../src/config.js';

process.env.DEVIN_CONNECT = '1';
process.env.DEVIN_CONNECT_TOKEN = 'env-token-shape-only';
delete process.env.WINDSURF_API_KEY;
delete process.env.WINDSURFAPI_STRICT_MODEL;

const SERVER_SIDE = [
  'file_search',
  'computer_use_preview',
  'mcp',
  'code_interpreter',
  'image_generation',
  'local_shell',
  'shell',
  'apply_patch',
];
const HOSTED = ['web_search', 'web_search_preview', 'web_search_preview_2025_03_11', 'web_search_2025_08_26'];
const VERSIONED = SERVER_SIDE.map((type) => `${type}_2025_03_11`);

const state = { hits: 0, lastParams: null };

function fakeRes() {
  const listeners = new Map();
  return {
    body: '',
    writableEnded: false,
    write(chunk) { this.body += String(chunk); return true; },
    end(chunk) {
      if (chunk) this.write(chunk);
      this.writableEnded = true;
      for (const cb of listeners.get('close') || []) cb();
    },
    on(event, cb) {
      if (!listeners.has(event)) listeners.set(event, []);
      listeners.get(event).push(cb);
      return this;
    },
    removeListener() {},
  };
}

__setConnectDeps({
  toChatCompletion: async (params) => {
    state.hits++;
    state.lastParams = params;
    return {
      status: 200,
      body: {
        id: 'x', object: 'chat.completion', created: 0, model: 'swe-1-6-slow',
        choices: [{ index: 0, message: { role: 'assistant', content: 'SERVED' }, finish_reason: 'stop' }],
      },
    };
  },
  streamChatCompletion: async (params, send) => {
    state.hits++;
    state.lastParams = params;
    send({ id: 'x', object: 'chat.completion.chunk', created: 0, model: 'swe-1-6-slow', choices: [{ index: 0, delta: { role: 'assistant', content: 'SERVED' } }] });
    send({ id: 'x', object: 'chat.completion.chunk', created: 0, model: 'swe-1-6-slow', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] });
    return { usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } };
  },
});

async function row(label, { tools, stream = false, typeName = null }) {
  state.hits = 0;
  state.lastParams = null;
  const logs = [];
  const origWarn = log.warn;
  const origInfo = log.info;
  log.warn = (...args) => { logs.push(args.join(' ')); };
  log.info = (...args) => { logs.push(args.join(' ')); };
  let result;
  try {
    result = await handleChatCompletions({
      model: 'swe-1-6-slow',
      stream,
      messages: [{ role: 'user', content: 'hi' }],
      ...(tools ? { tools } : {}),
    }, { callerKey: '' });
    if (result?.stream && typeof result.handler === 'function') {
      await result.handler(fakeRes());
    }
  } finally {
    log.warn = origWarn;
    log.info = origInfo;
  }
  const status = result?.stream ? 'STREAM-200' : (result?.status ?? 200);
  const error = result?.body?.error || {};
  const toolLogs = logs.filter((line) => /tool/i.test(line) && !/Probe\[/.test(line));
  const typeLogs = typeName ? logs.filter((line) => line.includes(typeName)) : [];
  const forwarded = state.lastParams ? JSON.stringify(state.lastParams.messages || []).includes('code_interpreter') : false;
  console.log([
    'ROW',
    label,
    `status=${status}`,
    `code=${error.code || '-'}`,
    `param=${error.param || '-'}`,
    `hits=${state.hits}`,
    `toolLogLines=${toolLogs.length}`,
    `typeNamedLogs=${typeLogs.length}`,
    `toolTextUpstream=${forwarded}`,
    `msg=${JSON.stringify(error.message || '').slice(0, 90)}`,
  ].join('\t'));
}

for (const type of [...SERVER_SIDE, ...VERSIONED, ...HOSTED]) {
  await row(`tools ${type}`, { tools: [{ type }], typeName: type });
  await row(`tools ${type} stream`, { tools: [{ type }], stream: true, typeName: type });
}

await row('namespace tools child code_interpreter', {
  tools: [{ type: 'namespace', name: 'ns', tools: [{ type: 'code_interpreter' }] }],
  typeName: 'code_interpreter',
});
await row('namespace children child mcp', {
  tools: [{ type: 'namespace', name: 'ns', tools: [], children: [{ type: 'mcp' }] }],
  typeName: 'mcp',
});
await row('function apply_patch (control)', {
  tools: [{ type: 'function', function: { name: 'apply_patch', parameters: { type: 'object', properties: {} } } }],
});
await row('custom apply_patch (control)', {
  tools: [{ type: 'custom', name: 'apply_patch' }],
  typeName: 'custom',
});
await row('unknown banana', { tools: [{ type: 'banana' }], typeName: 'banana' });
await row('unknown banana stream', { tools: [{ type: 'banana' }], stream: true, typeName: 'banana' });

__resetConnectDeps();