docker/gateguard-effectiveness/session.js

Rama main · réplica de lectura del repositorio original.

'use strict';

// Multi-turn session driver.
//
// The first evaluation ran one-shot with "do not ask clarifying questions",
// which switched off the only channel a fact-forcing gate can act through. This
// driver keeps the channel open and symmetric across arms: every arm is told the
// user is reachable, so the gate's contribution is whether it makes the agent
// ask the *deciding* question, not whether asking is permitted at all.
//
// Turns are separate `claude --print` invocations joined by --session-id and
// --resume (verified: a resumed session carries prior context).

const crypto = require('crypto');
const { spawnSync } = require('child_process');
const { parseStream, childEnv } = require('./lib');
const userSim = require('./user-sim');

const COLLAB_NOTE =
  'The person who asked for this is reachable. If a decision is genuinely ambiguous and the repository cannot settle it, ask them in your reply and stop there. Otherwise carry the work out.';

const DEFAULT_USER_TURNS = 3;

/** Session args; unlike lib.claudeArgs these allow resume, which persistence would block. */
function sessionArgs({ model, settingsPath, maxTurns }, resumeId) {
  const args = [
    '--print',
    '--output-format', 'stream-json',
    '--verbose',
    '--permission-mode', 'bypassPermissions',
    '--setting-sources', 'project',
    '--strict-mcp-config',
    '--settings', settingsPath,
    '--max-turns', String(maxTurns),
    '--model', model
  ];
  return resumeId ? [...args, '--resume', resumeId] : [...args, '--session-id', crypto.randomUUID()];
}

function sessionIdOf(args) {
  const at = args.indexOf('--session-id');
  return at === -1 ? null : args[at + 1];
}

/** Last assistant text of a turn: what the user would actually read. */
function finalText(stream) {
  return stream.texts.length ? stream.texts[stream.texts.length - 1] : '';
}

function emptyUsage() {
  return { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, costUsd: 0, turns: 0, durationMs: 0 };
}

function addUsage(total, result) {
  const u = (result && result.usage) || {};
  const n = value => (Number.isFinite(value) ? value : 0);
  total.inputTokens += n(u.input_tokens) + n(u.cache_creation_input_tokens);
  total.cachedInputTokens += n(u.cache_read_input_tokens);
  total.outputTokens += n(u.output_tokens);
  total.costUsd += result && Number.isFinite(result.total_cost_usd) ? result.total_cost_usd : 0;
  total.turns += result && Number.isFinite(result.num_turns) ? result.num_turns : 0;
  total.durationMs += result && Number.isFinite(result.duration_ms) ? result.duration_ms : 0;
  return total;
}

/**
 * Runs one trial as a conversation: the agent works, the simulated user answers
 * only what it is actually asked, and the exchange stops when the agent stops
 * asking or the user-turn budget runs out.
 *
 * Returns conversation facts only. Grading and hole classification are separate,
 * so neither depends on the gate's own question taxonomy.
 */
function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, options) {
  const {
    executable = 'claude', model, maxTurns, timeoutMs, userTurns = DEFAULT_USER_TURNS,
    execute = spawnSync, judge = userSim.answer, judgeModel = userSim.JUDGE_MODEL
  } = options;

  // The config directory is deliberately NOT isolated. Resuming needs session
  // persistence, and the obvious fix - a per-trial CLAUDE_CONFIG_DIR - puts the
  // credentials out of reach: every turn then returns "Not logged in". The
  // persisted transcripts are cleaned up by the runner instead.
  const env = childEnv(stateDir);
  const usage = emptyUsage();
  const exchange = [];
  const disclosed = new Set();
  const denialsPerTurn = [];
  let gateDenials = 0;
  let editCalls = 0;
  let shellCalls = 0;
  const tools = {};
  let judgeFailed = false;
  let asked = false;
  let resumeId = null;
  let input = `${prompt}\n\n${COLLAB_NOTE}\n`;
  let lastStatus = null;
  let timedOut = false;
  let providerError = false;
  let providerMessage = null;

  for (let turn = 0; turn <= userTurns; turn++) {
    const args = sessionArgs({ model, settingsPath, maxTurns }, resumeId);
    if (!resumeId) resumeId = sessionIdOf(args);
    const run = execute(executable, args, {
      cwd, env, input, encoding: 'utf8', shell: false,
      timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 64 * 1024 * 1024
    });
    lastStatus = run.status;
    // A timeout on any turn stands: `a || b ? c : false` cleared it on the next clean turn.
    timedOut = timedOut || /ETIMEDOUT|SIGKILL/.test(String(run.error && run.error.code));
    const stream = parseStream(run.stdout);
    // Same test as lib.js: a missing or is_error result is a provider failure,
    // not a trial. Recorded per turn so a failure on any turn invalidates the
    // conversation rather than being graded as a finding.
    if (!stream.result || stream.result.is_error === true) {
      providerError = true;
      providerMessage = String((stream.result && stream.result.result)
        || [...stream.texts].reverse().find(text => text.trim() !== '') || '').slice(0, 200) || null;
    }
    addUsage(usage, stream.result);
    denialsPerTurn.push(stream.gateDenials);
    gateDenials += stream.gateDenials;
    editCalls += stream.editCalls;
    shellCalls += stream.shellCalls;
    for (const [name, count] of Object.entries(stream.tools)) tools[name] = (tools[name] || 0) + count;

    const message = finalText(stream);
    exchange.push({ role: 'agent', text: message });
    if (turn === userTurns) break;

    const reply = judge(intent, message, { executable, model: judgeModel, execute });
    judgeFailed = judgeFailed || reply.judgeFailed;
    if (!reply.isQuestion) break;
    asked = true;
    for (const id of reply.disclosed) disclosed.add(id);
    exchange.push({ role: 'user', text: reply.reply });
    input = reply.reply;
  }

  const decisive = userSim.decisiveFacts(intent);
  return {
    exchange,
    userTurns: exchange.filter(entry => entry.role === 'user').length,
    asked,
    disclosed: [...disclosed],
    disclosedDecisive: decisive.length > 0 && decisive.every(id => disclosed.has(id)),
    gateDenials,
    // Per turn, because the gate marks a file checked once per session: a zero
    // after a non-zero is the latch, not the gate choosing to stay quiet.
    denialsPerTurn,
    editCalls,
    shellCalls,
    tools,
    judgeFailed,
    exitStatus: lastStatus,
    timedOut,
    providerError,
    providerMessage,
    ...usage
  };
}

module.exports = { runConversation, sessionArgs, sessionIdOf, finalText, COLLAB_NOTE, DEFAULT_USER_TURNS };