src/devin-connect-metrics.js

Rama master · réplica de lectura del repositorio original.

/**
 * DEVIN_CONNECT operational counters — in-memory, lifetime-of-process.
 *
 * WHY: the DEVIN_CONNECT recovery machinery (re-login, failover, liveness,
 * cooldowns) only emitted log lines. During an incident there was no way to
 * answer "how many tokens died?", "how often are we failing over?", or "is
 * auto-relogin even succeeding?" without grepping logs. These counters make
 * that state queryable (dashboard /connect-metrics) so it can be watched and
 * alarmed on.
 *
 * Deliberately NOT persisted: they're health/rate signals, not billing — a
 * restart resetting them is correct (you want "since this process started").
 * Counting is cheap and side-effect-free; callers bump on the event path.
 */

const _counters = {
  // re-login (same-account token recovery via stored credentials)
  relogin_ok: 0,
  relogin_fail: 0,
  // cross-account failover hops taken after a dead/unrecoverable token
  failover_hops: 0,
  // requests that exhausted all failover hops / pooled accounts
  failover_exhausted: 0,
  // cross-account failover hops taken because THIS account hit a quota/rate
  // dry-well (QUOTA_EXHAUSTED / RATE_LIMITED) while the pool still had a healthy
  // account — the request moves on instead of surfacing 402/429 to the client (R1)
  quota_failover_hops: 0,
  // cross-account failover hops taken because the upstream refused THIS selector
  // for THIS account ("This model is no longer available for your account").
  // Sibling of quota_failover_hops, deliberately separate: a spike here means the
  // upstream retired a model for part of the pool, not that anyone is out of
  // credit. No errorCount / re-login / reauth is involved (see
  // finalizeConnectAccount's SELECTOR_NOT_ELIGIBLE arm).
  selector_failover_hops: 0,
  // upstream refused a specific selector for one account. Scoped to the
  // (account, selector) pair: the token is alive and every other selector keeps
  // working, so this counter moving is NOT an account-health signal. It counts the
  // REFUSAL, not the routing latch: with no eligible peer the latch is deliberately
  // skipped (the account must keep surfacing the honest 403) and the refusal still
  // counts here.
  selector_ineligible: 0,
  // session tokens observed dead (UNAUTHORIZED, not revived on same account)
  dead_tokens: 0,
  // requests rejected because the whole pool was rate-limited/unavailable
  pool_exhausted: 0,
  // streaming requests replayed after a pre-emit transient blip
  transient_replays: 0,
  // accounts cooled down for running out of credit/quota
  quota_exhausted: 0,
  // requests where the MODEL was temporarily at capacity ("high demand"); a
  // transient soft cooldown, not an account fault (P0 #56/#57)
  capacity_throttled: 0,
  // upstream returned a transient backend fault ("an internal error occurred
  // (trace ID/error ID: ...)"), often in a 401/403 shell, on a provably-alive
  // token; classified UPSTREAM_INTERNAL → streak-quarantine, NOT a re-login or
  // entitlement wall (#56/#57 shape, internal-error class)
  upstream_internal: 0,
  // the upstream request VALIDATOR refused the request (gRPC invalid_argument /
  // failed_precondition) — an unsupported image / model capability (measured
  // 2026-10-25, evidence/AccountMatrix.md §7.1) or a malformed message/tool
  // shape. Surfaced to the client as 400 invalid_request_error, never retried,
  // and NEVER an account penalty: the token is alive and the account is healthy.
  // A spike means callers are sending a shape the selected model cannot take;
  // the client-facing message names supports_images in /v1/models.
  request_rejected: 0,
  // upstream returned an unclassified error, or the gRPC `internal` class that
  // classifyUpstreamError documents as a PERMANENT CLIENT mistake (short
  // fingerprint / gzipped body). Neither is an account fault, so it is recorded
  // as a health event only — never an errorCount eviction (same treatment as
  // CONTENT_BLOCKED). A spike here means callers are sending bad requests.
  upstream_error: 0,
  // upstream socket ended mid-answer with no end-of-stream frame → the reply was
  // TRUNCATED. A transport fault (ECONNRESET class), not an account fault, so it
  // is a health event only — never an errorCount eviction. A spike here means the
  // network path to the upstream is dropping connections, and callers are seeing
  // `response.incomplete` / retried turns rather than silent half answers.
  stream_truncated: 0,
  // upstream stall / transport / configuration faults (idle TIMEOUT, absolute
  // DEADLINE_EXCEEDED, NO_TOKEN). Health events only — never an errorCount
  // eviction, since none of them says the ACCOUNT is unhealthy. A spike here means
  // the upstream is stalling or the deployment is misconfigured, not that the pool
  // is dying.
  transport_fault: 0,
  // an error code finalizeConnectAccount did not classify. DENY-BY-DEFAULT: the
  // blame chain is an allow-list of named verdicts, so an unknown code takes the
  // health-window path (reportInternalError) and never charges the errorCount
  // budget. This counter is the visibility that the old silent `else reportError`
  // lacked: a spike means devin-connect.js (or Node's socket layer — EPROTO,
  // HPE_*, ERR_SSL_* are produced by https.request itself) emitted a code with no
  // explicit blame decision. Add an arm rather than widening an enumeration.
  unclassified_fault: 0,
  // the LOCAL frame queue hit its cap because the consumer was slower than the
  // upstream stream (QUEUE_OVERFLOW in devin-connect.js). Not an account fault —
  // the upstream and its token are healthy — so it carries no errorCount penalty.
  // A spike means DEVIN_CONNECT_MAX_QUEUE_FRAMES/BYTES or a downstream writer is
  // the bottleneck, not the pool.
  queue_overflow: 0,
  // Inline images dropped with an image-only ASSISTANT turn (#272). The drop is
  // deliberate — the upstream ignores #10 on role=assistant (measured 2026-10-21)
  // — but it used to be silent. A non-zero value means a client is replaying
  // images on assistant turns and those images will not reach the model.
  assistant_images_dropped: 0,
  // Image blocks dropped because the live catalog EXPLICITLY declares
  // `supports_images=false` for the resolved selector (C3-01). /v1/models has
  // advertised that bit since 3.9.23; this counter is the request-path proof
  // that it now governs forwarding. The text turn is preserved, so a non-zero
  // value means the client sent images to a model the upstream calls text-only
  // and the caller got an answer without them. Deliberately NOT an errorCount
  // signal: the request succeeds, and a catalog bit must never degrade account
  // health. Absence of the bit is unknown and never counted (see
  // getConnectSelectorImageCapability).
  images_dropped_capability: 0,
  // Turns where the model NARRATED a tool call in visible text, then finished
  // with `stop` and zero tool calls (#236 "form A"). Log-only detector: nothing
  // is retried or rewritten, because the shape cannot be told apart from a
  // legitimate prose answer without a real capture. A spike here is that capture.
  prose_tool_intent: 0,
  // Remote http(s) image_url blocks resolved to data URLs before the connect
  // wire was built (inlineRemoteImages). `inlined` moving means clients use
  // hosted image URLs; a matching rise in `fetch_failed` means those fetches are
  // being refused/blocked/oversized and the image was dropped (text still sent).
  remote_images_inlined: 0,
  remote_image_fetch_failed: 0,
  // A remote image_url reached the SYNCHRONOUS wire builder unresolved, i.e. a
  // direct buildGetChatMessageRequest() caller skipped streamChat's resolver.
  // The image is not sent; the log line names the host. Production should stay 0.
  remote_image_unresolved: 0,
  // Requests that asked for `parallel_tool_calls:false` (OpenAI) /
  // `tool_choice.disable_parallel_tool_use` (Anthropic) while the #11
  // `disable_parallel_tool_calls` passthrough was OFF, i.e. the hint was accepted
  // and served but not honoured — the upstream may return several calls in one
  // turn. Counted per request, because the log line for it is once per process.
  //
  // Non-zero is EXPECTED, not a fault: Codex CLI 0.164 sends the hint on every
  // request (evidence/ClientSandboxes.md §3.3 C1). It is the signal that would
  // justify calibrating #11 (DEVIN_CONNECT_TOOL_CHOICE=1 or a tag override).
  // Deliberately NOT an errorCount signal: nothing failed, and a caller preference
  // must never degrade account health.
  parallel_tool_calls_hint_ignored: 0,
  // liveness probe pre-emptively recovered a dying token
  liveness_recovered: 0,
  // SessionReauth: accounts benched because a free GetUserStatus probe confirmed
  // the session token was retired, and the ways back out. A `reauth_required`
  // that never finds a matching `reauth_ok` is a token nobody re-authenticated.
  reauth_required: 0,
  reauth_cleared: 0,
  reauth_ok: 0,
  reauth_fail: 0,
  // router-model (adaptive/arena-*) AssignModel resolution failed → fell back
  assign_model_failed: 0,
  // credential store was read back through a resilience tier (JSON repair or
  // per-record regex salvage) instead of a clean parse — a single corrupt byte
  // would otherwise wipe the whole fleet's stored relogin credentials (C6)
  cred_store_repaired: 0,
  // an account-gone cleanup (conversation-pool invalidateFor / detached
  // evictForAccount / user-JWT epoch bump) rejected or threw (G14). The
  // cleanup is fire-and-forget by design, but a failure leaves stale
  // pool/detached/JWT state behind — this counter plus the
  // `account cleanup "<op>" failed` warn makes that observable instead of
  // silent. A non-zero value means an operator should check whether a
  // removed/banned account is still resumable.
  account_cleanup_failed: 0,
};
let _startedAt = Date.now();

/** Increment a named counter by n (default 1). Unknown names are ignored. */
export function bumpConnect(name, n = 1) {
  if (Object.prototype.hasOwnProperty.call(_counters, name)) _counters[name] += n;
}

/**
 * Snapshot of all counters plus credential-store decrypt health (folded in so
 * a single endpoint answers "is recovery healthy?"). uptimeMs lets a scraper
 * compute rates without a separate clock.
 */
export function getConnectMetrics() {
  let cred = { decryptFailures: 0, lastDecryptError: null };
  try {
    // Lazy require to avoid a load-time cycle (credentials.js → config → ...).
    // Synchronous access via the module's getter when it's already loaded.
    const m = _credHealthRef;
    if (m) cred = m();
  } catch { /* leave defaults */ }
  return {
    uptimeMs: Date.now() - _startedAt,
    ..._counters,
    credDecryptFailures: cred.decryptFailures,
    credLastDecryptError: cred.lastDecryptError,
  };
}

// The credentials module registers its health getter here at import time so we
// avoid a static import cycle while still surfacing cred health in one place.
let _credHealthRef = null;
export function __registerCredHealth(fn) { _credHealthRef = fn || null; }

/** Reset counters (tests / dashboard DELETE). */
export function resetConnectMetrics() {
  for (const k of Object.keys(_counters)) _counters[k] = 0;
  _startedAt = Date.now();
}