butler/transport/error_classifier.py
Rama main · réplica de lectura del repositorio original.
"""API error classification for Butler LLM failover and recovery.
Maps exceptions to recovery actions (retry, compress, fallback, abort)
with a priority-ordered classification pipeline. Based on Hermes
``agent/error_classifier.py`` with provider-specific patterns.
"""
from __future__ import annotations
import enum
import json
import re
from dataclasses import dataclass, field
from typing import Any, Dict, Optional
class FailoverReason(str, enum.Enum):
# Authentication / authorization
auth = "auth"
auth_permanent = "auth_permanent"
# Billing / quota
billing = "billing"
rate_limit = "rate_limit"
upstream_rate_limit = "upstream_rate_limit"
# Server-side
overloaded = "overloaded"
server_error = "server_error"
# Transport
timeout = "timeout"
ssl_cert_verification = "ssl_cert_verification"
# Context / payload
context_overflow = "context_overflow"
payload_too_large = "payload_too_large"
image_too_large = "image_too_large"
# Model / provider policy
model_not_found = "model_not_found"
provider_policy_blocked = "provider_policy_blocked"
content_policy_blocked = "content_policy_blocked"
# Request format
format_error = "format_error"
multimodal_tool_content_unsupported = "multimodal_tool_content_unsupported"
# Provider-specific
thinking_signature = "thinking_signature"
long_context_tier = "long_context_tier"
llama_cpp_grammar_pattern = "llama_cpp_grammar_pattern"
# Catch-all
unknown = "unknown"
@dataclass
class ClassifiedError:
reason: FailoverReason
status_code: Optional[int] = None
provider: Optional[str] = None
model: Optional[str] = None
message: str = ""
error_context: Dict[str, Any] = field(default_factory=dict)
retryable: bool = True
should_compress: bool = False
should_rotate_credential: bool = False
should_fallback: bool = False
@property
def is_auth(self) -> bool:
return self.reason in {FailoverReason.auth, FailoverReason.auth_permanent}
# ── Error patterns ──────────────────────────────────────────────────────────────
_BILLING_PATTERNS = (
"insufficient credits", "insufficient_quota", "insufficient balance",
"credit balance", "credits exhausted", "payment required",
"billing hard limit", "account is deactivated", "out of funds",
"balance_depleted", "model_not_supported_on_free_tier",
)
_RATE_LIMIT_PATTERNS = (
"rate limit", "rate_limit", "too many requests", "throttled",
"resource_exhausted", "throttlingexception", "servicequotaexceededexception",
)
_OVERLOADED_PATTERNS = (
"overloaded", "temporarily overloaded", "service is temporarily overloaded",
"server is overloaded", "at capacity", "over capacity",
)
_CONTEXT_OVERFLOW_PATTERNS = (
"context length", "context window", "token limit", "too many tokens",
"maximum context", "prompt is too long", "max_model_len",
"上下文长度", "超过最大长度", "context length exceeded",
"max_tokens", "prompt length", "input is too long",
)
_PAYLOAD_TOO_LARGE_PATTERNS = (
"request entity too large", "payload too large", "error code: 413",
)
_IMAGE_TOO_LARGE_PATTERNS = (
"image exceeds", "image too large", "image_too_large",
"image size exceeds", "image dimensions exceed",
)
_MODEL_NOT_FOUND_PATTERNS = (
"is not a valid model", "invalid model", "model not found",
"model_not_found", "does not exist", "unknown model", "unsupported model",
)
_REQUEST_VALIDATION_PATTERNS = (
"unknown parameter", "unsupported parameter", "unrecognized request argument",
"invalid_request_error",
)
_PROVIDER_POLICY_BLOCKED_PATTERNS = (
"no endpoints available matching your guardrail",
"no endpoints available matching your data policy",
)
_CONTENT_POLICY_BLOCKED_PATTERNS = (
"flagged for possible cybersecurity risk", "trusted access for cyber",
"violates our usage policies", "violates openai's usage policies",
"your request was flagged by", "prompt was flagged by our safety",
"content_filter", "responsibleaipolicyviolation",
)
_AUTH_PATTERNS = (
"invalid api key", "invalid_api_key", "authentication", "unauthorized",
"forbidden", "invalid token", "token expired", "token revoked", "access denied",
)
_MULTIMODAL_TOOL_CONTENT_PATTERNS = (
"text is not set", "tool message content must be a string",
"tool content must be a string", "expected string, got list",
"expected string, got array", "tool_call.content must be string",
)
_SSL_CERT_VERIFY_PATTERNS = (
"certificate verify failed", "certificate_verify_failed",
"unable to get local issuer certificate", "self-signed certificate",
"certificate has expired", "hostname mismatch",
)
_SSL_TRANSIENT_PATTERNS = (
"bad record mac", "ssl alert", "tls alert", "ssl handshake failure",
"[ssl:", "bad_record_mac",
)
_SERVER_DISCONNECT_PATTERNS = (
"server disconnected", "peer closed connection", "connection reset by peer",
"network connection lost", "unexpected eof", "incomplete chunked read",
)
_TRANSPORT_ERROR_TYPES = frozenset({
"ReadTimeout", "ConnectTimeout", "PoolTimeout",
"ConnectError", "RemoteProtocolError", "ConnectionError",
"ConnectionResetError", "ConnectionAbortedError", "BrokenPipeError",
"TimeoutError", "ReadError", "ServerDisconnectedError",
"SSLError", "SSLZeroReturnError", "APIConnectionError", "APITimeoutError",
})
def _extract_status_code(error: Exception) -> Optional[int]:
for attr in ("status_code", "http_status", "code"):
val = getattr(error, attr, None)
if isinstance(val, int):
return val
m = re.search(r"\b(4\d{2}|5\d{2})\b", str(error))
if m:
return int(m.group(1))
return None
def _extract_error_body(error: Exception) -> Any:
for attr in ("body", "response", "error"):
val = getattr(error, attr, None)
if isinstance(val, (dict, str)):
if isinstance(val, str):
try:
return json.loads(val)
except (json.JSONDecodeError, TypeError):
return {"message": val}
return val
return {}
def _extract_error_code(body: dict) -> str:
if not isinstance(body, dict):
return ""
err = body.get("error", body)
if isinstance(err, dict):
return str(err.get("code", "") or "")
return ""
def _extract_message(error: Exception, body: Any) -> str:
msg = str(error)[:500]
if isinstance(body, dict):
err = body.get("error", body)
if isinstance(err, dict):
body_msg = err.get("message")
if body_msg and body_msg not in msg:
msg = f"{msg} | {str(body_msg)[:200]}"
return msg
def _error_text(error: Exception) -> str:
"""Build comprehensive error message string for pattern matching."""
_raw_msg = str(error).lower()
_body_msg = ""
_metadata_msg = ""
body = _extract_error_body(error)
if isinstance(body, dict):
_err_obj = body.get("error", {})
if isinstance(_err_obj, dict):
_body_msg = str(_err_obj.get("message") or "").lower()
_metadata = _err_obj.get("metadata", {})
if isinstance(_metadata, dict):
_raw_json = _metadata.get("raw") or ""
if isinstance(_raw_json, str) and _raw_json.strip():
try:
_inner = json.loads(_raw_json)
if isinstance(_inner, dict):
_inner_err = _inner.get("error", {})
if isinstance(_inner_err, dict):
_metadata_msg = str(_inner_err.get("message") or "").lower()
except (json.JSONDecodeError, TypeError):
pass
if not _body_msg:
_body_msg = str(body.get("message") or "").lower()
parts = [_raw_msg]
if _body_msg and _body_msg not in _raw_msg:
parts.append(_body_msg)
if _metadata_msg and _metadata_msg not in _raw_msg and _metadata_msg not in _body_msg:
parts.append(_metadata_msg)
return " ".join(parts)
def _is_openrouter_upstream_error(body: dict, provider: str) -> bool:
if provider.lower() != "openrouter":
return False
if not isinstance(body, dict):
return False
err = body.get("error", {})
return isinstance(err, dict) and "provider returned error" in str(err.get("message", "")).lower()
def _extract_upstream_provider_name(body: dict) -> Optional[str]:
if not isinstance(body, dict):
return None
err = body.get("error", {})
if isinstance(err, dict):
meta = err.get("metadata", {})
if isinstance(meta, dict):
return str(meta.get("upstream", "") or "")
return None
def classify_api_error(
error: Exception,
*,
provider: str = "",
model: str = "",
approx_tokens: int = 0,
context_length: int = 200000,
num_messages: int = 0,
) -> ClassifiedError:
"""Classify an API error into a structured recovery recommendation.
Priority-ordered pipeline:
1. Provider-specific patterns (content policy, thinking signatures)
2. HTTP status code + message-aware refinement
3. Error code classification
4. Message pattern matching
5. SSL/TLS transient alert patterns → retry as timeout
6. Server disconnect + large session → context overflow
7. Transport error heuristics
8. Fallback: unknown (retryable with backoff)
"""
status_code = _extract_status_code(error)
error_type = type(error).__name__
if status_code is None and error_type == "RateLimitError":
status_code = 429
body = _extract_error_body(error)
error_code = _extract_error_code(body)
error_msg = _error_text(error)
provider_lower = provider.strip().lower()
model.strip().lower()
def _result(reason: FailoverReason, **overrides: Any) -> ClassifiedError:
defaults = {
"reason": reason,
"status_code": status_code,
"provider": provider,
"model": model,
"message": _extract_message(error, body),
}
defaults.update(overrides)
return ClassifiedError(**defaults)
# ── 1. Provider-specific patterns (highest priority) ──────────────────────
# Content policy / safety-filter block (deterministic, don't retry)
if any(p in error_msg for p in _CONTENT_POLICY_BLOCKED_PATTERNS):
return _result(FailoverReason.content_policy_blocked, retryable=False, should_fallback=True)
# Anthropic thinking block recovery (400)
if (
status_code == 400
and "thinking" in error_msg
and ("signature" in error_msg or "cannot be modified" in error_msg)
):
return _result(FailoverReason.thinking_signature, retryable=True, should_compress=False)
# Anthropic long-context tier gate (429)
if status_code == 429 and "extra usage" in error_msg and "long context" in error_msg:
return _result(FailoverReason.long_context_tier, retryable=True, should_compress=True)
# llama.cpp grammar pattern error (400)
if (
status_code == 400
and ("error parsing grammar" in error_msg or "json-schema-to-grammar" in error_msg)
):
return _result(FailoverReason.llama_cpp_grammar_pattern, retryable=True, should_compress=False)
# ── 2. HTTP status code classification ─────────────────────────────────────
if status_code is not None:
if status_code == 401:
return _result(
FailoverReason.auth,
retryable=False,
should_rotate_credential=True,
should_fallback=True,
)
if status_code == 403:
if any(p in error_msg for p in _BILLING_PATTERNS) or "spending limit" in error_msg:
return _result(FailoverReason.billing, retryable=False, should_fallback=True)
return _result(FailoverReason.auth, retryable=False, should_fallback=True)
if status_code == 402:
return _result(FailoverReason.billing, retryable=False, should_fallback=True)
if status_code == 404:
if any(p in error_msg for p in _BILLING_PATTERNS):
return _result(FailoverReason.billing, retryable=False, should_fallback=True)
if any(p in error_msg for p in _PROVIDER_POLICY_BLOCKED_PATTERNS):
return _result(FailoverReason.provider_policy_blocked, retryable=False)
if any(p in error_msg for p in _MODEL_NOT_FOUND_PATTERNS):
return _result(FailoverReason.model_not_found, should_fallback=True, retryable=False)
return _result(FailoverReason.unknown, retryable=True)
if status_code == 413:
return _result(FailoverReason.payload_too_large, retryable=True, should_compress=True)
if status_code == 429:
if any(p in error_msg for p in _OVERLOADED_PATTERNS):
return _result(FailoverReason.overloaded, retryable=True)
if _is_openrouter_upstream_error(body, provider_lower):
upstream_provider = _extract_upstream_provider_name(body)
ctx = {"upstream_provider": upstream_provider} if upstream_provider else {}
return _result(
FailoverReason.upstream_rate_limit,
retryable=True,
should_rotate_credential=False,
should_fallback=True,
error_context=ctx,
)
return _result(FailoverReason.rate_limit, retryable=True, should_fallback=True)
if status_code == 400:
if any(p in error_msg for p in _IMAGE_TOO_LARGE_PATTERNS):
return _result(FailoverReason.image_too_large, retryable=True, should_compress=True)
if any(p in error_msg for p in _MULTIMODAL_TOOL_CONTENT_PATTERNS):
return _result(FailoverReason.multimodal_tool_content_unsupported, retryable=True)
if any(p in error_msg for p in _REQUEST_VALIDATION_PATTERNS):
return _result(FailoverReason.format_error, retryable=False, should_fallback=True)
if any(p in error_msg for p in _CONTEXT_OVERFLOW_PATTERNS):
return _result(FailoverReason.context_overflow, retryable=True, should_compress=True)
return _result(FailoverReason.format_error, retryable=False)
if status_code in {500, 502}:
if any(p in error_msg for p in _REQUEST_VALIDATION_PATTERNS):
return _result(FailoverReason.format_error, retryable=False, should_fallback=True)
if any(p in error_msg for p in _CONTEXT_OVERFLOW_PATTERNS):
return _result(FailoverReason.context_overflow, retryable=True, should_compress=True)
return _result(FailoverReason.server_error, retryable=True)
if status_code in {503, 529}:
if any(p in error_msg for p in _CONTEXT_OVERFLOW_PATTERNS):
return _result(FailoverReason.context_overflow, retryable=True, should_compress=True)
return _result(FailoverReason.overloaded, retryable=True)
if status_code == 408:
return _result(FailoverReason.timeout, retryable=True)
# ── 3. Error code classification ───────────────────────────────────────────
if error_code:
if error_code.lower() in {"invalid_request_error", "unknown_parameter", "unsupported_parameter"}:
return _result(FailoverReason.format_error, retryable=False)
if error_code.lower() in {"insufficient_quota", "billing_not_active", "payment_required"}:
return _result(FailoverReason.billing, retryable=False, should_fallback=True)
# ── 4. Message pattern matching (no status code) ───────────────────────────
if any(p in error_msg for p in _CONTEXT_OVERFLOW_PATTERNS):
return _result(FailoverReason.context_overflow, should_compress=True, retryable=True)
if any(p in error_msg for p in _BILLING_PATTERNS):
return _result(FailoverReason.billing, retryable=False, should_fallback=True)
if any(p in error_msg for p in _RATE_LIMIT_PATTERNS):
return _result(FailoverReason.rate_limit, retryable=True, should_fallback=True)
if any(p in error_msg for p in _OVERLOADED_PATTERNS):
return _result(FailoverReason.overloaded, retryable=True)
if any(p in error_msg for p in _MODEL_NOT_FOUND_PATTERNS):
return _result(FailoverReason.model_not_found, should_fallback=True, retryable=False)
if any(p in error_msg for p in _PAYLOAD_TOO_LARGE_PATTERNS):
return _result(FailoverReason.payload_too_large, should_compress=True, retryable=True)
if any(p in error_msg for p in _AUTH_PATTERNS):
return _result(FailoverReason.auth, should_fallback=True, retryable=True)
if "timed out" in error_msg or "deadline exceeded" in error_msg:
return _result(FailoverReason.timeout, retryable=True)
# ── 5. SSL certificate verification failures → fail fast ───────────────────
if any(p in error_msg for p in _SSL_CERT_VERIFY_PATTERNS):
return _result(FailoverReason.ssl_cert_verification, retryable=False, should_fallback=False)
# ── 5b. SSL/TLS transient errors → retry as timeout ────────────────────────
if any(p in error_msg for p in _SSL_TRANSIENT_PATTERNS):
return _result(FailoverReason.timeout, retryable=True)
# ── 6. Server disconnect + large session → context overflow ────────────────
is_disconnect = any(p in error_msg for p in _SERVER_DISCONNECT_PATTERNS)
if is_disconnect and not status_code:
is_large = approx_tokens > context_length * 0.6 or (
context_length <= 256000 and (approx_tokens > 120000 or num_messages > 200)
)
if is_large:
return _result(FailoverReason.context_overflow, retryable=True, should_compress=True)
return _result(FailoverReason.timeout, retryable=True)
# ── 7. Transport / timeout heuristics ──────────────────────────────────────
if error_type in _TRANSPORT_ERROR_TYPES or isinstance(error, (TimeoutError, ConnectionError, OSError)):
return _result(FailoverReason.timeout, retryable=True)
# ── 8. Fallback: unknown ────────────────────────────────────────────────────
return _result(FailoverReason.unknown, retryable=True)