internal/llm/capability_llm_client.go
Rama main · réplica de lectura del repositorio original.
package llm
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"github.com/yeomyeonggeori/blueclaw/internal/capability"
)
type CapabilityLLMClient struct {
CapabilityClient capability.Client
ModelName string
ModelTier string
ExecutionMode string
}
type capabilityStructuredResponseRequestDocument struct {
Model string `json:"model,omitempty"`
ModelTier string `json:"modelTier,omitempty"`
ExecutionMode string `json:"executionMode"`
Context *RequestContext `json:"context,omitempty"`
SessionID string `json:"sessionID,omitempty"`
Messages []Message `json:"messages"`
StructuredOutputSchema capabilityStructuredOutputSchema `json:"structuredOutputSchema"`
GenerationOptions *GenerationOptions `json:"generationOptions,omitempty"`
}
type capabilityTextResponseRequestDocument struct {
Model string `json:"model,omitempty"`
ModelTier string `json:"modelTier,omitempty"`
ExecutionMode string `json:"executionMode"`
Context *RequestContext `json:"context,omitempty"`
SessionID string `json:"sessionID,omitempty"`
Messages []Message `json:"messages"`
}
type capabilityChatCompletionRequestDocument struct {
Model string `json:"model,omitempty"`
ModelTier string `json:"modelTier,omitempty"`
ExecutionMode string `json:"executionMode"`
Context *RequestContext `json:"context,omitempty"`
SessionID string `json:"sessionID,omitempty"`
Messages []ChatCompletionMessage `json:"messages"`
Tools []ChatCompletionTool `json:"tools,omitempty"`
ToolChoice json.RawMessage `json:"toolChoice,omitempty"`
ParallelToolCalls bool `json:"parallelToolCalls"`
GenerationOptions *GenerationOptions `json:"generationOptions,omitempty"`
}
type capabilityChatCompletionResponseDocument struct {
ChatCompletionResponse
reportedWireExchange
UsedFallback bool `json:"usedFallback,omitempty"`
FallbackReason string `json:"fallbackReason,omitempty"`
}
type capabilityStructuredOutputSchema struct {
Name string `json:"name"`
Document json.RawMessage `json:"document"`
IsStrictlyEnforced bool `json:"isStrictlyEnforced"`
}
type capabilityStructuredResponseDocument struct {
ProviderName string `json:"provider"`
UpstreamProvider string `json:"upstreamProvider,omitempty"`
ModelName string `json:"model"`
Content string `json:"content"`
SelectedBackend string `json:"selectedBackend"`
FinishReason string `json:"finishReason,omitempty"`
ConstraintMode string `json:"constraintMode,omitempty"`
Usage Usage `json:"usage"`
UsedFallback bool `json:"usedFallback,omitempty"`
FallbackReason string `json:"fallbackReason,omitempty"`
reportedWireExchange
}
func (capabilityLLMClient CapabilityLLMClient) GenerateResponse(responseContext context.Context, prompt string) (string, error) {
return capabilityLLMClient.generateResponse(responseContext, prompt, capabilityLLMClient.executionMode())
}
func (capabilityLLMClient CapabilityLLMClient) GenerateRecoveryResponse(responseContext context.Context, prompt string) (string, error) {
autoContext, cancelAuto := recoveryAttemptContext(responseContext)
response, errorValue := capabilityLLMClient.generateResponse(autoContext, prompt, "auto")
cancelAuto()
if errorValue == nil && strings.TrimSpace(response) != "" {
return response, nil
}
if capabilityLLMClient.executionMode() == "device" {
return response, errorValue
}
deviceContext, cancelDevice := recoveryAttemptContext(responseContext)
defer cancelDevice()
return capabilityLLMClient.generateResponse(deviceContext, prompt, "device")
}
func (capabilityLLMClient CapabilityLLMClient) GenerateLocalRecoveryResponse(responseContext context.Context, prompt string) (string, error) {
deviceContext, cancelDevice := recoveryAttemptContext(responseContext)
defer cancelDevice()
return capabilityLLMClient.generateResponse(deviceContext, prompt, "device")
}
func (capabilityLLMClient CapabilityLLMClient) GenerateRecoveryChatCompletion(responseContext context.Context, request ChatCompletionRequest) (ChatCompletionResponse, error) {
if capabilityLLMClient.executionMode() == "device" {
return capabilityLLMClient.generateRecoveryChatAttempt(responseContext, request, "device")
}
response, errorValue := capabilityLLMClient.generateRecoveryChatAttempt(responseContext, request, "auto")
if errorValue == nil {
return response, nil
}
if contextError := responseContext.Err(); contextError != nil {
return response, contextError
}
return capabilityLLMClient.generateRecoveryChatAttempt(responseContext, request, "device")
}
func (capabilityLLMClient CapabilityLLMClient) generateRecoveryChatAttempt(responseContext context.Context, request ChatCompletionRequest, executionMode string) (ChatCompletionResponse, error) {
autoContext, cancelAuto := recoveryAttemptContext(responseContext)
response, errorValue := capabilityLLMClient.generateChatCompletion(autoContext, request, executionMode)
cancelAuto()
if errorValue != nil {
return response, errorValue
}
if executionMode == "device" && response.SelectedBackend != "device" {
return response, errors.New("device recovery chat returned a non-device backend")
}
_, errorValue = RecoveryChatCompletionText(response)
return response, errorValue
}
func (capabilityLLMClient CapabilityLLMClient) GenerateLocalRecoveryChatCompletion(responseContext context.Context, request ChatCompletionRequest) (ChatCompletionResponse, error) {
return capabilityLLMClient.generateRecoveryChatAttempt(responseContext, request, "device")
}
func recoveryAttemptContext(responseContext context.Context) (context.Context, context.CancelFunc) {
baseContext := context.Background()
requestContext := RequestContextFromContext(responseContext)
if requestContext != (RequestContext{}) {
baseContext = ContextWithRequestContext(baseContext, requestContext)
}
if deadline, hasDeadline := responseContext.Deadline(); hasDeadline {
return context.WithDeadline(baseContext, deadline)
}
return context.WithCancel(baseContext)
}
const llmTransientRetryCount = 4
func isTransientLLMTransportError(errorValue error) bool {
if errorValue == nil {
return false
}
message := strings.ToLower(errorValue.Error())
for _, marker := range []string{"eof", "connection refused", "connection reset", "broken pipe", "no such host", "i/o timeout", "server closed", "connection closed", "no route to host"} {
if strings.Contains(message, marker) {
return true
}
}
return false
}
func (capabilityLLMClient CapabilityLLMClient) postJSONWithRetry(responseContext context.Context, path string, requestDocument any, responseDocument any) error {
var errorValue error
for attempt := 0; attempt < llmTransientRetryCount; attempt++ {
if attempt > 0 {
select {
case <-responseContext.Done():
return responseContext.Err()
case <-time.After(time.Duration(attempt) * 500 * time.Millisecond):
}
}
errorValue = capabilityLLMClient.CapabilityClient.PostJSON(responseContext, path, requestDocument, responseDocument)
if errorValue == nil || !isTransientLLMTransportError(errorValue) {
return errorValue
}
}
return errorValue
}
func (capabilityLLMClient CapabilityLLMClient) generateResponse(responseContext context.Context, prompt string, executionMode string) (string, error) {
if capabilityLLMClient.CapabilityClient.HTTPClient == nil {
return "", errors.New("capability llm http client is not configured")
}
var responseDocument capabilityStructuredResponseDocument
errorValue := capabilityLLMClient.postJSONWithRetry(
responseContext,
"/v1/llm/text",
capabilityTextResponseRequestDocument{
Model: capabilityLLMClient.ModelName,
ModelTier: capabilityLLMClient.ModelTier,
ExecutionMode: executionMode,
Context: requestContextPointer(responseContext),
SessionID: sessionIDOf(responseContext),
Messages: []Message{{
Role: "user",
Content: prompt,
}},
},
&responseDocument,
)
if errorValue != nil {
return "", errorValue
}
responseDocument.record(responseContext)
return responseDocument.Content, nil
}
func (capabilityLLMClient CapabilityLLMClient) GenerateStructuredResponse(responseContext context.Context, structuredResponseRequest StructuredResponseRequest) (StructuredResponse, error) {
if capabilityLLMClient.CapabilityClient.HTTPClient == nil {
return StructuredResponse{}, errors.New("capability llm http client is not configured")
}
requestDocument, errorValue := capabilityLLMClient.buildStructuredRequestDocument(responseContext, structuredResponseRequest)
if errorValue != nil {
return StructuredResponse{}, errorValue
}
var responseDocument capabilityStructuredResponseDocument
errorValue = capabilityLLMClient.postJSONWithRetry(responseContext, "/v1/llm/structured", requestDocument, &responseDocument)
if errorValue != nil {
return StructuredResponse{}, errorValue
}
responseDocument.record(responseContext)
providerName := strings.TrimSpace(responseDocument.ProviderName)
if providerName == "" {
providerName = "capabilityLLM"
}
modelName := strings.TrimSpace(responseDocument.ModelName)
if modelName == "" {
modelName = capabilityLLMClient.ModelName
}
return StructuredResponse{
Transport: "capability",
ProviderName: providerName,
UpstreamProvider: responseDocument.UpstreamProvider,
ModelName: modelName,
Content: responseDocument.Content,
SelectedBackend: responseDocument.SelectedBackend,
FinishReason: responseDocument.FinishReason,
ConstraintMode: responseDocument.ConstraintMode,
Usage: responseDocument.Usage,
UsedFallback: responseDocument.UsedFallback,
FallbackReason: responseDocument.FallbackReason,
}, nil
}
func (capabilityLLMClient CapabilityLLMClient) GenerateChatCompletion(responseContext context.Context, request ChatCompletionRequest) (ChatCompletionResponse, error) {
return capabilityLLMClient.generateChatCompletion(responseContext, request, capabilityLLMClient.executionMode())
}
func (capabilityLLMClient CapabilityLLMClient) generateChatCompletion(responseContext context.Context, request ChatCompletionRequest, executionMode string) (ChatCompletionResponse, error) {
if capabilityLLMClient.CapabilityClient.HTTPClient == nil {
return ChatCompletionResponse{}, errors.New("capability llm http client is not configured")
}
requestDocument := capabilityChatCompletionRequestDocument{
Model: requestedModelNameOrDefault(request, capabilityLLMClient.ModelName),
ModelTier: capabilityLLMClient.ModelTier,
ExecutionMode: executionMode,
Context: requestContextPointer(responseContext),
SessionID: sessionIDOf(responseContext),
Messages: append([]ChatCompletionMessage{}, request.Messages...),
Tools: append([]ChatCompletionTool{}, request.Tools...),
ToolChoice: append(json.RawMessage{}, request.ToolChoice...),
ParallelToolCalls: request.ParallelToolCalls,
GenerationOptions: generationOptionsPointer(request.GenerationOptions),
}
var responseDocument capabilityChatCompletionResponseDocument
errorValue := capabilityLLMClient.postJSONWithRetry(responseContext, "/v1/llm/chat", requestDocument, &responseDocument)
if errorValue != nil {
return ChatCompletionResponse{}, errorValue
}
responseDocument.record(responseContext)
response := responseDocument.ChatCompletionResponse
response.UsedFallback = responseDocument.UsedFallback
response.FallbackReason = responseDocument.FallbackReason
if strings.TrimSpace(response.ProviderName) == "" {
response.ProviderName = "capabilityLLM"
}
if strings.TrimSpace(response.ModelName) == "" {
response.ModelName = capabilityLLMClient.ModelName
}
if response.Message.ToolCalls == nil {
response.Message.ToolCalls = []ChatCompletionToolCall{}
}
response.Transport = "capability"
return response, nil
}
func (capabilityLLMClient CapabilityLLMClient) buildStructuredRequestDocument(responseContext context.Context, structuredResponseRequest StructuredResponseRequest) (capabilityStructuredResponseRequestDocument, error) {
jsonSchemaDocument, errorValue := normalizeStructuredOutputSchema(structuredResponseRequest.StructuredOutputSchema)
if errorValue != nil {
return capabilityStructuredResponseRequestDocument{}, errorValue
}
return capabilityStructuredResponseRequestDocument{
Model: capabilityLLMClient.ModelName,
ModelTier: capabilityLLMClient.ModelTier,
ExecutionMode: capabilityLLMClient.executionMode(),
Context: requestContextPointer(responseContext),
SessionID: sessionIDOf(responseContext),
Messages: append([]Message{}, structuredResponseRequest.Messages...),
GenerationOptions: generationOptionsPointer(structuredResponseRequest.GenerationOptions),
StructuredOutputSchema: capabilityStructuredOutputSchema{
Name: structuredResponseRequest.StructuredOutputSchema.Name,
Document: jsonSchemaDocument,
IsStrictlyEnforced: structuredResponseRequest.StructuredOutputSchema.IsStrictlyEnforced,
},
}, nil
}
func requestedModelNameOrDefault(request ChatCompletionRequest, defaultModelName string) string {
if requestedModelName := strings.TrimSpace(request.ModelName); requestedModelName != "" {
return requestedModelName
}
return defaultModelName
}
func generationOptionsPointer(options GenerationOptions) *GenerationOptions {
if options.Seed == nil && options.Temperature == nil && options.MaxTokens == nil {
return nil
}
return &options
}
// The turn's conversation is the longest-lived identity the model call can
// see: the task run id is minted inside the turn and the request context
// carries no other run identity. Capabilityd pins one upstream provider and one
// prompt cache per id, so the turns of a conversation share both.
func sessionIDOf(ctx context.Context) string {
return strings.TrimSpace(RequestContextFromContext(ctx).ConversationID)
}
func requestContextPointer(ctx context.Context) *RequestContext {
requestContext := RequestContextFromContext(ctx)
if requestContext.RequesterPersonID == "" &&
requestContext.RequesterEmail == "" &&
requestContext.RequesterName == "" &&
requestContext.RequesterPlatformUserID == "" &&
requestContext.ConversationID == "" &&
requestContext.Platform == "" {
return nil
}
return &requestContext
}
func (capabilityLLMClient CapabilityLLMClient) executionMode() string {
executionMode := strings.TrimSpace(capabilityLLMClient.ExecutionMode)
if executionMode == "" {
return "auto"
}
return executionMode
}