internal/llm/fallback_language_model_provider_test.go

Rama main · réplica de lectura del repositorio original.

package llm

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"log/slog"
	"net/http"
	"strings"
	"testing"
	"time"

	"github.com/yeomyeonggeori/blueclaw/internal/capability"
)

type correctableStructuredOutputError struct {
	Code                string
	AllowLegacyFallback bool
	Diagnostic          StructuredOutputDiagnostic
}

func (errorValue correctableStructuredOutputError) Error() string {
	return errorValue.Code
}

func (errorValue correctableStructuredOutputError) StructuredOutputCorrection() (StructuredOutputCorrection, bool) {
	if errorValue.AllowLegacyFallback {
		return StructuredOutputCorrection{}, false
	}
	return StructuredOutputCorrection{Code: errorValue.Code, Diagnostic: errorValue.Diagnostic}, true
}

func (errorValue correctableStructuredOutputError) StructuredOutputDiagnostic() (StructuredOutputDiagnostic, bool) {
	return errorValue.Diagnostic, errorValue.Diagnostic.Category != ""
}

type staticLanguageModelProvider struct {
	response                StructuredResponse
	error                   error
	responseCalls           *int
	structuredResponseCalls *int
}

type recoveryChatLanguageModelProvider struct {
	response   ChatCompletionResponse
	error      error
	chatCalls  *int
	localCalls *int
	localReply ChatCompletionResponse
	localError error
}

func (provider recoveryChatLanguageModelProvider) GenerateResponse(context.Context, string) (string, error) {
	return "", provider.error
}

func (provider recoveryChatLanguageModelProvider) GenerateStructuredResponse(context.Context, StructuredResponseRequest) (StructuredResponse, error) {
	return StructuredResponse{}, provider.error
}

func (provider recoveryChatLanguageModelProvider) GenerateRecoveryChatCompletion(context.Context, ChatCompletionRequest) (ChatCompletionResponse, error) {
	if provider.chatCalls != nil {
		(*provider.chatCalls)++
	}
	return provider.response, provider.error
}

func (provider recoveryChatLanguageModelProvider) GenerateLocalRecoveryChatCompletion(context.Context, ChatCompletionRequest) (ChatCompletionResponse, error) {
	if provider.localCalls != nil {
		(*provider.localCalls)++
	}
	return provider.localReply, provider.localError
}

func (staticLanguageModelProvider staticLanguageModelProvider) GenerateResponse(responseContext context.Context, prompt string) (string, error) {
	_ = responseContext
	_ = prompt
	if staticLanguageModelProvider.responseCalls != nil {
		(*staticLanguageModelProvider.responseCalls)++
	}
	if staticLanguageModelProvider.error != nil {
		return "", staticLanguageModelProvider.error
	}

	return staticLanguageModelProvider.response.Content, nil
}

func (staticLanguageModelProvider staticLanguageModelProvider) GenerateStructuredResponse(responseContext context.Context, structuredResponseRequest StructuredResponseRequest) (StructuredResponse, error) {
	_ = responseContext
	_ = structuredResponseRequest
	if staticLanguageModelProvider.structuredResponseCalls != nil {
		(*staticLanguageModelProvider.structuredResponseCalls)++
	}
	if staticLanguageModelProvider.error != nil {
		return StructuredResponse{}, staticLanguageModelProvider.error
	}

	return staticLanguageModelProvider.response, nil
}

func TestFallbackLanguageModelProviderUsesFallbackAfterPrimaryFailure(t *testing.T) {
	fallbackLanguageModelProvider := FallbackLanguageModelProvider{
		PrimaryProvider: staticLanguageModelProvider{
			error: errors.New("primary failed"),
		},
		FallbackProvider: staticLanguageModelProvider{
			response: StructuredResponse{
				ProviderName: "litert-lm",
				ModelName:    "/models/gemma-4-E4B-it.litertlm",
				Content:      `{"answer":"fallback"}`,
			},
		},
	}

	structuredResponse, errorValue := fallbackLanguageModelProvider.GenerateStructuredResponse(
		context.Background(),
		StructuredResponseRequest{
			Messages: []Message{
				{
					Role:    "user",
					Content: "hello",
				},
			},
			StructuredOutputSchema: StructuredOutputSchema{
				Name:               "response",
				Document:           `{"type":"object","properties":{"answer":{"type":"string"}},"required":["answer"],"additionalProperties":false}`,
				IsStrictlyEnforced: true,
			},
		},
	)
	if errorValue != nil {
		t.Fatalf("expected fallback provider to succeed: %v", errorValue)
	}
	if !structuredResponse.UsedFallback {
		t.Fatal("expected fallback provider to mark used fallback")
	}
	if structuredResponse.ProviderName != "litert-lm" {
		t.Fatalf("expected fallback provider name, got %q", structuredResponse.ProviderName)
	}
}

func TestFallbackLanguageModelProviderReturnsCorrectableStructuredErrorWithoutFallback(t *testing.T) {
	var fallbackCalls int
	primaryError := correctableStructuredOutputError{
		Code: "structured_output_invalid",
		Diagnostic: StructuredOutputDiagnostic{
			Category: StructuredOutputDiagnosticSchemaValidation,
			ValidationIssues: []StructuredOutputValidationIssue{{
				FieldPath: "/title",
				Code:      StructuredOutputValidationRequired,
			}},
		},
	}
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: staticLanguageModelProvider{error: primaryError},
		FallbackProvider: staticLanguageModelProvider{
			response:                StructuredResponse{Content: "fallback"},
			structuredResponseCalls: &fallbackCalls,
		},
	}

	response, errorValue := provider.GenerateStructuredResponse(context.Background(), StructuredResponseRequest{})

	var returnedError correctableStructuredOutputError
	if response.Content != "" || !errors.As(errorValue, &returnedError) || returnedError.Code != primaryError.Code {
		t.Fatalf("expected original correctable error, got %#v and %v", response, errorValue)
	}
	if fallbackCalls != 0 {
		t.Fatalf("expected caller correction before tier fallback, got %d fallback calls", fallbackCalls)
	}
}

func TestFallbackLanguageModelProviderUsesFallbackForEmptyCompletion(t *testing.T) {
	var fallbackCalls int
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: staticLanguageModelProvider{error: correctableStructuredOutputError{
			Code: "structured_output_invalid",
			Diagnostic: StructuredOutputDiagnostic{
				Category:     StructuredOutputDiagnosticEmptyCompletion,
				FinishReason: StructuredOutputDiagnosticFinishStop,
			},
		}},
		FallbackProvider: staticLanguageModelProvider{
			response:                StructuredResponse{Content: "fallback"},
			structuredResponseCalls: &fallbackCalls,
		},
	}

	response, errorValue := provider.GenerateStructuredResponse(context.Background(), StructuredResponseRequest{})

	if errorValue != nil || response.Content != "fallback" || !response.UsedFallback {
		t.Fatalf("expected an empty completion to fall back to the next tier, got %#v and %v", response, errorValue)
	}
	if fallbackCalls != 1 {
		t.Fatalf("expected one fallback call, got %d", fallbackCalls)
	}
}

func TestFallbackLanguageModelProviderUsesFallbackForNonCorrectableLegacyError(t *testing.T) {
	var fallbackCalls int
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: staticLanguageModelProvider{error: correctableStructuredOutputError{
			Code:                "structured_output_invalid",
			AllowLegacyFallback: true,
			Diagnostic: StructuredOutputDiagnostic{
				Category: StructuredOutputDiagnosticSchemaValidation,
			},
		}},
		FallbackProvider: staticLanguageModelProvider{
			response:                StructuredResponse{Content: "fallback"},
			structuredResponseCalls: &fallbackCalls,
		},
	}

	response, errorValue := provider.GenerateStructuredResponse(context.Background(), StructuredResponseRequest{})

	if errorValue != nil || response.Content != "fallback" || !response.UsedFallback {
		t.Fatalf("expected existing non-correctable fallback semantics, got %#v and %v", response, errorValue)
	}
	if fallbackCalls != 1 {
		t.Fatalf("expected one fallback call, got %d", fallbackCalls)
	}
}

func TestFallbackLanguageModelProviderUsesRecoveryChatFallbackAfterPrimaryFailure(t *testing.T) {
	primaryCalls := 0
	fallbackCalls := 0
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: recoveryChatLanguageModelProvider{
			error:     errors.New("primary chat failed"),
			chatCalls: &primaryCalls,
		},
		FallbackProvider: recoveryChatLanguageModelProvider{
			response: ChatCompletionResponse{
				FinishReason: "stop",
				Message:      ChatCompletionMessage{Role: "assistant", Content: "fallback recovery chat"},
			},
			chatCalls: &fallbackCalls,
		},
	}

	response, errorValue := provider.GenerateRecoveryChatCompletion(context.Background(), ChatCompletionRequest{})
	if errorValue != nil || response.Message.Content != "fallback recovery chat" {
		t.Fatalf("expected recovery chat fallback, got %+v, %v", response, errorValue)
	}
	if !response.UsedFallback || primaryCalls != 1 || fallbackCalls != 1 {
		t.Fatalf("expected one primary and fallback call with fallback marker, got response=%+v primary=%d fallback=%d", response, primaryCalls, fallbackCalls)
	}
}

func TestFallbackLanguageModelProviderUsesRecoveryChatFallbackAfterBlankPrimarySuccess(t *testing.T) {
	var logBuffer bytes.Buffer
	primaryCalls := 0
	fallbackCalls := 0
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: recoveryChatLanguageModelProvider{
			response: ChatCompletionResponse{
				FinishReason: "stop",
				Message:      ChatCompletionMessage{Role: "assistant"},
			},
			chatCalls: &primaryCalls,
		},
		FallbackProvider: recoveryChatLanguageModelProvider{
			response: ChatCompletionResponse{
				FinishReason: "stop",
				Message:      ChatCompletionMessage{Role: "assistant", Content: "fallback recovery chat"},
			},
			chatCalls: &fallbackCalls,
		},
		Logger: slog.New(slog.NewTextHandler(&logBuffer, nil)),
	}

	response, errorValue := provider.GenerateRecoveryChatCompletion(context.Background(), ChatCompletionRequest{})
	if errorValue != nil || response.Message.Content != "fallback recovery chat" {
		t.Fatalf("expected blank primary success to use fallback, got %+v, %v", response, errorValue)
	}
	if !response.UsedFallback || primaryCalls != 1 || fallbackCalls != 1 {
		t.Fatalf("expected one primary and fallback call, got response=%+v primary=%d fallback=%d", response, primaryCalls, fallbackCalls)
	}
	if !strings.Contains(logBuffer.String(), "recovery chat completion is empty") {
		t.Fatalf("expected semantic failure log, got %q", logBuffer.String())
	}
}

func TestFallbackLanguageModelProviderLogsNilErrorSafely(t *testing.T) {
	var logBuffer bytes.Buffer
	provider := FallbackLanguageModelProvider{Logger: slog.New(slog.NewTextHandler(&logBuffer, nil))}

	provider.logFallback("recovery_chat", nil)

	if !strings.Contains(logBuffer.String(), `error="unknown error"`) {
		t.Fatalf("expected nil error to be logged safely, got %q", logBuffer.String())
	}
}

func TestFallbackLanguageModelProviderDoesNotUseRecoveryChatFallbackAfterCancellation(t *testing.T) {
	responseContext, cancel := context.WithCancel(context.Background())
	cancel()
	primaryCalls := 0
	fallbackCalls := 0
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: recoveryChatLanguageModelProvider{
			error:     context.Canceled,
			chatCalls: &primaryCalls,
		},
		FallbackProvider: recoveryChatLanguageModelProvider{
			response: ChatCompletionResponse{
				FinishReason: "stop",
				Message:      ChatCompletionMessage{Role: "assistant", Content: "fallback recovery chat"},
			},
			chatCalls: &fallbackCalls,
		},
	}

	response, errorValue := provider.GenerateRecoveryChatCompletion(responseContext, ChatCompletionRequest{})
	if response.Message.Content != "" || !errors.Is(errorValue, context.Canceled) {
		t.Fatalf("expected canceled recovery chat without fallback, got %+v and %v", response, errorValue)
	}
	if primaryCalls != 1 || fallbackCalls != 0 {
		t.Fatalf("expected only primary recovery chat call, got primary=%d fallback=%d", primaryCalls, fallbackCalls)
	}
}

func TestFallbackLanguageModelProviderLogsAndDescendsThroughTiers(t *testing.T) {
	var logBuffer bytes.Buffer
	logger := slog.New(slog.NewTextHandler(&logBuffer, &slog.HandlerOptions{Level: slog.LevelWarn}))

	chain := FallbackLanguageModelProvider{
		PrimaryProvider: staticLanguageModelProvider{error: errors.New("high tier unavailable")},
		FallbackProvider: FallbackLanguageModelProvider{
			PrimaryProvider:  staticLanguageModelProvider{error: errors.New("medium tier unavailable")},
			FallbackProvider: staticLanguageModelProvider{response: StructuredResponse{ModelName: "low-model"}},
			PrimaryLabel:     "medium",
			FallbackLabel:    "low",
			Logger:           logger,
		},
		PrimaryLabel:  "high",
		FallbackLabel: "medium",
		Logger:        logger,
	}

	structuredResponse, errorValue := chain.GenerateStructuredResponse(context.Background(), StructuredResponseRequest{})
	if errorValue != nil {
		t.Fatalf("expected descent to reach a working tier: %v", errorValue)
	}
	if structuredResponse.ModelName != "low-model" {
		t.Fatalf("expected lowest tier to answer, got %q", structuredResponse.ModelName)
	}
	if !structuredResponse.UsedFallback {
		t.Fatal("expected response to be marked as used fallback")
	}

	logOutput := logBuffer.String()
	if !strings.Contains(logOutput, "failedTier=high") || !strings.Contains(logOutput, "failedTier=medium") {
		t.Fatalf("expected a log line per failed tier, got: %s", logOutput)
	}
}

func TestFallbackLanguageModelProviderDoesNotFallbackAfterCancellation(t *testing.T) {
	responseContext, cancel := context.WithCancel(context.Background())
	cancel()
	var responseCalls int
	var structuredResponseCalls int
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: staticLanguageModelProvider{error: context.Canceled},
		FallbackProvider: staticLanguageModelProvider{
			response:                StructuredResponse{Content: "fallback"},
			responseCalls:           &responseCalls,
			structuredResponseCalls: &structuredResponseCalls,
		},
	}

	response, errorValue := provider.GenerateResponse(responseContext, "hello")
	if response != "" || !errors.Is(errorValue, context.Canceled) {
		t.Fatalf("expected cancellation without fallback response, got %q and %v", response, errorValue)
	}
	structuredResponse, errorValue := provider.GenerateStructuredResponse(responseContext, StructuredResponseRequest{})
	if structuredResponse.Content != "" || !errors.Is(errorValue, context.Canceled) {
		t.Fatalf("expected cancellation without fallback structured response, got %#v and %v", structuredResponse, errorValue)
	}
	if responseCalls != 0 || structuredResponseCalls != 0 {
		t.Fatalf("expected no fallback calls after cancellation, got response=%d structured=%d", responseCalls, structuredResponseCalls)
	}
}

func TestFallbackLanguageModelProviderDoesNotFallbackAfterDeadline(t *testing.T) {
	responseContext, cancel := context.WithTimeout(context.Background(), 0)
	defer cancel()
	var responseCalls int
	var structuredResponseCalls int
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: staticLanguageModelProvider{error: context.DeadlineExceeded},
		FallbackProvider: staticLanguageModelProvider{
			response:                StructuredResponse{Content: "fallback"},
			responseCalls:           &responseCalls,
			structuredResponseCalls: &structuredResponseCalls,
		},
	}

	response, errorValue := provider.GenerateResponse(responseContext, "hello")
	if response != "" || !errors.Is(errorValue, context.DeadlineExceeded) {
		t.Fatalf("expected deadline without fallback response, got %q and %v", response, errorValue)
	}
	structuredResponse, errorValue := provider.GenerateStructuredResponse(responseContext, StructuredResponseRequest{})
	if structuredResponse.Content != "" || !errors.Is(errorValue, context.DeadlineExceeded) {
		t.Fatalf("expected deadline without fallback structured response, got %#v and %v", structuredResponse, errorValue)
	}
	if responseCalls != 0 || structuredResponseCalls != 0 {
		t.Fatalf("expected no fallback calls after deadline, got response=%d structured=%d", responseCalls, structuredResponseCalls)
	}
}

type chatLanguageModelProvider struct {
	response  ChatCompletionResponse
	error     error
	chatCalls *int
}

func (provider chatLanguageModelProvider) GenerateResponse(context.Context, string) (string, error) {
	return "", provider.error
}

func (provider chatLanguageModelProvider) GenerateStructuredResponse(context.Context, StructuredResponseRequest) (StructuredResponse, error) {
	return StructuredResponse{}, provider.error
}

func (provider chatLanguageModelProvider) GenerateChatCompletion(context.Context, ChatCompletionRequest) (ChatCompletionResponse, error) {
	if provider.chatCalls != nil {
		(*provider.chatCalls)++
	}
	return provider.response, provider.error
}

func TestFallbackLanguageModelProviderUsesChatFallbackAfterPrimaryFailure(t *testing.T) {
	primaryCalls := 0
	fallbackCalls := 0
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: chatLanguageModelProvider{
			error:     errors.New("chat finish reason contract violated"),
			chatCalls: &primaryCalls,
		},
		FallbackProvider: chatLanguageModelProvider{
			response: ChatCompletionResponse{
				FinishReason: "tool_calls",
				Message:      ChatCompletionMessage{Role: "assistant", Content: "fallback chat"},
			},
			chatCalls: &fallbackCalls,
		},
	}

	response, errorValue := provider.GenerateChatCompletion(context.Background(), ChatCompletionRequest{})
	if errorValue != nil || response.Message.Content != "fallback chat" {
		t.Fatalf("expected chat fallback, got %+v, %v", response, errorValue)
	}
	if !response.UsedFallback || primaryCalls != 1 || fallbackCalls != 1 {
		t.Fatalf("expected one primary and one fallback chat call, got primary=%d fallback=%d response=%+v", primaryCalls, fallbackCalls, response)
	}
	if response.FallbackReason != "chat finish reason contract violated" {
		t.Fatalf("expected the primary failure reason on the fallback response, got %q", response.FallbackReason)
	}
}

func TestFallbackLanguageModelProviderDoesNotUseChatFallbackAfterCancellation(t *testing.T) {
	primaryCalls := 0
	fallbackCalls := 0
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: chatLanguageModelProvider{
			error:     context.Canceled,
			chatCalls: &primaryCalls,
		},
		FallbackProvider: chatLanguageModelProvider{
			response:  ChatCompletionResponse{Message: ChatCompletionMessage{Role: "assistant", Content: "fallback chat"}},
			chatCalls: &fallbackCalls,
		},
	}
	cancelledContext, cancel := context.WithCancel(context.Background())
	cancel()

	_, errorValue := provider.GenerateChatCompletion(cancelledContext, ChatCompletionRequest{})
	if errorValue == nil || fallbackCalls != 0 {
		t.Fatalf("expected cancellation to stop without fallback, got error=%v fallbackCalls=%d", errorValue, fallbackCalls)
	}
}

func TestFallbackLanguageModelProviderLaddersAfterProviderCallDeadline(t *testing.T) {
	primaryCalls := 0
	fallbackCalls := 0
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: chatLanguageModelProvider{
			error:     context.DeadlineExceeded,
			chatCalls: &primaryCalls,
		},
		FallbackProvider: chatLanguageModelProvider{
			response: ChatCompletionResponse{
				FinishReason: "stop",
				Message:      ChatCompletionMessage{Role: "assistant", Content: "escalated chat"},
			},
			chatCalls: &fallbackCalls,
		},
	}

	response, errorValue := provider.GenerateChatCompletion(context.Background(), ChatCompletionRequest{})
	if errorValue != nil || response.Message.Content != "escalated chat" {
		t.Fatalf("expected provider call deadline to escalate to fallback tier, got %+v, %v", response, errorValue)
	}
	if !response.UsedFallback || primaryCalls != 1 || fallbackCalls != 1 {
		t.Fatalf("expected one primary and one fallback call, got primary=%d fallback=%d", primaryCalls, fallbackCalls)
	}
}

func TestFallbackLanguageModelProviderStopsWhenCallerBudgetExpired(t *testing.T) {
	fallbackCalls := 0
	provider := FallbackLanguageModelProvider{
		PrimaryProvider: chatLanguageModelProvider{error: context.DeadlineExceeded},
		FallbackProvider: chatLanguageModelProvider{
			response:  ChatCompletionResponse{Message: ChatCompletionMessage{Role: "assistant", Content: "escalated chat"}},
			chatCalls: &fallbackCalls,
		},
	}
	expiredContext, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
	defer cancel()

	_, errorValue := provider.GenerateChatCompletion(expiredContext, ChatCompletionRequest{})
	if errorValue == nil || fallbackCalls != 0 {
		t.Fatalf("expected expired caller budget to stop without fallback, got error=%v fallbackCalls=%d", errorValue, fallbackCalls)
	}
}

func TestAChatCapabilitydAnswersAsAProviderFailureFallsBackToTheNextModel(t *testing.T) {
	requestedModels := []string{}
	httpClient := fakeCapabilityHTTPClient{handler: func(request *http.Request) (*http.Response, error) {
		var document struct {
			Model string `json:"model"`
		}
		if errorValue := json.NewDecoder(request.Body).Decode(&document); errorValue != nil {
			t.Fatalf("expected a chat request document: %v", errorValue)
		}
		requestedModels = append(requestedModels, document.Model)
		if document.Model == "primary-model" {
			return jsonCapabilityResponse(http.StatusBadGateway, "openrouter provider Wafer ended the primary-model completion with finish_reason=error"), nil
		}
		return jsonCapabilityResponse(http.StatusOK, `{"finishReason":"stop","provider":"capabilityLLM","model":"degraded-model","message":{"role":"assistant","content":"degraded answer"}}`), nil
	}}
	capabilityClient := capability.Client{Endpoint: "http://internkim-capability", HTTPClient: httpClient}
	provider := FallbackLanguageModelProvider{
		PrimaryProvider:  CapabilityLLMClient{CapabilityClient: capabilityClient, ModelName: "primary-model", ExecutionMode: "remote"},
		FallbackProvider: CapabilityLLMClient{CapabilityClient: capabilityClient, ModelName: "degraded-model", ExecutionMode: "remote"},
	}

	response, errorValue := provider.GenerateChatCompletion(context.Background(), ChatCompletionRequest{
		Messages: []ChatCompletionMessage{{Role: "user", Content: "check"}},
	})

	if errorValue != nil || response.Message.Content != "degraded answer" || !response.UsedFallback {
		t.Fatalf("expected the degraded model to answer after the provider failed, got %+v %v", response, errorValue)
	}
	if len(requestedModels) != 2 || requestedModels[0] != "primary-model" || requestedModels[1] != "degraded-model" {
		t.Fatalf("expected one primary ask and one degraded ask, got %v", requestedModels)
	}
	if !strings.Contains(response.FallbackReason, "Wafer") {
		t.Fatalf("expected the provider's failure carried as the fallback reason, got %q", response.FallbackReason)
	}
}