internal/gateway/request_overrides_test.go
Rama main · réplica de lectura del repositorio original.
package gateway
import (
"context"
"io"
"net/http"
"strings"
"testing"
"github.com/goccy/go-json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/enterpilot/gomodel/internal/core"
"github.com/enterpilot/gomodel/internal/requestparams"
)
// effortResolverStub forces a reasoning effort per concrete attempt model.
type effortResolverStub struct {
efforts map[string]string
params map[string]requestparams.Set
defaults map[string]requestparams.Set
requested []core.RequestedModelSelector
}
func (r *effortResolverStub) ResolveModel(requested core.RequestedModelSelector) (core.ModelSelector, bool, error) {
selector, err := requested.Normalize()
return selector, false, err
}
func (r *effortResolverStub) ResolveRequestOverrides(_ context.Context, resolution *core.RequestModelResolution, attempt core.ModelSelector) requestparams.Overrides {
r.requested = append(r.requested, resolution.Requested)
overrides := requestparams.Overrides{ReasoningEffort: r.efforts[attempt.QualifiedModel()]}
if r.params != nil {
overrides.Params = r.params[attempt.QualifiedModel()]
}
if r.defaults != nil {
overrides.Defaults = r.defaults[attempt.QualifiedModel()]
}
return overrides
}
// effortCaptureProvider fails chat and Responses calls to "openai/primary"
// and records every request it receives.
type effortCaptureProvider struct {
providerTypeResolverStub
chats []*core.ChatRequest
responses []*core.ResponsesRequest
}
func (p *effortCaptureProvider) ChatCompletion(_ context.Context, req *core.ChatRequest) (*core.ChatResponse, error) {
p.chats = append(p.chats, req)
if req.Model == "primary" {
return nil, core.NewProviderError("openai", http.StatusInternalServerError, "primary down", nil)
}
return &core.ChatResponse{Model: req.Model, Choices: []core.Choice{{Message: core.ResponseMessage{Role: "assistant", Content: "ok"}}}}, nil
}
func (p *effortCaptureProvider) StreamChatCompletion(_ context.Context, req *core.ChatRequest) (io.ReadCloser, error) {
p.chats = append(p.chats, req)
if req.Model == "primary" {
return nil, core.NewProviderError("openai", http.StatusInternalServerError, "primary down", nil)
}
return io.NopCloser(strings.NewReader("data: [DONE]\n\n")), nil
}
func (p *effortCaptureProvider) Responses(_ context.Context, req *core.ResponsesRequest) (*core.ResponsesResponse, error) {
p.responses = append(p.responses, req)
if req.Model == "primary" {
return nil, core.NewProviderError("openai", http.StatusInternalServerError, "primary down", nil)
}
return &core.ResponsesResponse{Model: req.Model}, nil
}
func effortFixture() (*InferenceOrchestrator, *effortCaptureProvider, *effortResolverStub, *core.Workflow) {
provider := &effortCaptureProvider{}
resolver := &effortResolverStub{efforts: map[string]string{
"openai/primary": "medium",
"openai/backup": "low",
}}
orchestrator := NewInferenceOrchestrator(InferenceConfig{
Provider: provider,
ModelResolver: resolver,
FailoverResolver: failoverResolverFunc(func(*core.RequestModelResolution, core.Operation) []core.ModelSelector {
return []core.ModelSelector{{Provider: "openai", Model: "backup"}}
}),
})
workflow := &core.Workflow{
Endpoint: core.DescribeEndpoint("POST", "/v1/chat/completions"),
Resolution: &core.RequestModelResolution{
Requested: core.NewRequestedModelSelector("team-model", ""),
ResolvedSelector: core.ModelSelector{Provider: "openai", Model: "primary"},
AliasApplied: true,
},
Policy: &core.ResolvedWorkflowPolicy{Features: core.WorkflowFeatures{Failover: true}},
}
return orchestrator, provider, resolver, workflow
}
func TestChatReasoningEffortOverridePerAttempt(t *testing.T) {
orchestrator, provider, resolver, workflow := effortFixture()
original := &core.ChatRequest{
Model: "primary",
Provider: "openai",
Reasoning: &core.Reasoning{Effort: "high"},
ExtraFields: core.UnknownJSONFieldsFromMap(map[string]json.RawMessage{"reasoning_effort": json.RawMessage(`"high"`), "seed": json.RawMessage(`7`)}),
}
_, _, err := orchestrator.DispatchChatCompletion(context.Background(), workflow, original)
require.NoError(t, err)
require.Len(t, provider.chats, 2)
for i, want := range []string{"medium", "low"} {
sent := provider.chats[i]
require.NotNil(t, sent.Reasoning)
assert.Equal(t, want, sent.Reasoning.Effort, "attempt %d", i)
assert.Nil(t, sent.ExtraFields.Lookup("reasoning_effort"), "attempt %d keeps the client's flat effort", i)
assert.JSONEq(t, `7`, string(sent.ExtraFields.Lookup("seed")), "attempt %d", i)
}
assert.Equal(t, "high", original.Reasoning.Effort, "client request must not be mutated")
assert.JSONEq(t, `"high"`, string(original.ExtraFields.Lookup("reasoning_effort")))
for _, requested := range resolver.requested {
assert.Equal(t, "team-model", requested.Model)
}
}
func TestResponsesReasoningEffortOverridePerAttempt(t *testing.T) {
orchestrator, provider, _, workflow := effortFixture()
workflow.Endpoint = core.DescribeEndpoint("POST", "/v1/responses")
_, _, err := orchestrator.DispatchResponses(context.Background(), workflow, &core.ResponsesRequest{Model: "primary", Provider: "openai", Input: "hi"})
require.NoError(t, err)
require.Len(t, provider.responses, 2)
for i, want := range []string{"medium", "low"} {
require.NotNil(t, provider.responses[i].Reasoning)
assert.Equal(t, want, provider.responses[i].Reasoning.Effort, "attempt %d", i)
}
}
// An effort override replaces the effort, not a summary the client explicitly
// requested: reasoning.summary the client sent must survive onto the attempt.
func TestResponsesReasoningEffortOverrideKeepsClientSummary(t *testing.T) {
original := &core.ResponsesRequest{
Model: "gpt-5.5",
Input: "hi",
Reasoning: &core.Reasoning{Effort: "high", Summary: "detailed"},
}
overridden := ResponsesRequestWithReasoningEffort(original, "low")
require.NotNil(t, overridden.Reasoning)
assert.Equal(t, "low", overridden.Reasoning.Effort)
assert.Equal(t, "detailed", overridden.Reasoning.Summary)
without := ResponsesRequestWithReasoningEffort(&core.ResponsesRequest{Model: "gpt-5.5"}, "low")
require.NotNil(t, without.Reasoning)
assert.Empty(t, without.Reasoning.Summary)
}
func TestReasoningEffortOverrideAbsentKeepsRequest(t *testing.T) {
orchestrator, provider, resolver, workflow := effortFixture()
resolver.efforts = nil
original := &core.ChatRequest{Model: "backup", Provider: "openai", Reasoning: &core.Reasoning{Effort: "high"}}
_, _, err := orchestrator.DispatchChatCompletion(context.Background(), workflow, original)
require.NoError(t, err)
require.Len(t, provider.chats, 1)
assert.Same(t, original, provider.chats[0])
}
func TestReasoningEffortOverrideForPrimary(t *testing.T) {
orchestrator, _, _, workflow := effortFixture()
assert.Equal(t, "medium", orchestrator.RequestOverrides(context.Background(), workflow).ReasoningEffort)
workflow.Resolution.ResolvedSelector = core.ModelSelector{Provider: "openai", Model: "other"}
assert.True(t, orchestrator.RequestOverrides(context.Background(), workflow).IsZero())
assert.True(t, NewInferenceOrchestrator(InferenceConfig{}).RequestOverrides(context.Background(), workflow).IsZero())
}
// The prepared request must carry the forced effort, since the response cache
// keys on it: two aliases forcing different efforts on one model must not
// share an entry.
func TestPrepareChatRequestAppliesReasoningEffortOverride(t *testing.T) {
orchestrator, _, _, workflow := effortFixture()
workflow.Mode = core.ExecutionModeTranslated
original := &core.ChatRequest{Model: "team-model", Reasoning: &core.Reasoning{Effort: "high"}}
prepared, err := orchestrator.PrepareChatRequest(context.Background(), original, RequestMeta{
Endpoint: workflow.Endpoint,
Workflow: workflow,
})
require.NoError(t, err)
require.NotNil(t, prepared.Request.Reasoning)
assert.Equal(t, "medium", prepared.Request.Reasoning.Effort)
assert.Equal(t, "primary", prepared.Request.Model)
}
func TestReasoningEffortOverrideFailoverRestoresClientEffort(t *testing.T) {
orchestrator, provider, resolver, workflow := effortFixture()
delete(resolver.efforts, "openai/backup")
clientExtra := core.UnknownJSONFieldsFromMap(map[string]json.RawMessage{"reasoning_effort": json.RawMessage(`"high"`)})
prepared, err := orchestrator.PrepareChatRequest(context.Background(), &core.ChatRequest{
Model: "primary",
Provider: "openai",
Messages: []core.Message{{Role: "user", Content: "hi"}},
Reasoning: &core.Reasoning{Effort: "high"},
ExtraFields: clientExtra,
}, RequestMeta{Workflow: workflow})
require.NoError(t, err)
_, _, err = orchestrator.DispatchChatCompletion(prepared.Context, prepared.Workflow, prepared.Request)
require.NoError(t, err)
require.Len(t, provider.chats, 2)
assert.Equal(t, "medium", provider.chats[0].Reasoning.Effort)
require.NotNil(t, provider.chats[1].Reasoning)
assert.Equal(t, "high", provider.chats[1].Reasoning.Effort, "fallback without an override keeps the client's effort")
assert.JSONEq(t, `"high"`, string(provider.chats[1].ExtraFields.Lookup("reasoning_effort")))
}
func TestReasoningEffortOverrideDropsGoogleThinkingEffort(t *testing.T) {
tests := []struct {
name string
extraBody string
want string
}{
{name: "budget only drops extra_body", extraBody: `{"google":{"thinking_config":{"thinking_budget":24576}}}`},
{name: "keeps include_thoughts", extraBody: `{"google":{"thinking_config":{"thinking_level":"high","include_thoughts":true}}}`, want: `{"google":{"thinking_config":{"include_thoughts":true}}}`},
{name: "keeps other members", extraBody: `{"google":{"thinking_config":{"thinkingBudget":1},"cached_content":"c"},"x":1}`, want: `{"google":{"cached_content":"c"},"x":1}`},
{name: "unrelated extra_body unchanged", extraBody: `{"google":{"cached_content":"c"}}`, want: `{"google":{"cached_content":"c"}}`},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := &core.ChatRequest{Model: "m", ExtraFields: core.UnknownJSONFieldsFromMap(map[string]json.RawMessage{
"extra_body": json.RawMessage(tt.extraBody),
"seed": json.RawMessage(`7`),
})}
for _, fields := range []core.UnknownJSONFields{
ChatRequestWithReasoningEffort(req, "none").ExtraFields,
ResponsesRequestWithReasoningEffort(&core.ResponsesRequest{ExtraFields: req.ExtraFields}, "none").ExtraFields,
} {
assert.JSONEq(t, `7`, string(fields.Lookup("seed")))
if tt.want == "" {
assert.Nil(t, fields.Lookup("extra_body"))
continue
}
assert.JSONEq(t, tt.want, string(fields.Lookup("extra_body")))
}
assert.JSONEq(t, tt.extraBody, string(req.ExtraFields.Lookup("extra_body")), "client request must not be mutated")
})
}
}
func rawJSON(s string) json.RawMessage { return json.RawMessage(s) }
func TestChatRequestParamsPerAttempt(t *testing.T) {
orchestrator, provider, resolver, workflow := effortFixture()
resolver.efforts = nil
resolver.params = map[string]requestparams.Set{
"openai/primary": {"chat_template_kwargs": rawJSON(`{"enable_thinking":true}`), "top_k": rawJSON("20")},
"openai/backup": {"top_k": nil},
}
resolver.defaults = map[string]requestparams.Set{
"openai/primary": {"temperature": rawJSON("1")},
"openai/backup": {"temperature": rawJSON("0.5")},
}
temperature := 0.2
original := &core.ChatRequest{
Model: "primary",
Provider: "openai",
Temperature: &temperature,
ExtraFields: core.UnknownJSONFieldsFromMap(map[string]json.RawMessage{"top_k": rawJSON("5"), "seed": rawJSON("7")}),
}
_, _, err := orchestrator.DispatchChatCompletion(context.Background(), workflow, original)
require.NoError(t, err)
require.Len(t, provider.chats, 2)
first := provider.chats[0]
require.NotNil(t, first.Temperature)
assert.Equal(t, 0.2, *first.Temperature, "a default never replaces the client's value")
assert.JSONEq(t, `20`, string(first.ExtraFields.Lookup("top_k")))
assert.JSONEq(t, `{"enable_thinking":true}`, string(first.ExtraFields.Lookup("chat_template_kwargs")))
assert.JSONEq(t, `7`, string(first.ExtraFields.Lookup("seed")))
second := provider.chats[1]
assert.Nil(t, second.ExtraFields.Lookup("top_k"), "the fallback strips top_k")
assert.Nil(t, second.ExtraFields.Lookup("chat_template_kwargs"), "the primary's params do not leak into the fallback")
assert.JSONEq(t, `7`, string(second.ExtraFields.Lookup("seed")))
assert.Equal(t, 0.2, *original.Temperature, "client request must not be mutated")
assert.JSONEq(t, `5`, string(original.ExtraFields.Lookup("top_k")))
}
func TestChatRequestDefaultsFillOmittedKeysPerAttempt(t *testing.T) {
orchestrator, provider, resolver, workflow := effortFixture()
resolver.efforts = nil
resolver.defaults = map[string]requestparams.Set{
"openai/primary": {"temperature": rawJSON("1"), "top_k": rawJSON("20")},
"openai/backup": {"temperature": rawJSON("0.5")},
}
_, _, err := orchestrator.DispatchChatCompletion(context.Background(), workflow, &core.ChatRequest{Model: "primary", Provider: "openai"})
require.NoError(t, err)
require.Len(t, provider.chats, 2)
require.NotNil(t, provider.chats[0].Temperature)
assert.Equal(t, float64(1), *provider.chats[0].Temperature)
assert.JSONEq(t, `20`, string(provider.chats[0].ExtraFields.Lookup("top_k")))
require.NotNil(t, provider.chats[1].Temperature)
assert.Equal(t, 0.5, *provider.chats[1].Temperature, "the fallback gets its own default")
assert.Nil(t, provider.chats[1].ExtraFields.Lookup("top_k"))
}
// The gateway adds stream_options.include_usage after preparation; a
// fallback rebuilt from the client's request must keep it.
func TestRequestParamsFailoverKeepsStreamUsageFlag(t *testing.T) {
orchestrator, provider, resolver, workflow := effortFixture()
resolver.efforts = nil
resolver.params = map[string]requestparams.Set{"openai/primary": {"top_k": rawJSON("20")}}
prepared, err := orchestrator.PrepareChatRequest(context.Background(), &core.ChatRequest{
Model: "primary", Provider: "openai", Stream: true,
Messages: []core.Message{{Role: "user", Content: "hi"}},
}, RequestMeta{Workflow: workflow})
require.NoError(t, err)
require.Nil(t, prepared.Request.StreamOptions, "the client sent no stream options")
// Usage enforcement adds the flag to the prepared request after the
// overrides ran, as the server and the stream dispatcher do.
prepared.Request.StreamOptions = &core.StreamOptions{IncludeUsage: true}
result, err := orchestrator.StreamChatCompletion(prepared.Context, prepared.Workflow, prepared.Request)
require.NoError(t, err)
require.NoError(t, result.Stream.Close())
require.Len(t, provider.chats, 2)
for i, sent := range provider.chats {
require.NotNil(t, sent.StreamOptions, "attempt %d", i)
assert.True(t, sent.StreamOptions.IncludeUsage, "attempt %d", i)
assert.True(t, sent.Stream, "attempt %d", i)
}
assert.Nil(t, provider.chats[1].ExtraFields.Lookup("top_k"))
}
func TestRequestParamsFailoverRebuildsFromClientRequest(t *testing.T) {
orchestrator, provider, resolver, workflow := effortFixture()
resolver.efforts = nil
resolver.params = map[string]requestparams.Set{"openai/primary": {"temperature": rawJSON("1"), "top_k": rawJSON("20")}}
temperature := 0.2
prepared, err := orchestrator.PrepareChatRequest(context.Background(), &core.ChatRequest{
Model: "primary",
Provider: "openai",
Temperature: &temperature,
Messages: []core.Message{{Role: "user", Content: "hi"}},
}, RequestMeta{Workflow: workflow})
require.NoError(t, err)
require.NotNil(t, prepared.Request.Temperature)
assert.Equal(t, float64(1), *prepared.Request.Temperature, "the prepared request carries the forced value for the cache key")
assert.JSONEq(t, `20`, string(prepared.Request.ExtraFields.Lookup("top_k")))
_, _, err = orchestrator.DispatchChatCompletion(prepared.Context, prepared.Workflow, prepared.Request)
require.NoError(t, err)
require.Len(t, provider.chats, 2)
assert.Same(t, prepared.Request, provider.chats[0], "the prepared primary attempt is sent as is")
fallback := provider.chats[1]
assert.Equal(t, "backup", fallback.Model)
require.NotNil(t, fallback.Temperature)
assert.Equal(t, 0.2, *fallback.Temperature, "the fallback keeps the client's temperature")
assert.Nil(t, fallback.ExtraFields.Lookup("top_k"))
assert.Equal(t, prepared.Request.Messages, fallback.Messages)
}
func TestResponsesRequestParamsPerAttempt(t *testing.T) {
orchestrator, provider, resolver, workflow := effortFixture()
workflow.Endpoint = core.DescribeEndpoint("POST", "/v1/responses")
resolver.params = map[string]requestparams.Set{"openai/backup": {"max_tokens": rawJSON("64")}}
_, _, err := orchestrator.DispatchResponses(context.Background(), workflow, &core.ResponsesRequest{Model: "primary", Provider: "openai", Input: "hi"})
require.NoError(t, err)
require.Len(t, provider.responses, 2)
assert.Nil(t, provider.responses[0].MaxOutputTokens)
assert.Equal(t, "medium", provider.responses[0].Reasoning.Effort)
require.NotNil(t, provider.responses[1].MaxOutputTokens)
assert.Equal(t, 64, *provider.responses[1].MaxOutputTokens, "max_tokens lands on max_output_tokens")
assert.Equal(t, "low", provider.responses[1].Reasoning.Effort, "effort and params apply together")
}
func TestRequestParamsApplyErrorIsAGatewayFault(t *testing.T) {
orchestrator, _, resolver, workflow := effortFixture()
resolver.efforts = nil
broken := requestparams.Set{"temperature": rawJSON(`"hot"`)}
resolver.params = map[string]requestparams.Set{"openai/primary": broken, "openai/backup": broken}
_, _, err := orchestrator.DispatchChatCompletion(context.Background(), workflow, &core.ChatRequest{Model: "primary", Provider: "openai"})
require.Error(t, err)
var gatewayErr *core.GatewayError
require.ErrorAs(t, err, &gatewayErr)
assert.Equal(t, http.StatusInternalServerError, gatewayErr.StatusCode)
assert.Equal(t, core.ErrorTypeInternal, gatewayErr.Type)
}