internal/responsecache/semantic_test.go
Rama main · réplica de lectura del repositorio original.
package responsecache
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/enterpilot/gomodel/config"
"github.com/enterpilot/gomodel/internal/auditlog"
"github.com/enterpilot/gomodel/internal/core"
"github.com/enterpilot/gomodel/internal/echotest"
)
// mockEmbedder is an Embedder implementation for testing that returns a fixed vector.
type mockEmbedder struct {
vector []float32
err error
calls int
}
func (m *mockEmbedder) Embed(_ context.Context, _ string) ([]float32, error) {
m.calls++
return m.vector, m.err
}
func (m *mockEmbedder) Close() error { return nil }
func (m *mockEmbedder) Identity() string { return "test\x00mock-model" }
func newTestSemanticMiddleware(threshold float64, maxConvMessages int, excludeSystem bool) (*semanticCacheMiddleware, *MapVecStore, *mockEmbedder) {
store := NewMapVecStore()
emb := &mockEmbedder{vector: []float32{1, 0, 0}}
cfg := config.SemanticCacheConfig{
SimilarityThreshold: threshold,
TTL: new(3600),
MaxConversationMessages: new(maxConvMessages),
ExcludeSystemPrompt: excludeSystem,
}
m := newSemanticCacheMiddleware(emb, store, cfg, nil)
return m, store, emb
}
func serveSemanticRequest(t *testing.T, m *semanticCacheMiddleware, body []byte, guardrailsHash string) *httptest.ResponseRecorder {
t.Helper()
c, rec := echotest.Post(t, "/v1/chat/completions", body)
if guardrailsHash != "" {
req := c.Request()
c.SetRequest(req.WithContext(core.WithGuardrailsHash(req.Context(), guardrailsHash)))
}
err := m.Handle(&echoExchange{c: c}, body, func() error {
return c.JSON(http.StatusOK, map[string]string{"answer": "42"})
})
require.NoError(t, err)
return rec
}
func TestSemanticCacheMiddleware_CacheHit(t *testing.T) {
m, store, _ := newTestSemanticMiddleware(0.90, 10, false)
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"What is 2+2?"}]}`)
rec1 := serveSemanticRequest(t, m, body, "")
require.Empty(t, rec1.Header().Get("X-Cache"))
m.wg.Wait()
require.Equal(t, 1, store.Len())
rec2 := serveSemanticRequest(t, m, body, "")
require.Equal(t, "HIT (semantic)", rec2.Header().Get("X-Cache"))
}
func TestSemanticCacheMiddleware_ParaphraseFinalUserSharesNamespace(t *testing.T) {
m, store, emb := newTestSemanticMiddleware(0.90, 10, false)
emb.vector = []float32{1, 0, 0}
body1 := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"What is 2+2?"}]}`)
body2 := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"What's two plus two?"}]}`)
serveSemanticRequest(t, m, body1, "")
m.wg.Wait()
require.Equal(t, 1, store.Len())
rec := serveSemanticRequest(t, m, body2, "")
require.Equal(t, "HIT (semantic)", rec.Header().Get("X-Cache"))
}
func TestSemanticCacheMiddleware_MultiTurnParaphraseLastUserHit(t *testing.T) {
m, store, emb := newTestSemanticMiddleware(0.90, 10, false)
emb.vector = []float32{1, 0, 0}
body1 := []byte(`{"model":"gpt-4","messages":[
{"role":"user","content":"Remember the number 7."},
{"role":"assistant","content":"OK."},
{"role":"user","content":"What is 2+2?"}
]}`)
body2 := []byte(`{"model":"gpt-4","messages":[
{"role":"user","content":"Remember the number 7."},
{"role":"assistant","content":"OK."},
{"role":"user","content":"What's two plus two?"}
]}`)
serveSemanticRequest(t, m, body1, "")
m.wg.Wait()
require.Equal(t, 1, store.Len())
rec := serveSemanticRequest(t, m, body2, "")
require.Equal(t, "HIT (semantic)", rec.Header().Get("X-Cache"))
}
func TestSemanticCacheMiddleware_MultimodalAttachmentIsolatesNamespace(t *testing.T) {
m, store, emb := newTestSemanticMiddleware(0.90, 10, false)
emb.vector = []float32{1, 0, 0}
body1 := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":[
{"type":"text","text":"describe the image"},
{"type":"image_url","image_url":{"url":"https://example.com/a.png"}}
]}]}`)
body2 := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":[
{"type":"text","text":"describe the image"},
{"type":"image_url","image_url":{"url":"https://example.com/b.png"}}
]}]}`)
serveSemanticRequest(t, m, body1, "")
m.wg.Wait()
require.Equal(t, 1, store.Len())
rec := serveSemanticRequest(t, m, body2, "")
require.NotEqual(t, "HIT (semantic)", rec.Header().Get("X-Cache"))
}
func TestSemanticCacheMiddleware_CacheMissOnLowScore(t *testing.T) {
store := NewMapVecStore()
emb := &mockEmbedder{}
m := newSemanticCacheMiddleware(emb, store, config.SemanticCacheConfig{
SimilarityThreshold: 0.99,
TTL: new(3600),
MaxConversationMessages: new(10),
}, nil)
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`)
emb.vector = []float32{1, 0, 0}
serveSemanticRequest(t, m, body, "")
m.wg.Wait()
emb.vector = []float32{0, 1, 0}
rec := serveSemanticRequest(t, m, body, "")
require.NotEqual(t, "HIT (semantic)", rec.Header().Get("X-Cache"))
}
func TestSemanticCacheMiddleware_ParamsHashIsolation_Temperature(t *testing.T) {
m, store, emb := newTestSemanticMiddleware(0.90, 10, false)
emb.vector = []float32{1, 0, 0}
temp1 := 0.5
temp2 := 1.0
body1, _ := json.Marshal(map[string]any{
"model": "gpt-4",
"temperature": temp1,
"messages": []map[string]string{{"role": "user", "content": "same prompt"}},
})
body2, _ := json.Marshal(map[string]any{
"model": "gpt-4",
"temperature": temp2,
"messages": []map[string]string{{"role": "user", "content": "same prompt"}},
})
serveSemanticRequest(t, m, body1, "")
m.wg.Wait()
require.Equal(t, 1, store.Len())
rec := serveSemanticRequest(t, m, body2, "")
require.NotEqual(t, "HIT (semantic)", rec.Header().Get("X-Cache"))
}
func TestComputeParamsHash_StreamIncludeUsageChangesHash(t *testing.T) {
base := []byte(`{"model":"gpt-4","stream":true,"messages":[{"role":"user","content":"same prompt"}]}`)
withUsage := []byte(`{"model":"gpt-4","stream":true,"stream_options":{"include_usage":true},"messages":[{"role":"user","content":"same prompt"}]}`)
plan := &core.Workflow{
Mode: core.ExecutionModeTranslated,
ProviderType: "openai",
Resolution: &core.RequestModelResolution{
ResolvedSelector: core.ModelSelector{Provider: "openai", Model: "gpt-4"},
},
}
first := computeParamsHash(base, "/v1/chat/completions", plan, "", "", false)
second := computeParamsHash(withUsage, "/v1/chat/completions", plan, "", "", false)
require.NotEqual(t, second, first)
}
func TestComputeParamsHash_StreamModeChangesHash(t *testing.T) {
base := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"same prompt"}]}`)
streaming := []byte(`{"model":"gpt-4","stream":true,"messages":[{"role":"user","content":"same prompt"}]}`)
plan := &core.Workflow{
Mode: core.ExecutionModeTranslated,
ProviderType: "openai",
Resolution: &core.RequestModelResolution{
ResolvedSelector: core.ModelSelector{Provider: "openai", Model: "gpt-4"},
},
}
first := computeParamsHash(base, "/v1/chat/completions", plan, "", "", false)
second := computeParamsHash(streaming, "/v1/chat/completions", plan, "", "", false)
require.NotEqual(t, second, first)
}
func TestSemanticCacheMiddleware_GuardrailsHashIsolation(t *testing.T) {
m, store, emb := newTestSemanticMiddleware(0.90, 10, false)
emb.vector = []float32{1, 0, 0}
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"medical question"}]}`)
serveSemanticRequest(t, m, body, "guardrails-v1-hash")
m.wg.Wait()
require.Equal(t, 1, store.Len())
rec := serveSemanticRequest(t, m, body, "guardrails-v2-hash")
require.NotEqual(t, "HIT (semantic)", rec.Header().Get("X-Cache"))
}
func TestSemanticCacheMiddleware_DynamicGuardrailsIsolation(t *testing.T) {
m, store, emb := newTestSemanticMiddleware(0.90, 10, false)
emb.vector = []float32{1, 0, 0}
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"user PII data"}]}`)
serveSemanticRequest(t, m, body, "hash-user-a-pii-rule")
m.wg.Wait()
require.Equal(t, 1, store.Len())
rec := serveSemanticRequest(t, m, body, "hash-user-b-different-pii-rule")
require.NotEqual(t, "HIT (semantic)", rec.Header().Get("X-Cache"))
}
func TestSemanticCacheMiddleware_ConversationThreshold_Skipped(t *testing.T) {
m, store, emb := newTestSemanticMiddleware(0.90, 2, false)
emb.vector = []float32{1, 0, 0}
longConvBody := []byte(`{"model":"gpt-4","messages":[
{"role":"user","content":"msg1"},
{"role":"assistant","content":"resp1"},
{"role":"user","content":"msg2"},
{"role":"assistant","content":"resp2"},
{"role":"user","content":"msg3"}
]}`)
serveSemanticRequest(t, m, longConvBody, "")
m.wg.Wait()
require.Equal(t, 0, store.Len())
}
func TestSemanticCacheMiddleware_ExcludeSystemPrompt(t *testing.T) {
m, store, emb := newTestSemanticMiddleware(0.90, 10, true)
emb.vector = []float32{1, 0, 0}
body := []byte(`{"model":"gpt-4","messages":[
{"role":"system","content":"You are a helpful assistant."},
{"role":"user","content":"what is 2+2"}
]}`)
serveSemanticRequest(t, m, body, "")
m.wg.Wait()
require.Equal(t, 1, store.Len())
rec := serveSemanticRequest(t, m, body, "")
require.Equal(t, "HIT (semantic)", rec.Header().Get("X-Cache"))
}
func TestSemanticCacheMiddleware_StreamingMissPopulatesStreamingSemanticCacheOnly(t *testing.T) {
m, store, _ := newTestSemanticMiddleware(0.90, 10, false)
streamBody := []byte(`{"model":"gpt-4","stream":true,"messages":[{"role":"user","content":"semantic-stream-cache"}]}`)
jsonBody := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"semantic-stream-cache"}]}`)
rawStream := []byte(
"data: {\"id\":\"chatcmpl-semantic-stream\",\"object\":\"chat.completion.chunk\",\"created\":1234567890,\"model\":\"gpt-4\",\"provider\":\"openai\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Semantic\"},\"finish_reason\":null}]}\n\n" +
"data: {\"id\":\"chatcmpl-semantic-stream\",\"object\":\"chat.completion.chunk\",\"created\":1234567890,\"model\":\"gpt-4\",\"provider\":\"openai\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" cache\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":2,\"total_tokens\":9}}\n\n" +
"data: [DONE]\n\n",
)
handlerCalls := 0
run := func(body []byte) *httptest.ResponseRecorder {
t.Helper()
c, rec := echotest.Post(t, "/v1/chat/completions", body)
err := m.Handle(&echoExchange{c: c}, body, func() error {
handlerCalls++
if isStreamingRequest(c.Request().URL.Path, body) {
c.Response().Header().Set("Content-Type", "text/event-stream")
c.Response().WriteHeader(http.StatusOK)
_, _ = c.Response().Write(rawStream)
return nil
}
return c.JSON(http.StatusOK, map[string]string{"mode": "json"})
})
require.NoError(t, err)
return rec
}
rec1 := run(streamBody)
require.Empty(t, rec1.Header().Get("X-Cache"))
m.wg.Wait()
require.Equal(t, 1, store.Len())
rec2 := run(jsonBody)
require.Empty(t, rec2.Header().Get("X-Cache"))
require.Equal(t, "application/json", rec2.Header().Get("Content-Type"))
require.Contains(t, rec2.Body.String(), `"mode":"json"`, "non-streaming miss body = %q, want JSON response", rec2.Body.String())
require.Equal(t, 2, handlerCalls)
m.wg.Wait()
require.Equal(t, 2, store.Len())
rec3 := run(streamBody)
require.Equal(t, "HIT (semantic)", rec3.Header().Get("X-Cache"))
require.Equal(t, "text/event-stream", rec3.Header().Get("Content-Type"))
require.Equal(t, rawStream, rec3.Body.Bytes())
require.Equal(t, 2, handlerCalls)
rec4 := run(jsonBody)
require.Equal(t, "HIT (semantic)", rec4.Header().Get("X-Cache"))
require.Equal(t, "application/json", rec4.Header().Get("Content-Type"))
require.Contains(t, rec4.Body.String(), `"mode":"json"`, "non-streaming semantic hit body = %q, want cached JSON response", rec4.Body.String())
require.Equal(t, 2, handlerCalls)
}
func TestSemanticCacheMiddleware_InvalidStreamingBodySkipsSemanticCacheWrite(t *testing.T) {
m, store, _ := newTestSemanticMiddleware(0.90, 10, false)
body := []byte(`{"model":"gpt-4","stream":true,"messages":[{"role":"user","content":"semantic-invalid-stream"}]}`)
invalidStream := []byte(
": keep-alive\n\n" +
"data: {\"id\":\"chatcmpl-semantic-invalid\",\"object\":\"chat.completion.chunk\",\"model\":\"gpt-4\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"partial\"},\"finish_reason\":null}]}\n\n",
)
handlerCalls := 0
run := func() *httptest.ResponseRecorder {
t.Helper()
c, rec := echotest.Post(t, "/v1/chat/completions", body)
err := m.Handle(&echoExchange{c: c}, body, func() error {
handlerCalls++
c.Response().Header().Set("Content-Type", "text/event-stream")
c.Response().WriteHeader(http.StatusOK)
_, _ = c.Response().Write(invalidStream)
return nil
})
require.NoError(t, err)
return rec
}
rec1 := run()
require.Empty(t, rec1.Header().Get("X-Cache"))
m.wg.Wait()
require.Equal(t, 0, store.Len())
rec2 := run()
require.Empty(t, rec2.Header().Get("X-Cache"))
require.Equal(t, 2, handlerCalls)
}
func TestSemanticCacheMiddleware_NoCacheControlSkip(t *testing.T) {
m, store, _ := newTestSemanticMiddleware(0.90, 10, false)
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"no-store test"}]}`)
c, _ := echotest.Post(t, "/v1/chat/completions", body, echotest.WithHeader("Cache-Control", "no-store"))
handlerCalled := false
err := m.Handle(&echoExchange{c: c}, body, func() error {
handlerCalled = true
return c.JSON(http.StatusOK, map[string]string{"r": "1"})
})
require.NoError(t, err)
m.wg.Wait()
require.Equal(t, 0, store.Len())
require.True(t, handlerCalled)
}
func TestSemanticCacheMiddleware_HeaderThresholdOverride(t *testing.T) {
// Against the cached {1, 0, 0}, near has cosine similarity ~0.9986 and
// far ~0.9507; the configured threshold is 0.99.
near := []float32{0.95, 0.05, 0}
far := []float32{0.95, 0.31, 0}
tests := []struct {
name string
vector []float32
header string
wantCache string
}{
{name: "near hits at configured threshold", vector: near, wantCache: CacheHeaderSemantic},
{name: "higher header tightens matching", vector: near, header: "0.9999", wantCache: ""},
{name: "far misses at configured threshold", vector: far, wantCache: ""},
{name: "lower header cannot loosen matching", vector: far, header: "0.50", wantCache: ""},
{name: "tiny header cannot loosen matching", vector: far, header: "0.0001", wantCache: ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
store := NewMapVecStore()
emb := &mockEmbedder{}
m := newSemanticCacheMiddleware(emb, store, config.SemanticCacheConfig{
SimilarityThreshold: 0.99,
TTL: new(3600),
MaxConversationMessages: new(10),
}, nil)
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`)
emb.vector = []float32{1, 0, 0}
serveSemanticRequest(t, m, body, "")
m.wg.Wait()
emb.vector = tt.vector
var opts []echotest.Option
if tt.header != "" {
opts = append(opts, echotest.WithHeader("X-Cache-Semantic-Threshold", tt.header))
}
c, rec := echotest.Post(t, "/v1/chat/completions", body, opts...)
err := m.Handle(&echoExchange{c: c}, body, func() error {
return c.JSON(http.StatusOK, map[string]string{"r": "1"})
})
require.NoError(t, err)
assert.Equal(t, tt.wantCache, rec.Header().Get("X-Cache"))
})
}
}
func TestSemanticCacheMiddleware_TTLExpiry(t *testing.T) {
store := NewMapVecStore()
current := time.Now()
store.now = func() time.Time { return current }
emb := &mockEmbedder{vector: []float32{1, 0, 0}}
m := newSemanticCacheMiddleware(emb, store, config.SemanticCacheConfig{
SimilarityThreshold: 0.90,
TTL: new(1),
MaxConversationMessages: new(10),
}, nil)
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"expiry test"}]}`)
serveSemanticRequest(t, m, body, "")
m.wg.Wait()
require.Equal(t, 1, store.Len())
current = current.Add(2 * time.Second)
err := store.DeleteExpired(context.Background())
require.NoError(t, err)
require.Equal(t, 0, store.Len())
}
func TestMapVecStore_DeleteExpiredOnlyRemovesExpired(t *testing.T) {
store := NewMapVecStore()
ctx := context.Background()
_ = store.Insert(ctx, "key-expired", []float32{1, 0}, nil, "ph", -1*time.Second)
_ = store.Insert(ctx, "key-live", []float32{0, 1}, nil, "ph", time.Hour)
_ = store.DeleteExpired(ctx)
require.Equal(t, 1, store.Len())
results, _ := store.Search(ctx, []float32{0, 1}, "ph", 1)
require.NotEmpty(t, results)
require.Equal(t, "key-live", results[0].Key)
}
func TestShouldSkipAllCacheHeaders_CacheControl(t *testing.T) {
for _, cacheControl := range []string{"private, no-store, max-age=0", "private, no-cache, max-age=0"} {
header := http.Header{"Cache-Control": []string{cacheControl}}
require.True(t, shouldSkipAllCacheHeaders(header.Get), "Cache-Control: %s", cacheControl)
}
}
func TestSemanticCacheMiddleware_HitMarksAuditEntryCacheType(t *testing.T) {
m, _, _ := newTestSemanticMiddleware(0.90, 10, false)
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"semantic-cache-type"}]}`)
rec1 := serveSemanticRequest(t, m, body, "")
require.Empty(t, rec1.Header().Get("X-Cache"))
m.wg.Wait()
entry := &auditlog.LogEntry{ID: "semantic-audit-entry"}
c, rec := echotest.Post(t, "/v1/chat/completions", body, echotest.WithValue(string(auditlog.LogEntryKey), entry))
err := m.Handle(&echoExchange{c: c}, body, func() error {
return c.JSON(http.StatusOK, map[string]string{"answer": "42"})
})
require.NoError(t, err)
require.Equal(t, "HIT (semantic)", rec.Header().Get("X-Cache"))
require.Equal(t, auditlog.CacheTypeSemantic, entry.CacheType)
}