internal/registry/devin_models_test.go
Rama main · réplica de lectura del repositorio original.
package registry
import (
"context"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"time"
)
func TestValidateDevinModelsJSONAggregatesLightning(t *testing.T) {
data := []byte(`{
"devin": [
{"id": "swe-2-medium-lightning", "display_name": "SWE-2 Medium Lightning", "owned_by": "cognition"},
{"id": "swe-2-high-lightning", "display_name": "SWE-2 High Lightning", "owned_by": "cognition"},
{"id": "swe-2-max-lightning", "display_name": "SWE-2 Max Lightning", "owned_by": "cognition"}
]
}`)
models, err := ValidateDevinModelsJSON(data)
if err != nil {
t.Fatalf("expected valid, got error: %v", err)
}
if len(models) != 1 {
t.Fatalf("expected 3 lightning UIDs to aggregate into 1 model, got %d: %+v", len(models), models)
}
m := models[0]
if m.ID != "devin/swe-2-lightning" {
t.Errorf("aggregated ID = %q, want devin/swe-2-lightning", m.ID)
}
if m.DisplayName != "SWE-2 Lightning" {
t.Errorf("aggregated DisplayName = %q, want SWE-2 Lightning", m.DisplayName)
}
if m.Thinking == nil {
t.Fatal("expected Thinking levels on aggregated lightning model")
}
wantLevels := []string{"medium", "high", "max"}
if !reflect.DeepEqual(m.Thinking.Levels, wantLevels) {
t.Errorf("Thinking.Levels = %v, want %v", m.Thinking.Levels, wantLevels)
}
}
func TestValidateDevinModelsJSON(t *testing.T) {
t.Run("valid envelope devin", func(t *testing.T) {
data := []byte(`{
"devin": [
{
"id": "devin/swe-2",
"display_name": "SWE-2",
"owned_by": "cognition",
"context_length": 262000
}
]
}`)
models, err := ValidateDevinModelsJSON(data)
if err != nil {
t.Fatalf("expected valid, got error: %v", err)
}
if len(models) != 1 || models[0].ID != "devin/swe-2" {
t.Fatalf("unexpected models: %+v", models)
}
if models[0].Type != "devin" {
t.Errorf("expected type 'devin', got %q", models[0].Type)
}
})
t.Run("valid envelope models", func(t *testing.T) {
data := []byte(`{
"models": [
{
"id": "devin/glm-5-2",
"display_name": "GLM-5.2"
}
]
}`)
models, err := ValidateDevinModelsJSON(data)
if err != nil {
t.Fatalf("expected valid, got error: %v", err)
}
if len(models) != 1 || models[0].ID != "devin/glm-5-2" {
t.Fatalf("unexpected models: %+v", models)
}
})
t.Run("valid direct array", func(t *testing.T) {
data := []byte(`[
{
"id": "devin/deepseek-v4-flash",
"display_name": "DeepSeek V4 Flash"
}
]`)
models, err := ValidateDevinModelsJSON(data)
if err != nil {
t.Fatalf("expected valid, got error: %v", err)
}
if len(models) != 1 || models[0].ID != "devin/deepseek-v4-flash" {
t.Fatalf("unexpected models: %+v", models)
}
})
t.Run("clean id without devin prefix automatically namespaced", func(t *testing.T) {
data := []byte(`{
"devin": [
{
"id": "swe-2",
"display_name": "SWE-2"
}
]
}`)
models, err := ValidateDevinModelsJSON(data)
if err != nil {
t.Fatalf("expected valid, got error: %v", err)
}
if len(models) != 1 || models[0].ID != "devin/swe-2" {
t.Fatalf("expected auto-namespaced to devin/swe-2, got: %q", models[0].ID)
}
})
t.Run("empty payload", func(t *testing.T) {
_, err := ValidateDevinModelsJSON([]byte(` `))
if err == nil {
t.Fatal("expected error on empty payload, got nil")
}
})
t.Run("empty model id", func(t *testing.T) {
data := []byte(`{"devin": [{"id": "", "display_name": "No ID"}]}`)
_, err := ValidateDevinModelsJSON(data)
if err == nil {
t.Fatal("expected error on empty model id, got nil")
}
})
for name, data := range map[string][]byte{
"exact duplicate": []byte(`{"devin": [{"id": "devin/swe-2"}, {"id": "devin/swe-2"}]}`),
"case duplicate": []byte(`{"devin": [{"id": "devin/SWE-2"}, {"id": "devin/swe-2"}]}`),
} {
t.Run("duplicate model id/"+name, func(t *testing.T) {
_, err := ValidateDevinModelsJSON(data)
if err == nil {
t.Fatal("expected error on duplicate model id, got nil")
}
})
}
}
func TestEmbeddedDevinModelsLoadedOnStartup(t *testing.T) {
models := GetDevinModels()
if len(models) < 30 {
t.Fatalf("expected at least 30 embedded Devin models, got %d", len(models))
}
foundMap := make(map[string]bool)
for _, m := range models {
foundMap[m.ID] = true
}
expectedIDs := []string{
"devin/swe-2",
"devin/swe-2-lightning",
"devin/swe-1-6-slow",
"devin/glm-5-2",
"devin/glm-5-3",
"devin/deepseek-v4-flash",
"devin/deepseek-v4-1-flash-1017",
"devin/gemini-3-8-flash",
"devin/grok-4-6",
"devin/claude-fable-5-1",
"devin/gpt-6-astra",
}
for _, id := range expectedIDs {
if !foundMap[id] {
t.Errorf("expected embedded catalog to contain %q", id)
}
}
}
func TestDevinModelsRemoteFetchFallback(t *testing.T) {
// Test remote failure maintains existing embedded data
origURLs := devinModelsURLs
defer func() { devinModelsURLs = origURLs }()
// Point to failing server
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "server error", http.StatusInternalServerError)
}))
defer ts.Close()
devinModelsURLs = []string{ts.URL + "/devin_models.json"}
initialCount := len(GetDevinModels())
if initialCount == 0 {
t.Fatal("expected non-empty initial Devin models")
}
// Attempt refresh from failing remote
tryRefreshDevinModels(context.Background(), "test failing refresh")
afterCount := len(GetDevinModels())
if afterCount != initialCount {
t.Fatalf("expected catalog to remain intact with %d models, got %d", initialCount, afterCount)
}
// Point to succeeding server with valid update
validUpdate := []byte(`{
"devin": [
{
"id": "devin/custom-test-model",
"display_name": "Custom Test Model",
"owned_by": "custom"
}
]
}`)
tsValid := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(validUpdate)
}))
defer tsValid.Close()
devinModelsURLs = []string{tsValid.URL + "/devin_models.json"}
tryRefreshDevinModels(context.Background(), "test succeeding refresh")
updatedModels := GetDevinModels()
foundCustom := false
foundBuiltinSlow := false
for _, m := range updatedModels {
if m != nil && m.ID == "devin/custom-test-model" {
foundCustom = true
}
if m != nil && (m.ID == "devin/swe-1-6-slow" || m.ID == "swe-1-6-slow") {
foundBuiltinSlow = true
}
}
if !foundCustom {
t.Fatalf("expected catalog to be updated to include custom-test-model, got: %+v", updatedModels)
}
if !foundBuiltinSlow {
t.Fatalf("expected catalog to retain builtin swe-1-6-slow, got: %+v", updatedModels)
}
// Restore original embedded data for following tests
_, _ = loadDevinModelsFromBytes(embeddedDevinModelsJSON, "restore-embed")
}
func TestFetchDevinModelsFromRemote_ContextNotCanceledBeforeRead_Issue6095(t *testing.T) {
origURLs := devinModelsURLs
defer func() { devinModelsURLs = origURLs }()
headerFlushed := make(chan struct{})
sendBody := make(chan struct{})
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
close(headerFlushed)
<-sendBody
_, _ = w.Write([]byte(`{"devin": [{"id": "devin/swe-2"}]}`))
}))
defer ts.Close()
go func() {
<-headerFlushed
// Yield briefly to let client.Do return and advance to body reading before streaming body chunks.
time.Sleep(20 * time.Millisecond)
close(sendBody)
}()
devinModelsURLs = []string{ts.URL + "/devin_models.json"}
body, source := fetchDevinModelsFromRemote(context.Background())
if body == nil || source == "" {
t.Fatalf("expected successful fetch of devin models, got nil body (source=%q)", source)
}
expectedBody := `{"devin": [{"id": "devin/swe-2"}]}`
if string(body) != expectedBody {
t.Fatalf("expected body %q, got %q", expectedBody, string(body))
}
expectedSource := ts.URL + "/devin_models.json"
if source != expectedSource {
t.Fatalf("expected source %q, got %q", expectedSource, source)
}
}
func TestDevinSWE16Slow_Issue6111(t *testing.T) {
// Verify that devin/swe-1-6-slow is present in GetDevinModels
models := GetDevinModels()
var found *ModelInfo
for _, m := range models {
if m != nil && m.ID == "devin/swe-1-6-slow" {
found = m
break
}
}
if found == nil {
t.Fatalf("devin/swe-1-6-slow missing from GetDevinModels()")
}
// Verify LookupDevinModel with and without prefix
lookupWithPrefix := LookupDevinModel("devin/swe-1-6-slow")
if lookupWithPrefix == nil {
t.Errorf("LookupDevinModel(\"devin/swe-1-6-slow\") returned nil")
}
lookupWithoutPrefix := LookupDevinModel("swe-1-6-slow")
if lookupWithoutPrefix == nil {
t.Errorf("LookupDevinModel(\"swe-1-6-slow\") returned nil")
}
// Verify model metadata
if lookupWithPrefix != nil {
if lookupWithPrefix.ID != "devin/swe-1-6-slow" {
t.Errorf("expected ID 'devin/swe-1-6-slow', got %q", lookupWithPrefix.ID)
}
if lookupWithPrefix.DisplayName != "SWE-1.6 Slow" {
t.Errorf("expected DisplayName 'SWE-1.6 Slow', got %q", lookupWithPrefix.DisplayName)
}
if lookupWithPrefix.Type != "devin" {
t.Errorf("expected Type 'devin', got %q", lookupWithPrefix.Type)
}
if lookupWithPrefix.OwnedBy != "cognition" {
t.Errorf("expected OwnedBy 'cognition', got %q", lookupWithPrefix.OwnedBy)
}
}
// Verify persistence even if dynamic update loads payload without swe-1-6-slow
payloadWithoutSlow := []byte(`{"devin": [{"id": "devin/swe-2", "display_name": "SWE-2"}]}`)
if _, errUpdate := loadDevinModelsFromBytes(payloadWithoutSlow, "test-dynamic-update"); errUpdate != nil {
t.Fatalf("loadDevinModelsFromBytes failed: %v", errUpdate)
}
defer func() {
// Restore embedded models after test
if _, errRestore := loadDevinModelsFromBytes(embeddedDevinModelsJSON, "embed"); errRestore != nil {
t.Errorf("loadDevinModelsFromBytes restore failed: %v", errRestore)
}
}()
modelsAfterUpdate := GetDevinModels()
foundAfter := false
for _, m := range modelsAfterUpdate {
if m != nil && m.ID == "devin/swe-1-6-slow" {
foundAfter = true
break
}
}
if !foundAfter {
t.Errorf("devin/swe-1-6-slow missing from GetDevinModels() after dynamic update")
}
if LookupDevinModel("devin/swe-1-6-slow") == nil {
t.Errorf("LookupDevinModel(\"devin/swe-1-6-slow\") returned nil after dynamic update")
}
}
func TestWithDevinBuiltins(t *testing.T) {
// Test WithDevinBuiltins with nil slice
builtins := WithDevinBuiltins(nil)
if len(builtins) != 1 {
t.Fatalf("expected 1 builtin model, got %d", len(builtins))
}
if builtins[0].ID != "devin/swe-1-6-slow" {
t.Errorf("expected ID 'devin/swe-1-6-slow', got %q", builtins[0].ID)
}
// Test WithDevinBuiltins preserves existing models and injects swe-1-6-slow
existing := []*ModelInfo{
{ID: "devin/swe-2", DisplayName: "SWE-2"},
}
merged := WithDevinBuiltins(existing)
if len(merged) != 2 {
t.Fatalf("expected 2 models, got %d", len(merged))
}
// Test WithDevinBuiltins replaces duplicate
duplicate := []*ModelInfo{
{ID: "devin/swe-1-6-slow", DisplayName: "Old Display Name"},
}
replaced := WithDevinBuiltins(duplicate)
if len(replaced) != 1 {
t.Fatalf("expected 1 model after deduplication, got %d", len(replaced))
}
if replaced[0].DisplayName != "SWE-1.6 Slow" {
t.Errorf("expected DisplayName 'SWE-1.6 Slow', got %q", replaced[0].DisplayName)
}
}
func TestEmbeddedDevinModels_NoThinkingSuffixes_Issue6141(t *testing.T) {
models := GetDevinModels()
disallowedSuffixes := []string{
"-low-fast", "-medium-fast", "-high-fast", "-xhigh-fast", "-max-fast",
"-none-fast", "-minimal-lightning", "-low-lightning", "-medium-lightning",
"-high-lightning", "-xhigh-lightning", "-max-lightning", "-none-lightning",
"-low", "-medium", "-high", "-xhigh", "-max", "-none",
"-thinking-1m", "-thinking", "_none", "_minimal", "_low", "_medium",
"_high", "_xhigh", "_max", "_thinking",
}
var exposedThinkingVariants []string
for _, m := range models {
cleanID := strings.TrimPrefix(m.ID, "devin/")
for _, s := range disallowedSuffixes {
if strings.HasSuffix(cleanID, s) {
exposedThinkingVariants = append(exposedThinkingVariants, m.ID)
break
}
}
}
if len(exposedThinkingVariants) > 0 {
t.Fatalf("Devin catalog exposes thinking-parameter model variants (count=%d): %v", len(exposedThinkingVariants), exposedThinkingVariants)
}
}
func TestValidateDevinModelsJSON_AggregatesThinkingVariants_Issue6141(t *testing.T) {
payload := []byte(`{
"devin": [
{
"id": "claude-opus-5",
"display_name": "Claude Opus 5",
"owned_by": "anthropic",
"thinking": {"levels": ["medium"]}
},
{
"id": "claude-opus-5-low-fast",
"display_name": "Claude Opus 5 Low Fast",
"owned_by": "anthropic"
},
{
"id": "claude-opus-5-high-fast",
"display_name": "Claude Opus 5 High Fast",
"owned_by": "anthropic"
}
]
}`)
models, err := ValidateDevinModelsJSON(payload)
if err != nil {
t.Fatalf("ValidateDevinModelsJSON failed: %v", err)
}
if len(models) != 1 {
t.Fatalf("expected 1 aggregated model, got %d: %+v", len(models), models)
}
if models[0].ID != "devin/claude-opus-5" {
t.Errorf("expected ID devin/claude-opus-5, got %q", models[0].ID)
}
if models[0].Thinking == nil {
t.Fatalf("expected Thinking to be populated")
}
expectedLevels := []string{"low", "medium", "high"}
if !reflect.DeepEqual(models[0].Thinking.Levels, expectedLevels) {
t.Fatalf("expected thinking levels %v, got %v", expectedLevels, models[0].Thinking.Levels)
}
}