internal/app/application_test.go
Rama main · réplica de lectura del repositorio original.
//go:build !nobundledharness
package app
import (
"context"
"encoding/json"
"errors"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"time"
"github.com/yeomyeonggeori/blueclaw/internal/agentruntime"
"github.com/yeomyeonggeori/blueclaw/internal/capability"
"github.com/yeomyeonggeori/blueclaw/internal/config"
"github.com/yeomyeonggeori/blueclaw/internal/connectors"
"github.com/yeomyeonggeori/blueclaw/internal/defaultharness"
"github.com/yeomyeonggeori/blueclaw/internal/llm"
"github.com/yeomyeonggeori/blueclaw/internal/protocolidentity"
"github.com/yeomyeonggeori/blueclaw/internal/runtimecontrol"
"github.com/yeomyeonggeori/blueclaw/internal/task"
"github.com/yeomyeonggeori/blueprotocol/agentcontract"
)
func TestTaskIntakeControllerStartsUnquiesced(t *testing.T) {
controller := runtimecontrol.NewTaskIntakeController(nil)
if controller.IsQuiesced() {
t.Fatal("fresh task intake controller must start unquiesced")
}
}
func TestCapabilityToolDescriptorsPreserveResultContracts(t *testing.T) {
inputIntentSchema := json.RawMessage(`{"type":"object","properties":{"title":{"type":"string"}},"additionalProperties":false}`)
descriptors := capabilityToolDescriptors([]config.CapabilityToolDescriptor{{
Name: "task_add",
InputIntentSchema: inputIntentSchema,
ResultContract: &config.CapabilityToolResultContract{
Schema: json.RawMessage(`{"type":"object","properties":{"taskID":{"type":"string"}},"required":["taskID"],"additionalProperties":false}`),
Effects: []config.CapabilityResourceEffectContract{{
ObjectType: "task",
Effect: "created",
ResultField: "taskID",
EffectIdentity: "id",
}},
EvidenceCondition: &config.EvidenceCondition{
ResultField: "taskID",
Equals: json.RawMessage(`"task-1"`),
},
},
}})
if len(descriptors) != 1 || descriptors[0].ResultContract == nil {
t.Fatalf("expected mapped result contract, got %+v", descriptors)
}
if string(descriptors[0].InputIntentSchema) != string(inputIntentSchema) {
t.Fatalf("expected mapped input intent schema, got %s", descriptors[0].InputIntentSchema)
}
if len(descriptors[0].ResultContract.Effects) != 1 ||
descriptors[0].ResultContract.Effects[0].ObjectType != "task" ||
descriptors[0].ResultContract.Effects[0].Effect != "created" ||
descriptors[0].ResultContract.Effects[0].ResultField != "taskID" ||
descriptors[0].ResultContract.Effects[0].EffectIdentity != "id" {
t.Fatalf("expected mapped resource effect, got %+v", descriptors[0].ResultContract)
}
if descriptors[0].ResultContract.EvidenceCondition == nil ||
string(descriptors[0].ResultContract.EvidenceCondition.Equals) != `"task-1"` {
t.Fatalf("expected mapped evidence condition, got %+v", descriptors[0].ResultContract)
}
}
func TestResolveIntakeLanguageModelProviderUsesReliableTaskTierModel(t *testing.T) {
runtimeConfiguration := configuredModelTierRuntime("")
runtimeConfiguration.Agent.Intake.Enabled = true
runtimeConfiguration.Agent.Intake.ExecutionMode = "auto"
languageModelProvider, errorValue := resolveIntakeLanguageModelProvider(runtimeConfiguration, nil)
if errorValue != nil {
t.Fatal(errorValue)
}
fallbackLanguageModelProvider, isFallbackProvider := languageModelProvider.(llm.FallbackLanguageModelProvider)
if !isFallbackProvider {
t.Fatalf("expected fallback intake provider, got %T", languageModelProvider)
}
primaryClient, isPrimaryCapabilityClient := unwrapModelTier(fallbackLanguageModelProvider.PrimaryProvider).(llm.CapabilityLLMClient)
if !isPrimaryCapabilityClient {
t.Fatalf("expected primary capability intake provider, got %T", fallbackLanguageModelProvider.PrimaryProvider)
}
fallbackClient, isFallbackCapabilityClient := unwrapModelTier(fallbackLanguageModelProvider.FallbackProvider).(llm.CapabilityLLMClient)
if !isFallbackCapabilityClient {
t.Fatalf("expected fallback capability intake provider, got %T", fallbackLanguageModelProvider.FallbackProvider)
}
if primaryClient.ModelName != "vendor/medium" {
t.Fatalf("expected the medium tier model for intake, got %q", primaryClient.ModelName)
}
if fallbackClient.ModelName != "vendor/high" {
t.Fatalf("expected the high tier model as the intake fallback, got %q", fallbackClient.ModelName)
}
if primaryClient.ExecutionMode != "auto" || fallbackClient.ExecutionMode != "auto" {
t.Fatalf("expected automatic intake execution mode, got %q and %q", primaryClient.ExecutionMode, fallbackClient.ExecutionMode)
}
}
func TestMaximumXLowTierCapsTaskModelsAndUsesLowForImages(t *testing.T) {
runtimeConfiguration := configuredModelTierRuntime("xlow")
providers, errorValue := resolveTaskTierLanguageModelProviders(runtimeConfiguration, slog.New(slog.DiscardHandler))
if errorValue != nil {
t.Fatal(errorValue)
}
for providerName, provider := range map[string]llm.LanguageModelProvider{
"low": providers.Low, "xlow": providers.XLow, "medium": providers.Medium,
"high": providers.High, "xhigh": providers.XHigh, "max": providers.Max,
} {
modelNames := languageModelProviderNames(provider)
if !reflect.DeepEqual(modelNames, []string{"vendor/low", "vendor/xlow"}) {
t.Fatalf("expected %s provider to use only xlow text and low vision models, got %v", providerName, modelNames)
}
}
}
func TestMaximumLowTierCapsIntakeFallbacks(t *testing.T) {
runtimeConfiguration := configuredModelTierRuntime("low")
runtimeConfiguration.Agent.Intake.Enabled = true
provider, errorValue := resolveIntakeLanguageModelProvider(runtimeConfiguration, nil)
if errorValue != nil {
t.Fatal(errorValue)
}
modelNames := languageModelProviderNames(provider)
if !reflect.DeepEqual(modelNames, []string{"vendor/low", "vendor/xlow"}) {
t.Fatalf("expected intake to stay at or below low, got %v", modelNames)
}
}
func configuredModelTierRuntime(maximumModelTier string) config.RuntimeConfiguration {
runtimeConfiguration := config.RuntimeConfiguration{}
runtimeConfiguration.LanguageModel.MaximumModelTier = maximumModelTier
runtimeConfiguration.LanguageModel.Capability.MaxModel = "vendor/max"
runtimeConfiguration.LanguageModel.Capability.XHighModel = "vendor/xhigh"
runtimeConfiguration.LanguageModel.Capability.HighModel = "vendor/high"
runtimeConfiguration.LanguageModel.Capability.MediumModel = "vendor/medium"
runtimeConfiguration.LanguageModel.Capability.LowModel = "vendor/low"
runtimeConfiguration.LanguageModel.Capability.XLowModel = "vendor/xlow"
return runtimeConfiguration
}
func languageModelProviderNames(provider llm.LanguageModelProvider) []string {
modelNames := map[string]bool{}
collectLanguageModelProviderNames(provider, modelNames)
result := make([]string, 0, len(modelNames))
for modelName := range modelNames {
result = append(result, modelName)
}
sort.Strings(result)
return result
}
func collectLanguageModelProviderNames(provider llm.LanguageModelProvider, modelNames map[string]bool) {
if tieredProvider, isTieredProvider := provider.(interface {
UnderlyingProvider() llm.LanguageModelProvider
}); isTieredProvider {
collectLanguageModelProviderNames(tieredProvider.UnderlyingProvider(), modelNames)
return
}
switch typedProvider := provider.(type) {
case llm.CapabilityLLMClient:
modelNames[typedProvider.ModelName] = true
case llm.FallbackLanguageModelProvider:
collectLanguageModelProviderNames(typedProvider.PrimaryProvider, modelNames)
collectLanguageModelProviderNames(typedProvider.FallbackProvider, modelNames)
case llm.VisionFallbackProvider:
collectLanguageModelProviderNames(typedProvider.TextOnlyModel, modelNames)
collectLanguageModelProviderNames(typedProvider.VisionModel, modelNames)
}
}
func unwrapModelTier(provider llm.LanguageModelProvider) llm.LanguageModelProvider {
if tieredProvider, isTieredProvider := provider.(interface {
UnderlyingProvider() llm.LanguageModelProvider
}); isTieredProvider {
return tieredProvider.UnderlyingProvider()
}
return provider
}
func TestLoadAgentInstructionPromptUsesIdentitySoulAgentsAndSkills(t *testing.T) {
workspacePath := t.TempDir()
skillDirectoryPath := filepath.Join(workspacePath, ".agents", "skills", "browser")
if errorValue := os.MkdirAll(skillDirectoryPath, 0o755); errorValue != nil {
t.Fatalf("expected skill directory: %v", errorValue)
}
if errorValue := os.WriteFile(filepath.Join(workspacePath, "identity.json"), []byte(`{"schemaVersion": 1, "names": ["김인턴", "Intern Kim"], "handle": "internkim", "role": "the company's AI intern"}`), 0o600); errorValue != nil {
t.Fatalf("expected identity file: %v", errorValue)
}
if errorValue := os.WriteFile(filepath.Join(workspacePath, "soul.json"), []byte(`{"schemaVersion": 1, "values": ["Lead with the result."]}`), 0o600); errorValue != nil {
t.Fatalf("expected soul file: %v", errorValue)
}
if errorValue := os.WriteFile(filepath.Join(workspacePath, "AGENTS.md"), []byte("Use agent-browser for web automation."), 0o600); errorValue != nil {
t.Fatalf("expected agents file: %v", errorValue)
}
if errorValue := os.WriteFile(filepath.Join(skillDirectoryPath, "SKILL.md"), []byte("Run agent-browser snapshot -i after navigation."), 0o600); errorValue != nil {
t.Fatalf("expected skill file: %v", errorValue)
}
runtimeConfiguration := config.RuntimeConfiguration{}
runtimeConfiguration.Terminal.WorkspaceRootPath = workspacePath
instructionBundle := loadAgentInstructionBundle(runtimeConfiguration, agentruntime.NewCapabilityRegistry(capability.Client{}, nil))
for _, expectedFragment := range []string{"Your name is 김인턴.", "\"Intern Kim\"", "@internkim", "Your role: the company's AI intern", "What you hold to:\n- Lead with the result.", "Use agent-browser for web automation."} {
if !strings.Contains(instructionBundle.Prompt, expectedFragment) {
t.Fatalf("expected instruction prompt to contain %q, got %q", expectedFragment, instructionBundle.Prompt)
}
}
if len(instructionBundle.Skills) == 0 || instructionBundle.Skills[0].Name != "browser" {
t.Fatalf("expected skill metadata to be loaded: %+v", instructionBundle.Skills)
}
}
func TestLoadAgentIdentityReadsTheIdentityDocument(t *testing.T) {
workspacePath := t.TempDir()
if errorValue := os.WriteFile(filepath.Join(workspacePath, "identity.json"), []byte(`{"schemaVersion": 1, "names": ["김인턴"], "handle": "internkim"}`), 0o600); errorValue != nil {
t.Fatalf("expected identity file: %v", errorValue)
}
runtimeConfiguration := config.RuntimeConfiguration{}
runtimeConfiguration.Terminal.WorkspaceRootPath = workspacePath
agentIdentity := loadAgentIdentity(runtimeConfiguration)
if agentIdentity.Name != "김인턴" || agentIdentity.Handle != "internkim" {
t.Fatalf("expected the identity document's name and handle, got %+v", agentIdentity)
}
}
func TestAPersonaDocumentTheSchemaRefusesIsReportedAndLeftOut(t *testing.T) {
workspacePath := t.TempDir()
if errorValue := os.WriteFile(filepath.Join(workspacePath, "identity.json"), []byte(`{"schemaVersion": 1, "names": ["김인턴"], "handle": "internkim", "nickname": "kim"}`), 0o600); errorValue != nil {
t.Fatalf("expected identity file: %v", errorValue)
}
runtimeConfiguration := config.RuntimeConfiguration{}
runtimeConfiguration.Terminal.WorkspaceRootPath = workspacePath
instructions := loadAgentInstructions(runtimeConfiguration, agentruntime.NewCapabilityRegistry(capability.Client{}, nil))
if strings.Contains(instructions.Bundle.Prompt, "김인턴") {
t.Fatalf("expected the refused identity to stay out of the prompt, got %q", instructions.Bundle.Prompt)
}
if len(instructions.RejectedDocuments) != 1 || !strings.Contains(instructions.RejectedDocuments[0].Error.Error(), "nickname") {
t.Fatalf("expected the refusal to be reported with its reason, got %+v", instructions.RejectedDocuments)
}
if agentIdentity := loadAgentIdentity(runtimeConfiguration); agentIdentity.Name != "" {
t.Fatalf("expected no identity from a refused document, got %+v", agentIdentity)
}
}
func TestLoadAgentInstructionBundleDiscoversAddedUserSkill(t *testing.T) {
workspacePath := t.TempDir()
skillDirectoryPath := filepath.Join(workspacePath, ".agents", "skills", "research-helper")
if errorValue := os.MkdirAll(skillDirectoryPath, 0o755); errorValue != nil {
t.Fatalf("expected skill directory: %v", errorValue)
}
skillDocument := `---
name: research-helper
description: Help with research tasks and source lookup requests.
---
Research helper body.
`
if errorValue := os.WriteFile(filepath.Join(skillDirectoryPath, "SKILL.md"), []byte(skillDocument), 0o600); errorValue != nil {
t.Fatalf("expected skill file: %v", errorValue)
}
runtimeConfiguration := config.RuntimeConfiguration{}
runtimeConfiguration.Terminal.WorkspaceRootPath = workspacePath
instructionBundle := loadAgentInstructionBundle(runtimeConfiguration, agentruntime.NewCapabilityRegistry(capability.Client{}, nil))
if len(instructionBundle.Skills) != 1 || instructionBundle.Skills[0].Name != "research-helper" {
t.Fatalf("expected added user skill to be discovered, got %+v", instructionBundle.Skills)
}
if instructionBundle.Skills[0].Description != "Help with research tasks and source lookup requests." {
t.Fatalf("expected standard skill fields, got %+v", instructionBundle.Skills[0])
}
}
func TestDeriveAllowedToolNamesByProfileKeepsDomainCapabilitiesOutOfBaseline(t *testing.T) {
runtimeConfiguration := config.RuntimeConfiguration{}
runtimeConfiguration.AgentProfiles = []config.AgentProfileConfiguration{
{Name: "default", AllowedToolNames: []string{"bash"}},
}
runtimeConfiguration.Capabilities.ToolDescriptors = []config.CapabilityToolDescriptor{{Name: "event_add"}}
allowedToolNamesByProfile := deriveAllowedToolNamesByProfile(runtimeConfiguration)
defaultProfileToolNames := allowedToolNamesByProfile["default"]
if containsString(defaultProfileToolNames, "event_add") {
t.Fatalf("expected domain capability to stay out of profile baseline, got %+v", defaultProfileToolNames)
}
for _, expectedToolName := range []string{"bash", "file_deliver", "skill_search", "file_read", "write", "edit", "file_preview", "image_read"} {
if !containsString(defaultProfileToolNames, expectedToolName) {
t.Fatalf("expected baseline tool %q, got %+v", expectedToolName, defaultProfileToolNames)
}
}
if containsString(defaultProfileToolNames, "ask_confirm") {
t.Fatalf("expected runtime-owned confirmation to stay out of model tools, got %+v", defaultProfileToolNames)
}
if containsString(defaultProfileToolNames, "ask_input") {
t.Fatalf("expected typed user input to stay out of the baseline tools, got %+v", defaultProfileToolNames)
}
}
func TestApplicationChecksProtocolIdentityOnceAndStoresResult(t *testing.T) {
protocolVersion := "0.4.0"
aggregateProtocolHash := "58ff1977989bacbf2db3fdce08fd57c9b52f344ca747a3322f4e60bdf6052a78"
requestCount := 0
server := httptest.NewServer(http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) {
requestCount++
responseWriter.Header().Set("Content-Type", "application/json")
_, _ = responseWriter.Write([]byte(`{"status":"ok","protocolVersion":"` + protocolVersion + `","aggregateProtocolHash":"` + aggregateProtocolHash + `"}`))
}))
defer server.Close()
runtimeConfiguration := config.RuntimeConfiguration{}
runtimeConfiguration.Logging.DirectoryPath = t.TempDir()
runtimeConfiguration.Capabilities.Endpoint = server.URL
runtimeConfiguration.Capabilities.ProtocolVersion = protocolVersion
runtimeConfiguration.Capabilities.AggregateProtocolHash = aggregateProtocolHash
application := NewApplication(runtimeConfiguration, "", InboundOptions{}, WithBundledACPFactory(defaultharness.NewFactory))
application.protocolIdentityExpected = protocolidentity.Identity{
ProtocolVersion: protocolVersion,
AggregateProtocolHash: aggregateProtocolHash,
}
application.protocolIdentityChecker = protocolidentity.NewChecker(protocolidentity.Configuration{
CapabilityEndpoint: server.URL,
HTTPClient: server.Client(),
})
if errorValue := application.checkProtocolIdentity(); errorValue != nil {
t.Fatalf("expected protocol identity check to pass: %v", errorValue)
}
if errorValue := application.checkProtocolIdentity(); errorValue != nil {
t.Fatalf("expected repeated protocol identity check to reuse result: %v", errorValue)
}
if requestCount != 2 {
t.Fatalf("expected one registry warm-up and one capabilityd request, got %d", requestCount)
}
if !application.protocolIdentityStatus.Passed {
t.Fatalf("expected stored protocol identity result to pass: %+v", application.protocolIdentityStatus)
}
}
func TestApplicationAutoResumeLaunchesAtMostFiveInterruptedTaskRuns(t *testing.T) {
taskEventService := task.NewTaskEventService()
taskRunService := task.NewTaskRunService(taskEventService)
now := time.Now()
repository := &applicationAutoResumeRepository{}
for index := 0; index < 7; index++ {
taskRun := task.TaskRun{
TaskRunID: "task-" + string(rune('a'+index)),
RequesterPersonID: "person-1",
OriginConversationID: "conversation-1",
OriginReplyTargetID: "reply-1",
Status: task.TaskStatusInterrupted,
Prompt: "finish task",
CreatedAt: now.Add(time.Duration(index) * time.Minute),
UpdatedAt: now.Add(time.Duration(index) * time.Minute),
}
repository.taskRuns = append(repository.taskRuns, taskRun)
taskEventService.AppendTaskEvent(taskRun.TaskRunID, agentcontract.TaskEventTaskInterrupted, "runtime restarted")
}
taskRunService.UseRepository(repository)
resumer := &applicationAutoResumeResumer{}
application := &Application{
taskRunService: taskRunService,
interruptedTaskResumer: resumer,
interruptedTaskResumeDelay: 0,
}
application.resumeInterruptedTaskRuns(context.Background(), now.Add(time.Hour))
if len(resumer.taskRunIDs) != 5 {
t.Fatalf("resume count = %d, want 5", len(resumer.taskRunIDs))
}
for _, taskRunID := range resumer.taskRunIDs {
if !taskEventsContainApplicationEvent(taskRunService.ListTaskEvent(taskRunID), "task.auto_resume_attempted") {
t.Fatalf("expected auto-resume attempt event for %s", taskRunID)
}
}
skippedCount := 0
for _, taskRun := range repository.taskRuns {
if taskEventsContainApplicationEvent(taskRunService.ListTaskEvent(taskRun.TaskRunID), "task.auto_resume_skipped") {
skippedCount++
}
}
if skippedCount != 2 {
t.Fatalf("skipped count = %d, want 2", skippedCount)
}
}
type applicationAutoResumeResumer struct {
taskRunIDs []string
failedTaskRunIDs []string
}
func (resumer *applicationAutoResumeResumer) CanResumeInterruptedTaskRun(task.TaskRun) bool {
return true
}
func (resumer *applicationAutoResumeResumer) ResumeInterruptedTaskRun(_ context.Context, taskRun task.TaskRun) (connectors.ConnectorRuntimeResult, error) {
resumer.taskRunIDs = append(resumer.taskRunIDs, taskRun.TaskRunID)
return connectors.ConnectorRuntimeResult{Handled: true, TaskRunID: taskRun.TaskRunID}, nil
}
func (resumer *applicationAutoResumeResumer) FailUnresumedInterruptedTaskRun(_ context.Context, taskRun task.TaskRun, _ string) bool {
resumer.failedTaskRunIDs = append(resumer.failedTaskRunIDs, taskRun.TaskRunID)
return true
}
func TestApplicationFailsUnresumedInterruptedTaskRunsWithNotice(t *testing.T) {
taskEventService := task.NewTaskEventService()
taskRunService := task.NewTaskRunService(taskEventService)
now := time.Now()
repository := &applicationAutoResumeRepository{}
for index := 0; index < 2; index++ {
taskRun := task.TaskRun{
TaskRunID: "exhausted-" + string(rune('a'+index)),
RequesterPersonID: "person-1",
OriginConversationID: "conversation-1",
OriginReplyTargetID: "reply-1",
Status: task.TaskStatusInterrupted,
FailureReason: task.TaskInterruptReasonRuntimeRestart,
Prompt: "finish task",
CreatedAt: now,
UpdatedAt: now,
}
repository.taskRuns = append(repository.taskRuns, taskRun)
taskEventService.AppendTaskEvent(taskRun.TaskRunID, agentcontract.TaskEventTaskInterrupted, task.TaskInterruptReasonRuntimeRestart)
taskEventService.AppendTaskEvent(taskRun.TaskRunID, agentcontract.TaskEventTaskAutoResumeAttempted, `{"attemptCount":1}`)
}
taskRunService.UseRepository(repository)
resumer := &applicationAutoResumeResumer{}
application := &Application{
taskRunService: taskRunService,
interruptedTaskResumer: resumer,
interruptedTaskResumeDelay: 0,
}
application.resumeInterruptedTaskRuns(context.Background(), now.Add(time.Hour))
if len(resumer.taskRunIDs) != 0 {
t.Fatalf("resume count = %d, want 0 for attempt-exhausted crash interruptions", len(resumer.taskRunIDs))
}
if len(resumer.failedTaskRunIDs) != 2 {
t.Fatalf("failed count = %d, want 2", len(resumer.failedTaskRunIDs))
}
for _, taskRun := range repository.taskRuns {
if !taskEventsContainApplicationEvent(taskRunService.ListTaskEvent(taskRun.TaskRunID), "task.auto_resume_abandoned") {
t.Fatalf("expected auto-resume abandoned event for %s", taskRun.TaskRunID)
}
}
}
type applicationAutoResumeRepository struct {
taskRuns []task.TaskRun
}
func (repository *applicationAutoResumeRepository) SaveTaskRun(taskRun task.TaskRun) error {
repository.taskRuns = append(repository.taskRuns, taskRun)
return nil
}
func (repository *applicationAutoResumeRepository) StartTaskRunAttempt(task.TaskRun, task.TaskAttempt) error {
return nil
}
func (repository *applicationAutoResumeRepository) FinishTaskRunAttempt(task.TaskRun, task.TaskAttempt) error {
return nil
}
func (repository *applicationAutoResumeRepository) TransitionTaskRun(transition task.TaskRunTransition) (task.TaskRun, error) {
for index, taskRun := range repository.taskRuns {
if taskRun.TaskRunID != transition.TaskRunID {
continue
}
if !applicationTaskRunStatusAllowed(taskRun.Status, transition.FromStates) {
return task.TaskRun{}, task.ErrIllegalTransition{
TaskRunID: transition.TaskRunID,
CurrentStatus: taskRun.Status,
FromStates: append([]task.TaskStatus{}, transition.FromStates...),
ToState: transition.ToState,
}
}
taskRun.Status = transition.ToState
taskRun.UpdatedAt = transition.UpdatedAt
repository.taskRuns[index] = taskRun
return taskRun, nil
}
return task.TaskRun{}, errors.New("task run not found")
}
func applicationTaskRunStatusAllowed(status task.TaskStatus, allowedStatuses []task.TaskStatus) bool {
for _, allowedStatus := range allowedStatuses {
if status == allowedStatus {
return true
}
}
return false
}
func (repository *applicationAutoResumeRepository) FindTaskRun(taskRunID string) (task.TaskRun, bool, error) {
for _, taskRun := range repository.taskRuns {
if taskRun.TaskRunID == taskRunID {
return taskRun, true, nil
}
}
return task.TaskRun{}, false, nil
}
func (repository *applicationAutoResumeRepository) FindTaskAttempt(string) (task.TaskAttempt, bool, error) {
return task.TaskAttempt{}, false, nil
}
func (repository *applicationAutoResumeRepository) ListTaskRun() ([]task.TaskRun, error) {
return append([]task.TaskRun{}, repository.taskRuns...), nil
}
func (repository *applicationAutoResumeRepository) ListTaskRunByPersonID(personID string) ([]task.TaskRun, error) {
taskRuns := []task.TaskRun{}
for _, taskRun := range repository.taskRuns {
if taskRun.RequesterPersonID == personID {
taskRuns = append(taskRuns, taskRun)
}
}
return taskRuns, nil
}
func (repository *applicationAutoResumeRepository) DeleteTaskRun(string, []string) (bool, error) {
return false, nil
}
func (repository *applicationAutoResumeRepository) DeleteTaskRunsBefore(time.Time, []string) ([]string, error) {
return nil, nil
}
func taskEventsContainApplicationEvent(taskEvents []task.TaskEvent, name string) bool {
for _, taskEvent := range taskEvents {
if taskEvent.Name == name {
return true
}
}
return false
}
func TestResolveTaskTierLanguageModelProvidersEscalateLowToMedium(t *testing.T) {
runtimeConfiguration := configuredModelTierRuntime("")
providers, errorValue := resolveTaskTierLanguageModelProviders(runtimeConfiguration, slog.New(slog.DiscardHandler))
if errorValue != nil {
t.Fatal(errorValue)
}
lowProvider, isFallbackProvider := providers.Low.(llm.FallbackLanguageModelProvider)
if !isFallbackProvider {
t.Fatalf("expected low tier to escalate on failure, got %T", providers.Low)
}
if lowProvider.PrimaryLabel != "low" || lowProvider.FallbackLabel != "medium" {
t.Fatalf("expected low/medium labels, got %q/%q", lowProvider.PrimaryLabel, lowProvider.FallbackLabel)
}
xLowProvider, isFallbackProvider := providers.XLow.(llm.FallbackLanguageModelProvider)
if !isFallbackProvider {
t.Fatalf("expected xlow tier fallback provider, got %T", providers.XLow)
}
if _, xLowClimbsTheLadder := xLowProvider.FallbackProvider.(llm.FallbackLanguageModelProvider); !xLowClimbsTheLadder {
t.Fatalf("expected xlow to fall into the low-to-medium ladder, got %T", xLowProvider.FallbackProvider)
}
mediumProvider, isFallbackProvider := providers.Medium.(llm.FallbackLanguageModelProvider)
if !isFallbackProvider {
t.Fatalf("expected medium tier fallback provider, got %T", providers.Medium)
}
if _, mediumWouldRetryItself := mediumProvider.FallbackProvider.(llm.FallbackLanguageModelProvider); mediumWouldRetryItself {
t.Fatalf("expected medium to fall back to the bare low model without climbing back up, got %T", mediumProvider.FallbackProvider)
}
}
func TestResolveTaskTierLanguageModelProvidersKeepsBareLowWhenMediumMatchesLow(t *testing.T) {
runtimeConfiguration := configuredModelTierRuntime("")
runtimeConfiguration.LanguageModel.Capability.LowModel = "vendor/shared"
runtimeConfiguration.LanguageModel.Capability.MediumModel = "vendor/shared"
providers, errorValue := resolveTaskTierLanguageModelProviders(runtimeConfiguration, slog.New(slog.DiscardHandler))
if errorValue != nil {
t.Fatal(errorValue)
}
if _, isFallbackProvider := providers.Low.(llm.FallbackLanguageModelProvider); isFallbackProvider {
t.Fatal("expected bare low provider when the medium model equals the low model")
}
}
func TestEveryTaskTierPostsTheModelItsEndpointWasGiven(t *testing.T) {
receivedModelNames := make(chan string, 6)
server := httptest.NewServer(http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) {
requestDocument := struct {
Model string `json:"model"`
}{}
if errorValue := json.NewDecoder(request.Body).Decode(&requestDocument); errorValue != nil {
t.Errorf("expected a decodable completion request: %v", errorValue)
}
receivedModelNames <- requestDocument.Model
responseWriter.Header().Set("Content-Type", "application/json")
_, _ = responseWriter.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":"call-1","type":"function","function":{"name":"answer","arguments":"{}"}}]},"finish_reason":"tool_calls"}]}`))
}))
defer server.Close()
runtimeConfiguration := config.RuntimeConfiguration{}
runtimeConfiguration.LanguageModel.Tiers = map[string][]config.ModelEndpointConfiguration{}
for _, modelTier := range llm.ModelTiers {
runtimeConfiguration.LanguageModel.Tiers[modelTier] = []config.ModelEndpointConfiguration{
{Endpoint: server.URL, Model: "vendor/one"},
}
}
providers, errorValue := resolveTaskTierLanguageModelProviders(runtimeConfiguration, slog.New(slog.DiscardHandler))
if errorValue != nil {
t.Fatal(errorValue)
}
for tierName, provider := range map[string]llm.LanguageModelProvider{
"xlow": providers.XLow,
"low": providers.Low,
"medium": providers.Medium,
"high": providers.High,
"xhigh": providers.XHigh,
"max": providers.Max,
} {
if provider == nil {
t.Fatalf("expected a %s tier provider", tierName)
}
request := llm.StructuredResponseRequest{
Messages: []llm.Message{{Role: "user", Content: "hello"}},
StructuredOutputSchema: llm.StructuredOutputSchema{
Name: "answer",
Document: `{"type":"object","properties":{},"additionalProperties":false}`,
},
}
if _, errorValue := provider.GenerateStructuredResponse(context.Background(), request); errorValue != nil {
t.Fatalf("expected the %s tier to reach the endpoint: %v", tierName, errorValue)
}
receivedModelName := <-receivedModelNames
if receivedModelName != "vendor/one" {
t.Fatalf("the %s tier asked the endpoint for %q, which it does not serve", tierName, receivedModelName)
}
}
}
func TestApplicationServesHealthWhenProtocolIdentityDisagrees(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) {
responseWriter.Header().Set("Content-Type", "application/json")
_, _ = responseWriter.Write([]byte(`{"status":"ok","protocolVersion":"0.4.0","aggregateProtocolHash":"b4b5c630888e6de30e52ec88809c9356cf2fb42d7e1189215d3de0d44e60c775"}`))
}))
defer server.Close()
runtimeConfiguration := config.RuntimeConfiguration{}
runtimeConfiguration.Logging.DirectoryPath = t.TempDir()
runtimeConfiguration.Capabilities.Endpoint = server.URL
application := NewApplication(runtimeConfiguration, "", InboundOptions{}, WithBundledACPFactory(defaultharness.NewFactory))
application.httpServer.Addr = "127.0.0.1:0"
application.protocolIdentityExpected = protocolidentity.Identity{
ProtocolVersion: "0.4.0",
AggregateProtocolHash: "fccec45c4b3fc539159b3a293d61275ed2fc4ae738f371ec9122c1546b32a42f",
}
application.protocolIdentityChecker = protocolidentity.NewChecker(protocolidentity.Configuration{
CapabilityEndpoint: server.URL,
HTTPClient: server.Client(),
})
serveError := make(chan error, 1)
go func() { serveError <- application.Start() }()
t.Cleanup(func() { _ = application.httpServer.Close() })
select {
case errorValue := <-serveError:
t.Fatalf("a rejected protocol identity must not end the process: %v", errorValue)
case <-time.After(500 * time.Millisecond):
}
if application.protocolIdentityStatus.Passed {
t.Fatal("expected the stored protocol identity result to record the disagreement")
}
}