packages/ai/test/decision.test.ts
Rama dev · réplica de lectura del repositorio original.
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientRequest } from "effect/http"
import { ai, DecisionClient, DecisionUsage, Media } from "../src/effect.js"
import {
Baseten,
CloudflareWorkersAI,
OpenAI,
OpenCode,
OpenRouter,
TypeSafeAI,
VercelAIGateway,
} from "../src/providers.js"
import { it } from "./lib/effect.js"
import { dynamicResponse, json, type Handler } from "./lib/http.js"
const client = (handler: Handler) => DecisionClient.layer.pipe(Layer.provide(dynamicResponse(handler)))
const offline = client(() => Effect.die("an invalid decision request reached the network"))
const png = Media.bytes(new Uint8Array([1, 2, 3]), "image/png")
const questions = [
{
name: "department",
type: "choice",
instructions: "Which team should handle this?",
choices: [{ value: "billing", description: "Payments and refunds" }, { value: "technical" }],
},
{
name: "urgency",
type: "score",
instructions: "How urgent is this?",
levels: [{ label: "Can wait" }, { label: "Needs attention", description: "Same day" }, { label: "Blocking" }],
},
{ name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" },
] as const
const defect = {
name: "defect",
type: "predicate",
instructions: "Is the customer reporting a product defect?",
criteria: {
true: "The product itself is broken or faulty.",
false: "Preference, sizing, shipping, or pricing complaint.",
},
} as const
const systemOneAnswers = {
department: { type: "choice", choice: "billing", probabilities: { technical: 0.1, billing: 0.9 }, confidence: 0.8 },
urgency: {
type: "score",
score: 1.2,
probabilities: { "2": 0.2, "0": 0, "1": 0.8 },
legend: { "0": "Can wait", "1": "Needs attention: Same day", "2": "Blocking" },
confidence: 0.6,
},
refund: { type: "noul", noul: 0.97 },
}
const expectedAnswers = [
{
name: "department",
type: "choice",
choice: "billing",
probabilities: [
{ value: "billing", probability: 0.9 },
{ value: "technical", probability: 0.1 },
],
confidence: 0.8,
},
{
name: "urgency",
type: "score",
score: 1.2,
probabilities: [
{ value: 0, label: "Can wait", probability: 0 },
{ value: 1, label: "Needs attention", probability: 0.8 },
{ value: 2, label: "Blocking", probability: 0.2 },
],
confidence: 0.6,
},
{ name: "refund", type: "predicate", probability: 0.97 },
]
describe("Decision", () => {
it.effect("decides through System One with TypeSafe image assets", () =>
Effect.gen(function* () {
const response = yield* ai.decision.generate({
model: TypeSafeAI.configure({
apiKey: "test",
baseURL: "https://typesafe.test/v1/",
headers: { "x-default": "yes" },
http: { body: { deployment: "test" }, query: { api: "v1" } },
}).decision("jev-latest"),
input: [
{ type: "text", text: "Ticket:" },
{ type: "json", value: { ticket: "Please refund the duplicate charge." } },
{ type: "media", media: png },
],
questions,
providerOptions: { trace: { enabled: true } },
headers: { "x-request": "yes" },
http: { body: { request_metadata: "value" }, query: { trace: "1" } },
})
expect(response.answers).toEqual(expectedAnswers)
expect(response.usage?.totalTokens).toBe(36)
expect(response.providerMetadata).toEqual({
typesafe: { model: "jev-1.14.0", usage: { input_tokens: 30, output_tokens: 6 } },
})
}).pipe(
Effect.provide(
client((input) =>
Effect.gen(function* () {
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(request.url).toBe("https://typesafe.test/v1/systemone?api=v1&trace=1")
expect(request.headers.get("authorization")).toBe("Bearer test")
expect(request.headers.get("x-default")).toBe("yes")
expect(request.headers.get("x-request")).toBe("yes")
expect(request.headers.get("typesafe-danger-zone-images")).toBe("true")
expect(JSON.parse(input.text)).toEqual({
deployment: "test",
request_metadata: "value",
trace: { enabled: true },
model: "jev-latest",
state: ["Ticket:", { ticket: "Please refund the duplicate charge." }, "<image-1>"],
assets: { "<image-1>": { type: "image", format: "png", data_base64: "AQID" } },
questions: {
department: {
type: "choice",
instructions: "Which team should handle this?",
criteria: { billing: "Payments and refunds", technical: null },
},
urgency: {
type: "score",
instructions: "How urgent is this?",
criteria: ["Can wait", "Needs attention: Same day", "Blocking"],
},
refund: { type: "noul", instructions: "Is the customer asking for a refund?" },
},
})
return json(input, {
model: "jev-1.14.0",
answers: systemOneAnswers,
usage: { input_tokens: 30, output_tokens: 6 },
})
}),
),
),
),
)
it.effect("sends a lone input object as the System One state and maps boolean choices back", () =>
ai.decision
.generate({
model: OpenCode.configure({ apiKey: "opencode-key", baseURL: "https://opencode.test/v1" }).decision("jev-1.13"),
input: { ticket: "hello" },
questions: [
{
name: "greeting",
type: "choice",
instructions: "Is this a greeting?",
choices: [{ value: true }, { value: false }],
},
],
})
.pipe(
Effect.tap((response) =>
Effect.sync(() => {
const answer = response.answers[0]
if (answer.type !== "choice") throw new Error("expected a choice answer")
expect(answer.choice).toBe(true)
expect(answer.probabilities).toEqual([
{ value: true, probability: 0.99 },
{ value: false, probability: 0.01 },
])
expect(response.usage).toEqual(
new DecisionUsage({ inputTokens: 10, outputTokens: 2, totalTokens: 12, cost: 0.0004 }),
)
expect(response.providerMetadata).toEqual({
opencode: { model: "jev-1.13.0", usage: { input_tokens: 10, output_tokens: 2, cost: 0.0004 } },
})
}),
),
Effect.provide(
client((input) => {
expect(input.request.url).toBe("https://opencode.test/v1/systemone")
expect(input.request.headers.authorization).toBe("Bearer opencode-key")
expect(input.request.headers["typesafe-danger-zone-images"]).toBe("true")
expect(JSON.parse(input.text)).toEqual({
model: "jev-1.13",
state: { ticket: "hello" },
questions: {
greeting: {
type: "choice",
instructions: "Is this a greeting?",
criteria: { true: null, false: null },
},
},
})
return Effect.succeed(
json(input, {
model: "jev-1.13.0",
answers: {
greeting: {
type: "choice",
choice: "true",
probabilities: { false: 0.01, true: 0.99 },
confidence: 1,
},
},
usage: { input_tokens: 10, output_tokens: 2, cost: 0.0004 },
}),
)
}),
),
),
)
it.effect("rejects a choice with both a boolean and a string of the same text for System One", () =>
Effect.gen(function* () {
const error = yield* ai.decision
.generate({
model: TypeSafeAI.decision("jev-latest"),
input: { ticket: "hello" },
questions: [
{ name: "ok", type: "choice", instructions: "OK?", choices: [{ value: true }, { value: "true" }] },
],
})
.pipe(Effect.flip)
expect(error.reason._tag).toBe("InvalidRequest")
expect(error.message).toContain("ok")
}).pipe(Effect.provide(offline)),
)
it.effect("places media in the OpenRouter state and downloads URL media first", () =>
ai.decision
.generate({
model: OpenRouter.configure({ apiKey: "openrouter-key", baseURL: "https://openrouter.test/api/v1" }).decision(
"cloudflare/clef-omni",
),
input: [
{ type: "text", text: "Before:" },
{ type: "media", media: Media.url("https://assets.test/before.png", { mediaType: "image/png" }) },
{ type: "media", media: Media.base64("AAAA", "audio/wav") },
],
questions: [{ name: "red", type: "predicate", instructions: "Is the image mostly red?" }],
providerOptions: { user: "user-1", session_id: "session-1" },
})
.pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.answers).toEqual([{ name: "red", type: "predicate", probability: 0.98 }])
expect(response.providerMetadata?.openrouter).toEqual({
model: "cloudflare/clef-omni-20261001",
responseId: "gen-1",
provider: "Cloudflare",
})
}),
),
Effect.provide(
client((input) => {
if (input.request.method === "GET") {
expect(input.request.url).toBe("https://assets.test/before.png")
return Effect.succeed(
input.respond(new Uint8Array([1, 2, 3]), { headers: { "content-type": "image/png" } }),
)
}
expect(input.request.url).toBe("https://openrouter.test/api/v1/systemone")
expect(JSON.parse(input.text)).toEqual({
model: "cloudflare/clef-omni",
user: "user-1",
session_id: "session-1",
state: [
"Before:",
{ type: "image_url", image_url: { url: "data:image/png;base64,AQID" } },
{ type: "input_audio", input_audio: { data: "AAAA", format: "wav" } },
],
questions: { red: { type: "noul", instructions: "Is the image mostly red?" } },
})
return Effect.succeed(
json(input, {
id: "gen-1",
model: "cloudflare/clef-omni-20261001",
provider: "Cloudflare",
answers: { red: { type: "noul", noul: 0.98 } },
}),
)
}),
),
),
)
it.effect("puts media in a user message inside the Baseten state", () =>
ai.decision
.generate({
model: Baseten.configure({ apiKey: "baseten-key" }).decision("decision-models/baseten-mammoth-27b-latest"),
input: [
{ type: "text", text: "Photo:" },
{ type: "media", media: png },
{ type: "json", value: { source: "upload" } },
],
questions: [{ name: "red", type: "predicate", instructions: "Is the photo mostly red?" }],
})
.pipe(
Effect.tap((response) =>
Effect.sync(() => expect(response.answers).toEqual([{ name: "red", type: "predicate", probability: 0.99 }])),
),
Effect.provide(
client((input) => {
expect(input.request.url).toBe("https://inference.baseten.co/v1/systemone")
expect(JSON.parse(input.text)).toEqual({
model: "decision-models/baseten-mammoth-27b-latest",
state: {
messages: [
{
role: "user",
content: [
{ type: "text", text: "Photo:" },
{ type: "image_url", image_url: { url: "data:image/png;base64,AQID" } },
{ type: "text", text: '{"source":"upload"}' },
],
},
],
},
questions: { red: { type: "noul", instructions: "Is the photo mostly red?" } },
})
return Effect.succeed(json(input, { model: "mammoth", answers: { red: { type: "noul", noul: 0.99 } } }))
}),
),
),
)
it.effect("decides through OpenAI Decisions", () =>
ai.decision
.generate({
model: OpenAI.configure({
apiKey: "openai-key",
baseURL: "https://openai.test/v1",
queryParams: { api: "v1" },
}).decision("gpt-6-luna"),
input: [
{ type: "text", text: "Ticket:" },
{ type: "json", value: { ticket: "Please refund the duplicate charge." } },
{ type: "json", value: 42 },
{ type: "media", media: png },
{ type: "media", media: Media.url("https://assets.test/receipt.jpg", { mediaType: "image/jpeg" }) },
],
questions,
providerOptions: { safetyIdentifier: "user-1", promptCacheKey: "support-v1" },
})
.pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.answers).toEqual(expectedAnswers)
expect(response.usage?.inputTokens).toBe(40)
expect(response.providerMetadata).toEqual({
openai: { model: "gpt-6-luna", usage: { input_tokens: 40, output_tokens: 0, total_tokens: 40 } },
})
}),
),
Effect.provide(
client((input) => {
expect(input.request.url).toBe("https://openai.test/v1/decisions?api=v1")
expect(input.request.headers.authorization).toBe("Bearer openai-key")
expect(JSON.parse(input.text)).toEqual({
safety_identifier: "user-1",
prompt_cache_key: "support-v1",
model: "gpt-6-luna",
input: [
{
role: "user",
content: [
{ type: "input_text", text: "Ticket:" },
{ type: "input_json", data: { ticket: "Please refund the duplicate charge." } },
{ type: "input_text", text: "42" },
{ type: "input_image", image_url: "data:image/png;base64,AQID" },
{ type: "input_image", image_url: "https://assets.test/receipt.jpg" },
],
},
],
questions: [
{
name: "department",
type: "choice",
instructions: "Which team should handle this?",
choices: [{ value: "billing", description: "Payments and refunds" }, { value: "technical" }],
},
{
name: "urgency",
type: "score",
instructions: "How urgent is this?",
levels: [
{ label: "Can wait" },
{ label: "Needs attention", description: "Same day" },
{ label: "Blocking" },
],
},
{ name: "refund", type: "predicate", instructions: "Is the customer asking for a refund?" },
],
})
// Answers come back out of order; the protocol restores question order by name.
return Effect.succeed(
json(input, {
model: "gpt-6-luna",
answers: [
{ type: "predicate", name: "refund", probability: 0.97 },
{
type: "choice",
name: "department",
choice: "billing",
probabilities: [
{ value: "billing", probability: 0.9 },
{ value: "technical", probability: 0.1 },
],
confidence: 0.8,
},
{
type: "score",
name: "urgency",
score: 1.2,
probabilities: [
{ value: 0, label: "Can wait", probability: 0 },
{ value: 1, label: "Needs attention", probability: 0.8 },
{ value: 2, label: "Blocking", probability: 0.2 },
],
confidence: 0.6,
},
],
usage: { input_tokens: 40, output_tokens: 0, total_tokens: 40 },
}),
)
}),
),
),
)
it.effect("returns OpenAI refusals as per-question errors", () =>
ai.decision
.generate({
model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"),
input: [{ type: "text", text: "hello" }],
questions: [
{ name: "greeting", type: "predicate", instructions: "Is this a greeting?" },
{ name: "unsafe", type: "predicate", instructions: "Something the model declines" },
{ name: "explained", type: "predicate", instructions: "Something the model declines with a reason" },
],
})
.pipe(
Effect.tap((response) =>
Effect.sync(() =>
expect(response.answers).toEqual([
{ name: "greeting", type: "predicate", probability: 0.99 },
{ name: "unsafe", type: "error", reason: "refusal" },
{ name: "explained", type: "error", reason: "refusal", message: "I can't help with that." },
]),
),
),
Effect.provide(
client((input) =>
Effect.succeed(
json(input, {
model: "gpt-6-luna",
answers: [
{ type: "predicate", name: "greeting", probability: 0.99 },
{ type: "refusal", name: "unsafe" },
{ type: "refusal", name: "explained", message: "I can't help with that." },
],
}),
),
),
),
),
)
it.effect("rejects audio for OpenAI Decisions before network I/O", () =>
Effect.gen(function* () {
const error = yield* ai.decision
.generate({
model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"),
input: [{ type: "media", media: Media.base64("AAAA", "audio/wav") }],
questions: [{ name: "speech", type: "predicate", instructions: "Is someone speaking?" }],
})
.pipe(Effect.flip)
expect(error.reason._tag).toBe("UnsupportedOperation")
expect(error.message).toContain("audio")
}).pipe(Effect.provide(offline)),
)
it.effect("decides through Vercel AI Gateway's OpenAI Decisions endpoint", () =>
ai.decision
.generate({
model: VercelAIGateway.configure({ apiKey: "gateway-key", baseURL: "https://gateway.test/v1/" }).decision(
"typesafe-ai/jev",
),
input: [{ type: "text", text: "refund" }],
questions: [{ name: "refund", type: "predicate", instructions: "Is a refund requested?" }],
})
.pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.answers).toEqual([{ name: "refund", type: "predicate", probability: 0.98 }])
expect(response.providerMetadata).toEqual({
"vercel-ai-gateway": { model: "typesafe-ai/jev", usage: { input_tokens: 10, output_tokens: 0 } },
})
}),
),
Effect.provide(
client((input) => {
expect(input.request.url).toBe("https://gateway.test/v1/decisions")
expect(input.request.headers.authorization).toBe("Bearer gateway-key")
expect(JSON.parse(input.text)).toEqual({
model: "typesafe-ai/jev",
input: [{ role: "user", content: [{ type: "input_text", text: "refund" }] }],
questions: [{ name: "refund", type: "predicate", instructions: "Is a refund requested?" }],
})
return Effect.succeed(
json(input, {
model: "typesafe-ai/jev",
answers: [{ type: "predicate", name: "refund", probability: 0.98 }],
usage: { input_tokens: 10, output_tokens: 0 },
}),
)
}),
),
),
)
it.effect("sends predicate criteria as System One criteria", () =>
ai.decision
.generate({
model: TypeSafeAI.configure({ apiKey: "test" }).decision("jev-latest"),
input: [{ type: "text", text: "I returned the sweater because I prefer green." }],
questions: [defect],
})
.pipe(
Effect.tap((response) =>
Effect.sync(() =>
expect(response.answers).toEqual([{ name: "defect", type: "predicate", probability: 0.03 }]),
),
),
Effect.provide(
client((input) => {
expect(JSON.parse(input.text).questions).toEqual({
defect: {
type: "noul",
instructions: "Is the customer reporting a product defect?",
criteria: {
true: "The product itself is broken or faulty.",
false: "Preference, sizing, shipping, or pricing complaint.",
},
},
})
return Effect.succeed(
json(input, { model: "jev-latest", answers: { defect: { type: "noul", noul: 0.03 } } }),
)
}),
),
),
)
it.effect("appends predicate criteria to OpenAI Decisions instructions", () =>
ai.decision
.generate({
model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"),
input: [{ type: "text", text: "I returned the sweater because I prefer green." }],
questions: [defect, { ...defect, name: "only_true", criteria: { true: "The product is broken." } }],
})
.pipe(
Effect.provide(
client((input) => {
expect(JSON.parse(input.text).questions).toEqual([
{
name: "defect",
type: "predicate",
instructions:
"Is the customer reporting a product defect?\n\nTrue means: The product itself is broken or faulty.\nFalse means: Preference, sizing, shipping, or pricing complaint.",
},
{
name: "only_true",
type: "predicate",
instructions: "Is the customer reporting a product defect?\n\nTrue means: The product is broken.",
},
])
return Effect.succeed(
json(input, {
model: "gpt-6-luna",
answers: [
{ type: "predicate", name: "defect", probability: 0.03 },
{ type: "predicate", name: "only_true", probability: 0.04 },
],
}),
)
}),
),
),
)
it.effect("puts media in Clef's top-level lists on Cloudflare Run", () =>
ai.decision
.generate({
model: CloudflareWorkersAI.configure({ accountId: "test-account", apiKey: "cloudflare-key" }).decision(
"@cf/cloudflare/clef-omni",
),
input: [
{ type: "text", text: "Compare:" },
{ type: "media", media: png },
{ type: "media", media: Media.base64("AAAA", "audio/wav") },
],
questions: [{ name: "red", type: "predicate", instructions: "Is the image mostly red?" }],
})
.pipe(
Effect.tap((response) =>
Effect.sync(() => {
expect(response.answers).toEqual([{ name: "red", type: "predicate", probability: 0.9 }])
expect(response.providerMetadata).toEqual({ "cloudflare-workers-ai": { model: "clef-omni" } })
}),
),
Effect.provide(
client((input) => {
expect(input.request.url).toBe("https://api.cloudflare.com/client/v4/accounts/test-account/ai/run")
expect(JSON.parse(input.text)).toEqual({
model: "@cf/cloudflare/clef-omni",
input: {
state: "Compare:",
images: ["data:image/png;base64,AQID"],
audio: ["data:audio/wav;base64,AAAA"],
questions: { red: { type: "noul", instructions: "Is the image mostly red?" } },
},
})
return Effect.succeed(
json(input, { result: { model: "clef-omni", answers: { red: { type: "noul", noul: 0.9 } } } }),
)
}),
),
),
)
it.effect("surfaces Cloudflare Run errors", () =>
ai.decision
.generate({
model: CloudflareWorkersAI.configure({ accountId: "test-account", apiKey: "cloudflare-key" }).decision(
"@cf/cloudflare/clef",
),
input: [{ type: "text", text: "refund" }],
questions: [{ name: "refund", type: "predicate", instructions: "Is a refund requested?" }],
})
.pipe(
Effect.flip,
Effect.tap((error) =>
Effect.sync(() => {
expect(error.reason._tag).toBe("InvalidRequest")
expect(error.message).toContain("At least one question is required")
}),
),
Effect.provide(
client((input) =>
Effect.succeed(
json(
input,
{
errors: [
{
message:
"Model execution failed (User Input Error): Validation error at questions: At least one question is required",
code: 7003,
},
],
success: false,
result: {},
messages: [],
},
{ status: 400 },
),
),
),
),
),
)
it.effect("fails the call when an answer picks an option that wasn't offered", () =>
ai.decision
.generate({
model: OpenAI.configure({ apiKey: "openai-key" }).decision("gpt-6-luna"),
input: [{ type: "text", text: "refund" }],
questions: [{ name: "tone", type: "choice", instructions: "Tone?", choices: [{ value: "calm" }] }],
})
.pipe(
Effect.flip,
Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidProviderOutput"))),
Effect.provide(
client((input) =>
Effect.succeed(
json(input, {
model: "gpt-6-luna",
answers: [
{
type: "choice",
name: "tone",
choice: "angry",
probabilities: [{ value: "angry", probability: 1 }],
confidence: 1,
},
],
}),
),
),
),
),
)
it.effect("fails the call when System One leaves a question unanswered", () =>
ai.decision
.generate({
model: TypeSafeAI.configure({ apiKey: "test" }).decision("jev-latest"),
input: [{ type: "text", text: "refund" }],
questions: [
{ name: "refund", type: "predicate", instructions: "Refund?" },
{ name: "angry", type: "predicate", instructions: "Angry?" },
],
})
.pipe(
Effect.flip,
Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidProviderOutput"))),
Effect.provide(
client((input) =>
Effect.succeed(json(input, { model: "jev-1.14.0", answers: { refund: { type: "noul", noul: 0.9 } } })),
),
),
),
)
it.effect("rejects duplicate question names before network I/O", () =>
Effect.gen(function* () {
const error = yield* ai.decision
.generate({
model: TypeSafeAI.decision("jev-latest"),
input: [{ type: "text", text: "hello" }],
questions: [
{ name: "same", type: "predicate", instructions: "One?" },
{ name: "same", type: "predicate", instructions: "Two?" },
],
})
.pipe(Effect.flip)
expect(error.reason._tag).toBe("InvalidRequest")
}).pipe(Effect.provide(offline)),
)
})