packages/ai/test/cache-policy.test.ts
Rama dev · réplica de lectura del repositorio original.
import { describe, expect, test } from "bun:test"
import { Effect } from "effect"
import { ai, CacheHint, CompactionPart, Message, ProviderID, ToolCallPart } from "../src/effect.js"
import { Auth } from "../src/route.js"
import { compileRequest } from "../src/route/client.js"
import {
Alibaba,
AmazonBedrock,
AnthropicCompatible,
CloudflareAIGateway,
DigitalOcean,
GoogleVertexMessages,
Meta,
MiniMax,
Moonshot,
ZAICodingPlan,
} from "../src/providers.js"
import * as AnthropicMessages from "../src/protocols/anthropic-messages.js"
import * as Gemini from "../src/protocols/gemini.js"
import * as OpenAIChat from "../src/protocols/openai-chat.js"
import { applyCachePolicy } from "../src/cache-policy.js"
import { it } from "./lib/effect.js"
const anthropicModel = AnthropicMessages.route
.with({ endpoint: { baseURL: "https://api.anthropic.test/v1/" }, auth: Auth.header("x-api-key", "test") })
.model({ id: "claude-sonnet-4-5" })
const bedrockModel = AmazonBedrock.configure({
credentials: { region: "us-east-1", accessKeyId: "fixture", secretAccessKey: "fixture" },
}).model("anthropic.claude-3-5-sonnet-20241022-v2:0")
const openaiModel = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-4o-mini" })
const geminiModel = Gemini.route
.with({
endpoint: { baseURL: "https://generativelanguage.test/v1beta/" },
auth: Auth.header("x-goog-api-key", "test"),
})
.model({ id: "gemini-2.5-flash" })
describe("applyCachePolicy", () => {
it.effect("undefined cache resolves to 'auto' (the recommended default)", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: anthropicModel,
system: "You are concise.",
prompt: "hi",
}),
)
// A single system block is both the first and last boundary, so the auto
// policy deduplicates it and still marks the conversation tail.
expect(prepared.body).toMatchObject({
system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
})
}),
)
it.effect("'auto' marks the last tool, first and last system parts, and final message boundary on Anthropic", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: anthropicModel,
system: [
{ type: "text", text: "Base agent" },
{ type: "text", text: "Project instructions" },
],
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
messages: [
Message.user("first user"),
Message.assistant("assistant reply"),
Message.user("latest user message"),
],
cache: "auto",
}),
)
expect(prepared.body).toMatchObject({
tools: [{ name: "t1", cache_control: { type: "ephemeral" } }],
system: [
{ type: "text", text: "Base agent", cache_control: { type: "ephemeral" } },
{ type: "text", text: "Project instructions", cache_control: { type: "ephemeral" } },
],
messages: [
{ role: "user", content: [{ type: "text", text: "first user" }] },
{ role: "assistant", content: [{ type: "text", text: "assistant reply" }] },
{
role: "user",
content: [{ type: "text", text: "latest user message", cache_control: { type: "ephemeral" } }],
},
],
})
}),
)
it.effect("'auto' emits Anthropic cache markers on Vertex", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: GoogleVertexMessages.configure({ accessToken: "test", location: "global", project: "test" }).model(
"claude-opus-4-8",
),
system: "You are concise.",
tools: [{ name: "lookup", description: "Look up a value", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
}),
)
expect(prepared.body).toMatchObject({
tools: [{ name: "lookup", cache_control: { type: "ephemeral" } }],
system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
})
}),
)
it.effect("'auto' emits cache_control markers on DigitalOcean", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: DigitalOcean.configure({ apiKey: "test" }).model("anthropic-claude-fable-5.1"),
system: "You are concise.",
tools: [{ name: "lookup", description: "Look up a value", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
}),
)
expect(prepared.body).toMatchObject({
tools: [{ type: "function", function: { name: "lookup" }, cache_control: { type: "ephemeral" } }],
messages: [
{
role: "system",
content: [{ text: "You are concise.", cache_control: { type: "ephemeral" } }],
},
{
role: "user",
content: [{ text: "hi", cache_control: { type: "ephemeral" } }],
},
],
})
}),
)
it.effect("Alibaba chat caches the system and conversation tail without marking tools", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: Alibaba.configure({ region: "ap-southeast-1", apiKey: "test" }).chat("qwen3.8-max"),
system: "You are concise.",
tools: [{ name: "lookup", description: "Look up a value", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
}),
)
expect(prepared.body).toMatchObject({
tools: [{ type: "function", function: { name: "lookup" } }],
messages: [
{
role: "system",
content: [{ text: "You are concise.", cache_control: { type: "ephemeral" } }],
},
{ role: "user", content: [{ text: "hi", cache_control: { type: "ephemeral" } }] },
],
})
expect(prepared.body.tools?.[0]?.cache_control).toBeUndefined()
}),
)
it.effect("Alibaba chat omits automatic cache markers when cache is none", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: Alibaba.configure({ region: "ap-southeast-1", apiKey: "test" }).chat("qwen3.8-max"),
system: "You are concise.",
prompt: "hi",
cache: "none",
}),
)
expect(JSON.stringify(prepared.body)).not.toContain("cache_control")
}),
)
it.effect("Alibaba chat does not assume non-Qwen models support cache markers", () =>
Effect.gen(function* () {
const alibaba = Alibaba.configure({ region: "ap-southeast-1", apiKey: "test" })
for (const modelID of ["kimi-k3", "glm-5.2", "deepseek-v4-flash-0731", "MiniMax-M2.5"]) {
const prepared = yield* compileRequest(
ai.request({ model: alibaba.chat(modelID), system: "You are concise.", prompt: "hi" }),
)
expect(JSON.stringify(prepared.body)).not.toContain("cache_control")
}
}),
)
it.effect("'auto' emits Anthropic cache markers on Anthropic-compatible routes", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: AnthropicCompatible.configure({ apiKey: "test", baseURL: "https://messages.example.test/v1" }).model(
"compatible",
),
system: "You are concise.",
prompt: "hi",
}),
)
expect(prepared.route).toBe("anthropic-compatible-messages")
expect(prepared.body).toMatchObject({
system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
})
}),
)
const messagesModels = [
["alibaba-messages", Alibaba.configure({ region: "ap-southeast-1", apiKey: "test" }).messages("qwen3.8-max")],
[
"cloudflare-ai-gateway-messages",
CloudflareAIGateway.configure({ accountId: "test", gatewayId: "test", apiKey: "test" }).model(
"anthropic/claude-sonnet-4-6",
),
],
["meta-messages", Meta.configure({ apiKey: "test" }).messages("muse-spark-1.3")],
["minimax-messages", MiniMax.configure({ apiKey: "test" }).model("MiniMax-M3")],
["moonshot-messages", Moonshot.configure({ apiKey: "test" }).messages("kimi-k3")],
["zai-coding-messages", ZAICodingPlan.configure({ apiKey: "test" }).messages("glm-5.3")],
] as const
messagesModels.forEach(([route, model]) =>
it.effect(`'auto' emits cache markers on ${route}`, () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(ai.request({ model, system: "Sys", prompt: "hi" }))
expect(prepared.route).toBe(route)
expect(prepared.body).toMatchObject({
system: [{ type: "text", text: "Sys", cache_control: { type: "ephemeral" } }],
messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
})
}),
),
)
it.effect("'auto' is a no-op on OpenAI (implicit caching protocol)", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: openaiModel,
system: "Sys",
prompt: "hi",
cache: "auto",
}),
)
const body = prepared.body as { messages: Array<{ content: unknown }> }
// OpenAI doesn't accept cache_control on messages — policy must skip.
const flat = JSON.stringify(body)
expect(flat).not.toContain("cache_control")
expect(flat).not.toContain("cachePoint")
}),
)
it.effect("'auto' is a no-op on Gemini (out-of-band caching protocol)", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: geminiModel,
system: "Sys",
prompt: "hi",
cache: "auto",
}),
)
const flat = JSON.stringify(prepared.body)
expect(flat).not.toContain("cache_control")
expect(flat).not.toContain("cachePoint")
}),
)
it.effect("'auto' on Bedrock emits cachePoint markers in the right places", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: bedrockModel,
system: [
{ type: "text", text: "Base agent" },
{ type: "text", text: "Project instructions" },
],
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
messages: [Message.user("first user"), Message.assistant("reply"), Message.user("latest user")],
cache: "auto",
}),
)
expect(prepared.body).toMatchObject({
toolConfig: {
tools: [{ toolSpec: { name: "t1" } }, { cachePoint: { type: "default" } }],
},
system: [
{ text: "Base agent" },
{ cachePoint: { type: "default" } },
{ text: "Project instructions" },
{ cachePoint: { type: "default" } },
],
messages: [
{ role: "user", content: [{ text: "first user" }] },
{ role: "assistant", content: [{ text: "reply" }] },
{ role: "user", content: [{ text: "latest user" }, { cachePoint: { type: "default" } }] },
],
})
}),
)
it.effect("'none' disables auto placement even when manual hints exist", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: anthropicModel,
system: "Sys",
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
cache: "none",
}),
)
expect(prepared.body).toMatchObject({
tools: [{ name: "t1", cache_control: undefined }],
system: [{ type: "text", text: "Sys", cache_control: undefined }],
})
}),
)
it.effect("granular object form: tools-only marks just tools", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: anthropicModel,
system: "Sys",
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
cache: { tools: true },
}),
)
expect(prepared.body).toMatchObject({
tools: [{ name: "t1", cache_control: { type: "ephemeral" } }],
system: [{ type: "text", text: "Sys", cache_control: undefined }],
})
}),
)
it.effect("deduplicates tools before counting cache hints", () =>
Effect.gen(function* () {
const manual = new CacheHint({ type: "ephemeral" })
const duplicate = (description: string) => ({
name: "lookup",
description,
inputSchema: { type: "object" },
cache: manual,
})
const prepared = yield* compileRequest(
ai.request({
model: anthropicModel,
prompt: "Hello",
tools: [
duplicate("first"),
duplicate("second"),
duplicate("third"),
duplicate("fourth"),
{ name: "lookup", description: "final", inputSchema: { type: "object" } },
],
cache: { tools: true },
}),
)
expect(prepared.body.tools).toEqual([
expect.objectContaining({ name: "lookup", description: "final", cache_control: { type: "ephemeral" } }),
])
}),
)
it.effect("auto policy preserves manual CacheHints on other parts", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: anthropicModel,
system: [
{ type: "text", text: "first system", cache: new CacheHint({ type: "ephemeral", ttl: "1h" }) },
{ type: "text", text: "last system" },
],
prompt: "hi",
cache: "auto",
}),
)
const body = prepared.body as {
system: Array<{ text: string; cache_control?: unknown }>
messages: Array<{ content: Array<{ cache_control?: unknown }> }>
}
expect(body.system[0]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral" })
expect(body.messages[0]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
}),
)
it.effect("auto policy stays within the four-breakpoint cap when preserving manual hints", () =>
Effect.gen(function* () {
const request = ai.request({
model: anthropicModel,
system: [
{ type: "text", text: "Base agent" },
{
type: "text",
text: "Manual context",
cache: new CacheHint({ type: "ephemeral", ttl: "1h" }),
},
{ type: "text", text: "Project instructions" },
],
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
cache: "auto",
})
const applied = applyCachePolicy(request)
expect(applied.tools[0]?.cache).toBeDefined()
expect(applied.system.map((part) => part.cache !== undefined)).toEqual([true, true, true])
const tail = applied.messages[0]!.content[0]!
expect("cache" in tail ? tail.cache : undefined).toBeUndefined()
expect(applyCachePolicy(applied)).toBe(applied)
const prepared = yield* compileRequest(request)
const body = prepared.body as {
tools: Array<{ cache_control?: unknown }>
system: Array<{ cache_control?: unknown }>
messages: Array<{ content: Array<{ cache_control?: unknown }> }>
}
const marked = [
...body.tools.map((tool) => tool.cache_control),
...body.system.map((part) => part.cache_control),
...body.messages.flatMap((message) => message.content.map((part) => part.cache_control)),
].filter((cache) => cache !== undefined)
expect(marked).toHaveLength(4)
expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
}),
)
test("marks the final leaf inside a tool namespace", () => {
const request = ai.request({
model: anthropicModel,
prompt: "Hello",
tools: [
{
type: "namespace",
name: "crm",
tools: [
{ name: "lookup", description: "lookup", inputSchema: {} },
{ name: "orders", description: "orders", inputSchema: {} },
],
},
],
cache: { tools: true },
})
const applied = applyCachePolicy(request)
const namespace = applied.tools[0]
expect(namespace?.type).toBe("namespace")
if (namespace?.type !== "namespace") throw new Error("Expected namespace")
expect(namespace.tools[0]).not.toHaveProperty("cache")
expect(namespace.tools[1]).toHaveProperty("cache", { type: "ephemeral" })
})
it.effect("a ttl-only policy keeps automatic placement", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({ model: anthropicModel, system: "Sys", prompt: "hi", cache: { ttl: "1h" } }),
)
expect(prepared.body).toMatchObject({
system: [{ type: "text", text: "Sys", cache_control: { type: "ephemeral", ttl: "1h" } }],
messages: [
{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral", ttl: "1h" } }] },
],
})
}),
)
it.effect("ttl in the policy flows through to wire markers", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: anthropicModel,
system: "Sys",
prompt: "hi",
cache: { system: true, ttl: "1h" },
}),
)
expect(prepared.body).toMatchObject({
system: [{ type: "text", text: "Sys", cache_control: { type: "ephemeral", ttl: "1h" } }],
})
}),
)
it.effect("messages: { tail: 2 } marks the last 2 message boundaries", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: anthropicModel,
messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2"), Message.assistant("a2")],
cache: { messages: { tail: 2 } },
}),
)
const body = prepared.body as { messages: Array<{ content: Array<{ cache_control?: unknown }> }> }
expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
expect(body.messages[1]?.content[0]?.cache_control).toBeUndefined()
expect(body.messages[2]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
expect(body.messages[3]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
}),
)
it.effect("'latest-assistant' marks the last assistant message", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
ai.request({
model: anthropicModel,
messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2")],
cache: { messages: "latest-assistant" },
}),
)
const body = prepared.body as { messages: Array<{ content: Array<{ cache_control?: unknown }> }> }
expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
expect(body.messages[1]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
expect(body.messages[2]?.content[0]?.cache_control).toBeUndefined()
}),
)
test("skips non-cacheable parts when placing message breakpoints", () => {
const compaction = CompactionPart.make({ provider: ProviderID.make("anthropic"), text: "summary" })
const toolCall = ToolCallPart.make({ id: "tc_1", name: "lookup", input: {} })
const mixed = ai.request({
model: anthropicModel,
messages: [Message.assistant([toolCall, compaction])],
cache: { messages: "latest-assistant" },
})
const appliedMixed = applyCachePolicy(mixed)
expect(appliedMixed.messages[0]?.content[0]).toMatchObject({
type: "tool-call",
id: "tc_1",
cache: { type: "ephemeral" },
})
expect(appliedMixed.messages[0]?.content[1]).toEqual(compaction)
const compactionOnly = ai.request({
model: anthropicModel,
messages: [Message.assistant([compaction])],
cache: { messages: "latest-assistant" },
})
expect(applyCachePolicy(compactionOnly)).toBe(compactionOnly)
})
test("returns the same request reference when policy is a no-op (pure function)", () => {
const request = ai.request({
model: anthropicModel,
prompt: "hi",
cache: "none",
})
expect(applyCachePolicy(request)).toBe(request)
})
})