packages/codemode/test/codemode.test.ts
Rama dev · réplica de lectura del repositorio original.
import { describe, expect, test } from "bun:test"
import { Cause, Effect, Schema } from "effect"
import { CodeMode, Extension, Tool, toolError } from "./lib/index.js"
const run = (tool: Tool.Tool<never>) =>
Effect.runPromise(CodeMode.make({ tools: { host: { call: tool } } }).execute("return await tools.host.call({})"))
class HostError extends Schema.TaggedError<HostError>()("HostError", {
message: Schema.String,
}) {}
describe("CodeMode host failure boundary", () => {
test("preserves explicit tool failures", async () => {
const result = await run(
Tool.make({
description: "Fail",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.fail(toolError("Authorized request was refused")),
}),
)
expect(result.ok ? undefined : result.error).toStrictEqual({
kind: "ToolFailure",
message: "Authorized request was refused",
})
})
test("does not rewrite explicit tool failures", async () => {
const result = await run(
Tool.make({
description: "Fail",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.fail(toolError("File not found: /tmp/report.json")),
}),
)
expect(result.ok ? undefined : result.error).toStrictEqual({
kind: "ToolFailure",
message: "File not found: /tmp/report.json",
})
})
test("reports failures, defects, rejected Promises, and nested causes", async () => {
for (const failure of [
Effect.fail(new HostError({ message: "Connection refused" })),
Effect.die(new Error("Connection refused")),
Effect.promise(async () => {
throw new Error("Connection refused")
}),
Effect.fail(toolError("Request failed", new Error("Connection refused"))),
Effect.failCause(Cause.combine(Cause.fail("Request failed"), Cause.die("Connection refused"))),
]) {
const result = await run(
Tool.make({
description: "Fail internally",
input: Schema.Struct({}),
output: Schema.String,
execute: () => failure,
}),
)
expect(result.ok ? undefined : result.error).toStrictEqual({
kind: "ToolFailure",
message: expect.stringContaining("Connection refused"),
})
}
})
test("reports invalid host output", async () => {
const result = await run(
Tool.make({
description: "Return invalid output",
input: Schema.Struct({}),
output: Schema.Struct({ value: Schema.String }),
execute: () => Effect.succeed({ value: 1 } as unknown as { readonly value: string }),
}),
)
expect(result.ok ? undefined : result.error).toStrictEqual({
kind: "InvalidToolOutput",
message: "Invalid output from tool 'host.call': SchemaError(Expected string\n at [\"value\"])",
})
})
test("reports host output copying errors", async () => {
const result = await run(
Tool.make({
description: "Return hostile output",
input: Schema.Struct({}),
output: Schema.Unknown,
execute: () =>
Effect.succeed(
new Proxy(
{},
{
ownKeys: () => {
throw new Error("Cannot enumerate output")
},
},
),
),
}),
)
expect(result.ok ? undefined : result.error).toStrictEqual({
kind: "InvalidToolOutput",
message: "Invalid output from tool 'host.call': Error: Cannot enumerate output",
})
})
test("caught tool failures are Error values in-program", async () => {
const result = await Effect.runPromise(
CodeMode.make({
tools: {
host: {
call: Tool.make({
description: "Refuse",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.fail(toolError("Refused")),
}),
},
},
}).execute(`
try {
await tools.host.call({})
return "no"
} catch (e) {
return { isError: e instanceof Error, message: e.message }
}
`),
)
expect(result.ok).toBe(true)
if (result.ok) expect(result.value).toStrictEqual({ isError: true, message: "Refused" })
})
test("propagates host interruption instead of returning a diagnostic", async () => {
const exit = await Effect.runPromiseExit(
CodeMode.make({
tools: {
host: {
call: Tool.make({
description: "Interrupt",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.interrupt,
}),
},
},
}).execute("return await tools.host.call({})"),
)
expect(exit._tag).toBe("Failure")
if (exit._tag === "Failure") {
expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true)
}
})
})
describe("CodeMode call hooks", () => {
const ended = (result: CodeMode.CallResult) => {
if (result.status !== "failure") return result.status
return `failure:${result.error instanceof Error ? result.error.message : String(result.error)}`
}
// Every hook logs its event, so the order across tools and extensions is visible.
const observe = (events: Array<string>): CodeMode.Hooks => ({
"tool.before": (call) =>
Effect.sync(() => void events.push(`tool.before ${JSON.stringify({ name: call.name, input: call.input })}`)),
"tool.after": (call, result) => Effect.sync(() => void events.push(`tool.after ${call.name} ${ended(result)}`)),
"extension.before": (call) => Effect.sync(() => void events.push(`extension.before ${JSON.stringify(call)}`)),
"extension.after": (call, result) =>
Effect.sync(() => void events.push(`extension.after ${call.name} ${ended(result)}`)),
})
test("surround the tools actually invoked, with decoded input", async () => {
const events: Array<string> = []
const lookup = Tool.make({
description: "Look up a value",
input: Schema.Struct({ query: Schema.String }),
output: Schema.String,
execute: ({ query }) => (query === "boom" ? Effect.fail(toolError("Lookup refused")) : Effect.succeed(query)),
})
const runtime = CodeMode.make({ tools: { context: { lookup } }, hooks: observe(events) })
const success = await Effect.runPromise(
runtime.execute(`
if (false) await tools.context.lookup({ query: "not called" })
return await tools.context.lookup({ query: "deployment failure" })
`),
)
expect(success).toMatchObject({ ok: true, value: "deployment failure" })
const failure = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "boom" })`))
expect(failure.ok).toBe(false)
expect(events).toEqual([
'tool.before {"name":"context.lookup","input":{"query":"deployment failure"}}',
"tool.after context.lookup success",
'tool.before {"name":"context.lookup","input":{"query":"boom"}}',
"tool.after context.lookup failure:Lookup refused",
])
})
test("surround each call to an extension global, with its arguments, but not functions inside results", async () => {
const events: Array<string> = []
const runtime = CodeMode.make({
extensions: [
Extension.make({
name: "web",
globals: {
fetch: async (url: string, init?: { method?: string }) => ({
url,
method: init?.method ?? "GET",
json: () => 1,
}),
fail: () => {
throw new RangeError("nope")
},
},
}),
],
hooks: observe(events),
})
const result = await Effect.runPromise(
runtime.execute(`
const res = await fetch("https://a.test/", { method: "POST" })
res.json()
try { fail() } catch {}
return res.url
`),
)
expect(result).toMatchObject({ ok: true, value: "https://a.test/", toolCalls: [] })
expect(events).toEqual([
'extension.before {"extension":"web","name":"fetch","args":["https://a.test/",{"method":"POST"}]}',
"extension.after fetch success",
'extension.before {"extension":"web","name":"fail","args":[]}',
"extension.after fail failure:nope",
])
})
test("a failing before hook denies the call; the program catches the host's error", async () => {
const runtime = CodeMode.make({
tools: {
lookup: Tool.make({
description: "Look up",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("ok"),
}),
},
extensions: [Extension.make({ name: "web", globals: { fetch: async (url: string) => url } })],
hooks: { "extension.before": (call) => Effect.fail(new Error(`${call.name} is not allowed`)) },
})
expect(
await Effect.runPromise(
runtime.execute(`
const denied = await fetch("https://a.test/").catch((e) => [e instanceof Error, e.message])
return [denied, await tools.lookup({})]
`),
),
).toMatchObject({ ok: true, value: [[true, "fetch is not allowed"], "ok"] })
expect(await Effect.runPromise(runtime.execute(`await fetch("https://a.test/")`))).toMatchObject({
ok: false,
error: { kind: "ExecutionFailure", message: "Error: fetch is not allowed" },
})
const refused = CodeMode.make({
tools: {
lookup: Tool.make({
description: "Look up",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("ok"),
}),
},
hooks: { "tool.before": () => Effect.fail(toolError("lookup is not allowed")) },
})
expect(
await Effect.runPromise(
refused.execute(`try { await tools.lookup({}) } catch (e) { return [e.name, e.message] }`),
),
).toMatchObject({ ok: true, value: ["Error", "lookup is not allowed"], toolCalls: [{ name: "lookup" }] })
expect(await Effect.runPromise(refused.execute(`await tools.lookup({})`))).toMatchObject({
ok: false,
error: { kind: "ToolFailure", message: "lookup is not allowed" },
})
})
test("a before hook runs before the host function, so a denied call never reaches the host", async () => {
const called: Array<string> = []
const record = <A>(name: string, result: () => A) => {
called.push(name)
return result()
}
const runtime = CodeMode.make({
extensions: [
Extension.make({
name: "host",
globals: {
value: () => record("value", () => 1),
asyncFn: async () => record("asyncFn", () => 1),
promise: Extension.async(() => record("promise", () => Promise.resolve(1))),
effect: Extension.async(() => Effect.sync(() => record("effect", () => 1))),
},
}),
],
hooks: { "extension.before": (call) => Effect.fail(new Error(`${call.name} is not allowed`)) },
})
const run = async (code: string) => (await Effect.runPromise(runtime.execute(code))) as { value?: unknown }
// A denied call fails the way the global would: an async global rejects its promise, a sync one throws.
expect(
await run(`return (await Promise.allSettled([asyncFn(), promise(), effect()])).map((r) => r.reason.message)`),
).toMatchObject({ value: ["asyncFn is not allowed", "promise is not allowed", "effect is not allowed"] })
expect(await run(`try { value() } catch (e) { return e.message }`)).toMatchObject({ value: "value is not allowed" })
expect(called).toEqual([])
})
test("see a tool defect as the failure the program gets, and an interrupted extension call", async () => {
const events: Array<string> = []
const broken = Tool.make({
description: "Broken",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.die("broken"),
})
const runtime = CodeMode.make({
tools: { broken },
extensions: [Extension.make({ name: "slow", globals: { forever: () => new Promise(() => {}) } })],
hooks: observe(events),
})
expect(await Effect.runPromise(runtime.execute(`await tools.broken({})`))).toMatchObject({ ok: false })
expect(await Effect.runPromise(runtime.execute(`forever(); return "done"`))).toMatchObject({
ok: true,
value: "done",
})
expect(events).toEqual([
'tool.before {"name":"broken","input":{}}',
"tool.after broken failure:broken",
'extension.before {"extension":"slow","name":"forever","args":[]}',
"extension.after forever interrupted",
])
})
test("observes interrupted calls", async () => {
const events: Array<string> = []
const call = Tool.make({
description: "Interrupt",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.interrupt,
})
const exit = await Effect.runPromiseExit(
CodeMode.make({ tools: { host: { call } }, hooks: observe(events) }).execute("return await tools.host.call({})"),
)
expect(exit._tag).toBe("Failure")
expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"])
})
test("observes running calls interrupted during completion", async () => {
const events: Array<string> = []
const call = Tool.make({
description: "Pending",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.never,
})
const result = await Effect.runPromise(
CodeMode.make({ tools: { host: { call } }, hooks: observe(events) }).execute(
'tools.host.call({}); return "done"',
),
)
expect(result).toMatchObject({ ok: true, value: "done" })
expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"])
})
test("observes calls interrupted by the execution timeout", async () => {
const events: Array<string> = []
const call = Tool.make({
description: "Pending",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.never,
})
const result = await Effect.runPromise(
CodeMode.make({ tools: { host: { call } }, limits: { timeoutMs: 10 }, hooks: observe(events) }).execute(
"return await tools.host.call({})",
),
)
expect(result).toMatchObject({ ok: false, error: { kind: "TimeoutExceeded" } })
expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"])
})
})
describe("CodeMode console capture", () => {
test("captures console output as bounded result logs", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `
const returned = console.log("Thread info:", { name: "Demo", count: 2 })
console.warn("careful")
return returned
`,
}),
)
expect(result).toStrictEqual({
ok: true,
value: null,
logs: ['Thread info: {"name":"Demo","count":2}', "[warn] careful"],
toolCalls: [],
})
expect(Schema.decodeUnknownSync(CodeMode.Result)(JSON.parse(JSON.stringify(result)))).toStrictEqual(result)
})
test("keeps logs captured before failures", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `
console.log("before failure")
throw new Error("boom")
`,
}),
)
expect(result.ok ? undefined : result.logs).toStrictEqual(["before failure"])
expect(result.ok ? undefined : result.error.message).toBe("Error: boom")
})
test("prints NaN and Infinity literally instead of the JSON null", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `
console.log(NaN)
console.log(Infinity, -Infinity)
console.log({ ratio: NaN, bounds: [Infinity] })
return null
`,
}),
)
expect(result.ok).toBe(true)
expect(result.logs).toStrictEqual(["NaN", "Infinity -Infinity", '{"ratio":NaN,"bounds":[Infinity]}'])
})
test("renders CodeMode values nested inside logged containers", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `
console.log({ m: new Map([["a", 1]]), when: new Date(0), r: /ab/g, s: new Set([1, 2]) })
console.log([new Date(0)])
return null
`,
}),
)
expect(result.ok).toBe(true)
expect(result.logs).toStrictEqual([
'{"m":Map(1) [["a",1]],"when":1970-01-01T00:00:00.000Z,"r":/ab/g,"s":Set(2) [1,2]}',
"[1970-01-01T00:00:00.000Z]",
])
})
test("console formatting is total: cycles and opaque references render as markers", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `
const m = new Map()
m.set("self", m)
console.log({ box: m })
console.log({ fn: (x) => x, ok: 1 })
return null
`,
}),
)
expect(result.ok).toBe(true)
expect(result.logs).toStrictEqual(['{"box":Map(1) [["self",[Circular]]]}', '{"fn":[opaque reference],"ok":1}'])
})
test("console.table renders CodeMode value cells", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `
console.table([{ when: new Date(0), n: NaN }])
return null
`,
}),
)
expect(result.ok).toBe(true)
expect(result.logs).toStrictEqual(["(index)\twhen\tn\n0\t1970-01-01T00:00:00.000Z\tNaN"])
})
test("captures console.dir and console.table output", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `
console.dir({ nested: { ok: true } })
console.table([
{ name: "Kit", count: 1, hidden: "x" },
{ name: "Olive", count: 2, hidden: "y" }
], ["name", "count"])
return "done"
`,
}),
)
expect(result).toStrictEqual({
ok: true,
value: "done",
logs: ['{"nested":{"ok":true}}', "(index)\tname\tcount\n0\tKit\t1\n1\tOlive\t2"],
toolCalls: [],
})
})
})
describe("CodeMode output budget", () => {
test("absent maxOutputBytes means no truncation at all", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `console.log("z".repeat(50_000)); return "x".repeat(100_000)`,
}),
)
expect(result.ok).toBe(true)
if (!result.ok) return
expect(result.truncated).toBeUndefined()
expect(result.value).toBe("x".repeat(100_000))
expect(result.logs).toStrictEqual(["z".repeat(50_000)])
})
test("truncates an oversized result value with a marker instead of failing", async () => {
const limits: CodeMode.ExecutionLimits = { maxOutputBytes: 40 }
const result = await Effect.runPromise(
CodeMode.execute({
code: `return { data: "${"x".repeat(200)}" }`,
limits,
}),
)
expect(result.ok).toBe(true)
if (!result.ok) return
expect(result.truncated).toBe(true)
expect(typeof result.value).toBe("string")
expect(result.value).toMatch(
/^\{"data":"x+ \[result truncated: \d+ bytes exceeds the 40-byte output limit; return a smaller value\]$/,
)
expect(Schema.decodeUnknownSync(CodeMode.Result)(JSON.parse(JSON.stringify(result)))).toStrictEqual(result)
})
test("keeps leading logs within the remaining budget and marks the cut", async () => {
const limits: CodeMode.ExecutionLimits = { maxOutputBytes: 40 }
const result = await Effect.runPromise(
CodeMode.execute({
code: `
console.log("first line")
console.log("${"y".repeat(200)}")
return "ok"
`,
limits,
}),
)
expect(result.ok).toBe(true)
if (!result.ok) return
expect(result.value).toBe("ok")
expect(result.truncated).toBe(true)
expect(result.logs).toStrictEqual(["first line", "[logs truncated: showing 1 of 2 lines]"])
})
test("does not mark results within the budget", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `
console.log("fits")
return { fits: true }
`,
}),
)
expect(result).toStrictEqual({
ok: true,
value: { fits: true },
logs: ["fits"],
toolCalls: [],
})
})
})
describe("CodeMode schema flexibility", () => {
test("accepts render-only JSON Schema input and omitted output", async () => {
const observed: Array<unknown> = []
const call = Tool.make({
description: "Call an adapter-described tool",
input: {
type: "object",
properties: { id: { type: "string" }, count: { type: "number" } },
required: ["id"],
},
execute: (input) =>
Effect.sync(() => {
observed.push(input)
return { echoed: input }
}),
})
const runtime = CodeMode.make({ tools: { adapter: { call } } })
expect(runtime.catalog).toStrictEqual([
{
path: "adapter.call",
description: "Call an adapter-described tool",
signature: "tools.adapter.call({\n id: string,\n count?: number,\n}): Promise<void>",
},
])
// JSON Schema is render-only: mistyped input passes through unvalidated.
const result = await Effect.runPromise(runtime.execute(`return await tools.adapter.call({ id: 42 })`))
expect(result.ok).toBe(true)
if (result.ok) expect(result.value).toBeNull()
expect(observed).toStrictEqual([{ id: 42 }])
})
test("outbound tool arguments follow JSON serialization semantics", async () => {
const observed: Array<unknown> = []
const call = Tool.make({
description: "Observe raw input",
input: { type: "object" },
execute: (input) =>
Effect.sync(() => {
observed.push(input)
return "ok"
}),
})
const runtime = CodeMode.make({ tools: { adapter: { call } } })
const result = await Effect.runPromise(
runtime.execute(
`return await tools.adapter.call({ q: undefined, limit: 0 / 0, rate: 1 / 0, items: [1, undefined, 2], holes: [1, , 3] })`,
),
)
expect(result.ok).toBe(true)
const received = observed[0] as Record<string, unknown>
expect(received).toStrictEqual({ limit: null, rate: null, items: [1, null, 2], holes: [1, null, 3] })
// The undefined-valued property is dropped like JSON.stringify, not delivered as undefined.
expect(Object.hasOwn(received, "q")).toBe(false)
})
test("dropping undefined values lets optionalKey schemas accept conditional arguments", async () => {
const observed: Array<unknown> = []
const find = Tool.make({
description: "Find things",
input: Schema.Struct({ query: Schema.optionalKey(Schema.String), limit: Schema.optionalKey(Schema.Number) }),
execute: (input) =>
Effect.sync(() => {
observed.push(input)
return "ok"
}),
})
const runtime = CodeMode.make({ tools: { things: { find } } })
// The `cond ? value : undefined` idiom: optionalKey rejects a present undefined, so the
// JSON boundary must drop the key before the schema decodes.
const result = await Effect.runPromise(
runtime.execute(`return await tools.things.find({ query: undefined, limit: 5 })`),
)
expect(result.ok).toBe(true)
expect(observed).toStrictEqual([{ limit: 5 }])
const search = await Effect.runPromise(runtime.execute(`return (await search({ query: undefined })).items.length`))
expect(search.ok).toBe(true)
})
test("renders JSON Schema outputs and $defs references", async () => {
const lookup = Tool.make({
description: "Look up a user",
input: { type: "object", properties: { login: { type: "string" } }, required: ["login"] },
output: {
$ref: "#/$defs/User",
$defs: {
User: {
type: "object",
properties: { login: { type: "string" }, id: { type: "number" } },
required: ["login", "id"],
},
},
},
execute: () => Effect.succeed({ login: "kit", id: 7 }),
})
const runtime = CodeMode.make({ tools: { users: { lookup } } })
expect(runtime.catalog).toStrictEqual([
{
path: "users.lookup",
description: "Look up a user",
signature: "tools.users.lookup({\n login: string,\n}): Promise<{\n login: string,\n id: number,\n}>",
},
])
const result = await Effect.runPromise(runtime.execute(`return await tools.users.lookup({ login: "kit" })`))
expect(result.ok).toBe(true)
if (result.ok) expect(result.value).toStrictEqual({ login: "kit", id: 7 })
})
test("Effect Schema output without an input transform renders void when omitted", async () => {
const ping = Tool.make({
description: "Ping",
input: Schema.Struct({ host: Schema.String }),
execute: () => Effect.succeed("pong"),
})
const runtime = CodeMode.make({ tools: { net: { ping } } })
expect(runtime.catalog[0]?.signature).toBe("tools.net.ping({\n host: string,\n}): Promise<void>")
const result = await Effect.runPromise(runtime.execute(`return await tools.net.ping({ host: "example.test" })`))
expect(result.ok).toBe(true)
if (result.ok) expect(result.value).toBeNull()
})
})
describe("CodeMode public contract", () => {
const lookup = Tool.make({
description: "Look up an order by ID",
input: Schema.Struct({ id: Schema.String }),
output: Schema.Struct({ id: Schema.String, status: Schema.String }),
execute: ({ id }) => Effect.succeed({ id, status: "open" }),
})
const tools = { orders: { lookup } }
const source = `return await tools.orders.lookup({ id: "order_42" })`
test("keeps one-shot and reusable execution equivalent", async () => {
const runtime = CodeMode.make({ tools })
const [oneShot, reusable] = await Promise.all([
Effect.runPromise(CodeMode.execute({ tools, code: source })),
Effect.runPromise(runtime.execute(source)),
])
expect(reusable).toStrictEqual(oneShot)
const input: CodeMode.Input = { code: source }
expect(Schema.decodeUnknownSync(CodeMode.Input)(input)).toStrictEqual(input)
expect(Schema.decodeUnknownSync(CodeMode.Result)(JSON.parse(JSON.stringify(reusable)))).toStrictEqual(reusable)
})
test("a reused execution Effect starts from a clean slate", async () => {
const echo = Tool.make({
description: "echo",
input: Schema.Struct({}),
output: Schema.Number,
execute: () => Effect.succeed(1),
})
const effect = CodeMode.execute({
tools: { host: { echo } },
code: `console.log("hi"); return await tools.host.echo({})`,
limits: { maxToolCalls: 1 },
})
const first = await Effect.runPromise(effect)
const second = await Effect.runPromise(effect)
// Per-execution state (tool-call budget and audit list, logs, timeout bookkeeping) must
// bind at run time, so the second run neither exhausts the budget nor leaks run 1's logs.
expect(first).toStrictEqual(second)
expect(second).toStrictEqual({ ok: true, value: 1, logs: ["hi"], toolCalls: [{ name: "host.echo" }] })
})
test("describes the catalog and keeps the search built-in registered", async () => {
const runtime = CodeMode.make({ tools })
expect(runtime.catalog).toStrictEqual([
{
path: "orders.lookup",
description: "Look up an order by ID",
signature: "tools.orders.lookup({\n id: string,\n}): Promise<{\n id: string,\n status: string,\n}>",
},
])
const result = await Effect.runPromise(runtime.execute(`return search({ query: "order" })`))
expect(result.ok).toBe(true)
if (result.ok) {
expect(result.value).toStrictEqual({
items: [
{
path: "tools.orders.lookup",
description: "Look up an order by ID",
signature: "tools.orders.lookup({\n id: string,\n}): Promise<{\n id: string,\n status: string,\n}>",
},
],
remaining: 0,
next: null,
})
}
})
test("a freeform tool takes its one string field directly and still accepts the object", async () => {
const input = Schema.Struct({ url: Schema.String.annotate({ description: "Page URL" }) })
const fetchPage = Tool.make({
description: "Fetch a page",
input,
freeform: true,
output: Schema.String,
execute: ({ url }) => Effect.succeed(url),
})
const plain = Tool.make({
description: "Fetch a page",
input,
output: Schema.String,
execute: ({ url }) => Effect.succeed(url),
})
const runtime = CodeMode.make({ tools: { fetchPage, plain } })
expect(runtime.catalog.map((tool) => tool.signature)).toEqual([
"tools.fetchPage(/** Page URL */ url: string): Promise<string>",
"tools.plain({\n /** Page URL */\n url: string,\n}): Promise<string>",
])
expect(
await Effect.runPromise(
runtime.execute('return [await tools.fetchPage("a"), await tools.fetchPage({ url: "b" })]'),
),
).toMatchObject({ ok: true, value: ["a", "b"] })
expect(await Effect.runPromise(runtime.execute('return await tools.plain("a")'))).toMatchObject({ ok: false })
})
test("renders equivalent catalogs identically regardless of tool insertion order", () => {
const alpha = Tool.make({
description: "Alpha tool",
input: Schema.Struct({}),
output: Schema.Void,
execute: () => Effect.void,
})
const zeta = Tool.make({
description: "Zeta tool",
input: Schema.Struct({}),
output: Schema.Void,
execute: () => Effect.void,
})
const first = CodeMode.make({ tools: { zeta: { zeta, alpha }, alpha: { zeta, alpha } } })
const second = CodeMode.make({ tools: { alpha: { alpha, zeta }, zeta: { alpha, zeta } } })
expect(first.catalog).toStrictEqual(second.catalog)
expect(first.catalog.map((tool) => tool.path)).toEqual(["alpha.alpha", "alpha.zeta", "zeta.alpha", "zeta.zeta"])
})
test("renders bracket notation for tool names that are not JavaScript identifiers", async () => {
const resolveLibrary = Tool.make({
description: "Resolve a library ID",
input: Schema.Struct({ libraryName: Schema.String }),
output: Schema.String,
execute: ({ libraryName }) => Effect.succeed(`/resolved/${libraryName}`),
})
const runtime = CodeMode.make({ tools: { context7: { "resolve-library-id": resolveLibrary } } })
expect(runtime.catalog).toStrictEqual([
{
path: "context7.resolve-library-id",
description: "Resolve a library ID",
signature: 'tools.context7["resolve-library-id"]({\n libraryName: string,\n}): Promise<string>',
},
])
const search = await Effect.runPromise(runtime.execute(`return search({ query: "resolve library id" })`))
expect(search.ok).toBe(true)
if (search.ok) {
expect(search.value).toStrictEqual({
items: [
{
path: 'tools.context7["resolve-library-id"]',
description: "Resolve a library ID",
signature: 'tools.context7["resolve-library-id"]({\n libraryName: string,\n}): Promise<string>',
},
],
remaining: 0,
next: null,
})
}
const call = await Effect.runPromise(
runtime.execute(`return await tools.context7["resolve-library-id"]({ libraryName: "TypeScript" })`),
)
expect(call.ok).toBe(true)
if (call.ok) expect(call.value).toBe("/resolved/TypeScript")
const exact = await Effect.runPromise(
runtime.execute(`return search({ query: 'tools.context7["resolve-library-id"]' })`),
)
expect(exact.ok).toBe(true)
if (exact.ok) expect(exact.value).toMatchObject({ remaining: 0, next: null })
})
test("uses one ranked search returning complete tools for large catalogs", async () => {
const upload = Tool.make({
description: "Upload one readable local file to the current Discord thread",
input: Schema.Struct({ path: Schema.String }),
output: Schema.Struct({ sent: Schema.Boolean }),
execute: () => Effect.succeed({ sent: true }),
})
const generate = Tool.make({
description: "Generate an image and upload it to the current Discord thread",
input: Schema.Struct({ prompt: Schema.String }),
output: Schema.Struct({ sent: Schema.Boolean }),
execute: () => Effect.succeed({ sent: true }),
})
const runtime = CodeMode.make({
tools: { thread: { uploadFile: upload, generateImage: generate }, orders: { lookup } },
})
const result = await Effect.runPromise(
runtime.execute(`
return search({
query: "send message attachment upload file to current Discord thread",
limit: 2
})
`),
)
expect(result.ok).toBe(true)
if (!result.ok) return
expect(result.value).toStrictEqual({
items: [
{
path: "tools.thread.uploadFile",
description: "Upload one readable local file to the current Discord thread",
signature: "tools.thread.uploadFile({\n path: string,\n}): Promise<{\n sent: boolean,\n}>",
},
{
path: "tools.thread.generateImage",
description: "Generate an image and upload it to the current Discord thread",
signature: "tools.thread.generateImage({\n prompt: string,\n}): Promise<{\n sent: boolean,\n}>",
},
],
remaining: 0,
next: null,
})
expect(result.toolCalls).toStrictEqual([{ name: "search" }])
const variants = await Effect.runPromise(
runtime.execute(`
return [
search({ query: "file" }),
search({ query: "image" })
]
`),
)
expect(variants.ok).toBe(true)
if (variants.ok) {
expect((variants.value as Array<{ items: Array<{ path: string }> }>)[0]?.items[0]?.path).toBe(
"tools.thread.uploadFile",
)
expect((variants.value as Array<{ items: Array<{ path: string }> }>)[1]?.items[0]?.path).toBe(
"tools.thread.generateImage",
)
}
})
test("search is a counted tool call: it burns maxToolCalls and is observed", async () => {
const observed: Array<CodeMode.ToolInvocation> = []
const limited = CodeMode.make({
tools,
limits: { maxToolCalls: 1 },
hooks: { "tool.before": (call) => Effect.sync(() => void observed.push(call)) },
})
const result = await Effect.runPromise(limited.execute(`search({}); return search({})`))
expect(result.ok).toBe(false)
if (!result.ok) expect(result.error.kind).toBe("ToolCallLimitExceeded")
expect(observed).toEqual([{ name: "search", input: {}, key: expect.any(String) }])
})
test("search is an opaque, shadowable global like other built-ins", async () => {
const runtime = CodeMode.make({ tools })
expect(await Effect.runPromise(runtime.execute(`return typeof search`))).toMatchObject({ value: "function" })
// A program-level declaration shadows the global, as JS module scope does.
const shadowed = await Effect.runPromise(runtime.execute(`const search = () => "local"; return search()`))
expect(shadowed.ok).toBe(true)
if (shadowed.ok) expect(shadowed.value).toBe("local")
// The reference itself vanishes at the data boundary, as functions do in JSON.stringify.
const escaped = await Effect.runPromise(runtime.execute(`return { search }`))
expect(escaped).toMatchObject({ ok: true, value: {} })
})
test("search defaults to 10 results and resolves exact tool paths", async () => {
const tool = (index: number) =>
Tool.make({
description: `Numbered tool ${index}`,
input: Schema.Struct({ id: Schema.String }),
output: Schema.String,
execute: () => Effect.succeed("ok"),
})
const runtime = CodeMode.make({
tools: {
many: Object.fromEntries(Array.from({ length: 14 }, (_, index) => [`tool${index}`, tool(index)])),
},
})
const browse = await Effect.runPromise(runtime.execute(`return search({})`))
expect(browse.ok).toBe(true)
if (browse.ok) {
const value = browse.value as {
items: Array<{ path: string }>
remaining: number
next: { offset: number } | null
}
expect(value.items).toHaveLength(10)
expect(value.remaining).toBe(4)
expect(value.next).toStrictEqual({ offset: 10 })
}
for (const query of ["many.tool13", "tools.many.tool13"]) {
const exact = await Effect.runPromise(runtime.execute(`return search({ query: ${JSON.stringify(query)} })`))
expect(exact.ok).toBe(true)
if (exact.ok) {
expect(exact.value).toStrictEqual({
items: [
{
path: "tools.many.tool13",
description: "Numbered tool 13",
signature: "tools.many.tool13({\n id: string,\n}): Promise<string>",
},
],
remaining: 0,
next: null,
})
}
}
})
test("scopes search to one namespace and browses it alphabetically", async () => {
const simple = (description: string) =>
Tool.make({
description,
input: Schema.Struct({ id: Schema.String }),
output: Schema.String,
execute: () => Effect.succeed("ok"),
})
const runtime = CodeMode.make({
tools: {
github: { list_issues: simple("List issues"), create_issue: simple("Create an issue") },
linear: { list_issues: simple("List Linear issues") },
},
})
// Empty query + namespace browses just that namespace, alphabetical by path.
const browse = await Effect.runPromise(runtime.execute(`return search({ query: "", namespace: "github" })`))
expect(browse.ok).toBe(true)
if (browse.ok) {
const value = browse.value as { items: Array<{ path: string }>; remaining: number }
expect(value.remaining).toBe(0)
expect(value.items.map((item) => item.path)).toStrictEqual([
"tools.github.create_issue",
"tools.github.list_issues",
])
}
// A query + namespace ranks within that namespace only.
const scoped = await Effect.runPromise(runtime.execute(`return search({ query: "issues", namespace: "linear" })`))
expect(scoped.ok).toBe(true)
if (scoped.ok) {
const value = scoped.value as { items: Array<{ path: string }>; remaining: number }
expect(value.remaining).toBe(0)
expect(value.items[0]?.path).toBe("tools.linear.list_issues")
}
const prefixed = await Effect.runPromise(runtime.execute(`return search({ query: "", namespace: "tools.github" })`))
expect(prefixed.ok).toBe(true)
if (prefixed.ok) expect((prefixed.value as { items: Array<unknown> }).items).toHaveLength(2)
const invalid = await Effect.runPromise(runtime.execute(`return search({ query: "issues", namespace: 7 })`))
expect(invalid.ok).toBe(false)
if (!invalid.ok) expect(invalid.error.kind).toBe("InvalidToolInput")
})
test("matches input parameter names and partial-word substrings", async () => {
const upload = Tool.make({
description: "Send a document to the workspace",
input: {
type: "object",
properties: { attachment: { type: "string", description: "Local path of the payload to send" } },
required: ["attachment"],
},
execute: () => Effect.succeed("ok"),
})
const other = Tool.make({
description: "Rename the workspace",
input: Schema.Struct({ name: Schema.String }),
output: Schema.String,
execute: () => Effect.succeed("ok"),
})
const runtime = CodeMode.make({ tools: { files: { upload, other } } })
// "attachment" appears in neither path nor description - only in the input schema's
// property names, which the searchable text includes.
const byParameter = await Effect.runPromise(runtime.execute(`return search({ query: "attachment" })`))
expect(byParameter.ok).toBe(true)
if (byParameter.ok) {
const value = byParameter.value as { items: Array<{ path: string }>; remaining: number }
expect(value.remaining).toBe(0)
expect(value.items[0]?.path).toBe("tools.files.upload")
}
// Substring matching: a partial word ("docum") still hits the description.
const bySubstring = await Effect.runPromise(runtime.execute(`return search({ query: "docum" })`))
expect(bySubstring.ok).toBe(true)
if (bySubstring.ok) {
const value = bySubstring.value as { items: Array<{ path: string }>; remaining: number }
expect(value.remaining).toBe(0)
expect(value.items[0]?.path).toBe("tools.files.upload")
}
})
test("a query term matches whole words, not substrings", async () => {
const simple = (description: string) =>
Tool.make({
description,
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("ok"),
})
const runtime = CodeMode.make({
tools: {
// Declared so that alphabetical order would put the substring match first.
cloudflare: { get_timezones: simple("List timezones"), get_zones: simple("List zones") },
},
})
const ranked = await Effect.runPromise(runtime.execute(`return search({ query: "zones" })`))
expect(ranked.ok).toBe(true)
if (ranked.ok) {
const value = ranked.value as { items: Array<{ path: string }> }
expect(value.items.map((item) => item.path)).toStrictEqual(["tools.cloudflare.get_zones"])
}
})
test("search indexes are shared only between runtimes with the same tool text", async () => {
const simple = (description: string) =>
Tool.make({
description,
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("ok"),
})
const top = async (runtime: ReturnType<typeof CodeMode.make>, query: string) => {
const result = await Effect.runPromise(runtime.execute(`return search({ query: ${JSON.stringify(query)} })`))
expect(result.ok).toBe(true)
return result.ok ? (result.value as { items: Array<{ path: string }> }).items[0]?.path : undefined
}
// Same tools registered in a different order index identically.
const first = CodeMode.make({ tools: { mail: { send: simple("Send an email"), archive: simple("Archive a thread") } } })
const reordered = CodeMode.make({ tools: { mail: { archive: simple("Archive a thread"), send: simple("Send an email") } } })
// Same paths and count, different descriptions: must not reuse the other index.
const changed = CodeMode.make({ tools: { mail: { send: simple("Archive a thread"), archive: simple("Send an email") } } })
expect(await top(first, "email")).toBe("tools.mail.send")
expect(await top(reordered, "email")).toBe("tools.mail.send")
expect(await top(changed, "email")).toBe("tools.mail.archive")
expect(await top(first, "archive thread")).toBe("tools.mail.archive")
})
test("search ranks by rare words, prefers short names and matches partial words", async () => {
const simple = (description: string) =>
Tool.make({
description,
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("ok"),
})
const runtime = CodeMode.make({
tools: {
gmail: { list_labels: simple("List the labels in the mailbox"), list_messages: simple("List messages") },
slack: {
list_channels: simple("List channels"),
list_users: simple("List users"),
list_user_groups: simple("List user groups"),
get_schedules: simple("Get scheduled messages"),
},
},
})
const paths = async (query: Record<string, unknown>) => {
const result = await Effect.runPromise(runtime.execute(`return search(${JSON.stringify(query)})`))
expect(result.ok).toBe(true)
return result.ok ? (result.value as { items: Array<{ path: string }> }).items.map((item) => item.path) : []
}
// "list" is in every tool; "labels" decides.
expect((await paths({ query: "list gmail labels" }))[0]).toBe("tools.gmail.list_labels")
// Both match "users"; the shorter, more general name wins.
expect((await paths({ query: "users", namespace: "slack" }))[0]).toBe("tools.slack.list_users")
// A partial word of 4+ letters that matches nothing whole finds the words it starts with.
expect(await paths({ query: "schedu" })).toStrictEqual(["tools.slack.get_schedules"])
// Short plurals and stop-word-only queries still match.
const short = CodeMode.make({
tools: { cloud: { describe_vm: simple("Describe a virtual machine"), list_all_issues: simple("List issues") } },
})
const top = async (query: string) => {
const result = await Effect.runPromise(short.execute(`return search(${JSON.stringify({ query })})`))
return result.ok ? (result.value as { items: Array<{ path: string }> }).items.map((item) => item.path) : []
}
expect((await top("vms"))[0]).toBe("tools.cloud.describe_vm")
expect(await top("all")).toStrictEqual(["tools.cloud.list_all_issues"])
// A namespace filter is a hard filter.
expect(await paths({ query: "labels", namespace: "slack" })).toStrictEqual([])
})
test("a plural query term matches singular-only tool text", async () => {
const simple = (description: string) =>
Tool.make({
description,
input: Schema.Struct({ id: Schema.String }),
output: Schema.String,
execute: () => Effect.succeed("ok"),
})
const runtime = CodeMode.make({
tools: {
// Neither path nor description contains "issues" - only the singular "issue".
tracker: { fetch_all: simple("Fetch every open issue in the project") },
github: { list_issues: simple("List issues") },
misc: { rename: simple("Rename the workspace") },
},
})
// "issues" still finds the singular-only tool (term OR singular(term) per field)...
const plural = await Effect.runPromise(runtime.execute(`return search({ query: "issues", namespace: "tracker" })`))
expect(plural.ok).toBe(true)
if (plural.ok) {
const value = plural.value as { items: Array<{ path: string }>; remaining: number }
expect(value.remaining).toBe(0)
expect(value.items[0]?.path).toBe("tools.tracker.fetch_all")
}
// ...while a true "issues" path match still outranks the singular-only description match.
const ranked = await Effect.runPromise(runtime.execute(`return search({ query: "issues" })`))
expect(ranked.ok).toBe(true)
if (ranked.ok) {
const value = ranked.value as { items: Array<{ path: string }>; remaining: number }
expect(value.remaining).toBe(0)
expect(value.items.map((item) => item.path)).toStrictEqual([
"tools.github.list_issues",
"tools.tracker.fetch_all",
])
}
})
test("empty query lists everything alphabetically by path", async () => {
const simple = (description: string) =>
Tool.make({
description,
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("ok"),
})
// Deliberately declared out of alphabetical order.
const runtime = CodeMode.make({
tools: {
zeta: { last: simple("Last") },
alpha: { beta: simple("Middle"), aardvark: simple("First") },
},
})
const browse = await Effect.runPromise(runtime.execute(`return search({})`))
expect(browse.ok).toBe(true)
if (browse.ok) {
const value = browse.value as { items: Array<{ path: string }>; remaining: number; next: unknown }
expect(value.items.map((item) => item.path)).toStrictEqual([
"tools.alpha.aardvark",
"tools.alpha.beta",
"tools.zeta.last",
])
expect(value.remaining).toBe(0)
expect(value.next).toBeNull()
}
const middle = await Effect.runPromise(runtime.execute(`return search({ limit: 1, offset: 1 })`))
expect(middle.ok).toBe(true)
if (middle.ok) {
expect(middle.value).toMatchObject({
items: [{ path: "tools.alpha.beta" }],
remaining: 1,
next: { offset: 2 },
})
}
const exhausted = await Effect.runPromise(runtime.execute(`return search({ limit: 1, offset: 3 })`))
expect(exhausted.ok).toBe(true)
if (exhausted.ok) expect(exhausted.value).toStrictEqual({ items: [], remaining: 0, next: null })
})
test("decodes tool input and output before exposing either side", async () => {
const observed: Array<unknown> = []
const transformed = Tool.make({
description: "Double a number",
input: Schema.Struct({ value: Schema.NumberFromString }),
output: Schema.NumberFromString,
execute: ({ value }) =>
Effect.sync(() => {
observed.push(value)
return String(value * 2)
}),
})
const runtime = CodeMode.make({
tools: { math: { double: transformed } },
hooks: { "tool.before": (call) => Effect.sync(() => void observed.push(call.input)) },
})
const success = await Effect.runPromise(runtime.execute(`return await tools.math.double({ value: "21" })`))
expect(success).toStrictEqual({ ok: true, value: 42, toolCalls: [{ name: "math.double" }] })
expect(observed).toStrictEqual([{ value: 21 }, 21])
const invalid = await Effect.runPromise(runtime.execute(`return await tools.math.double({ value: 21 })`))
expect(invalid.ok).toBe(false)
if (invalid.ok) return
expect(invalid.error.kind).toBe("InvalidToolInput")
expect(observed).toStrictEqual([{ value: 21 }, 21])
})
test("returns JSON-safe data: undefined vanishes as in JSON.stringify, and a bare undefined is null", async () => {
const result = await Effect.runPromise(
CodeMode.execute({
code: `return { top: undefined, nested: [1, undefined] }`,
}),
)
expect(result).toStrictEqual({
ok: true,
value: { nested: [1, null] },
toolCalls: [],
})
expect(await Effect.runPromise(CodeMode.execute({ code: `return undefined` }))).toMatchObject({ value: null })
expect(Schema.decodeUnknownSync(CodeMode.Result)(JSON.parse(JSON.stringify(result)))).toStrictEqual(result)
})
test("returns the final top-level expression when return is omitted", async () => {
const result = await Effect.runPromise(CodeMode.execute({ code: `1; 2` }))
expect(result).toStrictEqual({ ok: true, value: 2, toolCalls: [] })
})
test("does not implicitly return expressions nested in control flow", async () => {
const result = await Effect.runPromise(CodeMode.execute({ code: `if (true) { 2 }` }))
expect(result).toStrictEqual({ ok: true, value: null, toolCalls: [] })
})
test("returns null when the final top-level statement is not an expression", async () => {
const result = await Effect.runPromise(CodeMode.execute({ code: `1; const value = 2` }))
expect(result).toStrictEqual({ ok: true, value: null, toolCalls: [] })
})
test("rejects invalid configuration and search limits", async () => {
expect(() => CodeMode.execute({ code: "return 1", limits: { timeoutMs: 0 } })).toThrow(RangeError)
expect(() => CodeMode.execute({ code: "return 1", limits: { timeoutMs: Number.POSITIVE_INFINITY } })).toThrow(
RangeError,
)
expect(() => CodeMode.execute({ code: "return 1", limits: { maxToolCalls: -1 } })).toThrow(RangeError)
expect(() => CodeMode.execute({ code: "return 1", limits: { maxOutputBytes: -1 } })).toThrow(RangeError)
const result = await Effect.runPromise(
CodeMode.make({ tools }).execute(`return search({ query: "order", limit: 0.5 })`),
)
expect(result.ok).toBe(false)
if (result.ok) return
expect(result.error.kind).toBe("InvalidToolInput")
for (const offset of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1, "1"]) {
const invalidOffset = await Effect.runPromise(
CodeMode.make({ tools }).execute(`return search({ query: "order", offset: ${JSON.stringify(offset)} })`),
)
expect(invalidOffset.ok).toBe(false)
if (!invalidOffset.ok) expect(invalidOffset.error.kind).toBe("InvalidToolInput")
}
})
test("enforces the tool-call limit as a diagnostic", async () => {
const result = await Effect.runPromise(CodeMode.execute({ tools, code: source, limits: { maxToolCalls: 0 } }))
expect(result.ok).toBe(false)
if (!result.ok) expect(result.error.kind).toBe("ToolCallLimitExceeded")
})
test("timeoutMs and maxToolCalls have no defaults: absent means unlimited", async () => {
// 150 tool calls would have exceeded the old default cap of 100; with no limits
// provided, there is no cap and no timeout - budgets are host policy.
const counter = Tool.make({
description: "Count invocations",
input: Schema.Struct({}),
output: Schema.Number,
execute: () => Effect.succeed(1),
})
const result = await Effect.runPromise(
CodeMode.execute({
tools: { host: { count: counter } },
code: `
let total = 0
for (let i = 0; i < 150; i += 1) total += await tools.host.count({})
return total
`,
}),
)
expect(result).toMatchObject({ ok: true, value: 150 })
if (result.ok) expect(result.toolCalls.length).toBe(150)
})
test("the timeout interrupts a busy loop without any operation budget", async () => {
// Regression: timeout interruption must not depend on interpreter-side work accounting.
// The Effect fiber runtime auto-yields between interpreter steps, so a pure `while
// (true) {}` loop is interrupted by `timeoutMs` alone.
const startedAt = Date.now()
const result = await Effect.runPromise(CodeMode.execute({ code: "while (true) {}", limits: { timeoutMs: 200 } }))
const elapsedMs = Date.now() - startedAt
expect(result.ok).toBe(false)
if (!result.ok) {
expect(result.error.kind).toBe("TimeoutExceeded")
expect(result.error.message).toContain("timed out after 200ms")
}
expect(elapsedMs).toBeLessThan(3_000)
})
})