scripts/smoke-ai.ts
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env npx tsx
/**
* Smoke test for the configured AI backend, against the real API.
*
* Unit tests cover which provider and model id get resolved from the
* environment; they cannot tell you whether the resulting request is one the
* backend accepts. This script sends real traffic over all three shapes the
* app actually uses, so a credential or parameter problem surfaces here rather
* than in front of a user:
*
* 1. A plain `messages.create` on both agent model ids.
* 2a. A streamed turn with a tool, the shape the chat loop sends.
* 2b. Adaptive thinking, an effort level and a cached system prompt: the
* rest of that parameter set, probed separately because one turn cannot
* falsify both tool use and thinking at once (see the note there).
* 3. Document extraction end to end, when given a file.
*
* Works against whichever backend lib/ai/provider.ts resolves (AWS Bedrock or
* the direct Anthropic API), so it doubles as the acceptance check when
* switching between them.
*
* Usage:
* npx tsx scripts/smoke-ai.ts
* npx tsx scripts/smoke-ai.ts ./some-receipt.pdf # also runs step 3
*/
import { config } from 'dotenv'
config({ path: '.env.local' })
import { readFile } from 'node:fs/promises'
import { basename, extname } from 'node:path'
import {
aiCredentialPrefix,
hasAiCredentials,
resolveAiProvider,
} from '../src/lib/ai/provider'
type ComposerModule = typeof import('../src/lib/agent/composer/client')
type ExtractionModule = typeof import('../src/extensions/general/invoice-inbox/lib/extract-invoice-fields')
let getAnthropic: ComposerModule['getAnthropic']
let EFFORT_DEEP: ComposerModule['EFFORT_DEEP']
let MAX_TOKENS_DEEP: ComposerModule['MAX_TOKENS_DEEP']
let OPUS_MODEL: ComposerModule['OPUS_MODEL']
let SONNET_MODEL: ComposerModule['SONNET_MODEL']
let extractInvoiceFields: ExtractionModule['extractInvoiceFields']
let failures = 0
function fail(step: string, err: unknown): void {
const message = err instanceof Error ? err.message : String(err)
console.error(` x ${step}: ${message}`)
failures++
}
/** Step 1: the simplest possible request, once per agent model id. */
async function ping(model: string): Promise<void> {
const start = Date.now()
try {
const resp = await getAnthropic().messages.create({
model,
max_tokens: 10,
messages: [{ role: 'user', content: 'Säg "hej" på svenska.' }],
})
const text = resp.content
.filter((b) => b.type === 'text')
.map((b) => (b as { type: 'text'; text: string }).text)
.join('')
console.log(` ok ${model}: ${Date.now() - start}ms: "${text.trim()}"`)
} catch (err) {
fail(model, err)
}
}
/**
* Step 2: the chat loop's parameter set.
*
* Split in two because one turn cannot falsify both things at once. A question
* that needs a tool gets answered by calling the tool, and adaptive thinking
* correctly declines to reason about it, so a zero thinking-block count there
* means nothing. Each probe therefore asks for one behaviour and nothing else.
*/
async function streamingWithTool(): Promise<void> {
const start = Date.now()
try {
const stream = getAnthropic().messages.stream({
model: SONNET_MODEL,
max_tokens: MAX_TOKENS_DEEP,
system: 'Du är en svensk redovisningsassistent. Använd verktyget när du behöver ett kontosaldo.',
tools: [
{
name: 'get_account_balance',
description: 'Hämtar saldot för ett BAS-konto i innevarande räkenskapsår.',
input_schema: {
type: 'object',
properties: {
account: { type: 'string', description: 'BAS-kontonummer, till exempel 1930.' },
},
required: ['account'],
},
},
],
messages: [{ role: 'user', content: 'Vad är saldot på konto 1930?' }],
})
let deltas = 0
stream.on('text', () => {
deltas++
})
const message = await stream.finalMessage()
const toolUse = message.content.filter((b) => b.type === 'tool_use').length
console.log(
` ok verktyg+streaming: ${Date.now() - start}ms, stop=${message.stop_reason}, ` +
`textdeltan=${deltas}, tool_use=${toolUse}`
)
if (toolUse === 0) {
console.warn(' varning: inget tool_use-block, verktyget kan ha ignorerats')
}
} catch (err) {
fail('verktyg+streaming', err)
}
}
/**
* Step 2b: adaptive thinking, effort and prompt caching.
*
* The question needs several dependent steps (reverse charge, then a partial
* deduction, then which boxes move), because adaptive thinking is supposed to
* skip reasoning it does not need: asking something easy cannot distinguish
* "declined to think" from "parameter ignored".
*
* The system prompt is padded past the 1024-token minimum cacheable prefix.
* Below it the API caches nothing and reports no error, so a short prompt
* makes the cache counters read zero no matter whether caching works.
*/
async function thinkingTurn(): Promise<void> {
const start = Date.now()
const filler = 'Svara alltid på svenska och hänvisa till BAS-konton med nummer. '.repeat(120)
try {
const stream = getAnthropic().messages.stream({
model: SONNET_MODEL,
max_tokens: MAX_TOKENS_DEEP,
thinking: { type: 'adaptive', display: 'summarized' },
output_config: { effort: EFFORT_DEEP },
system: [
{
type: 'text',
text: `Du är en svensk redovisningsassistent. ${filler}`,
cache_control: { type: 'ephemeral', ttl: '1h' },
},
],
messages: [
{
role: 'user',
content:
'Ett svenskt momsregistrerat bolag köper en konsulttjänst från Tyskland för 10 000 kr. ' +
'Bolaget har blandad verksamhet med 60 procent avdragsrätt. Hur bokförs affären, ' +
'och vilka rutor i momsdeklarationen påverkas?',
},
],
})
const message = await stream.finalMessage()
const thinkingBlocks = message.content.filter((b) => b.type === 'thinking')
const thinkingChars = thinkingBlocks
.map((b) => (b as { type: 'thinking'; thinking: string }).thinking?.length ?? 0)
.reduce((a, b) => a + b, 0)
const cacheWrite = message.usage.cache_creation_input_tokens ?? 0
const cacheRead = message.usage.cache_read_input_tokens ?? 0
console.log(
` ok thinking+cache: ${Date.now() - start}ms, stop=${message.stop_reason}, ` +
`thinking-block=${thinkingBlocks.length}, thinking-tecken=${thinkingChars}, ` +
`cache write/read=${cacheWrite}/${cacheRead}`
)
if (thinkingBlocks.length === 0) {
console.warn(
' varning: inget thinking-block pa en fraga som kraver flera steg. ' +
'Chattens "Tankte…"-block skulle da aldrig fyllas.'
)
} else if (thinkingChars === 0) {
console.warn(
' varning: thinking-block utan text, sa display:"summarized" slog inte igenom.'
)
}
if (cacheWrite === 0 && cacheRead === 0) {
console.warn(' varning: ingenting cachat trots prefix over minimigransen')
}
} catch (err) {
fail('thinking+cache', err)
}
}
/** Step 3: the real extraction path, including prompt, media block and JSON parse. */
async function extraction(path: string): Promise<void> {
const start = Date.now()
const mimeByExt: Record<string, string> = {
'.pdf': 'application/pdf',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.png': 'image/png',
'.webp': 'image/webp',
'.gif': 'image/gif',
}
const mimeType = mimeByExt[extname(path).toLowerCase()]
if (!mimeType) {
fail('extraction', `okänd filändelse för ${path}, stöds: ${Object.keys(mimeByExt).join(', ')}`)
return
}
try {
const buffer = await readFile(path)
const { data, rawText } = await extractInvoiceFields({
buffer,
mimeType,
fileName: basename(path),
})
// extractInvoiceFields never throws: a null rawText means the call was
// skipped or the reply did not parse, which is exactly the silent failure
// this script exists to make loud.
if (!rawText) {
fail('extraction', 'tomt resultat (nycklar saknas, filtyp stöds inte, eller JSON-parsen föll)')
return
}
// documentKind arrived with the receipt-aware extraction work. Read it
// defensively so this script still compiles against a checkout from
// before that landed: the whole point of it is to be runnable anywhere
// the app runs, including an older self-hosted deployment.
const kind = (data as { documentKind?: string | null }).documentKind ?? '-'
console.log(
` ok extraction: ${Date.now() - start}ms, leverantör="${data.supplier.name ?? '-'}", ` +
`nummer=${data.invoice.invoiceNumber ?? '-'}, datum=${data.invoice.invoiceDate ?? '-'}, ` +
`typ=${kind}`
)
} catch (err) {
fail('extraction', err)
}
}
async function main(): Promise<void> {
// These modules resolve their model ids at import time. Load them only after
// dotenv has populated the environment so .env.local provider and model
// overrides are exercised by the smoke test.
const [composer, extractionModule] = await Promise.all([
import('../src/lib/agent/composer/client'),
import('../src/extensions/general/invoice-inbox/lib/extract-invoice-fields'),
])
getAnthropic = composer.getAnthropic
EFFORT_DEEP = composer.EFFORT_DEEP
MAX_TOKENS_DEEP = composer.MAX_TOKENS_DEEP
OPUS_MODEL = composer.OPUS_MODEL
SONNET_MODEL = composer.SONNET_MODEL
extractInvoiceFields = extractionModule.extractInvoiceFields
const provider = resolveAiProvider()
const explicitProvider = (process.env.AI_PROVIDER ?? '').trim().toLowerCase()
console.log(`Leverantör: ${provider}`)
console.log(`Nyckel: ${hasAiCredentials() ? `${aiCredentialPrefix()}…` : 'SAKNAS'}`)
if (provider === 'bedrock') console.log(`Region: ${process.env.AWS_REGION || 'eu-north-1'}`)
console.log(`Modeller: ${SONNET_MODEL} / ${OPUS_MODEL}`)
if (!hasAiCredentials() && explicitProvider !== 'bedrock') {
console.error(
'\nInga synliga nycklar. Sätt ANTHROPIC_API_KEY, eller AWS_ACCESS_KEY_ID +\n' +
'AWS_SECRET_ACCESS_KEY för Bedrock. (Bedrock via instansprofil/IRSA syns\n' +
'inte härifrån: kör i så fall vidare med AI_PROVIDER=bedrock.)'
)
process.exitCode = 1
return
}
console.log('\n1. Enkla anrop, båda modellerna')
await ping(SONNET_MODEL)
if (OPUS_MODEL !== SONNET_MODEL) await ping(OPUS_MODEL)
console.log('\n2a. Streaming med verktyg')
await streamingWithTool()
console.log('\n2b. Adaptive thinking, effort och prompt-cache')
await thinkingTurn()
const path = process.argv[2]
console.log('\n3. Dokumenttolkning')
if (path) await extraction(path)
else console.log(' - skipping, no file given (npx tsx scripts/smoke-ai.ts <file>)')
console.log(failures === 0 ? '\nAllt grönt.' : `\n${failures} steg föll.`)
if (failures > 0) process.exitCode = 1
}
main().catch((err) => {
console.error(err)
process.exit(1)
})