tests/backend-alert-resilience.unit.test.ts
Rama main · réplica de lectura del repositorio original.
import { describe, expect, it, vi } from 'vitest'
import { X_UPLOAD_HANDLER_RETRYABLE } from '../supabase/functions/_backend/files/util.ts'
import { existInEnv, getEnv } from '../supabase/functions/_backend/utils/utils.ts'
function createTestContext() {
return {
get(key: string) {
if (key === 'requestId')
return 'req-test'
if (key === 'fileId')
return 'orgs/test-org/apps/test-app/test.zip'
return undefined
},
} as any
}
describe('backend alert resilience helpers', () => {
it('creates upload durable object stub with runtime maxRetries 0', async () => {
const { filesTestUtils } = await import('../supabase/functions/_backend/files/files.ts')
let getArgs: unknown[] | null = null
const ns = {
idFromName: (name: string) => ({ id: name }),
get: (id: unknown, options?: unknown) => {
getArgs = [id, options]
return { fetch: () => Promise.resolve(new Response()) }
},
}
const stub = filesTestUtils.getAttachmentUploadDurableObjectStub(ns as any, ns.idFromName('test-key') as any)
expect(stub).toBeDefined()
expect(getArgs).not.toBeNull()
expect(getArgs![1]).toEqual(filesTestUtils.DO_STUB_DISABLE_RUNTIME_RETRY)
})
it('clears stale thrown errors after a later retry succeeds', async () => {
const { retryWithBackoff } = await import('../supabase/functions/_backend/utils/retry.ts')
let attempts = 0
const outcome = await retryWithBackoff(async () => {
attempts += 1
if (attempts === 1) {
throw new Error('temporary queue failure')
}
return { ok: true }
}, {
attempts: 3,
baseDelayMs: 1,
})
expect(attempts).toBe(2)
expect(outcome.result).toEqual({ ok: true })
expect(outcome.lastError).toBeUndefined()
expect(outcome.attempts).toBe(2)
})
it('retries retryable durable object resets for replayable upload requests', async () => {
const { filesTestUtils } = await import('../supabase/functions/_backend/files/files.ts')
const handler = {
fetch: vi.fn(),
} as any
let attempts = 0
handler.fetch.mockImplementation(async () => {
attempts += 1
if (attempts === 1) {
throw Object.assign(
new Error('cannot access storage because object has moved to a different machine'),
{
retryable: true,
durableObjectReset: true,
},
)
}
return new Response(null, { status: 204 })
})
const response = await filesTestUtils.fetchUploadHandlerWithRetry(
createTestContext(),
handler,
new Request('http://localhost/files/upload/attachments/test.zip', {
method: 'HEAD',
}),
)
expect(response.status).toBe(204)
expect(handler.fetch).toHaveBeenCalledTimes(2)
})
it.concurrent('retries retryable durable object responses for empty-body upload creation requests', async () => {
const { filesTestUtils } = await import('../supabase/functions/_backend/files/files.ts')
const handler = {
fetch: vi.fn(),
} as any
handler.fetch
.mockImplementationOnce(async (request: Request) => {
expect(request.body).toBeNull()
return new Response(JSON.stringify({
error: 'durable_object_temporarily_unavailable',
}), {
status: 503,
headers: {
[X_UPLOAD_HANDLER_RETRYABLE]: '1',
},
})
})
.mockImplementationOnce(async (request: Request) => {
expect(request.body).toBeNull()
return new Response(null, { status: 201 })
})
const response = await filesTestUtils.fetchUploadHandlerWithRetry(
createTestContext(),
handler,
new Request('http://localhost/files/upload/attachments/test.zip', {
method: 'POST',
headers: {
'Content-Length': '0',
'Tus-Resumable': '1.0.0',
},
}),
)
expect(response.status).toBe(201)
expect(handler.fetch).toHaveBeenCalledTimes(2)
})
it.concurrent('returns retryable response after exhausting replayable network upload creation failures', async () => {
const { filesTestUtils } = await import('../supabase/functions/_backend/files/files.ts')
const handler = {
fetch: vi.fn(async () => {
throw Object.assign(new Error('Network connection lost.'), {
retryable: true,
})
}),
} as any
const response = await filesTestUtils.fetchUploadHandlerWithRetry(
createTestContext(),
handler,
new Request('http://localhost/files/upload/attachments/test.zip', {
method: 'POST',
headers: {
'Content-Length': '0',
'Tus-Resumable': '1.0.0',
'Upload-Length': '2500',
},
}),
)
expect(response.status).toBe(503)
expect(response.headers.get('Retry-After')).toBe('1')
expect(response.headers.get('Tus-Resumable')).toBe('1.0.0')
await expect(response.json()).resolves.toEqual({
error: 'upload_retryable',
message: 'Upload temporarily unavailable. Retry the upload request.',
})
expect(handler.fetch).toHaveBeenCalledTimes(3)
})
it.concurrent('retries Cloudflare internal errors and returns a retryable response once exhausted', async () => {
const { filesTestUtils } = await import('../supabase/functions/_backend/files/files.ts')
const handler = {
fetch: vi.fn(async () => {
// Cloudflare raises this as a bare error with no retryable flags.
throw new Error('internal error; reference = 0123456789abcdef')
}),
} as any
const response = await filesTestUtils.fetchUploadHandlerWithRetry(
createTestContext(),
handler,
new Request('http://localhost/files/upload/attachments/test.zip', {
method: 'POST',
headers: {
'Content-Length': '0',
'Tus-Resumable': '1.0.0',
'Upload-Length': '2500',
},
}),
)
expect(response.status).toBe(503)
expect(response.headers.get('Retry-After')).toBe('1')
expect(handler.fetch).toHaveBeenCalledTimes(3)
})
it.concurrent('forwards a replayable zero-byte TUS creation-with-upload body', async () => {
const { filesTestUtils } = await import('../supabase/functions/_backend/files/files.ts')
const forwardedBodyLengths: number[] = []
const handler = {
fetch: vi.fn(async (request: Request) => {
expect(request.body).not.toBeNull()
forwardedBodyLengths.push((await request.arrayBuffer()).byteLength)
if (forwardedBodyLengths.length === 1) {
return new Response(JSON.stringify({
error: 'durable_object_temporarily_unavailable',
}), {
status: 503,
headers: {
[X_UPLOAD_HANDLER_RETRYABLE]: '1',
},
})
}
return new Response(null, { status: 201 })
}),
} as any
const response = await filesTestUtils.fetchUploadHandlerWithRetry(
createTestContext(),
handler,
new Request('http://localhost/files/upload/attachments/test.zip', {
method: 'POST',
headers: {
'Content-Length': '0',
'Content-Type': 'application/offset+octet-stream',
'Tus-Resumable': '1.0.0',
'Upload-Length': '0',
},
body: new ArrayBuffer(0),
}),
)
expect(response.status).toBe(201)
expect(handler.fetch).toHaveBeenCalledTimes(2)
expect(forwardedBodyLengths).toEqual([0, 0])
})
it.concurrent('forwards a replayable zero-byte TUS patch body', async () => {
const { filesTestUtils } = await import('../supabase/functions/_backend/files/files.ts')
const forwardedBodyLengths: number[] = []
const handler = {
fetch: vi.fn(async (request: Request) => {
expect(request.body).not.toBeNull()
forwardedBodyLengths.push((await request.arrayBuffer()).byteLength)
if (forwardedBodyLengths.length === 1) {
return new Response(JSON.stringify({
error: 'durable_object_temporarily_unavailable',
}), {
status: 503,
headers: {
[X_UPLOAD_HANDLER_RETRYABLE]: '1',
},
})
}
return new Response(null, { status: 204 })
}),
} as any
const response = await filesTestUtils.fetchUploadHandlerWithRetry(
createTestContext(),
handler,
new Request('http://localhost/files/upload/attachments/test.zip', {
method: 'PATCH',
headers: {
'Content-Length': '0',
'Content-Type': 'Application/Offset+Octet-Stream',
'Tus-Resumable': '1.0.0',
'Upload-Offset': '0',
},
body: new ArrayBuffer(0),
}),
)
expect(response.status).toBe(204)
expect(handler.fetch).toHaveBeenCalledTimes(2)
expect(forwardedBodyLengths).toEqual([0, 0])
})
it.concurrent('recovers upload offset after a retryable durable object patch response', async () => {
const { filesTestUtils } = await import('../supabase/functions/_backend/files/files.ts')
const handler = {
fetch: vi.fn()
.mockResolvedValueOnce(new Response(JSON.stringify({
error: 'durable_object_temporarily_unavailable',
}), {
status: 503,
headers: {
[X_UPLOAD_HANDLER_RETRYABLE]: '1',
},
}))
.mockResolvedValueOnce(new Response(null, {
status: 200,
headers: {
'Upload-Offset': '5242880',
'Tus-Resumable': '1.0.0',
},
})),
} as any
const response = await filesTestUtils.fetchUploadHandlerWithRetry(
createTestContext(),
handler,
new Request('http://localhost/files/upload/attachments/test.zip', {
method: 'PATCH',
headers: {
'Content-Type': 'application/offset+octet-stream',
'Upload-Offset': '0',
'Tus-Resumable': '1.0.0',
},
body: 'chunk-data',
}),
)
expect(response.status).toBe(409)
expect(response.headers.get('Upload-Offset')).toBe('5242880')
expect(handler.fetch).toHaveBeenCalledTimes(2)
})
it('returns retryable response for streaming upload bodies when a durable object moves', async () => {
const { filesTestUtils } = await import('../supabase/functions/_backend/files/files.ts')
const handler = {
fetch: vi.fn(),
} as any
handler.fetch.mockImplementation(async () => {
throw Object.assign(
new Error('cannot access storage because object has moved to a different machine'),
{
retryable: true,
durableObjectReset: true,
},
)
})
const response = await filesTestUtils.fetchUploadHandlerWithRetry(
createTestContext(),
handler,
new Request('http://localhost/files/upload/attachments/test.zip', {
method: 'PATCH',
body: 'chunk-data',
}),
)
expect(response.status).toBe(503)
expect(response.headers.get('Retry-After')).toBe('1')
expect(response.headers.get('Tus-Resumable')).toBe('1.0.0')
await expect(response.json()).resolves.toEqual({
error: 'upload_retryable',
message: 'Upload temporarily unavailable. Retry the upload request.',
})
expect(handler.fetch).toHaveBeenCalledTimes(1)
})
it('retries transient PostgREST 502 responses for cron_stat_app', async () => {
const { cronStatAppTestUtils } = await import('../supabase/functions/_backend/triggers/cron_stat_app.ts')
let attempts = 0
const result = await cronStatAppTestUtils.runSupabaseResultWithRetry(
createTestContext(),
'test_postgrest_retry',
async () => {
attempts += 1
if (attempts === 1) {
return {
data: null,
error: {
name: 'PostgrestError',
message: 'error code: 502',
},
}
}
return {
data: { ok: true },
error: null,
}
},
)
expect(attempts).toBe(2)
expect(result.data).toEqual({ ok: true })
})
it('retries transient PostgREST top-level 502 statuses for cron_stat_app', async () => {
const { cronStatAppTestUtils } = await import('../supabase/functions/_backend/triggers/cron_stat_app.ts')
let attempts = 0
const result = await cronStatAppTestUtils.runSupabaseResultWithRetry(
createTestContext(),
'test_postgrest_top_level_status_retry',
async () => {
attempts += 1
if (attempts === 1) {
return {
data: null,
error: {
name: 'PostgrestError',
message: 'temporary upstream failure',
},
status: 502,
}
}
return {
data: { ok: true },
error: null,
status: 200,
}
},
)
expect(attempts).toBe(2)
expect(result.data).toEqual({ ok: true })
})
it.concurrent('retries transient PostgREST statuses for on_manifest_create updates', async () => {
const { onManifestCreateTestUtils } = await import('../supabase/functions/_backend/triggers/on_manifest_create.ts')
let attempts = 0
await onManifestCreateTestUtils.runManifestUpdateWithRetry(
createTestContext(),
async () => {
attempts += 1
if (attempts === 1) {
return {
error: {
name: 'PostgrestError',
message: 'error code: 502',
},
status: 502,
}
}
return {
error: null,
status: 204,
}
},
)
expect(attempts).toBe(2)
})
it.concurrent('returns empty strings when env bindings are missing from the context', () => {
const context = {
req: {
header: () => undefined,
},
} as any
expect(existInEnv(context, 'ENVIRONMENT')).toBe(false)
expect(getEnv(context, 'ENVIRONMENT')).toBe('')
})
it('skips stale cron_stat_app jobs when the app no longer exists', async () => {
vi.resetModules()
vi.doMock('../supabase/functions/_backend/utils/hono.ts', async (importOriginal) => {
const actual = await importOriginal<typeof import('../supabase/functions/_backend/utils/hono.ts')>()
return {
...actual,
middlewareAPISecret: async (_c: any, next: any) => {
await next()
},
}
})
vi.doMock('../supabase/functions/_backend/utils/pg.ts', async (importOriginal) => {
const actual = await importOriginal<typeof import('../supabase/functions/_backend/utils/pg.ts')>()
return {
...actual,
closeClient: () => undefined,
getPgClient: async () => ({
query: async () => ({ rows: [] }),
}),
}
})
const { app: cronStatApp } = await import('../supabase/functions/_backend/triggers/cron_stat_app.ts')
const response = await cronStatApp.fetch(
new Request('http://localhost/', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
appId: 'missing-app',
orgId: 'test-org',
}),
}),
{} as any,
{ waitUntil: () => undefined } as any,
)
expect(response.status).toBe(200)
expect(await response.json()).toEqual({
reason: 'app_not_found',
status: 'skipped',
})
})
it('passes the queue retry budget to dispatched trigger requests', async () => {
const { http_post_helper, MAX_QUEUE_READS } = await import('../supabase/functions/_backend/triggers/queue_consumer.ts')
const originalFetch = globalThis.fetch
let dispatchedHeaders: Record<string, string> = {}
const fetchMock = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
dispatchedHeaders = init?.headers as Record<string, string>
return new Response(JSON.stringify({ status: 'ok' }), { status: 200 })
})
globalThis.fetch = fetchMock as typeof fetch
try {
await http_post_helper({
env: { API_SECRET: 'testsecret' },
get: (key: string) => key === 'requestId' ? 'request-id' : undefined,
} as any, 'cron_stat_app', 'cloudflare', {
appId: 'com.example.app',
orgId: 'org-id',
}, 'cf-id', {
msgId: 1271329,
queueName: 'cron_stat_app',
readCount: 1,
}, 'https://api.capgo.app/triggers/cron_stat_app')
}
finally {
globalThis.fetch = originalFetch
}
expect(fetchMock).toHaveBeenCalledOnce()
expect(dispatchedHeaders['x-capgo-queue-name']).toBe('cron_stat_app')
expect(dispatchedHeaders['x-capgo-queue-read-count']).toBe('1')
expect(dispatchedHeaders['x-capgo-queue-max-reads']).toBe(String(MAX_QUEUE_READS))
})
})