tests/integration/security-hardening.test.ts
Rama main · réplica de lectura del repositorio original.
import { afterEach, test } from "bun:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Db } from "../../src/core/db";
import {
authenticate,
createApiKey,
LAST_USED_RESOLUTION_MS,
organizationStatement,
} from "../../src/core/auth";
import { migrate } from "../../src/core/migrations";
import {
createSecretCipher,
migrateSigningSecrets,
prepareSigningSecrets,
} from "../../src/core/secrets";
import {
dispatchWebhook,
isUnsafeWebhookAddress,
validateWebhookDestination,
type WebhookSecurity,
} from "../../src/core/webhook-security";
import { drainWebhookQueue } from "../../src/core/webhooks";
import { createSqliteDb, openDatabase } from "../../src/runtime/bun/sqlite";
import { pinnedLookup } from "../../src/runtime/bun/webhooks";
const KEY_ONE = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
const KEY_TWO = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE";
const directories: string[] = [];
function database(): { db: Db; close(): void } {
const directory = mkdtempSync(join(tmpdir(), "titen-security-"));
directories.push(directory);
const handle = openDatabase(join(directory, "titen.db"));
return { db: createSqliteDb(handle), close: () => handle.close() };
}
afterEach(() => {
for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true });
});
test("last_used_at is coarsened so a read does not cost a durable write", async () => {
const { db, close } = database();
await migrate(db);
const issued = new Date("2026-08-01T09:00:00.000Z");
const key = await createApiKey({
orgId: "org_key_coarsen",
principalId: "agent_key_coarsen",
principalKind: "agent",
label: "coarsened key",
scopes: ["context:compile"],
maxTrust: "asserted",
notBefore: issued,
expiresAt: null,
}, issued);
await db.batch([organizationStatement("org_key_coarsen", "Coarsen", issued), key.statement]);
const request = new Request("http://titen.test/v1/context/compile", {
headers: { authorization: `Bearer ${key.key}` },
});
const stored = async () => (await db.all<{ last_used_at: string | null }>(
"SELECT last_used_at FROM api_keys WHERE id = ?", [key.id],
))[0]!.last_used_at;
const first = new Date("2026-08-01T10:00:00.000Z");
await authenticate(db, request, first);
assert.equal(await stored(), first.toISOString(), "the first use records itself");
// Anything inside the interval leaves the row untouched, which is what keeps
// the UPDATE from writing a WAL frame: SQLite skips the page when the CASE
// resolves to the value already stored.
await authenticate(db, request, new Date(first.getTime() + LAST_USED_RESOLUTION_MS - 1));
assert.equal(await stored(), first.toISOString(), "a use inside the interval writes nothing");
const later = new Date(first.getTime() + LAST_USED_RESOLUTION_MS + 1);
await authenticate(db, request, later);
assert.equal(await stored(), later.toISOString(), "a use past the interval refreshes it");
// Monotonic under a backwards clock step larger than the interval.
await authenticate(db, request, new Date(first.getTime() - LAST_USED_RESOLUTION_MS * 10));
assert.equal(await stored(), later.toISOString(), "it never moves into the past");
close();
});
test("API key lifecycle uses exact boundaries and monotonic atomic usage", async () => {
const { db, close } = database();
await migrate(db);
const issued = new Date("2026-08-01T09:00:00.000Z");
const key = await createApiKey({
orgId: "org_key_lifecycle",
principalId: "agent_key_lifecycle",
principalKind: "agent",
label: "bounded key",
scopes: ["context:compile"],
maxTrust: "asserted",
notBefore: new Date("2026-08-01T10:00:00.000Z"),
expiresAt: new Date("2026-08-01T11:00:00.000Z"),
}, issued);
await db.batch([organizationStatement("org_key_lifecycle", "Lifecycle", issued), key.statement]);
const request = new Request("http://titen.test/v1/context/compile", {
headers: { authorization: `Bearer ${key.key}` },
});
await assert.rejects(() => authenticate(db, request, new Date("2026-08-01T09:59:59.999Z")));
assert.equal((await authenticate(db, request, new Date("2026-08-01T10:00:00.000Z"))).keyId, key.id);
await authenticate(db, request, new Date("2026-08-01T10:30:00.000Z"));
await authenticate(db, request, new Date("2026-08-01T10:20:00.000Z"));
assert.equal((await db.all<{ last_used_at: string }>(
"SELECT last_used_at FROM api_keys WHERE id = ?", [key.id],
))[0]!.last_used_at, "2026-08-01T10:30:00.000Z");
await assert.rejects(() => authenticate(db, request, new Date("2026-08-01T11:00:00.000Z")));
await assert.rejects(() => db.batch([{
sql: "UPDATE api_keys SET expires_at = ? WHERE id = ?",
params: ["2026-08-01T12:00:00.000Z", key.id],
}]), /API_KEY_LIFECYCLE_IMMUTABLE/u);
await db.batch([{
sql: "UPDATE api_keys SET revoked_at = ? WHERE id = ?",
params: ["2026-08-01T10:31:00.000Z", key.id],
}]);
await assert.rejects(() => authenticate(db, request, new Date("2026-08-01T10:32:00.000Z")));
close();
});
test("signing secrets migrate from plaintext, rotate, and fail closed without the key", async () => {
const { db, close } = database();
await migrate(db);
await db.batch([
{ sql: "INSERT INTO organizations(id, name, created_at) VALUES ('org_1', 'Org', '2026-07-30T00:00:00Z')" },
{
sql: `INSERT INTO webhooks(id, org_id, url, secret_hash, secret, events, status, created_at)
VALUES ('whk_1', 'org_1', 'https://hooks.example.com/a', 'hash', ?, '*', 'active', '2026-07-30T00:00:00Z')`,
params: ['{"legacy":"plaintext-secret"}'],
},
{
sql: `INSERT INTO federation_peers(id, org_id, name, endpoint, shared_secret_hash, shared_secret, direction, status, created_at)
VALUES ('fpeer_1', 'org_1', 'Peer', 'https://peer.example.com', 'hash', ?, 'push', 'active', '2026-07-30T00:00:00Z')`,
params: ["federation-plaintext-secret"],
},
...Array.from({ length: 55 }, (_, index) => ({
sql: `INSERT INTO webhooks(id, org_id, url, secret_hash, secret, events, status, created_at)
VALUES (?, 'org_1', ?, 'hash', ?, '*', 'paused', '2026-07-30T00:00:00Z')`,
params: [
`whk_page_${String(index).padStart(2, "0")}`,
`https://hooks.example.com/page/${index}`,
`page-plaintext-secret-${index}`,
],
})),
]);
assert.equal(await prepareSigningSecrets(db, undefined), false);
const first = createSecretCipher({ active: "v1", keys: { v1: KEY_ONE } });
assert.equal(await migrateSigningSecrets(db, first), 57);
const encrypted = await db.all<{ secret: string }>("SELECT secret FROM webhooks WHERE id = 'whk_1'");
assert.match(encrypted[0]!.secret, /^titen-secret:v1:/);
assert.ok(!encrypted[0]!.secret.includes("plaintext-secret"));
assert.equal(await first.decrypt(encrypted[0]!.secret, "webhook:whk_1"), '{"legacy":"plaintext-secret"}');
await assert.rejects(() => first.decrypt(encrypted[0]!.secret, "webhook:other"));
const rotating = createSecretCipher({ active: "v2", keys: { v1: KEY_ONE, v2: KEY_TWO } });
assert.equal(await migrateSigningSecrets(db, rotating), 57);
const rotated = await db.all<{ secret: string }>("SELECT secret FROM webhooks WHERE id = 'whk_1'");
assert.match(rotated[0]!.secret, /"kid":"v2"/);
const wrong = createSecretCipher({ active: "v2", keys: { v2: KEY_ONE } });
assert.equal(await prepareSigningSecrets(db, wrong), false);
assert.equal(await prepareSigningSecrets(db, createSecretCipher({ active: "v2", keys: { v2: KEY_TWO } })), true);
await db.batch([{
sql: `INSERT INTO webhooks(id, org_id, url, secret_hash, secret, events, status, created_at)
VALUES ('whk_null', 'org_1', 'https://hooks.example.com/null', 'hash', NULL, '*', 'active', '2026-07-30T00:00:00Z')`,
}, {
sql: `INSERT INTO webhooks(id, org_id, url, secret_hash, secret, events, status, created_at)
VALUES ('whk_paused_null', 'org_1', 'https://hooks.example.com/paused-null', 'hash', NULL, '*', 'paused', '2026-07-30T00:00:00Z')`,
}, {
sql: `INSERT INTO federation_peers(id, org_id, name, endpoint, shared_secret_hash, shared_secret, direction, status, created_at)
VALUES ('fpeer_null', 'org_1', 'Null Peer', 'https://null-peer.example.com', 'hash', NULL, 'push', 'active', '2026-07-30T00:00:00Z')`,
}, {
sql: `INSERT INTO federation_peers(id, org_id, name, endpoint, shared_secret_hash, shared_secret, direction, status, created_at)
VALUES ('fpeer_suspended_null', 'org_1', 'Suspended Null Peer', 'https://suspended-null-peer.example.com', 'hash', NULL, 'push', 'suspended', '2026-07-30T00:00:00Z')`,
}]);
assert.equal(await prepareSigningSecrets(db, rotating), true, "unrecoverable active NULL secrets are terminalized");
assert.deepEqual(
await db.all<{ id: string; status: string }>("SELECT id, status FROM webhooks WHERE id LIKE 'whk_%null' ORDER BY id"),
[
{ id: "whk_null", status: "disabled" },
{ id: "whk_paused_null", status: "paused" },
],
);
assert.deepEqual(
await db.all<{ id: string; status: string }>("SELECT id, status FROM federation_peers WHERE id LIKE 'fpeer_%null' ORDER BY id"),
[
{ id: "fpeer_null", status: "suspended" },
{ id: "fpeer_suspended_null", status: "suspended" },
],
);
assert.equal((await db.all<{ status: string }>("SELECT status FROM webhooks WHERE id = 'whk_1'"))[0]!.status, "active");
assert.equal((await db.all<{ status: string }>("SELECT status FROM federation_peers WHERE id = 'fpeer_1'"))[0]!.status, "active");
close();
});
test("webhook validation rejects local addresses, rebinding, redirects, and unpinned connections", async () => {
for (const address of ["127.0.0.1", "169.254.169.254", "10.0.0.1", "::1", "::ffff:127.0.0.1", "2001:db8::1"])
assert.equal(isUnsafeWebhookAddress(address), true, address);
let answers = ["93.184.216.34"];
let dispatches = 0;
const security: WebhookSecurity = {
allowedHostnames: ["hooks.example.com"],
resolve: async () => answers,
dispatch: async ({ addresses }) => {
dispatches += 1;
return { response: new Response(null, { status: 200 }), connectedAddress: addresses[0]! };
},
};
for (const url of ["https://127.0.0.1/x", "https://2130706433/x", "https://[::ffff:127.0.0.1]/x"])
await assert.rejects(() => validateWebhookDestination(url, security));
await assert.rejects(() => validateWebhookDestination("https://other.example.com/x", security));
answers = ["169.254.169.254"];
await assert.rejects(() => dispatchWebhook("https://hooks.example.com/x", security, {}));
assert.equal(dispatches, 0, "rebinding must be rejected before outbound I/O");
answers = ["93.184.216.34"];
security.dispatch = async ({ addresses }) => ({
response: new Response(null, { status: 302 }),
connectedAddress: addresses[0]!,
});
await assert.rejects(() => dispatchWebhook("https://hooks.example.com/x", security, {}));
security.dispatch = async () => ({ response: new Response(null, { status: 200 }), connectedAddress: "1.1.1.1" });
await assert.rejects(() => dispatchWebhook("https://hooks.example.com/x", security, {}));
});
test("the production pinned lookup supports Node's all-address callback contract", () => {
const lookup = pinnedLookup("93.184.216.34");
lookup("hooks.example.com", { all: true }, (error, addresses) => {
assert.equal(error, null);
assert.deepEqual(addresses, [{ address: "93.184.216.34", family: 4 }]);
});
lookup("hooks.example.com", {}, (error, address, family) => {
assert.equal(error, null);
assert.equal(address, "93.184.216.34");
assert.equal(family, 4);
});
});
async function seededDelivery(db: Db, secret: string, ids = ["dlv_1"]): Promise<void> {
await db.batch([
{ sql: "INSERT INTO organizations(id, name, created_at) VALUES ('org_1', 'Org', '2026-07-30T00:00:00Z')" },
{
sql: `INSERT INTO webhooks(id, org_id, principal_id, url, secret_hash, secret, events, status, created_at)
VALUES ('whk_1', 'org_1', 'actor', 'https://hooks.example.com/a', 'hash', ?, '*', 'active', '2026-07-30T00:00:00Z')`,
params: [secret],
},
...ids.flatMap((id, index) => [
{
sql: `INSERT INTO events(id, org_id, kind, actor_id, resource_type, resource_id, payload, created_at)
VALUES (?, 'org_1', 'test.event', 'actor', 'test', ?, '{}', '2026-07-30T00:00:01Z')`,
params: [`evt_${index}`, `resource_${index}`],
},
{
sql: `INSERT INTO webhook_deliveries(id, webhook_id, event_id, status, attempts, next_retry_at, created_at)
VALUES (?, 'whk_1', ?, 'pending', 0, '2026-07-30T00:00:02Z', '2026-07-30T00:00:02Z')`,
params: [id, `evt_${index}`],
},
]),
]);
}
test("webhook claims are exclusive, retries keep one delivery ID, and disable terminalizes siblings", async () => {
const { db, close } = database();
await migrate(db);
const cipher = createSecretCipher({ active: "v1", keys: { v1: KEY_ONE } });
await seededDelivery(db, await cipher.encrypt("shared-secret-value", "webhook:whk_1"), ["dlv_1", "dlv_2"]);
await db.batch([{ sql: "UPDATE webhook_deliveries SET next_retry_at = '2026-07-30T02:00:00Z' WHERE id = 'dlv_2'" }]);
let calls = 0;
const deliveries: string[] = [];
const attempts: string[] = [];
const security: WebhookSecurity = {
allowedHostnames: ["hooks.example.com"],
timeoutMs: 50,
resolve: async () => ["93.184.216.34"],
dispatch: async ({ addresses, init }) => {
calls += 1;
deliveries.push(new Headers(init.headers).get("x-titen-delivery")!);
attempts.push(new Headers(init.headers).get("x-titen-attempt")!);
return { response: new Response(null, { status: calls === 1 ? 500 : 200 }), connectedAddress: addresses[0]! };
},
};
const now = new Date("2026-07-30T00:01:00Z");
await Promise.all(Array.from({ length: 20 }, () => drainWebhookQueue({ db, orgId: "org_1", now, limit: 1, security, cipher })));
assert.equal(calls, 1, "only one concurrent claimant may dispatch a row");
await db.batch([{ sql: "UPDATE webhook_deliveries SET next_retry_at = ? WHERE id = 'dlv_1'", params: [now.toISOString()] }]);
await drainWebhookQueue({ db, orgId: "org_1", now, limit: 1, security, cipher });
assert.deepEqual(deliveries, ["dlv_1", "dlv_1"]);
assert.notEqual(attempts[0], attempts[1]);
await db.batch([
{ sql: "UPDATE webhooks SET failure_count = 9 WHERE id = 'whk_1'" },
{ sql: "UPDATE webhook_deliveries SET status = 'pending', attempts = 0, next_retry_at = ? WHERE id = 'dlv_2'", params: [now.toISOString()] },
]);
security.dispatch = async ({ addresses }) => ({ response: new Response(null, { status: 500 }), connectedAddress: addresses[0]! });
await drainWebhookQueue({ db, orgId: "org_1", now, limit: 1, security, cipher });
assert.equal((await db.all<{ status: string }>("SELECT status FROM webhooks WHERE id = 'whk_1'"))[0]!.status, "disabled");
assert.deepEqual(await db.all("SELECT id FROM webhook_deliveries WHERE webhook_id = 'whk_1' AND status = 'pending'"), []);
close();
});
test("each webhook attempt signs its own start time and holds a lease through the request", async () => {
const { db, close } = database();
await migrate(db);
const cipher = createSecretCipher({ active: "v1", keys: { v1: KEY_ONE } });
const ids = Array.from({ length: 6 }, (_, index) => `dlv_${index}`);
await seededDelivery(db, await cipher.encrypt("shared-secret-value", "webhook:whk_1"), ids);
// A healthy but slow receiver: every request takes 4 s of an injected clock.
let clock = Date.parse("2026-07-30T00:01:00Z");
const now = () => new Date(clock);
const sent: Array<{ delivery: string; timestamp: number; startedAt: number; lease: string }> = [];
let second: Promise<unknown> | undefined;
const security: WebhookSecurity = {
allowedHostnames: ["hooks.example.com"],
timeoutMs: 10_000,
resolve: async () => ["93.184.216.34"],
dispatch: async ({ addresses, init }) => {
const headers = new Headers(init.headers);
const delivery = headers.get("x-titen-delivery")!;
const [row] = await db.all<{ lease_expires_at: string }>(
"SELECT lease_expires_at FROM webhook_deliveries WHERE id = ?", [delivery],
);
sent.push({ delivery, timestamp: Number(headers.get("x-titen-timestamp")), startedAt: clock, lease: row!.lease_expires_at });
// A second drain starts while the fifth request is in flight, 16 s into the pass.
if (sent.length === 5) second = drainWebhookQueue({ db, orgId: "org_1", now, limit: 50, security, cipher });
await new Promise((resolve) => setTimeout(resolve, 1));
clock += 4_000;
return { response: new Response(null, { status: 200 }), connectedAddress: addresses[0]! };
},
};
await drainWebhookQueue({ db, orgId: "org_1", now, limit: 50, security, cipher });
await second;
for (const attempt of sent) {
assert.ok(Math.abs(attempt.timestamp * 1000 - attempt.startedAt) < 1000, `${attempt.delivery} signs its own start time`);
assert.ok(Date.parse(attempt.lease) >= attempt.startedAt + 10_000, `${attempt.delivery} lease outlasts the dispatch timeout`);
}
assert.equal(sent.length, 6, "a concurrent drain never repeats an in-flight delivery");
assert.deepEqual([...new Set(sent.map((attempt) => attempt.delivery))].sort(), ids);
const [hook] = await db.all<{ status: string; failure_count: number }>("SELECT status, failure_count FROM webhooks WHERE id = 'whk_1'");
assert.deepEqual(hook, { status: "active", failure_count: 0 });
close();
});
test("webhook retries stop when the subscriber loses event access", async () => {
const { db, close } = database();
await migrate(db);
const cipher = createSecretCipher({ active: "v1", keys: { v1: KEY_ONE } });
await db.batch([
{ sql: "INSERT INTO organizations(id, name, created_at) VALUES ('org_1', 'Org', '2026-07-30T00:00:00Z')" },
{ sql: "INSERT INTO workspaces(id, org_id, name, created_at) VALUES ('workspace_1', 'org_1', 'Team', '2026-07-30T00:00:00Z')" },
{
sql: `INSERT INTO memberships(id, org_id, workspace_id, principal_id, principal_kind, role, created_at)
VALUES ('membership_1', 'org_1', 'workspace_1', 'subscriber', 'agent', 'member', '2026-07-30T00:00:00Z')`,
},
{
sql: `INSERT INTO observations
(id, org_id, subject_id, actor_id, kind, content, content_hash, source_type,
trust, visibility, ingested_at, workspace_id)
VALUES ('obs_1', 'org_1', 'subject_1', 'writer', 'tool_result', 'secret', 'hash', 'tool',
'verified', 'team', '2026-07-30T00:00:01Z', 'workspace_1')`,
},
{
sql: `INSERT INTO events(id, org_id, kind, actor_id, resource_type, resource_id, payload, created_at)
VALUES ('evt_1', 'org_1', 'observation.appended', 'writer', 'observation', 'obs_1', '{}',
'2026-07-30T00:00:01Z')`,
},
{
sql: `INSERT INTO webhooks(id, org_id, principal_id, url, secret_hash, secret, events, status, created_at)
VALUES ('whk_1', 'org_1', 'subscriber', 'https://hooks.example.com/a', 'hash', ?, '*', 'active',
'2026-07-30T00:00:00Z')`,
params: [await cipher.encrypt("shared-secret-value", "webhook:whk_1")],
},
{
sql: `INSERT INTO webhook_deliveries
(id, webhook_id, event_id, status, attempts, next_retry_at, created_at, lease_token, lease_expires_at)
VALUES ('dlv_1', 'whk_1', 'evt_1', 'pending', 1, '2026-07-30T00:01:00Z',
'2026-07-30T00:00:02Z', 'stale-lease', '2026-07-30T00:02:00Z')`,
},
{ sql: "UPDATE memberships SET removed_at = '2026-07-30T00:00:30Z' WHERE id = 'membership_1'" },
]);
let calls = 0;
const security: WebhookSecurity = {
allowedHostnames: ["hooks.example.com"],
resolve: async () => ["93.184.216.34"],
dispatch: async ({ addresses }) => {
calls += 1;
return { response: new Response(null, { status: 200 }), connectedAddress: addresses[0]! };
},
};
const result = await drainWebhookQueue({
db,
orgId: "org_1",
now: new Date("2026-07-30T00:01:00Z"),
limit: 1,
security,
cipher,
});
assert.deepEqual(result, { attempted: 0, delivered: 0 });
assert.equal(calls, 0, "revoked membership must block outbound retry I/O");
assert.deepEqual(
(await db.all<{ status: string; next_retry_at: string | null; lease_token: string | null; lease_expires_at: string | null }>(
"SELECT status, next_retry_at, lease_token, lease_expires_at FROM webhook_deliveries WHERE id = 'dlv_1'",
))[0],
{ status: "expired", next_retry_at: null, lease_token: null, lease_expires_at: null },
);
close();
});
test("an expired lease is recovered and a timed-out dispatch returns to retry state", async () => {
const { db, close } = database();
await migrate(db);
const cipher = createSecretCipher({ active: "v1", keys: { v1: KEY_ONE } });
await seededDelivery(db, await cipher.encrypt("shared-secret-value", "webhook:whk_1"));
await db.batch([{
sql: `UPDATE webhook_deliveries SET lease_token = 'crashed', lease_expires_at = '2026-07-30T00:00:30Z'
WHERE id = 'dlv_1'`,
}]);
let calls = 0;
const security: WebhookSecurity = {
allowedHostnames: ["hooks.example.com"],
timeoutMs: 20,
resolve: async () => ["93.184.216.34"],
dispatch: async ({ init }) => {
calls += 1;
await new Promise((_, reject) => init.signal?.addEventListener("abort", () => reject(init.signal?.reason), { once: true }));
throw new Error("unreachable");
},
};
const started = Date.now();
await drainWebhookQueue({ db, orgId: "org_1", now: new Date("2026-07-30T00:01:00Z"), limit: 1, security, cipher });
assert.equal(calls, 1);
assert.ok(Date.now() - started < 500, "outbound duration must be bounded");
const row = (await db.all<{ status: string; attempts: number; lease_token: string | null; next_retry_at: string | null }>(
"SELECT status, attempts, lease_token, next_retry_at FROM webhook_deliveries WHERE id = 'dlv_1'",
))[0]!;
assert.equal(row.status, "pending");
assert.equal(row.attempts, 1);
assert.equal(row.lease_token, null);
assert.ok(row.next_retry_at);
close();
});
test("the fifth failed attempt becomes terminal", async () => {
const { db, close } = database();
await migrate(db);
const cipher = createSecretCipher({ active: "v1", keys: { v1: KEY_ONE } });
await seededDelivery(db, await cipher.encrypt("shared-secret-value", "webhook:whk_1"));
await db.batch([{ sql: "UPDATE webhook_deliveries SET attempts = 4 WHERE id = 'dlv_1'" }]);
const security: WebhookSecurity = {
allowedHostnames: ["hooks.example.com"],
resolve: async () => ["93.184.216.34"],
dispatch: async ({ addresses }) => ({ response: new Response(null, { status: 500 }), connectedAddress: addresses[0]! }),
};
await drainWebhookQueue({ db, orgId: "org_1", now: new Date("2026-07-30T00:01:00Z"), limit: 1, security, cipher });
const row = (await db.all<{ status: string; attempts: number; next_retry_at: string | null }>(
"SELECT status, attempts, next_retry_at FROM webhook_deliveries WHERE id = 'dlv_1'",
))[0]!;
assert.deepEqual(row, { status: "failed", attempts: 5, next_retry_at: null });
close();
});