tests/e2e/automation-control-plane-observability.spec.ts
Rama main · réplica de lectura del repositorio original.
import { expect, test } from "@playwright/test";
import type { ObservabilityJob } from "@/lib/types";
import { enableScopedE2EAuth, scopedPath } from "./helpers/auth";
import { emptyAutomationFailuresResponse } from "./helpers/activation-fixtures";
import {
fulfillJson,
mockBaseChrome,
modelId,
} from "./helpers/automation-control-plane-fixtures";
test("observability centers runtime runs and exposes repair/requeue actions", async ({
page,
}) => {
let repairCount = 0;
let requeueCount = 0;
let cancelCount = 0;
const jobRequests: URL[] = [];
const callRequests: URL[] = [];
let jobs: ObservabilityJob[] = [];
await enableScopedE2EAuth(page);
await mockBaseChrome(page);
await page.route("**/api/observability/stats*", (route) =>
fulfillJson(route, {
summary: {
total_calls: 4,
total_tokens: 12000,
total_cost: 0.32,
cost_today: 0.08,
reconciliation_pending: 0,
avg_duration_ms: 1820,
success_rate: 75,
calls_today: 2,
tokens_today: 4000,
sandbox_time_ms: 600000,
sandbox_active: 1,
sandbox_total: 2,
job_runs_total: 3,
job_runs_running: 1,
job_runs_pending: 1,
job_runs_stale_pending: 1,
job_runs_failed_in_range: 1,
job_runs_repaired_in_range: 1,
job_runs_concluded_in_range: 3,
job_runs_success_rate_in_range: 66.7,
suppressed_in_range: 2,
deferred_in_range: 1,
start_failed_in_range: 1,
limit_allowed_in_range: 0,
limit_denied_in_range: 0,
oldest_pending_age_ms: 420000,
},
by_model: [],
by_type: [],
})
);
await page.route("**/api/observability/jobs?*", async (route) => {
jobRequests.push(new URL(route.request().url()));
const payload = {
jobs: [
{
id: "job-failed",
assignment_id: "assignment-1",
trigger_id: null,
workflow_run_id: "wf_failed",
retry_of_job_run_id: null,
status: "failed",
created_at: "2026-03-21T19:00:00.000Z",
started_at: "2026-03-21T19:00:10.000Z",
completed_at: "2026-03-21T19:01:00.000Z",
input_tokens: 100,
output_tokens: 200,
duration_ms: 50000,
cost_usd: null,
error:
"This Mogplex account cannot use hosted AI because it has no credit. Add funds or choose a plan in Settings > Billing. Or add an AI Gateway or provider key in Settings > API Keys.",
start_attempts: 1,
last_start_attempt_at: "2026-03-21T19:00:00.000Z",
last_start_error: null,
last_start_source: "cron",
metadata: { repo_full_name: "acme/demo-app" },
source_kind: "assignment",
source_type: "cron_refactor",
repo: { id: "repo-1", full_name: "acme/demo-app" },
agent: { id: "agent-1", name: "Refactor Bot", slug: "refactor-bot" },
latest_dispatch_event: {
id: "dispatch-failed",
event_kind: "control",
outcome: "failed",
reason: "START_RUNTIME_FAILED",
metadata: null,
created_at: "2026-03-21T19:01:00.000Z",
},
latest_ai_call: {
id: "call-1",
status: "failed",
model: modelId,
total_tokens: 300,
tool_calls_count: 4,
started_at: "2026-03-21T19:00:10.000Z",
},
cancel_requested_at: null,
cancelled_at: null,
cancel_reason: null,
cancel_error: null,
cancelable: false,
repairable: false,
requeueable: true,
},
{
id: "job-pending",
assignment_id: null,
trigger_id: "trigger-1",
workflow_run_id: null,
retry_of_job_run_id: null,
status: "pending",
created_at: "2026-03-21T18:00:00.000Z",
started_at: null,
completed_at: null,
input_tokens: null,
output_tokens: null,
duration_ms: null,
cost_usd: null,
error: null,
start_attempts: 2,
last_start_attempt_at: "2026-03-21T18:05:00.000Z",
last_start_error: "workflow unavailable",
last_start_source: "repair",
metadata: { repo_full_name: "acme/demo-app" },
source_kind: "trigger",
source_type: "mention",
repo: { id: "repo-1", full_name: "acme/demo-app" },
agent: { id: "agent-2", name: "Mention Bot", slug: "mention-bot" },
latest_ai_call: null,
latest_dispatch_event: {
id: "dispatch-pending",
event_kind: "start",
outcome: "deferred",
reason: "REPO_CONCURRENCY_LIMIT",
metadata: { start_source: "repair" },
created_at: "2026-03-21T18:05:00.000Z",
},
cancel_requested_at: null,
cancelled_at: null,
cancel_reason: null,
cancel_error: null,
cancelable: true,
repairable: true,
requeueable: false,
},
{
id: "job-review-no-findings",
assignment_id: "assignment-2",
trigger_id: null,
workflow_run_id: "wf_review",
retry_of_job_run_id: null,
status: "success",
created_at: "2026-03-21T17:00:00.000Z",
started_at: "2026-03-21T17:00:05.000Z",
completed_at: "2026-03-21T17:00:40.000Z",
input_tokens: 50,
output_tokens: 60,
duration_ms: 35000,
cost_usd: null,
error: null,
start_attempts: 1,
last_start_attempt_at: "2026-03-21T17:00:00.000Z",
last_start_error: null,
last_start_source: "webhook",
metadata: { repo_full_name: "acme/demo-app" },
source_kind: "assignment",
source_type: "pr_review",
repo: { id: "repo-1", full_name: "acme/demo-app" },
agent: { id: "agent-3", name: "Reviewer Bot", slug: "reviewer-bot" },
latest_dispatch_event: {
id: "dispatch-review",
event_kind: "control",
outcome: "completed",
reason: "PR_REVIEW_NO_FINDINGS",
metadata: {
review_outcome: "PR_REVIEW_NO_FINDINGS",
review_outcome_label: "No findings",
},
created_at: "2026-03-21T17:00:40.000Z",
},
latest_ai_call: null,
cancel_requested_at: null,
cancelled_at: null,
cancel_reason: null,
cancel_error: null,
cancelable: false,
repairable: false,
requeueable: false,
},
] satisfies ObservabilityJob[],
total: 30,
page: Number(new URL(route.request().url()).searchParams.get("page")),
limit: 25,
};
jobs = payload.jobs;
await fulfillJson(route, payload);
});
await page.route(/\/api\/observability\/jobs\/[^/?]+(?:\?.*)?$/, (route) => {
const id = new URL(route.request().url()).pathname.split("/").at(-1);
const job = jobs.find((candidate) => candidate.id === id);
return fulfillJson(route, {
run: job
? {
...job,
dispatch_events: job.latest_dispatch_event
? [job.latest_dispatch_event]
: [],
ai_calls: [],
review_findings: [],
}
: null,
});
});
await page.route("**/api/observability/calls?*", (route) => {
callRequests.push(new URL(route.request().url()));
return fulfillJson(route, {
calls: [
{
id: "call-live",
user_id: "user-1",
type: "chat",
model: modelId,
input_tokens: null,
output_tokens: null,
total_tokens: null,
duration_ms: null,
started_at: "2026-03-21T19:10:00.000Z",
completed_at: null,
status: "streaming",
error: null,
conversation_id: "conversation-1",
job_run_id: null,
repo_id: "repo-1",
runtime_command_id: null,
tool_calls_count: 0,
tool_calls: [],
metadata: { repo: "acme/demo-app" },
},
{
id: "call-1",
user_id: "user-1",
type: "cron_refactor",
model: modelId,
input_tokens: 100,
output_tokens: 200,
total_tokens: 300,
duration_ms: 50000,
started_at: "2026-03-21T19:00:10.000Z",
completed_at: "2026-03-21T19:01:00.000Z",
status: "failed",
error: "lint failed",
conversation_id: null,
job_run_id: "job-failed",
repo_id: "repo-1",
runtime_command_id: null,
tool_calls_count: 4,
tool_calls: [],
metadata: { repo: "acme/demo-app" },
},
],
total: 75,
page: Number(new URL(route.request().url()).searchParams.get("page")),
limit: 50,
});
});
await page.route("**/api/observability/automation-events?*", (route) =>
fulfillJson(route, {
events: [
{
id: "event-1",
job_run_id: "job-pending",
assignment_id: null,
trigger_id: "trigger-1",
repo_id: "repo-1",
installation_id: 101,
source_kind: "trigger",
source_type: "mention",
event_kind: "start",
outcome: "deferred",
reason: "REPO_CONCURRENCY_LIMIT",
metadata: { start_source: "repair" },
created_at: "2026-03-21T18:05:00.000Z",
repo: { id: "repo-1", full_name: "acme/demo-app" },
agent: { id: "agent-2", name: "Mention Bot", slug: "mention-bot" },
},
],
total: 1,
page: 1,
limit: 25,
})
);
await page.route(
/\/api\/observability\/automation-failures(?:\?.*)?$/,
(route) => fulfillJson(route, emptyAutomationFailuresResponse)
);
await page.route(
"**/api/observability/jobs/job-pending/repair",
async (route) => {
repairCount += 1;
await fulfillJson(route, {
ok: true,
started: true,
status: "pending",
runtimeProvider: "workflow",
runtimeRunId: "wf_repair",
workflowRunId: "wf_repair",
});
}
);
await page.route(
"**/api/observability/jobs/job-pending/cancel",
async (route) => {
cancelCount += 1;
await fulfillJson(
route,
{ error: "Job run is no longer cancelable" },
409
);
}
);
await page.route(
"**/api/observability/jobs/job-failed/requeue",
async (route) => {
requeueCount += 1;
await fulfillJson(route, {
ok: true,
started: true,
status: "pending",
runtimeProvider: "workflow",
runtimeRunId: "wf_retry",
workflowRunId: "wf_retry",
jobRunId: "job-retry",
});
}
);
await page.goto(scopedPath("observability"));
await page.waitForLoadState("networkidle");
await expect(
page.getByRole("heading", { name: "Automation health" })
).toBeVisible();
await expect(
page.getByLabel("Automation health").getByText("Issues recorded")
).toBeVisible();
await expect(page.getByText("Run success")).toBeVisible();
await expect(page.getByText("66.7%", { exact: true })).toBeVisible();
await expect(page.getByText("Operational signals")).toBeVisible();
await expect(page.getByText("Prevented", { exact: true })).toBeVisible();
await expect(page.getByText("2 events not run")).toBeVisible();
await expect(page.getByText("1 start attempt retried")).toBeVisible();
await expect(page.getByText("1 failed start attempt")).toBeVisible();
await expect(page.getByText("Run history", { exact: true })).toBeVisible();
await expect(page.getByText("1 run remains failed")).toBeVisible();
await expect(page.getByText("1 delayed run", { exact: true })).toBeVisible();
const recoveryRequestPromise = page.waitForRequest((request) => {
const url = new URL(request.url());
return (
url.pathname === "/api/observability/jobs" &&
url.searchParams.get("status") === "pending" &&
url.searchParams.get("only_repairable") === "true"
);
});
await page.getByText("1 delayed run", { exact: true }).click();
const recoveryRequest = new URL((await recoveryRequestPromise).url());
expect(recoveryRequest.searchParams.get("from")).toBeNull();
expect(recoveryRequest.searchParams.get("to")).toBeNull();
await expect(
page.getByText("Current active work across all dates.")
).toBeVisible();
const pendingRequestPromise = page.waitForRequest((request) => {
const url = new URL(request.url());
return (
url.pathname === "/api/observability/jobs" &&
url.searchParams.get("status") === "pending" &&
!url.searchParams.has("only_repairable")
);
});
await page.getByRole("button", { name: "Inspect 1 pending run" }).click();
const pendingRequest = new URL((await pendingRequestPromise).url());
expect(pendingRequest.searchParams.get("from")).toBeNull();
expect(pendingRequest.searchParams.get("to")).toBeNull();
await page
.getByRole("button", { name: "Inspect 2 prevented events" })
.click();
// The prevented-events drill-down switches the table tab group to the
// Pressure tab with the outcome filter applied.
await expect(
page.getByRole("tab", { name: /^Automation events/ })
).toHaveAttribute("aria-selected", "true");
await expect(page.getByLabel("Outcome")).toHaveValue("suppressed");
await page.getByRole("tab", { name: /^Runs/ }).click();
await expect(page.getByLabel("Run status")).toBeVisible();
await page.getByRole("button", { name: "Refactor Bot", exact: true }).click();
const inspector = page.getByRole("region", {
name: "Run details",
exact: true,
});
await expect(
page.getByText(
"This Mogplex account cannot use hosted AI because it has no credit. Add funds or choose a plan in Settings > Billing. Or add an AI Gateway or provider key in Settings > API Keys."
)
).toBeVisible();
await expect(
inspector.getByRole("button", { name: "Retry as new run", exact: true })
).toBeVisible();
await inspector
.getByRole("button", { name: "Retry as new run", exact: true })
.click();
await inspector
.getByRole("button", { name: "Confirm retry as new run" })
.click();
await expect.poll(() => requeueCount).toBe(1);
await inspector.getByRole("button", { name: "Back to runs" }).click();
await page.getByRole("button", { name: "Mention Bot", exact: true }).click();
await expect(
inspector.getByText("Repository run limit reached")
).toBeVisible();
await inspector
.getByRole("button", { name: "Cancel run", exact: true })
.click();
await inspector.getByRole("button", { name: "Confirm cancel run" }).click();
await expect(
page
.getByRole("alert")
.filter({ hasText: "Job run is no longer cancelable" })
).toBeVisible();
await inspector
.getByRole("button", { name: "Repair run", exact: true })
.click();
await inspector.getByRole("button", { name: "Confirm repair run" }).click();
await expect.poll(() => repairCount).toBe(1);
await inspector.getByRole("button", { name: "Back to runs" }).click();
await page.getByRole("button", { name: "Reviewer Bot", exact: true }).click();
await expect(
inspector.getByText(
"The review completed with no findings. This does not mean the PR has been merged."
)
).toBeVisible();
await inspector.getByRole("button", { name: "Back to runs" }).click();
expect(cancelCount).toBe(1);
expect(repairCount).toBe(1);
expect(requeueCount).toBe(1);
// Only the active tab's panel is mounted, so getByRole("tabpanel") always
// resolves to the table currently shown.
const activePanel = page.getByRole("tabpanel");
await activePanel.getByRole("button", { name: "Next" }).click();
await expect(activePanel.getByText("Showing 26–30 of 30")).toBeVisible();
await page.getByRole("tab", { name: /^Usage/ }).click();
await activePanel.getByRole("button", { name: "Next" }).click();
await expect(activePanel.getByText("Showing 51–75 of 75")).toBeVisible();
await page.getByRole("tab", { name: /^Runs/ }).click();
await page.getByLabel("Run status").selectOption("");
const previousJobsFrom = jobRequests.at(-1)?.searchParams.get("from");
const previousCallsFrom = callRequests.at(-1)?.searchParams.get("from");
expect(previousCallsFrom).toBe(callRequests[0]?.searchParams.get("from"));
const changedJobsRequest = page.waitForRequest((request) => {
const url = new URL(request.url());
return (
url.pathname === "/api/observability/jobs" &&
url.searchParams.get("from") !== previousJobsFrom
);
});
const changedCallsRequest = page.waitForRequest((request) => {
const url = new URL(request.url());
return (
url.pathname === "/api/observability/calls" &&
url.searchParams.get("from") !== previousCallsFrom
);
});
await page.getByLabel("Date range").selectOption("today");
const [jobsRequest, callsRequest] = await Promise.all([
changedJobsRequest,
changedCallsRequest,
]);
expect(new URL(jobsRequest.url()).searchParams.get("page")).toBe("1");
expect(new URL(callsRequest.url()).searchParams.get("page")).toBe("1");
});