scripts/testing/control-omb-ui.e2e.test.ts
Rama main · réplica de lectura del repositorio original.
// Drives the real <App/> through `control-omb ui` and reads outcomes from the
// accessibility tree. Needs the pinned agent-browser binary: runs when one
// resolves (tools directory, OMB_AGENT_BROWSER_PATH or PATH) or OMB_UI_E2E=1
// asks for the verified download; otherwise skipped.
import { spawn } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { afterAll, describe, expect, it, vi } from "vitest";
import { resolveAgentBrowserBinary } from "../../server/browser-engine.ts";
import { removeTempDir, waitForExit } from "../../server/testing/cleanup.ts";
import { runControlOmb } from "../control-omb.ts";
import { UI_TOOLS_DIR } from "./control-omb-ui.ts";
import { fixtureApi } from "./preview-fixture.ts";
const ROOT = fileURLToPath(new URL("../..", import.meta.url));
const CLI = join(ROOT, "scripts", "control-omb.ts");
const forced = process.env.OMB_UI_E2E === "1";
const binary = resolveAgentBrowserBinary({ dataDir: UI_TOOLS_DIR, env: process.env });
const enabled = forced || Boolean(binary);
if (!enabled) {
console.log(`skipping control-omb ui e2e: no agent-browser binary resolves (looked in ${UI_TOOLS_DIR}, OMB_AGENT_BROWSER_PATH and PATH); set OMB_UI_E2E=1 to install the pinned release`);
}
const run = enabled ? it : it.skip;
// A forced run may still have Chrome for Testing to download even when
// agent-browser is cached.
const LAUNCH_TIMEOUT_MS = forced ? 600_000 : 180_000;
// Synthetic provider outcomes exercise the UI, not the commands themselves.
const TOOL_CALLS = JSON.stringify([
{ name: "Bash", input: { command: "pnpm control:omb doctor" }, ok: true },
{ name: "Bash", input: { command: "pnpm control:omb ui click --name Missing" }, ok: false },
{ name: "Bash", input: { command: "pnpm control:omb ui flag --set features.showToolCalls=true --dry-run" }, ok: true },
]);
const REPLY = "hello from fake claude"; // the fake engine's default reply text
// OMB_UI_EVIDENCE_DIR keeps the screenshot (CI uploads it); otherwise it is temporary.
const evidenceDir = process.env.OMB_UI_EVIDENCE_DIR ? resolve(ROOT, process.env.OMB_UI_EVIDENCE_DIR) : mkdtempSync(join(tmpdir(), "omb-ui-evidence-"));
const ownsEvidenceDir = !process.env.OMB_UI_EVIDENCE_DIR;
interface Launched {
child: ReturnType<typeof spawn>;
info: { ui: string; url: string; previewUrl: string; botId: string; dataDir: string; logPath: string };
stderr: () => string;
}
function launch(args: string[]): Promise<Launched> {
return new Promise((done, fail) => {
// Own process group: a timeout must also take down whatever it is running.
const child = spawn(process.execPath, ["--experimental-strip-types", CLI, "ui", "launch", ...args], {
cwd: ROOT, env: process.env, stdio: ["ignore", "pipe", "pipe"], detached: process.platform !== "win32",
});
const killGroup = (signal: NodeJS.Signals) => {
if (child.pid && process.platform !== "win32") {
try { process.kill(-child.pid, signal); return; } catch { /* group already gone */ }
}
child.kill(signal);
};
let stdout = "";
let stderr = "";
let settled = false;
const timer = setTimeout(() => {
if (settled) return;
settled = true;
killGroup("SIGINT");
setTimeout(() => killGroup("SIGKILL"), 10_000).unref();
fail(new Error(`ui launch printed no handle within ${LAUNCH_TIMEOUT_MS}ms\nstderr:\n${stderr}`));
}, LAUNCH_TIMEOUT_MS);
child.stderr!.on("data", (chunk: Buffer) => { stderr += String(chunk); });
child.stdout!.on("data", (chunk: Buffer) => {
stdout += String(chunk);
if (settled) return;
// A tool printing on stdout before the handle still parses from the
// first line that starts the object.
const start = stdout.startsWith("{") ? 0 : stdout.indexOf("\n{") + 1;
if (start <= 0 && !stdout.startsWith("{")) return;
try {
const info = JSON.parse(stdout.slice(start));
settled = true;
clearTimeout(timer);
done({ child, info, stderr: () => stderr });
} catch {
}
});
child.once("close", (code) => {
if (settled) return;
settled = true;
clearTimeout(timer);
fail(new Error(`ui launch exited ${code} before printing a handle\nstdout:\n${stdout}\nstderr:\n${stderr}`));
});
});
}
const ui = (verb: string, handle: string, ...args: string[]) =>
runControlOmb(["ui", verb, "--ui", handle, ...args]) as Promise<Record<string, any>>;
/** Interactive refs by accessible name, from a snapshot's `refs` table. */
const refsNamed = (snapshot: Record<string, any>, name: string, role?: string) =>
Object.entries(snapshot.refs as Record<string, { name: string; role: string }>)
.filter(([, element]) => element.name === name && (!role || element.role === role))
.map(([id]) => `@${id}`);
describe("control-omb ui drives the real renderer", () => {
let launched: Launched | undefined;
afterAll(async () => {
if (launched && launched.child.exitCode === null && launched.child.signalCode === null) {
await waitForExit(launched.child, { signal: "SIGINT", graceMs: 30_000 });
}
if (ownsEvidenceDir) await removeTempDir(evidenceDir);
});
run("shows the matched text inside a searched message", async () => {
launched = await launch([]);
const { info } = launched;
await ui("type", info.ui, "--name", "Message Pepper", "--text", "Find the striped zebra");
await ui("press", info.ui, "--keys", "Enter");
await ui("wait-settle", info.ui, "--timeout", "60");
// The chat's ⋯ items live in the bot panel, which the chat header's pill opens.
await ui("click", info.ui, "--name", "Open Pepper's profile");
await ui("click", info.ui, "--name", "Find in conversation");
await ui("type", info.ui, "--name", "Find in this conversation", "--text", "zebra");
const highlighted = async () => (await ui("eval", info.ui, "--js", "[...CSS.highlights.get('search-result-text') ?? []][0]?.toString() ?? null")).result;
await expect.poll(async () => (await ui("snapshot", info.ui)).snapshot, { timeout: 10_000 }).toContain("1 of 1");
expect((await ui("snapshot", info.ui)).snapshot).not.toContain("Show less");
mkdirSync(evidenceDir, { recursive: true });
await ui("screenshot", info.ui, "--out", join(evidenceDir, "search-result.png"));
await expect.poll(async () => (await ui("eval", info.ui, "--js", "(() => { const row = [...document.querySelectorAll('[data-mid]')].find(el => el.textContent.includes('Find the striped zebra')); const bubble = row?.querySelector('[data-chat-bubble]'); return !!bubble && row.querySelector('.ring-2') === bubble && bubble.getBoundingClientRect().width < row.lastElementChild.getBoundingClientRect().width; })()")).result, { timeout: 10_000 }).toBe(true);
await expect.poll(highlighted, { timeout: 10_000 }).toBe("zebra");
await ui("type", info.ui, "--name", "Message Pepper", "--text", `${"hay ".repeat(160)}saffron`);
await ui("press", info.ui, "--keys", "Enter");
await ui("wait-settle", info.ui, "--timeout", "60");
await ui("click", info.ui, "--name", "Find in this conversation");
await ui("eval", info.ui, "--js", "document.querySelector('input[aria-label=\"Find in this conversation\"]').select(); true");
await ui("press", info.ui, "--keys", "Backspace");
await ui("type", info.ui, "--name", "Find in this conversation", "--text", "saffron");
await expect.poll(highlighted, { timeout: 10_000 }).toBe("saffron");
expect((await ui("eval", info.ui, "--js", "[...document.querySelectorAll('.chat-text')].find(el => el.textContent.includes('saffron'))?.classList.contains('max-h-40')")).result).toBe(false);
await expect.poll(async () => (await ui("eval", info.ui, "--js", "(() => { const range = [...CSS.highlights.get('search-result-text')][0]; const bounds = range.startContainer.parentElement.closest('.overflow-y-auto').getBoundingClientRect(); const match = range.getBoundingClientRect(); return match.top >= bounds.top && match.bottom <= bounds.bottom; })()")).result).toBe(true);
await ui("click", info.ui, "--name", "Find in this conversation");
await ui("eval", info.ui, "--js", "document.querySelector('input[aria-label=\"Find in this conversation\"]').select(); true");
await ui("press", info.ui, "--keys", "Backspace");
await ui("type", info.ui, "--name", "Find in this conversation", "--text", "fake claude");
await expect.poll(highlighted, { timeout: 10_000 }).toBe("fake claude");
await ui("press", info.ui, "--keys", "Escape");
// The sidebar's search opens from its button into a field of the same name.
await ui("click", info.ui, "--name", "Search bots and messages");
const [searchField] = refsNamed(await ui("snapshot", info.ui, "--interactive"), "Search bots and messages", "textbox");
expect(searchField).toBeDefined();
await ui("type", info.ui, "--ref", searchField, "--text", "zebra");
await expect.poll(async () => (await ui("eval", info.ui, "--js", "[...document.querySelectorAll('mark')].some(mark => mark.textContent === 'zebra')")).result, { timeout: 10_000 }).toBe(true);
await ui("eval", info.ui, "--js", "[...document.querySelectorAll('mark')].find(mark => mark.textContent === 'zebra').closest('button').click(); true");
await expect.poll(highlighted, { timeout: 10_000 }).toBe("zebra");
await waitForExit(launched.child, { signal: "SIGINT", graceMs: 30_000 });
expect(launched.child.exitCode).toBe(0);
}, LAUNCH_TIMEOUT_MS + 90_000);
run("saves a key on Enter, checks it once saved, shows it masked, and keeps a refused or failed draft", async () => {
launched = await launch([]);
const { info } = launched;
await fixtureApi(info.url)("PUT", "/api/config", {
openaiCompat: { key: "fixture-saved-key", url: "http://127.0.0.1:1/v1" },
});
const evaluate = async (js: string) => (await ui("eval", info.ui, "--js", js)).result;
const click = (name: string) => ui("click", info.ui, "--name", name);
const row = `document.querySelector('[data-api-key-row="openaiCompat"]')`;
const input = `${row}?.querySelector('[data-key-input]')`;
const verdict = () => evaluate(`${row}.querySelector('[role="status"]')?.textContent`);
const rowText = () => evaluate(`${row}.textContent`);
const masked = () => evaluate(`${row}.querySelector('[data-key-masked]')?.textContent ?? null`);
const replace = () => evaluate(`${row}.querySelector('[data-key-replace]').click(); true`);
const type = async (text: string) => {
await click("OpenAI-compatible API key");
await evaluate(`${input}.select(); true`);
await ui("press", info.ui, "--keys", "Backspace");
if (text) await ui("type", info.ui, "--name", "OpenAI-compatible API key", "--text", text);
};
const enter = () => ui("press", info.ui, "--keys", "Enter");
await evaluate(`(() => {
const original = window.fetch.bind(window);
window.keyTests = [];
window.keySaves = 0;
window.rejectKeySave = false;
window.fetch = (url, init = {}) => {
if (String(url) === '/api/keys/test') {
window.keyTests.push(JSON.parse(init.body));
return Promise.resolve(Response.json({ ok: true, check: 'models', models: ['fixture-model'] }));
}
if (String(url) === '/api/config' && init.method === 'PUT' && String(init.body).includes('"openaiCompat"')) {
window.keySaves++;
if (window.rejectKeySave) return Promise.resolve(Response.json({ error: 'Fixture save rejected' }, { status: 503 }));
}
return original(url, init);
};
return true;
})()`);
await click("You");
await click("Settings");
// Settings is a lazy chunk: it paints once the chunk arrives. Keys sit on the AI page, under API keys.
await expect.poll(() => evaluate(`(() => {
document.querySelector('[data-settings-page="ai"]')?.click();
document.querySelector('[data-settings-disclosure="ai-keys"] button[aria-expanded="false"]')?.click();
return Boolean(${row}?.checkVisibility());
})()`), { timeout: 10_000 }).toBe(true);
// The servers section opens on its own, because a key is saved there.
await expect.poll(masked, { timeout: 10_000 }).toContain("••••••••-key");
await expect.poll(() => evaluate(`${row}.querySelector('[data-key-check]')?.disabled ?? null`), { timeout: 10_000 }).toBe(false);
await evaluate(`${row}.querySelector('[data-key-check]').click(); true`);
await expect.poll(() => evaluate("window.keyTests")).toEqual([{ provider: "openaiCompat" }]);
await expect.poll(verdict).toBe("Saved key: Model catalog reachable: fixture-model. Authentication and chat not verified.");
await replace();
await type(" fixture-draft-key ");
await enter();
await expect.poll(() => evaluate(`${input} ?? null`), { timeout: 10_000 }).toBe(null);
await expect.poll(masked, { timeout: 10_000 }).toContain("••••••••-key");
await expect.poll(() => evaluate("window.keyTests")).toEqual([{ provider: "openaiCompat" }, { provider: "openaiCompat" }]);
expect(await evaluate("window.keySaves")).toBe(1);
await replace();
await type("not a key at all");
await enter();
await expect.poll(rowText, { timeout: 10_000 }).toContain("That doesn't look like an API key");
expect(await evaluate("window.keySaves")).toBe(1);
mkdirSync(evidenceDir, { recursive: true });
await ui("screenshot", info.ui, "--out", join(evidenceDir, "provider-key-refused.png"));
await type("fixture-replacement-key");
await evaluate("window.rejectKeySave = true");
await enter();
await expect.poll(rowText, { timeout: 10_000 }).toContain("Didn't work: Fixture save rejected");
await expect.poll(() => evaluate(`${input}?.value ?? null`), { timeout: 10_000 }).toBe("fixture-replacement-key");
await evaluate("window.rejectKeySave = false");
await click("OpenAI-compatible API key");
await enter();
await expect.poll(() => evaluate(`${input} ?? null`), { timeout: 10_000 }).toBe(null);
await expect.poll(() => evaluate("window.keyTests.length")).toBe(3);
await expect.poll(verdict).toBe("Saved key: Model catalog reachable: fixture-model. Authentication and chat not verified.");
await waitForExit(launched.child, { signal: "SIGINT", graceMs: 30_000 });
expect(launched.child.exitCode).toBe(0);
expect(existsSync(info.dataDir)).toBe(false);
}, LAUNCH_TIMEOUT_MS + 180_000);
run("sends a turn from the composer and shows the reply and the scripted tool chip", async () => {
launched = await launch(["--tool-calls", TOOL_CALLS]);
const { info } = launched;
expect(info.ui).toBe(join(info.dataDir, "ui.json"));
expect(existsSync(info.ui)).toBe(true);
const handle = JSON.parse(readFileSync(info.ui, "utf8"));
expect(handle).toMatchObject({ url: info.url, previewUrl: info.previewUrl, botId: info.botId, home: info.dataDir, session: `omb-ui-${new URL(info.url).port}` });
expect(existsSync(handle.binary)).toBe(true);
const dry = await ui("flag", info.ui, "--set", "features.showToolCalls=true", "--dry-run");
expect(dry).toMatchObject({ ok: true, dryRun: true, patch: { features: { showToolCalls: true } } });
const flagged = await ui("flag", info.ui, "--set", "features.showToolCalls=true");
expect(flagged).toMatchObject({ ok: true, features: { showToolCalls: true } });
expect(flagged.features).toMatchObject({ skillAuthoring: true });
const savedBot = async () => (await fetch(`${info.url}/api/bots`).then((response) => response.json())).bots.find((bot: any) => bot.id === info.botId);
const originalBot = await savedBot();
const originalModel = originalBot.modelSelection.model;
const models = await runControlOmb(["models", "--url", info.url]) as any;
const options = models.instances.find((instance: any) => instance.instanceId === originalBot.modelSelection.instanceId).models.options;
const originalLabel = options.find((option: any) => option.id === originalModel).label;
// A model the installed CLI is too old for is listed but cannot be picked.
const nextModel = options.find((option: any) => option.id !== originalModel && !option.needsCliUpdate);
// The picker sets the bot's model, which every chat of it uses.
const threadModel = async () => (await savedBot()).tasks.find((task: any) => task.threadId === originalBot.threadId).modelSelection.model;
await ui("click", info.ui, "--name", originalLabel);
await expect.poll(async () => (await ui("snapshot", info.ui)).snapshot, { timeout: 10_000 }).toContain("Pepper's model, used in all its chats");
mkdirSync(evidenceDir, { recursive: true });
await ui("screenshot", info.ui, "--out", join(evidenceDir, "model-picker.png"));
await ui("click", info.ui, "--name", nextModel.label);
await expect.poll(async () => (await savedBot()).modelSelection.model, { timeout: 10_000 }).toBe(nextModel.id);
expect(await threadModel()).toBe(nextModel.id);
// A model with reasoning levels keeps the menu open after the pick, so its level can be set next.
expect(await ui("eval", info.ui, "--js", "String(Boolean(document.querySelector('[data-model-picker-content]')))"))
.toMatchObject({ result: "true" });
const modelSnapshot = await ui("snapshot", info.ui, "--interactive");
const [originalRow] = refsNamed(modelSnapshot, originalLabel, "button");
expect(originalRow).toBeDefined();
await ui("click", info.ui, "--ref", originalRow);
await expect.poll(async () => (await savedBot()).modelSelection.model, { timeout: 10_000 }).toBe(originalModel);
expect(await threadModel()).toBe(originalModel);
expect((await savedBot()).approvalMode).toBe(originalBot.approvalMode);
// The menu is still open after that pick; close it before using the composer.
await ui("press", info.ui, "--keys", "Escape");
await expect.poll(async () => (await ui("eval", info.ui, "--js", "String(Boolean(document.querySelector('[data-model-picker-content]')))")).result,
{ timeout: 10_000 }).toBe("false");
const before = await ui("snapshot", info.ui, "--interactive");
expect(before.ok).toBe(true);
const [composer, ...moreComposers] = refsNamed(before, "Message Pepper", "textbox");
expect(composer).toMatch(/^@e\d+$/);
expect(moreComposers).toEqual([]);
expect(before.snapshot).not.toContain(REPLY);
const typed = await ui("type", info.ui, "--ref", composer, "--text", "hello");
expect(typed).toMatchObject({ ok: true, target: composer, typed: "hello" });
const pressed = await ui("press", info.ui, "--keys", "Enter");
expect(pressed).toMatchObject({ ok: true, pressed: "Enter" });
const settled = await ui("wait-settle", info.ui, "--timeout", "60");
expect(settled).toMatchObject({ ok: true, status: "settled", browser: { state: "networkidle" }, renderer: { rendered: true } });
expect((settled.bots as Array<{ busy: boolean }>).every((bot) => !bot.busy)).toBe(true);
const after = await ui("snapshot", info.ui);
expect(after.ok).toBe(true);
const tree = after.snapshot as string;
// The sidebar row previews the reply too; read the transcript landmark.
const transcriptStart = tree.indexOf('log "Conversation with Pepper"');
expect(transcriptStart).toBeGreaterThan(-1);
const transcript = tree.slice(transcriptStart);
expect(transcript).toContain('StaticText "hello"'); // the sent turn
expect(transcript).toContain(`StaticText "${REPLY}"`);
// The scripted Bash calls, in plain words, and the turn's count of them.
expect(transcript).toContain('StaticText "Run a command"');
expect(transcript).toContain("Did 3 tool calls");
expect(tree).not.toContain("Not logged in");
expect(tree).not.toContain("Execution timeline");
expect(await runControlOmb(["doctor", "--url", info.url])).toMatchObject({ ok: true });
await expect(ui("click", info.ui, "--name", "Deliberately missing QA control")).rejects.toThrow("no element is named");
// Planted with the same delay the real control has: a --name lookup that
// lands a tick early must still click it.
await ui("eval", info.ui, "--js", `(() => {
const late = document.createElement("button");
late.textContent = "Late QA control";
late.setAttribute("aria-label", "Late QA control");
// Keep this synthetic target inside the viewport: appending a normal
// flow sibling below the full-height app makes click scroll the app
// out of view, interfering with the real controls exercised next.
late.style.cssText = 'position:fixed;top:0;left:0;z-index:2147483647';
setTimeout(() => document.body.appendChild(late), 1500);
return "planted";
})()`);
expect(await ui("click", info.ui, "--name", "Late QA control")).toMatchObject({ ok: true });
await ui("eval", info.ui, "--js", `document.querySelector('[aria-label="Late QA control"]').remove()`);
mkdirSync(evidenceDir, { recursive: true });
const shotPath = join(evidenceDir, "chat-ui.png");
const shot = await ui("screenshot", info.ui, "--out", shotPath);
expect(shot).toMatchObject({ ok: true, path: shotPath });
const png = readFileSync(shotPath);
expect(png.length).toBeGreaterThan(1_000);
expect(png.subarray(0, 8)).toEqual(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]));
await ui("click", info.ui, "--name", "Open Pepper's profile");
await ui("click", info.ui, "--name", "Inspector");
// A local page fetches a screen when it opens, so the panel arrives just after the click.
const inspected = await vi.waitFor(async () => {
const snapshot = await ui("snapshot", info.ui);
expect(snapshot.snapshot).toContain('complementary "Inspector"');
return snapshot;
}, { timeout: 5_000, interval: 100 });
const runLog = (inspected.snapshot as string).slice((inspected.snapshot as string).indexOf('complementary "Inspector"'));
expect(runLog).toContain('tab "Run Log" [selected');
expect(runLog).toContain("pnpm control:omb doctor");
expect(runLog).toContain("pnpm control:omb ui click --name Missing");
expect(runLog).toContain('StaticText "Failed"');
expect(runLog).toContain("Copy redacted run log");
await ui("screenshot", info.ui, "--out", join(evidenceDir, "run-log.png"));
const [eventsTab] = refsNamed(inspected, "Events", "tab");
expect(eventsTab).toBeDefined();
await ui("click", info.ui, "--ref", eventsTab);
const events = await ui("snapshot", info.ui);
expect(events.snapshot).toContain("turn.started");
const [rawTab] = refsNamed(events, "Raw", "tab");
await ui("click", info.ui, "--ref", rawTab);
expect((await ui("snapshot", info.ui)).snapshot).toContain('tab "Raw" [selected');
await ui("click", info.ui, "--name", "Close the Inspector");
expect((await ui("snapshot", info.ui)).snapshot).not.toContain('complementary "Inspector"');
const logs = await ui("console", info.ui);
expect(logs.ok).toBe(true);
expect((logs.messages as Array<{ type: string; text: string }>).filter((message) => message.type === "error")).toEqual([]);
const title = await ui("eval", info.ui, "--js", "document.title");
expect(title).toMatchObject({ ok: true, result: "Isolated OpenMaus Chat" });
await waitForExit(launched.child, { signal: "SIGINT", graceMs: 30_000 });
expect(launched.child.exitCode).toBe(0);
expect(existsSync(info.dataDir)).toBe(false);
expect(existsSync(info.logPath)).toBe(true);
await expect(ui("snapshot", info.ui)).rejects.toThrow("could not read the ui handle");
}, LAUNCH_TIMEOUT_MS + 120_000);
});