Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/agent/tool-resource-class.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ const TOOL_RESOURCE_CLASS: Record<string, ResourceClass> = {
"browser.site_tools": "browser",
"browser.debug": "browser",
"browser.annotate": "browser",
"browser.scenario": "browser",

// os.shell.* — always approval-gated when approvals are on
"os.shell.run": "approval_gated",
Expand Down
1 change: 1 addition & 0 deletions src/llm/grammar/build-grammar.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1010,6 +1010,7 @@ describe("the base grammar's vocabulary", () => {
// Remove a name here when the grammar learns it.
const NOT_YET_IN_GRAMMAR = new Set([
"browser.annotate",
"browser.scenario",
"browser.screenshot",
"browser.site_tools",
"github.issue.comment",
Expand Down
7 changes: 5 additions & 2 deletions src/llm/provider/openai/openai-strict-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -490,11 +490,14 @@ describe("strict-tool conformance over every registered tool", () => {
// unable to write a check at all. `parseDelegateArgs` validates the
// shape at run time, as it always did. `browser.extract` takes the
// caller's own JSON Schema as `schema`; `browser.site_tools` passes
// `arguments` shaped by the site's own declared tool (ATO-521). Each
// is the tool's actual payload, so none can be closed.
// `arguments` shaped by the site's own declared tool (ATO-521);
// `browser.scenario` takes the values a recorded scenario's
// placeholders are filled with as `params` (ATO-960). Each is the
// tool's actual payload, so none can be closed.
expect(refused).toEqual([
"browser__extract",
"browser__site_tools",
"browser__scenario",
"os__http__request",
"mcp__prompt__get",
"verify__run",
Expand Down
4 changes: 4 additions & 0 deletions src/prompt/compact-tool-forms.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ const FORMS: Record<string, CompactToolForm> = {
summary: "The user pins comments on page elements: start, then read (selectors, styles, boxes), stop.",
tier: RARE,
},
"browser.scenario": {
summary: "Record browser steps (start, stop with a name), then run them again without the model; list, delete.",
tier: RARE,
},
"os.shell.run": {
summary:
"Run a shell command in the working directory (may require approval). `cmd` is required and must not be empty. Not for deleting (os.fs.trash) or moving files (os.fs.move). A long command comes back as a job: wait, kill or list jobs.",
Expand Down
12 changes: 12 additions & 0 deletions src/prompt/default-tool-args-schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,18 @@ const DEFAULT_TOOL_ARGS_SCHEMAS: ReadonlyMap<string, Schema> = new Map<
["action"],
),
],
[
"browser.scenario",
obj(
{
action: { type: "string", enum: ["start", "stop", "cancel", "run", "list", "delete"] },
name: stringSchema,
params: { type: "object" },
timeoutMs: numberSchema,
},
["action"],
),
],
[
"browser.scroll",
obj(
Expand Down
6 changes: 6 additions & 0 deletions src/prompt/default-tool-descriptors-a.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,12 @@ export const DEFAULT_TOOL_DESCRIPTORS_A: readonly ToolDescriptor[] = [
"Let the user mark up the open page: start shows an overlay where they click elements and type what to change; read returns each pin's comment, selector, box and styles (crops adds element pictures), grouped by page area; stop removes it.",
argsSchema: `{ action: "start" | "read" | "stop", crops?: boolean }`,
},
{
name: "browser.scenario",
summary:
"Record a browser task once and replay it without the model: start records this chat's navigate/search/click/type/act steps; stop saves them under `name` (`params` turns typed text into {{placeholders}}, e.g. { query: \"running shoes\" }); run replays a saved scenario, finding each element by role and name, with `params` filling the placeholders, and stops at the first step it cannot do; cancel, list, delete. Every replayed step asks for approval like the same call by you.",
argsSchema: `{ action: "start" | "stop" | "cancel" | "run" | "list" | "delete", name?: string, params?: object, timeoutMs?: number }`,
},
{
name: "os.shell.run",
summary:
Expand Down
10 changes: 8 additions & 2 deletions src/runtime/composition/runtime-tool-catalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import { resolveExtraApiKeys } from "../../config/resolve-llm-api-key.js";
import type { ToolOutputSpill } from "../../tools/tool-output-spill.js";
import { finishTool } from "../../tools/finish.js";
import { buildAskUserTool, replyTool } from "../../tools/conversation/index.js";
import { buildBrowserPageRenderer, buildBrowserTools, UserSiteApprovals, type BrowserExtractModelRef } from "../../tools/browser/index.js";
import { browserScenarioDir, buildBrowserPageRenderer, buildBrowserTools, UserSiteApprovals, type BrowserExtractModelRef } from "../../tools/browser/index.js";
import type { BrowserBackend } from "../../tools/browser/browser-backend.js";
import { registerOsScreenTools, registerOsTools, type DeclaredInputsRegistry, type ShellJobRegistry, type WebSearchKeyHelp } from "../../tools/os/index.js";
import { registerVerifyTools, runChecks } from "../../tools/verify/index.js";
Expand Down Expand Up @@ -163,7 +163,13 @@ export function createRuntimeToolRegistry(
userProfile: config.browser.connect === "user-chrome",
...(config.browser.connect === "user-chrome" ? { userSites: new UserSiteApprovals() } : {}),
};
for (const tool of buildBrowserTools(browserBackend, dangerous, domainPolicy, browserExtract)) {
for (const tool of buildBrowserTools(
browserBackend,
dangerous,
domainPolicy,
browserExtract,
browserScenarioDir(config.paths.stateDir),
)) {
toolRegistry.register(tool);
}
}
Expand Down
1 change: 1 addition & 0 deletions src/runtime/filter-disabled-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ const GATED_TOOLS = {
"browser.site_tools",
"browser.debug",
"browser.annotate",
"browser.scenario",
],
webSearch: ["os.web.search"],
vision: ["vision.describe"],
Expand Down
13 changes: 13 additions & 0 deletions src/tools/browser/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,19 @@ A page on a blocked host is never captured. A capture is discarded, never saved,
- Gates, before the backend acts: any action on a blocked host is refused. `upload` asks as `browser_secret` every time, with each file's name, size and full path in the preview; its paths are also checked like reads (`../read-scope/read-scope-targets.ts`), so the agent's own secret files and files outside the read scope ask first. `key` Enter / Space on a `ref` goes through the click / type gate above (Enter like `type` with Enter, Space like a click); without a `ref` the focused element cannot be read, so Enter / Space ask as `browser_commit`. Accepting a dialog asks as `browser_commit` (it may confirm a delete or a payment); dismissing does not. In the user's own Chrome these `browser_commit` prompts ask as `browser_secret`. select, hover, back, forward, wait and other keys do not ask.
- After select, key, back and forward the landed page is re-checked against the block list, like after a click.

## Scenarios: record once, replay without the model

`browser.scenario { action, name?, params?, timeoutMs? }` (`scenario.ts`, `scenario-recorder.ts`, `scenario-format.ts`, `scenario-store.ts`, ATO-960). The idea is Stagehand's action cache and browser-use's workflow recordings: a task the model did once is replayed step by step with no model call, each element found the way Playwright codegen writes locators (`getByRole(role, { name })`).

- `start` records this chat's successful `navigate`, `search`, `click`, `type` and `act` calls (the recorder wraps those tools; a chat that is not recording pays nothing). Before each call it reads the target ref's role and name from the page's ARIA snapshot, plus the page URL and snapshot digest. On `run`, a step that acts on a page stops with `origin_changed` when the browser is not on the origin (scheme, host, port) the step was recorded on, so a redirect or a changed site never gets the recorded text; the digest is kept for information only. A call that fails, is refused or is denied is not recorded. `upload` is never recorded (local files stay out of scenarios); stop says so.
- `stop { name, params? }` saves the steps as `<stateDir>/browser-scenarios/<name>.json` (`{ version: 1, name, createdAt, params, steps }`, names `a-z0-9-_`, at most 200 steps). `params` turns typed text into a placeholder: `{ "query": "running shoes" }` makes each step that typed exactly "running shoes" type `{{query}}` at replay. `cancel` drops the recording.
- Text typed into a field the approval gate calls `browser_secret` (password, one-time code, card number, or a field that could not be read) is never written. The step names a secret parameter instead (from the field's label, e.g. `password`), and `run` refuses to start until it is supplied.
- `run { name, params?, timeoutMs? }` replays the steps in order. A step's element is looked up in the current snapshot by role and exact name, then by role and a name equal ignoring case and runs of spaces ("self-healing lite"); it waits up to `timeoutMs` (default 5 s, at most 30 s) for the element to appear. No match or more than one match stops the run with an error naming the step (`target_missing`, `target_ambiguous`), so the model or the user carries on from there. Falling back to the model on a miss is left for later.
- Every replayed step is a call to the same `browser.navigate` / `search` / `click` / `type` / `act` tool the model uses, so every gate above applies unchanged: `browser_commit` and `browser_secret` prompts (a secret parameter asks every time), blocked and allowed domains, offline mode, the landed-page re-check, the user's own Chrome rules. A denied prompt ends the run like a denied model call; a step that returns an error stops it with that tool's message.
- `list` shows the saved scenarios (a broken file is listed as unreadable); `delete { name }` removes one.
- On a schedule: a scheduled task whose message says "Run browser scenario `<name>`" replays it in a background turn (headless by default, see below). Prompts in that turn follow the task's usual approval handling, so a scenario that sends, posts or types a secret still needs a yes.
- Not yet: recording by hand in the desktop browser panel, and the model fallback on a miss.


## Extracting data

Expand Down
4 changes: 2 additions & 2 deletions src/tools/browser/browser-gate-test-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,10 @@ export class GateBackend implements BrowserBackend {
this.snapshots += 1;
return { url: this.url, title: "Page", text: "- secret page text [ref=e1]", digest: "d", refs: ["e1"] };
}
async hasRef(): Promise<boolean> {
async hasRef(_ref: string): Promise<boolean> {
return this.refPresent;
}
async describeRef(): Promise<RefDescription | null> {
async describeRef(_ref: string): Promise<RefDescription | null> {
if (this.describeThrows) throw new Error("evaluate timed out");
return this.element;
}
Expand Down
1 change: 1 addition & 0 deletions src/tools/browser/browser-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,7 @@ describe("browser tools on a fake backend", () => {
"browser.extract",
"browser.navigate",
"browser.read_aria",
"browser.scenario",
"browser.screenshot",
"browser.scroll",
"browser.search",
Expand Down
40 changes: 35 additions & 5 deletions src/tools/browser/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ import {
type BrowserDomainPolicy,
} from "./browser-domain-policy.js";
import { AriaSnapshotStore } from "./aria-view.js";
import { ScenarioRecorder } from "./scenario-recorder.js";
import { ScenarioStore } from "./scenario-store.js";
import { buildBrowserScenarioTool } from "./scenario.js";

export type {
AnnotateReadInput,
Expand Down Expand Up @@ -165,6 +168,12 @@ export { PageDebugLog, redactDebugUrl } from "./page-debug-log.js";
export type { ConsoleEntry, DebugLogKind, NetworkEntry } from "./page-debug-log.js";
export { BROWSER_ANNOTATE_TOOL, buildBrowserAnnotateTool } from "./annotate.js";
export type { BrowserAnnotateDeps } from "./annotate.js";
export { BROWSER_SCENARIO_TOOL, buildBrowserScenarioTool } from "./scenario.js";
export type { BrowserScenarioDeps, ScenarioReplayTools } from "./scenario.js";
export { RECORDED_BROWSER_TOOLS, ScenarioRecorder } from "./scenario-recorder.js";
export { BROWSER_SCENARIO_DIR, browserScenarioDir, ScenarioStore } from "./scenario-store.js";
export { findTarget, parseScenario, SCENARIO_VERSION } from "./scenario-format.js";
export type { Scenario, ScenarioStep, ScenarioTarget } from "./scenario-format.js";

/**
* Build the complete set of browser tools for the registry. The caller is
Expand All @@ -173,28 +182,49 @@ export type { BrowserAnnotateDeps } from "./annotate.js";
* and payment input, and hosts outside the allow-list; `policy` carries
* `browser.blockedDomains` / `browser.allowedDomains`. `extractModel` is
* the model `browser.extract` calls, bound by the runtime once it exists.
* `scenarioDir` is where `browser.scenario` keeps its files
* (`browserScenarioDir(stateDir)`); without it scenarios live in memory.
*/
export function buildBrowserTools(
backend: BrowserBackend,
dangerous: DangerousToolOptions,
policy: BrowserDomainPolicy = EMPTY_DOMAIN_POLICY,
extractModel: BrowserExtractModelRef = {},
scenarioDir?: string,
): ToolDefinition[] {
// One per tool set: the last snapshot of each tab, for read_aria's diff view.
const store = new AriaSnapshotStore();
// The tools a scenario replays. The model gets them wrapped by the
// recorder; a replay calls them directly, gates and all (ATO-960).
const replay = {
navigate: buildBrowserNavigateTool(backend, dangerous, policy, store),
search: buildBrowserSearchTool(backend, dangerous, policy, store),
click: buildBrowserClickTool(backend, dangerous, policy),
type: buildBrowserTypeTool(backend, dangerous, policy),
act: buildBrowserActTool(backend, dangerous, policy),
};
const recorder = new ScenarioRecorder(backend);
return [
buildBrowserNavigateTool(backend, dangerous, policy, store),
buildBrowserClickTool(backend, dangerous, policy),
buildBrowserTypeTool(backend, dangerous, policy),
recorder.wrap(replay.navigate),
recorder.wrap(replay.click),
recorder.wrap(replay.type),
buildBrowserReadAriaTool(backend, policy, store, dangerous),
buildBrowserSearchTool(backend, dangerous, policy, store),
recorder.wrap(replay.search),
buildBrowserTabsTool(backend, dangerous, policy),
buildBrowserScrollTool(backend),
buildBrowserScreenshotTool(backend, policy, undefined, dangerous),
buildBrowserActTool(backend, dangerous, policy),
recorder.wrap(replay.act),
buildBrowserExtractTool(backend, policy, extractModel, dangerous),
buildBrowserSiteToolsTool(backend, dangerous, policy),
buildBrowserDebugTool(backend, policy),
buildBrowserAnnotateTool(backend, policy, dangerous),
buildBrowserScenarioTool({
backend,
recorder,
store: new ScenarioStore(scenarioDir),
tools: replay,
policy,
ariaStore: store,
}),
];
}
105 changes: 105 additions & 0 deletions src/tools/browser/scenario-format.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import { describe, expect, it } from "vitest";
import {
findTarget,
normalizeScenarioName,
paramNameFrom,
parseScenario,
targetOfRef,
} from "./scenario-format.js";

/** The scenario file format and the role + name lookup (ATO-960). */

const PAGE = [
"url: https://shop.example/",
"title: Shop",
"",
'- searchbox "Search products" [ref=e3]',
'- button "Search" [ref=e4]',
'- link "Learn more" [ref=e5]',
"- 'button \"Pay: $5\" [ref=e6]'",
'- button "Add" [ref=e7]',
'- button "Add" [ref=e8]',
"- text: plain text, no ref",
].join("\n");

describe("targetOfRef", () => {
it("reads the role and name of a ref, including a quoted YAML key", () => {
expect(targetOfRef(PAGE, "e4")).toEqual({ role: "button", name: "Search" });
expect(targetOfRef(PAGE, "e6")).toEqual({ role: "button", name: "Pay: $5" });
expect(targetOfRef(PAGE, "e99")).toBeNull();
});
});

describe("findTarget", () => {
it("finds an exact role and name", () => {
expect(findTarget(PAGE, { role: "button", name: "Search" })).toEqual({ kind: "found", ref: "e4", how: "exact" });
});

it("falls back to a name that differs only in case and spaces", () => {
expect(findTarget(PAGE, { role: "searchbox", name: " search PRODUCTS " })).toEqual({
kind: "found",
ref: "e3",
how: "normalized",
});
});

it("does not match the right name with the wrong role", () => {
expect(findTarget(PAGE, { role: "link", name: "Search" })).toEqual({ kind: "missing" });
});

it("reports several matches instead of guessing", () => {
expect(findTarget(PAGE, { role: "button", name: "Add" })).toEqual({ kind: "ambiguous", count: 2 });
});
});

describe("names", () => {
it("normalizes scenario names and refuses path-like ones", () => {
expect(normalizeScenarioName("Check prices")).toBe("check-prices");
expect(normalizeScenarioName("daily_report")).toBe("daily_report");
expect(normalizeScenarioName("../etc/passwd")).toBeNull();
expect(normalizeScenarioName("a/b")).toBeNull();
expect(normalizeScenarioName("")).toBeNull();
expect(normalizeScenarioName(42)).toBeNull();
});

it("makes a parameter name from a field label, unique", () => {
expect(paramNameFrom("Password", new Set())).toBe("password");
expect(paramNameFrom("Password", new Set(["password"]))).toBe("password_2");
expect(paramNameFrom("2FA code!", new Set())).toBe("fa_code");
expect(paramNameFrom("", new Set())).toBe("value");
});
});

describe("parseScenario", () => {
const valid = {
version: 1,
name: "find-shoes",
createdAt: "2026-11-18T00:00:00.000Z",
params: [{ name: "password", secret: true }],
steps: [
{ kind: "navigate", url: "https://shop.example/" },
{ kind: "type", target: { role: "textbox", name: "Password" }, param: "password" },
{ kind: "act", action: { action: "key", key: "Enter" }, target: { role: "textbox", name: "Password" } },
],
};

it("reads a valid file and derives the params from the steps", () => {
const scenario = parseScenario(valid);
expect(scenario.steps).toHaveLength(3);
expect(scenario.params).toEqual([{ name: "password", secret: true }]);
});

it.each([
["an unknown version", { ...valid, version: 2 }],
["no steps", { ...valid, steps: [] }],
["an unknown step kind", { ...valid, steps: [{ kind: "eval", code: "1" }] }],
["a type step with both text and param", { ...valid, steps: [{ kind: "type", target: { role: "textbox", name: "x" }, text: "a", param: "b" }] }],
["a type step with neither", { ...valid, steps: [{ kind: "type", target: { role: "textbox", name: "x" } }] }],
["an upload step", { ...valid, steps: [{ kind: "act", action: { action: "upload", paths: ["/etc/hosts"] } }] }],
["an act step with a stray ref", { ...valid, steps: [{ kind: "act", action: { action: "hover", ref: "e1" } }] }],
["a click without a target", { ...valid, steps: [{ kind: "click" }] }],
["a bad name", { ...valid, name: "../x" }],
])("refuses %s", (_label, file) => {
expect(() => parseScenario(file)).toThrow(/browser scenario file/);
});
});
Loading
Loading