replicas-engine 0.1.427 → 0.1.428

package/dist/src/index.js

@@ -504,7 +504,7 @@ var WORKSPACE_SIZES = ["small", "large"];
 var INVALID_WORKSPACE_SIZE_ERROR = `Invalid size: must be one of ${WORKSPACE_SIZES.join(", ")}`;
 
 // ../shared/src/e2b.ts
-var E2B_TEMPLATE_NAME = "replicas-sandbox-2026-07-11-v2";
+var E2B_TEMPLATE_NAME = "replicas-sandbox-2026-07-11-v3";
 
 // ../shared/src/runtime-env.ts
 function shellQuotePosix(value) {
@@ -764,13 +764,12 @@ replicas computer info
 # 2) Launch a browser on the workspace display.
 replicas computer launch chrome https://news.ycombinator.com
 
-# 3) Observe the settled screen and browser tab state before clicking.
-replicas computer observe /tmp/state.png
-replicas computer browser --snapshot
+# 3) Capture the settled viewport and native accessibility tree together.
+replicas computer browser-state /tmp/state.png --full
 
-# 4) Drive the UI.
-replicas computer click 521 700                    # click coordinates from the screenshot
-replicas computer scroll down --amount 5
+# 4) Batch actions that use refs from the same state, then capture the semantic diff.
+replicas computer browser-batch '[{"action":"fill","ref":17,"value":"hello"},{"action":"click","ref":42}]'
+replicas computer browser-state /tmp/after-click.png
 
 # 5) (Optional) Record a screencap to share back.
 replicas computer record start /tmp/demo.mp4 --fps 60
@@ -799,21 +798,33 @@ Waits briefly for the screen to stop changing, saves a 1:1 screenshot, and print
 
 Use this instead of hand-written \`sleep && screenshot\` loops after clicks, navigation, typing, or page loads. By default it saves a 100px coordinate-grid screenshot and waits up to 3s for 600ms of visual stability. Pass \`--raw\` if you need an unannotated 1:1 screenshot.
 
-### \`replicas computer browser\`
-Prints JSON for Chrome tabs launched through Replicas, including page titles and URLs. Pass \`--snapshot\` to include visible page text and interactive controls with DOM viewport bounding boxes.
+### \`replicas computer browser-state <path> [--full] [--timeout MS] [--stable-ms MS]\`
+Waits for the selected Chrome page to settle, saves its viewport screenshot, and prints a compact native Chrome accessibility tree in the same coordinate space. Actionable nodes have DOM-backed refs such as \`[ref=42] button "Save"\`; the tree includes accessible roles, names, values, states, visible bounds, and iframe content.
+
+The first call returns a full tree. Later calls for the same target and document return semantic changes; navigation or reload forces a new full tree. Pass \`--full\` to reset the baseline. Refs are valid for the current document.
+
+### \`replicas computer browser [--snapshot]\`
+Lists Chrome tabs. \`--snapshot\` returns the compatibility accessibility snapshot. Use \`browser-state\` for the primary action/observe loop.
 
-Use this alongside \`observe\` when testing web apps so you do not infer navigation, page state, or click targets from pixels alone. Snapshot coordinates are DOM viewport coordinates, not desktop click coordinates.
+### \`replicas computer browser-batch '<actions-json>'\`
+Runs up to 100 ordered browser actions through one Chrome session and stops at the first failure. Supported actions are \`click\`, \`fill\`, \`key\`, \`type\`, \`scroll\`, and \`wait\`. Prefer this when several actions use refs from the same \`browser-state\`; it avoids repeated process, tab-selection, and connection setup. Do not carry refs across navigation\u2014capture fresh state and start a new batch.
 
-### \`replicas computer browser-click <text> [--exact] [--index N]\`
-Clicks the first visible Chrome control whose text, label, placeholder, or href matches \`<text>\`. Use this for web buttons and links found via \`browser --snapshot\`; it is faster and less error-prone than converting DOM coordinates to desktop pixels.
+### \`replicas computer browser-click --ref ID [--button left|middle|right] [--double]\`
+Scrolls the referenced node into view and sends trusted Chrome mouse input at its current visible bounds. Prefer refs from \`browser-state\`. \`browser-click <text> [--exact] [--index N]\` remains a fallback when no fresh ref is available.
 
-### \`replicas computer browser-fill <field> <value> [--exact] [--index N]\`
-Fills the first visible Chrome field whose label, placeholder, name, or text matches \`<field>\`, then dispatches input/change events. Use this for web forms instead of clicking a field and typing through the desktop.
+### \`replicas computer browser-fill --ref ID <value>\`
+Focuses the referenced field, fills it with trusted Chrome input, and verifies the resulting value. Select controls use their semantic option value. \`browser-fill <field> <value> [--exact] [--index N]\` remains a text-matching fallback.
+
+### \`replicas computer browser-scroll <up|down|left|right> [--ref ID] [--amount PX]\`
+Sends trusted wheel input at a referenced element or the center of the viewport.
+
+### \`replicas computer browser-key <combo>\` / \`browser-type <text>\`
+Sends trusted keyboard input to the focused element in the selected Chrome target. Prefer \`browser-fill\` when setting a known field value.
 
 ### \`replicas computer browser-wait <text> [--mode any|text|title|url|control] [--exact] [--timeout MS]\`
 Waits until the active Chrome page matches text in the title, URL, body text, or visible controls. Use this after \`browser-click\` / \`browser-fill\` when you need web app state to settle without screenshot polling.
 
-For \`browser-click\`, \`browser-fill\`, and \`browser-wait\`, pass \`--id <id>\`, \`--title <text>\`, \`--url <text>\`, or \`--page <n>\` when multiple Chrome tabs are open. Run \`replicas computer browser\` first to list tabs. Prefer \`--id\` when a click may change the page title or URL.
+For browser state and actions, pass \`--target-id <id>\`, \`--title <text>\`, \`--url <text>\`, or \`--page <n>\` when multiple Chrome tabs are open. Prefer \`--target-id\` because titles and URLs can change after an action. The matched tab is brought to the foreground before each action, so driving multiple tabs or recording the screen switches which tab is visible.
 
 ### \`replicas computer click <x> <y> [--button N] [--double] [--modifiers ctrl+shift]\`
 Move to (x, y) and click. Coordinates can be absolute pixels or percentages such as \`50%\` \`50%\`. Default is left-click (button 1); pass \`--button 3\` for right-click. \`--modifiers\` holds keys during the click (e.g. ctrl-click a link to open in a new tab).
@@ -846,26 +857,29 @@ Spawns an app on the workspace display. Built-in aliases:
 Anything else gets \`exec\`'d verbatim, so \`replicas computer launch xeyes\` works if xeyes is installed.
 When opening a known page, prefer passing the URL directly: \`replicas computer launch chrome http://localhost:3000/\`.
 
+Chrome launches do not return until the requested page is controllable. The output includes its page ID; carry that ID with \`--target-id\` into \`browser-state\` and browser actions so unrelated tabs cannot steal task context.
+
 ### \`replicas computer record start <path> [--fps N]\`
-Starts an ffmpeg screen recorder. Output is post-processed when stopped: action windows stay at normal speed, idle gaps accelerate, click moments get eased camera zoom, and a synthetic cursor animates between logged mouse positions. The raw capture is fragmented MP4 while active (still playable if the workspace dies mid-record). Default 60fps; drop to 30 if the workspace is CPU-constrained.
+Starts an ffmpeg screen recorder and returns only after ffmpeg has written the first bytes. Output is post-processed when stopped: action windows stay at normal speed, idle gaps accelerate, click moments get eased camera zoom, and a synthetic cursor animates between logged mouse positions. The raw capture is fragmented MP4 while active (still playable if the workspace dies mid-record). Default 60fps; drop to 30 if the workspace is CPU-constrained.
 
 Only one recording at a time. Re-running \`start\` while one is active fails - call \`stop\` first.
 
 ### \`replicas computer record stop\`
-SIGINTs ffmpeg, waits for it to finalize the MP4, prints the output path. Upload it with \`replicas media upload <path>\` to share it.
+SIGINTs ffmpeg, verifies that the process has finalized the MP4, then prints the output path. A finalization timeout preserves the recording state so \`stop\` can be retried safely. Upload it with \`replicas media upload <path>\` to share it.
 
 ## Patterns
 
-### Action / observe loop
-You are blind between tool calls. After any action that changes the screen, observe before deciding the next coordinate:
+### Browser action / state loop
+For Chrome, use the native accessibility state before interpreting pixels or guessing coordinates:
 
 \`\`\`bash
-replicas computer click 521 700
-replicas computer observe /tmp/after-click.png
-# read /tmp/after-click.png and the JSON output, decide next click
+replicas computer launch chrome https://example.com
+replicas computer browser-state /tmp/before.png --full --target-id <page-id>
+replicas computer browser-batch '[{"action":"fill","ref":17,"value":"hello"},{"action":"click","ref":23}]' --target-id <page-id>
+replicas computer browser-state /tmp/after.png --target-id <page-id>
 \`\`\`
 
-The JSON \`stable\`, \`frames\`, and \`changes\` fields tell you whether something changed while you were waiting. If \`stable\` is false, observe again or increase \`--timeout\` before acting on coordinates.
+Read the screenshot and tree as one state. Batch ordered actions whose refs all come from that state, then inspect the next semantic diff. A batch is fail-fast and validates each ref against the cached document. Split at navigation or whenever a later action depends on newly rendered refs. Fall back to \`observe\` plus desktop coordinates for canvas content, browser chrome, non-Chrome apps, or controls missing from the accessibility tree.
 
 ### Typing into an address bar
 \`\`\`bash
@@ -874,10 +888,9 @@ replicas computer observe /tmp/browser-open.png
 replicas computer key ctrl+l
 replicas computer type "https://example.com"
 replicas computer key Return
-replicas computer observe /tmp/loaded.png
-replicas computer browser --snapshot
-replicas computer browser-fill "Search" "replicas"
-replicas computer browser-click "More information" --title "Example"
+replicas computer browser-state /tmp/loaded.png --full
+replicas computer browser-fill --ref 17 "replicas"
+replicas computer browser-click --ref 23 --title "Example"
 replicas computer browser-wait "Example Domain" --mode title --title "Example"
 \`\`\`
 
@@ -8437,7 +8450,7 @@ var DEFAULT_CODEX_ARGS = ["app-server", "--listen", "stdio://"];
 var MIN_CODEX_CLI_VERSION = "0.144.0";
 var CODEX_UPGRADE_TIMEOUT_MS = 12e4;
 var codexCliVersionEnsured = null;
-var ENGINE_PACKAGE_VERSION = "0.1.427";
+var ENGINE_PACKAGE_VERSION = "0.1.428";
 var INITIALIZE_METHOD = "initialize";
 var INITIALIZED_NOTIFICATION = "initialized";
 var ACCOUNT_LOGIN_START_METHOD = "account/login/start";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-engine",
-  "version": "0.1.427",
+  "version": "0.1.428",
   "description": "Lightweight API server for Replicas workspaces",
   "type": "module",
   "main": "dist/src/index.js",