replicas-mcp 0.2.9 → 0.2.10

package/dist/index.js

@@ -61,7 +61,7 @@ The CLI is the canonical surface. Chat transcripts that show \`DISPLAY=:99 xdoto
 \`\`\`bash
 # DO
 replicas computer key Return
-replicas computer screenshot /tmp/state.png
+replicas computer observe /tmp/state.png
 
 # DON'T
 DISPLAY=:99 xdotool key Return
@@ -79,9 +79,9 @@ replicas computer info
 # 2) Launch a browser on the workspace display.
 replicas computer launch chrome https://news.ycombinator.com
 
-# 3) Take a screenshot so you can see what's there.
-replicas computer screenshot /tmp/state.png
-# (Read the PNG yourself before deciding where to click.)
+# 3) Observe the settled screen and browser tab state before clicking.
+replicas computer observe /tmp/state.png
+replicas computer browser --snapshot
 
 # 4) Drive the UI.
 replicas computer click 521 700                    # click coordinates from the screenshot
@@ -102,16 +102,39 @@ Prints the live noVNC viewer URL (\`https://6080-<hash>.tryreplicas.com/\`) for
 If invoked very early in the workspace lifecycle, \`info\` will poll briefly while the engine finishes registering the preview, then error if it's still not available.
 
 ### \`replicas computer status\`
-Prints which desktop services are running and the active preview URL (if any). Useful for debugging when a tool call seems to be doing nothing.
+Checks and repairs the desktop bridge, then prints which desktop services are running and the active preview URL (if any). Useful for debugging when a tool call seems to be doing nothing.
 
-### \`replicas computer screenshot <path>\`
-Captures the current desktop to a PNG at the given path. Read the file (e.g. with your Read tool) to see what's on screen - coordinates from the screenshot drive subsequent \`click\` / \`move\` / \`drag\` calls.
+### \`replicas computer screenshot <path> [--raw] [--grid [px]]\`
+Captures the current desktop to a PNG at the given path.
+
+Use \`--raw\` for a 1:1 desktop capture with no branding, padding, or rounded corners. Use \`--grid\` for the same 1:1 capture plus a coordinate grid; the optional value sets grid spacing in pixels and defaults to 100. Use the default branded screenshot only when you plan to share the image with \`replicas media upload\`.
+
+### \`replicas computer observe <path> [--raw] [--grid [px]] [--timeout MS] [--stable-ms MS] [--poll-ms MS]\`
+Waits briefly for the screen to stop changing, saves a 1:1 screenshot, and prints JSON with screen dimensions, whether the screen stabilized, frame/change counts, mouse location, active window title, and visible window titles.
+
+Use this instead of hand-written \`sleep && screenshot\` loops after clicks, navigation, typing, or page loads. By default it saves a 100px coordinate-grid screenshot and waits up to 3s for 600ms of visual stability. Pass \`--raw\` if you need an unannotated 1:1 screenshot.
+
+### \`replicas computer browser\`
+Prints JSON for Chrome tabs launched through Replicas, including page titles and URLs. Pass \`--snapshot\` to include visible page text and interactive controls with DOM viewport bounding boxes.
+
+Use this alongside \`observe\` when testing web apps so you do not infer navigation, page state, or click targets from pixels alone. Snapshot coordinates are DOM viewport coordinates, not desktop click coordinates.
+
+### \`replicas computer browser-click <text> [--exact] [--index N]\`
+Clicks the first visible Chrome control whose text, label, placeholder, or href matches \`<text>\`. Use this for web buttons and links found via \`browser --snapshot\`; it is faster and less error-prone than converting DOM coordinates to desktop pixels.
+
+### \`replicas computer browser-fill <field> <value> [--exact] [--index N]\`
+Fills the first visible Chrome field whose label, placeholder, name, or text matches \`<field>\`, then dispatches input/change events. Use this for web forms instead of clicking a field and typing through the desktop.
+
+### \`replicas computer browser-wait <text> [--mode any|text|title|url|control] [--exact] [--timeout MS]\`
+Waits until the active Chrome page matches text in the title, URL, body text, or visible controls. Use this after \`browser-click\` / \`browser-fill\` when you need web app state to settle without screenshot polling.
+
+For \`browser-click\`, \`browser-fill\`, and \`browser-wait\`, pass \`--id <id>\`, \`--title <text>\`, \`--url <text>\`, or \`--page <n>\` when multiple Chrome tabs are open. Run \`replicas computer browser\` first to list tabs. Prefer \`--id\` when a click may change the page title or URL.
 
 ### \`replicas computer click <x> <y> [--button N] [--double] [--modifiers ctrl+shift]\`
-Move to (x, y) and click. Default is left-click (button 1); pass \`--button 3\` for right-click. \`--modifiers\` holds keys during the click (e.g. ctrl-click a link to open in a new tab).
+Move to (x, y) and click. Coordinates can be absolute pixels or percentages such as \`50%\` \`50%\`. Default is left-click (button 1); pass \`--button 3\` for right-click. \`--modifiers\` holds keys during the click (e.g. ctrl-click a link to open in a new tab).
 
 ### \`replicas computer move <x> <y>\`
-Move the mouse without clicking. Useful for hovering tooltips.
+Move the mouse without clicking. Coordinates can be absolute pixels or percentages. Useful for hovering tooltips.
 
 ### \`replicas computer type <text> [--delay MS]\`
 Type a literal string into the focused field. Default per-character delay is 12ms (~80 wpm) - feels human and avoids breaking apps that debounce input. Bump \`--delay 30\` for stricter apps.
@@ -122,10 +145,10 @@ For key combos (not literal text), use \`key\`. \`type "ctrl+l"\` will literally
 Press a single key or combo. Examples: \`Return\`, \`Escape\`, \`Tab\`, \`ctrl+l\`, \`ctrl+shift+t\`, \`alt+Left\`, \`Page_Down\`, \`Home\`. Syntax matches \`xdotool key\`.
 
 ### \`replicas computer scroll <up|down|left|right> [--amount N] [--x X --y Y]\`
-Scroll the wheel. Pass \`--x\` / \`--y\` to hover before scrolling (otherwise scrolls wherever the cursor currently is). Default amount is 3 wheel ticks.
+Scroll the wheel. Pass \`--x\` / \`--y\` to hover before scrolling (otherwise scrolls wherever the cursor currently is). Hover coordinates can be absolute pixels or percentages. Default amount is 3 wheel ticks.
 
 ### \`replicas computer drag <fromX> <fromY> <toX> <toY>\`
-Press left mouse at (fromX, fromY), drag to (toX, toY), release. For things like dragging a file onto an upload zone.
+Press left mouse at (fromX, fromY), drag to (toX, toY), release. Coordinates can be absolute pixels or percentages. For things like dragging a file onto an upload zone.
 
 ### \`replicas computer launch <app> [args...]\`
 Spawns an app on the workspace display. Built-in aliases:
@@ -148,31 +171,37 @@ SIGINTs ffmpeg, waits for it to finalize the MP4, prints the output path. Upload
 
 ## Patterns
 
-### Action / screenshot loop
-You are blind between tool calls. After any action that changes the screen, take a screenshot before deciding the next coordinate:
+### Action / observe loop
+You are blind between tool calls. After any action that changes the screen, observe before deciding the next coordinate:
 
 \`\`\`bash
 replicas computer click 521 700
-sleep 2  # let the page settle
-replicas computer screenshot /tmp/after-click.png
-# read /tmp/after-click.png, decide next click
+replicas computer observe /tmp/after-click.png
+# read /tmp/after-click.png and the JSON output, decide next click
 \`\`\`
 
-\`sleep\` is a regular shell sleep - there's no \`replicas computer wait\` command, but you can mix shell sleeps freely.
+The JSON \`stable\`, \`frames\`, and \`changes\` fields tell you whether something changed while you were waiting. If \`stable\` is false, observe again or increase \`--timeout\` before acting on coordinates.
 
 ### Typing into an address bar
 \`\`\`bash
 replicas computer launch chrome
-sleep 2
-replicas computer key ctrl+l                    # focus address bar
+replicas computer observe /tmp/browser-open.png
+replicas computer key ctrl+l
 replicas computer type "https://example.com"
 replicas computer key Return
-sleep 3                                          # wait for page load
-replicas computer screenshot /tmp/loaded.png
+replicas computer observe /tmp/loaded.png
+replicas computer browser --snapshot
+replicas computer browser-fill "Search" "replicas"
+replicas computer browser-click "More information" --title "Example"
+replicas computer browser-wait "Example Domain" --mode title --title "Example"
 \`\`\`
 
 ### Coordinates from screenshots
-The display is 1920\xD71080 by default. Screenshot pixels map 1:1 to click coordinates - if your Read tool shows a button at pixel (520, 700), click \`replicas computer click 520 700\`. **No translation needed.** Modern image-reading models often imagine the screenshot is at a different resolution; trust the \`xdpyinfo\` value (\`replicas computer status\` shows the real size).
+The display is 1920\xD71080 by default. For click planning, use \`observe\`, \`screenshot --raw\`, or \`screenshot --grid\`; those pixels map 1:1 to click coordinates. If the grid/raw screenshot shows a button at pixel (520, 700), click \`replicas computer click 520 700\`. The default screenshot is branded for sharing and has padding around the desktop, so do not use it for coordinates.
+
+Use percentages for broad, layout-relative targets when exact pixels are unnecessary: \`replicas computer click 50% 50%\` clicks the center of the screen, and \`replicas computer move 95% 5%\` moves near the top-right.
+
+Modern image-reading models often imagine the screenshot is at a different resolution. Trust the dimensions printed by \`observe\`, \`replicas computer screenshot ... --raw\` / \`--grid\`, and the \`xdpyinfo\` value shown by \`replicas computer status\`.
 
 ### Letting the user watch
 The Desktop tab is already live in the dashboard - the user can open it any time. If you're communicating with the user somewhere else (Slack, PR comment, etc.), grab the URL with \`replicas computer info\` and share it inline so they can watch you work.
@@ -190,8 +219,8 @@ Then embed the printed \`![\u2026](\u2026)\` line in your chat reply. See \`MEDI
 ## Failure modes
 
 - **"Desktop services script missing"**: workspace image is older than this skill. Tell the user - nothing you can do from the CLI side.
-- **\`xdotool ... failed: Can't open display\`**: Xvfb didn't come up. \`replicas computer status\` will show which service is dead. Re-running any CLI command auto-attempts to start it.
-- **Browser doesn't appear after \`launch chrome\`**: give it 1-2s, then screenshot. Chrome cold-start on the virtual display takes ~500ms but bigger pages take longer.
+- **\`xdotool ... failed: Can't open display\`**: Xvfb didn't come up. \`replicas computer status\` will show which service is dead and auto-repair the desktop bridge.
+- **Browser doesn't appear after \`launch chrome\`**: run \`replicas computer observe /tmp/state.png\`. Chrome cold-start on the virtual display takes ~500ms but bigger pages take longer.
 - **Live preview shows static / black screen**: the browser may have crashed. \`replicas computer status\` should show no Chrome process - re-launch.
 - **\`replicas computer info\` errors with "not registered"**: engine couldn't register the preview at startup (transient monolith error, or warming mode). Re-running the engine usually fixes it. Until it's registered, the Desktop tab will show a placeholder.
 
@@ -1355,6 +1384,7 @@ replicas automation create ... --disabled
 
 # Workspace lifecycle
 replicas automation create ... --lifecycle delete_when_done
+replicas automation create ... --lifecycle sleep_when_done
 replicas automation create ... --lifecycle delete_after_inactivity --auto-stop-minutes 30
 \`\`\`
 
@@ -2377,6 +2407,11 @@ function parseClaudeEvents(events, parentToolUseId) {
   const toolCallMap = /* @__PURE__ */ new Map();
   const taskMessageMap = /* @__PURE__ */ new Map();
   const partialIndexes = /* @__PURE__ */ new Map();
+  const completedStreamIds = /* @__PURE__ */ new Set();
+  const supersededStreamIds = /* @__PURE__ */ new Set();
+  let liveStreamId = null;
+  const assistantThinking = /* @__PURE__ */ new Map();
+  const assistantTextCounts = /* @__PURE__ */ new Map();
   const taskAccumulator = new TaskAccumulator();
   const taskSnapshot = () => taskAccumulator.getTasks().map((task) => ({
     text: task.subject,
@@ -2387,6 +2422,10 @@ function parseClaudeEvents(events, parentToolUseId) {
     if (event.type === CLAUDE_PARTIAL_MESSAGE_EVENT_TYPE) {
       const payload = coerceClaudePartialMessagePayload(event.payload);
       if (!payload) return;
+      if (liveStreamId !== null && liveStreamId !== payload.streamId) {
+        supersededStreamIds.add(liveStreamId);
+      }
+      liveStreamId = payload.streamId;
       const existing = partialIndexes.get(payload.streamId) ?? {};
       if (payload.thinking) {
         const reasoningMessage = {
@@ -2449,23 +2488,51 @@ function parseClaudeEvents(events, parentToolUseId) {
     if (event.type === "claude-assistant") {
       const messageId = event.payload.message?.id;
       const contentBlocks = normalizeContentBlocks(event.payload.message?.content);
-      contentBlocks.forEach((block, blockIndex) => {
+      const messageKey = messageId || event.timestamp;
+      const streamRefs = messageId ? partialIndexes.get(messageId) : void 0;
+      if (messageId) {
+        completedStreamIds.add(messageId);
+        if (liveStreamId !== null && liveStreamId !== messageId) {
+          supersededStreamIds.add(liveStreamId);
+        }
+        liveStreamId = null;
+      }
+      const thinkingBlocks = contentBlocks.flatMap(
+        (block) => block.type === "thinking" && typeof block.thinking === "string" ? [block.thinking] : []
+      );
+      if (thinkingBlocks.length > 0) {
+        const allThinking = [...assistantThinking.get(messageKey) ?? [], ...thinkingBlocks];
+        assistantThinking.set(messageKey, allThinking);
+        const reasoningMessage = {
+          id: `reasoning-${messageKey}-thinking`,
+          type: "reasoning",
+          content: allThinking.join("\n\n"),
+          status: "completed",
+          timestamp: event.timestamp
+        };
+        if (streamRefs?.reasoning !== void 0) {
+          messages[streamRefs.reasoning] = reasoningMessage;
+          streamRefs.reasoning = void 0;
+        } else {
+          upsertDisplayMessage(messages, reasoningMessage);
+        }
+      }
+      contentBlocks.forEach((block) => {
         if (block.type === "text" && block.text) {
-          upsertDisplayMessage(messages, {
-            id: `agent-${messageId || event.timestamp}-${blockIndex}`,
+          const textIndex = assistantTextCounts.get(messageKey) ?? 0;
+          assistantTextCounts.set(messageKey, textIndex + 1);
+          const agentMessage = {
+            id: `agent-${messageKey}-${textIndex}`,
             type: "agent",
             content: block.text,
             timestamp: event.timestamp
-          });
-        }
-        if (block.type === "thinking" && block.thinking) {
-          upsertDisplayMessage(messages, {
-            id: `reasoning-${messageId || event.timestamp}-thinking`,
-            type: "reasoning",
-            content: block.thinking,
-            status: "completed",
-            timestamp: event.timestamp
-          });
+          };
+          if (streamRefs?.agent !== void 0) {
+            messages[streamRefs.agent] = agentMessage;
+            streamRefs.agent = void 0;
+          } else {
+            upsertDisplayMessage(messages, agentMessage);
+          }
         }
         if (block.type === "tool_use" && block.id) {
           const toolName = block.name || "unknown";
@@ -2737,7 +2804,14 @@ function parseClaudeEvents(events, parentToolUseId) {
       }
     }
   });
-  return messages;
+  const staleIndexes = /* @__PURE__ */ new Set();
+  for (const streamId of [...completedStreamIds, ...supersededStreamIds]) {
+    const refs = partialIndexes.get(streamId);
+    if (refs?.reasoning !== void 0) staleIndexes.add(refs.reasoning);
+    if (refs?.agent !== void 0) staleIndexes.add(refs.agent);
+  }
+  if (staleIndexes.size === 0) return messages;
+  return messages.filter((_, index) => !staleIndexes.has(index));
 }
 
 // ../shared/src/display-message/parsers/codex-asp-parser.ts
@@ -2757,6 +2831,34 @@ function parseAgentEvents(events, agentType) {
   return parseClaudeEvents(events);
 }
 
+// ../shared/src/routes/workspaces.ts
+var WORKSPACE_STATUSES = ["active", "sleeping", "archived", "preparing", "error"];
+var VALID_LIFECYCLE_POLICIES = ["default", "delete_when_done", "sleep_when_done", "delete_after_inactivity"];
+var WORKSPACE_FILE_UPLOAD_MAX_SIZE_BYTES = 20 * 1024 * 1024;
+var WORKSPACE_FILE_CONTENT_MAX_SIZE_BYTES = 1 * 1024 * 1024;
+var WORKSPACE_STATUS_FILTERS = WORKSPACE_STATUSES.map((status) => status === "error" ? "failed" : status);
+var WORKSPACE_SORT_CHOICES = [
+  ["activity", "Last activity"],
+  ["created", "Created date"],
+  ["status", "Status"]
+];
+var WORKSPACE_SORT_OPTIONS = WORKSPACE_SORT_CHOICES.map(([option]) => option);
+var WORKSPACE_SORT_LABELS = Object.fromEntries(WORKSPACE_SORT_CHOICES);
+var WORKSPACE_SIDEBAR_ORGANIZATION_CHOICES = [
+  ["chronological", "Chronological"],
+  ["environments", "Environments"],
+  ["recent", "Recent environments"]
+];
+var WORKSPACE_SIDEBAR_ORGANIZATIONS = WORKSPACE_SIDEBAR_ORGANIZATION_CHOICES.map(([organization]) => organization);
+var WORKSPACE_SIDEBAR_ORGANIZATION_LABELS = Object.fromEntries(WORKSPACE_SIDEBAR_ORGANIZATION_CHOICES);
+var DEFAULT_STATUS_FILTERS = [...WORKSPACE_STATUS_FILTERS];
+var DEFAULT_WORKSPACE_FILTERS = {
+  statuses: [...DEFAULT_STATUS_FILTERS],
+  ownerIds: [],
+  activity: "any"
+};
+var ONE_DAY_MS = 24 * 60 * 60 * 1e3;
+
 // src/index.ts
 var API_URL = process.env.REPLICAS_API_URL || "https://api.tryreplicas.com";
 var API_KEY = process.env.REPLICAS_API_KEY;
@@ -2790,12 +2892,13 @@ function result(data, isError = false) {
 }
 var server = new McpServer({
   name: "replicas-mcp",
-  version: "0.2.9"
+  version: "0.2.10"
 });
 var registerTool = server.registerTool.bind(server);
 var codingAgentSchema = z.string().refine(isValidCodingAgentProvider, {
   message: `Coding agent must be ${getCodingAgentDisplayNames()}`
 });
+var lifecyclePolicySchema = z.enum(VALID_LIFECYCLE_POLICIES);
 registerTool(
   "create_replica",
   {
@@ -2807,7 +2910,7 @@ registerTool(
       repository_set_id: z.string().optional().describe("Repository set ID to use. Mutually exclusive with repository_ids."),
       coding_agent: codingAgentSchema.optional().describe("Coding agent to use (default: claude)"),
       model: z.string().optional().describe("Model to use for the coding agent"),
-      lifecycle_policy: z.enum(["default", "delete_when_done", "delete_after_inactivity"]).optional().describe("Lifecycle policy for the workspace")
+      lifecycle_policy: lifecyclePolicySchema.optional().describe("Lifecycle policy for the workspace")
     }
   },
   async (args) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-mcp",
-  "version": "0.2.9",
+  "version": "0.2.10",
   "license": "UNLICENSED",
   "type": "module",
   "main": "dist/index.js",

package/src/index.ts

@@ -14,6 +14,9 @@ import {
 import {
   type ReadReplicaHistoryResponse,
 } from '../../shared/src/routes/replica';
+import {
+  VALID_LIFECYCLE_POLICIES,
+} from '../../shared/src/routes/workspaces';
 
 const API_URL = process.env.REPLICAS_API_URL || 'https://api.tryreplicas.com';
 const API_KEY = process.env.REPLICAS_API_KEY;
@@ -68,13 +71,14 @@ type RegisterToolFn = (
 
 const server = new McpServer({
   name: 'replicas-mcp',
-  version: '0.2.9',
+  version: '0.2.10',
 });
 
 const registerTool = server.registerTool.bind(server) as RegisterToolFn;
 const codingAgentSchema = z.string().refine(isValidCodingAgentProvider, {
   message: `Coding agent must be ${getCodingAgentDisplayNames()}`,
 });
+const lifecyclePolicySchema = z.enum(VALID_LIFECYCLE_POLICIES);
 
 // ---------------------------------------------------------------------------
 // Replica CRUD
@@ -91,7 +95,7 @@ registerTool(
       repository_set_id: z.string().optional().describe('Repository set ID to use. Mutually exclusive with repository_ids.'),
       coding_agent: codingAgentSchema.optional().describe('Coding agent to use (default: claude)'),
       model: z.string().optional().describe('Model to use for the coding agent'),
-      lifecycle_policy: z.enum(['default', 'delete_when_done', 'delete_after_inactivity']).optional().describe('Lifecycle policy for the workspace'),
+      lifecycle_policy: lifecyclePolicySchema.optional().describe('Lifecycle policy for the workspace'),
     },
   },
   async (args) => {