replicas-engine 0.1.214 → 0.1.216

package/dist/src/index.js

@@ -276,8 +276,200 @@ function parsePosixEnvFile(content) {
   return result;
 }
 
+// ../shared/src/default-skills/replicas-agent/abilities/computer.ts
+var SECTION = `### Computer use (Linux desktop control)
+Drive the workspace's Linux desktop - open a browser, click, type, scroll, screenshot, record - and surface a live noVNC viewer to the user via the \`replicas computer\` CLI. Every Replicas workspace boots with Xvfb / openbox / x11vnc / noVNC pre-installed.
+
+**Reference:** \`references/COMPUTER-USE.md\`
+
+Use this when:
+- A task requires interacting with a website, web app, or desktop application that has no usable API
+- You want the user to watch the agent work - \`replicas computer start\` exposes a \`Desktop\` tab in the dashboard
+- You're testing UI changes in a browser before reporting them as done
+- You want to record a screen capture of a task as proof to share back`;
+var REFERENCE = `# Computer use (Linux desktop control)
+
+Every Replicas workspace boots with a full Linux desktop stack - Xvfb (1920\xD71080), openbox, tint2, x11vnc, noVNC, ffmpeg, and Google Chrome. You drive it through the \`replicas computer\` CLI.
+
+Use this for anything the user can't reasonably do via an API - clicking around web apps, filling forms, testing UI changes, dragging files between desktop apps, recording a walkthrough.
+
+## When to use it
+
+- **Prefer real APIs first.** If a task has a CLI or HTTP API (GitHub, Linear, Slack, Replicas itself), use that. Driving a UI is slower, flakier, and less auditable.
+- **Use it when there's no API**: testing a frontend you just changed, navigating a vendor portal, demonstrating a flow on video.
+- **Use it when the user wants to watch.** \`replicas computer start\` exposes a live \`Desktop\` tab in the dashboard - they can watch you work in real time.
+
+## Never use raw \`xdotool\` / \`scrot\` / \`ffmpeg\` directly
+
+The CLI is the canonical surface. Chat transcripts that show \`DISPLAY=:99 xdotool key Return\` are noisy and break when the underlying stack changes. Always call the CLI:
+
+\`\`\`bash
+# DO
+replicas computer key Return
+replicas computer screenshot /tmp/state.png
+
+# DON'T
+DISPLAY=:99 xdotool key Return
+scrot /tmp/state.png
+\`\`\`
+
+## Quickstart
+
+\`\`\`bash
+# 1) Make the desktop visible to the user (creates an authenticated noVNC
+#    preview on port 6080, prints the viewer URL, adds a "Desktop" tab to the
+#    dashboard).
+replicas computer start
+
+# 2) Launch a browser on the workspace display.
+replicas computer launch chrome
+
+# 3) Take a screenshot so you can see what's there.
+replicas computer screenshot /tmp/state.png
+# (Read the PNG yourself before deciding where to click.)
+
+# 4) Drive the UI.
+replicas computer key ctrl+l                       # focus address bar
+replicas computer type "https://news.ycombinator.com"
+replicas computer key Return
+replicas computer click 521 700                    # click coordinates from the screenshot
+replicas computer scroll down --amount 5
+
+# 5) (Optional) Record a screencap to share back.
+replicas computer record start /tmp/demo.mp4 --fps 60
+# ... do stuff ...
+replicas computer record stop
+replicas media upload /tmp/demo.mp4
+
+# 6) Tear down the live preview when done (services keep running for next time).
+replicas computer stop
+\`\`\`
+
+## Command reference
+
+### \`replicas computer start [--port N] [--display :N] [--size WxH]\`
+Ensures all desktop services are running and creates an authenticated noVNC preview. Prints the viewer URL (\`https://<port>-<hash>.tryreplicas.com/\`). The Replicas dashboard automatically shows a \`Desktop\` tab while this preview is live - point the user at it instead of pasting the URL.
+
+Idempotent - safe to call repeatedly. Use it as the first computer-use command in any session.
+
+### \`replicas computer stop [--port N]\`
+Tears down the noVNC preview (the \`Desktop\` tab disappears). The underlying Xvfb / openbox / x11vnc / browser keep running so the next \`start\` is instant.
+
+### \`replicas computer status\`
+Prints which desktop services are running and the active preview URL (if any). Useful for debugging when a tool call seems to be doing nothing.
+
+### \`replicas computer screenshot <path>\`
+Captures the current desktop to a PNG at the given path. Read the file (e.g. with your Read tool) to see what's on screen - coordinates from the screenshot drive subsequent \`click\` / \`move\` / \`drag\` calls.
+
+### \`replicas computer click <x> <y> [--button N] [--double] [--modifiers ctrl+shift]\`
+Move to (x, y) and click. Default is left-click (button 1); pass \`--button 3\` for right-click. \`--modifiers\` holds keys during the click (e.g. ctrl-click a link to open in a new tab).
+
+### \`replicas computer move <x> <y>\`
+Move the mouse without clicking. Useful for hovering tooltips.
+
+### \`replicas computer type <text> [--delay MS]\`
+Type a literal string into the focused field. Default per-character delay is 12ms (~80 wpm) - feels human and avoids breaking apps that debounce input. Bump \`--delay 30\` for stricter apps.
+
+For key combos (not literal text), use \`key\`. \`type "ctrl+l"\` will literally type the seven characters \`c t r l + l\`.
+
+### \`replicas computer key <combo>\`
+Press a single key or combo. Examples: \`Return\`, \`Escape\`, \`Tab\`, \`ctrl+l\`, \`ctrl+shift+t\`, \`alt+Left\`, \`Page_Down\`, \`Home\`. Syntax matches \`xdotool key\`.
+
+### \`replicas computer scroll <up|down|left|right> [--amount N] [--x X --y Y]\`
+Scroll the wheel. Pass \`--x\` / \`--y\` to hover before scrolling (otherwise scrolls wherever the cursor currently is). Default amount is 3 wheel ticks.
+
+### \`replicas computer drag <fromX> <fromY> <toX> <toY>\`
+Press left mouse at (fromX, fromY), drag to (toX, toY), release. For things like dragging a file onto an upload zone.
+
+### \`replicas computer launch <app> [args...]\`
+Spawns an app on the workspace display. Built-in aliases:
+- \`chrome\` - Google Chrome with a clean profile and sane flags
+- \`chromium\` - Chromium variant
+- \`firefox\` - Firefox
+- \`terminal\` - xfce4-terminal (modern, themed); \`xterm\` for the classic
+- \`notepad\` / \`editor\` - mousepad (lightweight GTK text editor)
+- \`files\` / \`filemanager\` - thunar (file manager)
+Anything else gets \`exec\`'d verbatim, so \`replicas computer launch xeyes\` works if xeyes is installed.
+
+### \`replicas computer record start <path> [--fps N]\`
+Starts an ffmpeg screen recorder. Output is a fragmented MP4 (still playable if the workspace dies mid-record). Default 60fps; drop to 30 if the workspace is CPU-constrained.
+
+Only one recording at a time. Re-running \`start\` while one is active fails - call \`stop\` first.
+
+### \`replicas computer record stop\`
+SIGINTs ffmpeg, waits for it to finalize the MP4, prints the output path. Upload it with \`replicas media upload <path>\` to share it.
+
+## Patterns
+
+### Action / screenshot loop
+You are blind between tool calls. After any action that changes the screen, take a screenshot before deciding the next coordinate:
+
+\`\`\`bash
+replicas computer click 521 700
+sleep 2  # let the page settle
+replicas computer screenshot /tmp/after-click.png
+# read /tmp/after-click.png, decide next click
+\`\`\`
+
+\`sleep\` is a regular shell sleep - there's no \`replicas computer wait\` command, but you can mix shell sleeps freely.
+
+### Typing into an address bar
+\`\`\`bash
+replicas computer key ctrl+l                    # focus address bar
+replicas computer type "https://example.com"
+replicas computer key Return
+sleep 3                                          # wait for page load
+replicas computer screenshot /tmp/loaded.png
+\`\`\`
+
+### Coordinates from screenshots
+The display is 1920\xD71080. Screenshot pixels map 1:1 to click coordinates - if your Read tool shows a button at pixel (520, 700), click \`replicas computer click 520 700\`. **No translation needed.** Modern image-reading models often imagine the screenshot is at a different resolution; trust the \`xdpyinfo\` value (\`replicas computer status\` shows the real size).
+
+### Letting the user watch
+Always start the desktop session with \`replicas computer start\` *before* doing anything visual, even if you don't need the URL yourself. The Desktop tab appears in their dashboard. They get to watch and intervene if needed.
+
+### Recording a deliverable
+For tasks the user wants proof of:
+\`\`\`bash
+replicas computer start                                 # makes it visible live too
+replicas computer record start /tmp/walkthrough.mp4
+# ... your work ...
+replicas computer record stop
+replicas media upload /tmp/walkthrough.mp4
+\`\`\`
+Then embed the printed \`![\u2026](\u2026)\` line in your chat reply. See \`MEDIA.md\`.
+
+### Cleaning up
+Call \`replicas computer stop\` when you're done with the visual demo so the live preview URL goes away. The services keep running so the next \`start\` is instant.
+
+## Failure modes
+
+- **"Desktop services script missing"**: workspace image is older than this skill. Tell the user - nothing you can do from the CLI side.
+- **\`xdotool ... failed: Can't open display\`**: Xvfb didn't come up. \`replicas computer status\` will show which service is dead. Re-running any CLI command auto-attempts to start it.
+- **Browser doesn't appear after \`launch chrome\`**: give it 1-2s, then screenshot. Chrome cold-start on the virtual display takes ~500ms but bigger pages take longer.
+- **Live preview shows static / black screen**: the browser may have crashed. \`replicas computer status\` should show no Chrome process - re-launch.
+
+## What gets baked in vs. lazy
+
+| Component | Where |
+|---|---|
+| \`xvfb\`, \`openbox\`, \`tint2\`, \`x11vnc\`, \`websockify\`, \`xdotool\`, \`scrot\`, \`ffmpeg\`, \`google-chrome\` | Baked into the workspace image |
+| Xvfb / openbox / tint2 / x11vnc / websockify processes | Started at workspace boot (\`replicas-start-desktop-services\`) |
+| noVNC preview URL (port 6080) | **Lazy** - created by \`replicas computer start\` |
+| Dashboard \`Desktop\` tab | Appears when the preview URL is live, disappears on \`stop\` |
+
+You can call any of the input tools (\`click\`, \`type\`, \`screenshot\`, etc.) without first calling \`start\` - they'll work since the daemons are running. \`start\` is only needed when you want the user to see the live stream.
+`;
+var COMPUTER_ABILITY = {
+  label: "Computer use",
+  description: "Drive the workspace's Linux desktop and stream a live noVNC viewer.",
+  bullet: "- Driving the workspace Linux desktop (open a browser, click, type, scroll, screenshot, record) and exposing a live noVNC viewer in the dashboard",
+  section: SECTION,
+  referenceFile: { name: "COMPUTER-USE.md", content: REFERENCE }
+};
+
 // ../shared/src/default-skills/replicas-agent/abilities/docker.ts
-var SECTION = `### Docker
+var SECTION2 = `### Docker
 Start and use the Docker daemon in Replicas workspaces. Docker is pre-installed but the daemon does not auto-start.
 
 **Reference:** \`references/DOCKER.md\`
@@ -286,7 +478,7 @@ Use this when:
 - You need to run \`docker\` or \`docker compose\` commands
 - You need to build or run Docker containers
 - Your task involves containerized services or Docker-based workflows`;
-var REFERENCE = `# Docker
+var REFERENCE2 = `# Docker
 
 Docker is pre-installed in Replicas workspaces, but the daemon does **not** auto-start. You must start it manually before running any \`docker\` or \`docker compose\` commands.
 
@@ -315,12 +507,12 @@ var DOCKER_ABILITY = {
   label: "Docker",
   description: "Start the daemon, build/run containers, drive `docker compose`.",
   bullet: "- Using Docker (starting the daemon, running containers, docker compose)",
-  section: SECTION,
-  referenceFile: { name: "DOCKER.md", content: REFERENCE }
+  section: SECTION2,
+  referenceFile: { name: "DOCKER.md", content: REFERENCE2 }
 };
 
 // ../shared/src/default-skills/replicas-agent/abilities/github.ts
-var SECTION2 = `### GitHub
+var SECTION3 = `### GitHub
 Use the pre-authenticated \`gh\` CLI for pull requests, issues, actions, and API calls.
 
 **Reference:** \`references/GITHUB.md\`
@@ -330,7 +522,7 @@ Use this when:
 - You need to interact with GitHub issues or actions
 - You need to use the GitHub API for advanced operations
 - You need to include images in PR descriptions`;
-var REFERENCE2 = `# GitHub Integration
+var REFERENCE3 = `# GitHub Integration
 
 This guide covers how to interact with GitHub from within your Replicas workspace.
 
@@ -456,12 +648,12 @@ var GITHUB_ABILITY = {
   label: "GitHub",
   description: "Pre-authenticated `gh` CLI for PRs, issues, releases, and GraphQL.",
   bullet: "- Interacting with GitHub (creating PRs, managing issues, using the API, etc.)",
-  section: SECTION2,
-  referenceFile: { name: "GITHUB.md", content: REFERENCE2 }
+  section: SECTION3,
+  referenceFile: { name: "GITHUB.md", content: REFERENCE3 }
 };
 
 // ../shared/src/default-skills/replicas-agent/abilities/google.ts
-var SECTION3 = `### Google Workspace (Docs, Sheets, Forms, Drive)
+var SECTION4 = `### Google Workspace (Docs, Sheets, Forms, Drive)
 Create and edit Google Docs, Sheets, and Forms via the Replicas gateway. Files are owned by Replicas \u2014 the integration cannot access pre-existing Google content created outside of it.
 
 **Reference:** \`references/GOOGLE.md\`
@@ -470,7 +662,7 @@ Use this when:
 - You need to create or edit a Google Doc, Sheet, or Form
 - You need to share, rename, move, or delete a Replicas-created Google file
 - You need to read responses from a Replicas-created Google Form`;
-var REFERENCE3 = `# Google Workspace (Docs, Sheets, Forms, Drive)
+var REFERENCE4 = `# Google Workspace (Docs, Sheets, Forms, Drive)
 
 This guide covers how to create and edit Google Docs, Sheets, and Forms \u2014 plus do basic Drive file operations \u2014 from inside a Replicas workspace, using the monolith as a gateway to Google's APIs.
 
@@ -731,12 +923,12 @@ var GOOGLE_ABILITY = {
   label: "Google Workspace",
   description: "Create / edit Docs, Sheets, Forms, and Drive files via the Replicas gateway.",
   bullet: "- Interacting with Google Workspace (creating and editing Docs, Sheets, and Forms, sharing files, reading form responses, etc.)",
-  section: SECTION3,
-  referenceFile: { name: "GOOGLE.md", content: REFERENCE3 }
+  section: SECTION4,
+  referenceFile: { name: "GOOGLE.md", content: REFERENCE4 }
 };
 
 // ../shared/src/default-skills/replicas-agent/abilities/linear.ts
-var SECTION4 = `### Linear
+var SECTION5 = `### Linear
 Fetch issues, update state, add comments, and search via the Linear GraphQL API.
 
 **Reference:** \`references/LINEAR.md\`
@@ -745,7 +937,7 @@ Use this when:
 - You encounter a Linear issue link and need to understand the task
 - You need to update an issue's state (e.g. mark as done)
 - You need to comment on or search for Linear issues`;
-var REFERENCE4 = `# Linear Integration
+var REFERENCE5 = `# Linear Integration
 
 This guide covers how to interact with Linear from within your Replicas workspace.
 
@@ -820,12 +1012,12 @@ var LINEAR_ABILITY = {
   label: "Linear",
   description: "Fetch issues, post comments, update states via the Linear GraphQL API.",
   bullet: "- Interacting with Linear (fetching issues, updating state, commenting, etc.)",
-  section: SECTION4,
-  referenceFile: { name: "LINEAR.md", content: REFERENCE4 }
+  section: SECTION5,
+  referenceFile: { name: "LINEAR.md", content: REFERENCE5 }
 };
 
 // ../shared/src/default-skills/replicas-agent/abilities/media.ts
-var SECTION5 = `### Media
+var SECTION6 = `### Media
 Share screenshots, screen recordings, generated diagrams, and audio clips inline in the Replicas chat (and as references in external messages).
 
 **Reference:** \`references/MEDIA.md\`
@@ -834,7 +1026,7 @@ Use this when:
 - You produce a screenshot, recording, generated image, or audio clip the user should see
 - You record video output (browser automation, screen capture) \u2014 including the recommended aspect ratio and FPS
 - You need to embed media in a Slack/Linear/GitHub message AND keep a referenceable copy in the Replicas dashboard`;
-var REFERENCE5 = `# Media (Screenshots, Recordings, Audio)
+var REFERENCE6 = `# Media (Screenshots, Recordings, Audio)
 
 This guide covers how to share screenshots, screen recordings, generated diagrams, and audio clips with the user inline in the Replicas chat.
 
@@ -948,12 +1140,12 @@ var MEDIA_ABILITY = {
   label: "Media",
   description: "Share screenshots, recordings, generated images, and audio clips.",
   bullet: "- Sharing media (screenshots, screen recordings, generated diagrams, audio clips) \u2014 including in your Replicas chat reply and to external platforms",
-  section: SECTION5,
-  referenceFile: { name: "MEDIA.md", content: REFERENCE5 }
+  section: SECTION6,
+  referenceFile: { name: "MEDIA.md", content: REFERENCE6 }
 };
 
 // ../shared/src/default-skills/replicas-agent/abilities/previews.ts
-var SECTION6 = `### Previews
+var SECTION7 = `### Previews
 Expose locally running services (web apps, APIs, databases) as public preview URLs so humans can interact with them directly.
 
 **Reference:** \`references/PREVIEWS.md\`
@@ -962,7 +1154,7 @@ Use this when:
 - You need to start a service that a human should view or interact with
 - The task involves UI work that benefits from human review
 - You are verifying frontend/backend integrations visually`;
-var REFERENCE6 = `# Preview URLs
+var REFERENCE7 = `# Preview URLs
 
 When you run services on ports \u2014 such as a web app, API server, or database \u2014 humans may want to interact with them directly. You can expose your locally running services as public preview URLs.
 
@@ -1044,12 +1236,12 @@ var PREVIEWS_ABILITY = {
   label: "Previews",
   description: "Expose locally running services on public preview URLs for humans.",
   bullet: "- Creating preview URLs for locally running services",
-  section: SECTION6,
-  referenceFile: { name: "PREVIEWS.md", content: REFERENCE6 }
+  section: SECTION7,
+  referenceFile: { name: "PREVIEWS.md", content: REFERENCE7 }
 };
 
 // ../shared/src/default-skills/replicas-agent/abilities/replicas.ts
-var SECTION7 = `### Replicas (in-workspace CLI)
+var SECTION8 = `### Replicas (in-workspace CLI)
 Take action *with* Replicas itself \u2014 manage automations, environments (variables, files), repos, and \`replicas.json\` config \u2014 using the pre-installed, pre-authenticated \`replicas\` CLI.
 
 **Reference:** \`references/REPLICAS.md\`
@@ -1059,7 +1251,7 @@ Use this when:
 - The user asks you to manage environments, environment variables, or environment files
 - The user asks "what envs / repos / automations do I have?"
 - The user asks you to scaffold a \`replicas.json\` / \`replicas.yaml\` in a repo`;
-var REFERENCE7 = `# Replicas (in-workspace CLI)
+var REFERENCE8 = `# Replicas (in-workspace CLI)
 
 This guide covers how to take action *with* Replicas itself from inside a Replicas workspace \u2014 managing automations, environments (and their variables/files), repos, previews, and the user's \`replicas.json\` config \u2014 using the pre-installed \`replicas\` CLI.
 
@@ -1253,13 +1445,13 @@ var REPLICAS_ABILITY = {
   description: "Teach the agent about Replicas itself \u2014 automations, environments, the in-workspace CLI.",
   // No bullet — help_instructions covers the `replicas` CLI surface in detail.
   bullet: "",
-  section: SECTION7,
-  referenceFile: { name: "REPLICAS.md", content: REFERENCE7 },
+  section: SECTION8,
+  referenceFile: { name: "REPLICAS.md", content: REFERENCE8 },
   locked: true
 };
 
 // ../shared/src/default-skills/replicas-agent/abilities/slack.ts
-var SECTION8 = `### Slack
+var SECTION9 = `### Slack
 Send messages, read threads, search conversations, and upload files via the Slack Web API.
 
 **Reference:** \`references/SLACK.md\`
@@ -1269,7 +1461,7 @@ Use this when:
 - You need to read or fetch a Slack conversation
 - You encounter a Slack message link and need to retrieve its content
 - The task asks you to notify, update, or communicate via Slack`;
-var REFERENCE8 = `# Slack Integration
+var REFERENCE9 = `# Slack Integration
 
 This guide covers how to interact with Slack from within your Replicas workspace.
 
@@ -1342,13 +1534,14 @@ var SLACK_ABILITY = {
   label: "Slack",
   description: "Send messages, read threads, search conversations, upload files.",
   bullet: "- Interacting with Slack (sending messages, reading threads, etc.)",
-  section: SECTION8,
-  referenceFile: { name: "SLACK.md", content: REFERENCE8 }
+  section: SECTION9,
+  referenceFile: { name: "SLACK.md", content: REFERENCE9 }
 };
 
 // ../shared/src/default-skills/replicas-agent/registry.ts
 var REPLICAS_AGENT_ABILITY_REGISTRY = {
   replicas: REPLICAS_ABILITY,
+  computer: COMPUTER_ABILITY,
   docker: DOCKER_ABILITY,
   github: GITHUB_ABILITY,
   google: GOOGLE_ABILITY,
@@ -1493,7 +1686,7 @@ function parseReplicasConfigString(content, filename) {
 }
 
 // ../shared/src/engine/environment.ts
-var DAYTONA_SNAPSHOT_ID = "26-05-2026-royal-york-v2";
+var DAYTONA_SNAPSHOT_ID = "26-05-2026-royal-york-v3";
 
 // ../shared/src/engine/types.ts
 var DEFAULT_CHAT_TITLES = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicas-engine",
-  "version": "0.1.214",
+  "version": "0.1.216",
   "description": "Lightweight API server for Replicas workspaces",
   "type": "module",
   "main": "dist/src/index.js",