surf-cli 2.0.0 → 2.1.0

package/README.md

@@ -170,16 +170,20 @@ surf scroll.bottom                  # Scroll to bottom
 
 ### Screenshots
 
-Screenshots are optimized for AI consumption by default:
+Screenshots auto-save to `/tmp` by default (optimized for AI agents):
 
 ```bash
-surf screenshot --output /tmp/shot.png      # Auto-resized to 1200px max
-surf screenshot --full --output /tmp/hd.png # Full resolution
-surf screenshot --annotate --output /tmp/labeled.png  # With element labels
-surf screenshot --fullpage --output /tmp/full.png     # Entire page
-surf snap                                   # Quick save to /tmp
+surf screenshot                             # Auto-saves to /tmp/surf-snap-*.png
+surf screenshot --output /tmp/shot.png      # Save to specific path
+surf screenshot --full --output /tmp/hd.png # Full resolution (skip resize)
+surf screenshot --annotate                  # With element labels
+surf screenshot --fullpage                  # Entire page
+surf screenshot --no-save                   # Return base64 + ID only (no file)
+surf snap                                   # Alias for screenshot
 ```
 
+To disable auto-save globally, set `autoSaveScreenshots: false` in `surf.json`.
+
 Actions like `click`, `type`, and `scroll` automatically capture a screenshot after execution - no extra command needed.
 
 ### Tabs

package/native/cli.cjs

@@ -234,7 +234,7 @@ const TOOLS = {
         examples: [{ cmd: "forward", desc: "Browser forward" }]
       },
       "screenshot": { 
-        desc: "Capture screenshot (auto-resized for LLM by default)", 
+        desc: "Capture screenshot (auto-saves to /tmp by default)", 
         args: [], 
         opts: { 
           output: "Save to file", 
@@ -243,14 +243,15 @@ const TOOLS = {
           fullpage: "Capture full page", 
           "max-height": "Max height for fullpage (default: 4000)",
           full: "Skip resize, save at full resolution",
-          "max-size": "Max dimension in px (default: 1200)" 
+          "max-size": "Max dimension in px (default: 1200)",
+          "no-save": "Don't auto-save, return base64 + ID (saves context)"
         },
         examples: [
-          { cmd: "screenshot --output /tmp/shot.png", desc: "Save to file (auto-resized)" },
-          { cmd: "screenshot --full --output /tmp/shot.png", desc: "Full resolution" },
-          { cmd: "screenshot --max-size 800 --output /tmp/small.png", desc: "Custom max size" },
-          { cmd: "screenshot --annotate --output /tmp/annotated.png", desc: "With element labels" },
-          { cmd: "snap", desc: "Auto-save to /tmp (resized)" },
+          { cmd: "screenshot", desc: "Auto-save to /tmp (default)" },
+          { cmd: "screenshot --output /tmp/shot.png", desc: "Save to specific file" },
+          { cmd: "screenshot --no-save", desc: "Return base64 without saving" },
+          { cmd: "screenshot --annotate", desc: "With element labels" },
+          { cmd: "snap", desc: "Alias for screenshot" },
         ]
       },
       "snap": { desc: "Alias for screenshot (auto-saves to /tmp)", args: [], alias: "screenshot" },
@@ -1672,7 +1673,7 @@ if (args.includes("--script")) {
   return;
 }
 
-const BOOLEAN_FLAGS = ["auto-capture", "json", "stream", "dry-run", "stop-on-error", "fail-fast", "clear", "submit", "all", "case-sensitive", "hard", "annotate", "fullpage", "reset", "no-screenshot", "full", "soft-fail", "has-body", "exclude-static", "v", "vv", "request", "by-tab", "har", "jsonl"];
+const BOOLEAN_FLAGS = ["auto-capture", "json", "stream", "dry-run", "stop-on-error", "fail-fast", "clear", "submit", "all", "case-sensitive", "hard", "annotate", "fullpage", "reset", "no-screenshot", "full", "soft-fail", "has-body", "exclude-static", "v", "vv", "request", "by-tab", "har", "jsonl", "no-save"];
 
 const AUTO_SCREENSHOT_TOOLS = ["click", "type", "key", "smart_type", "form.fill", "form_input", "drag", "hover", "scroll", "scroll.top", "scroll.bottom", "scroll.to", "dialog.accept", "dialog.dismiss", "js", "eval"];
 
@@ -1727,10 +1728,14 @@ if (REMOVED_COMMANDS[tool]) {
   process.exit(1);
 }
 
-const wasSnap = tool === "snap";
 tool = ALIASES[tool] || tool;
 
-if (wasSnap && !options.output && !options.savePath) {
+// Auto-save screenshots to temp file when no --output specified
+// This ensures agents always get a usable file path, not just an in-memory ID
+// Can be disabled with --no-save flag or autoSaveScreenshots: false in surf.json
+const config = loadConfig();
+const autoSaveEnabled = config.autoSaveScreenshots !== false && !options["no-save"];
+if (tool === "screenshot" && !options.output && !options.savePath && autoSaveEnabled) {
   options.savePath = `/tmp/surf-snap-${Date.now()}.png`;
 }
 
@@ -1887,7 +1892,7 @@ if (!noScreenshot && AUTO_SCREENSHOT_TOOLS.includes(tool)) {
 const outputPath = toolArgs.output;
 delete toolArgs.output;
 
-if ((tool === "screenshot" || tool === "snap") && outputPath) {
+if (tool === "screenshot" && outputPath) {
   if (typeof outputPath !== "string") {
     console.error("Error: --output requires a file path");
     process.exit(1);
@@ -2208,7 +2213,7 @@ async function handleResponse(response) {
     process.exit(0);
   }
 
-  if ((tool === "screenshot" || tool === "snap") && data?.base64 && (outputPath || toolArgs.savePath)) {
+  if (tool === "screenshot" && data?.base64 && (outputPath || toolArgs.savePath)) {
     const saveTo = outputPath || toolArgs.savePath;
     fs.writeFileSync(saveTo, Buffer.from(data.base64, "base64"));
     
@@ -2227,8 +2232,11 @@ async function handleResponse(response) {
     } else {
       console.log(`Saved to ${saveTo} (${origWidth}x${origHeight})`);
     }
-  } else if ((tool === "screenshot" || tool === "snap") && data?.message) {
+  } else if (tool === "screenshot" && data?.message) {
     console.log(data.message);
+    if (data.screenshotId) {
+      console.log(`[Screenshot ID: ${data.screenshotId}]`);
+    }
   } else if (tool === "tab.list") {
     const tabs = data?.tabs || data || [];
     if (Array.isArray(tabs)) {

package/native/config.cjs

@@ -8,6 +8,9 @@ let cachedConfig = null;
 let cachedConfigPath = null;
 
 const STARTER_CONFIG = {
+  // Set to false to disable auto-saving screenshots to /tmp
+  // When disabled, screenshots return base64 + ID instead of file path
+  autoSaveScreenshots: true,
   routes: {
     main: ["http://localhost:3000"]
   },

package/native/host-helpers.cjs

@@ -84,7 +84,17 @@ function formatToolContent(result, log = () => {}) {
     return text(result.output);
   }
 
-  if (result.screenshotId) {
+  // Screenshot saved to file (has path but no base64)
+  if (result.path && result.message && !result.base64) {
+    let msg = result.message;
+    if (result.screenshotId) {
+      msg += `\n[Screenshot ID: ${result.screenshotId} - use with upload_image]`;
+    }
+    return text(msg);
+  }
+
+  // Screenshot with inline base64 (MCP flow or no savePath)
+  if (result.screenshotId && result.base64) {
     const dims = result.width && result.height 
       ? `${result.width}x${result.height}` 
       : "unknown dimensions";
@@ -489,7 +499,7 @@ function mapToolToMessage(tool, args, tabId) {
     case "screenshot":
       return { 
         type: "EXECUTE_SCREENSHOT", 
-        savePath: a.savePath,
+        savePath: a.savePath || a.output,  // Accept both savePath (CLI) and output (MCP)
         annotate: a.annotate || false,
         fullpage: a.fullpage || false,
         maxHeight: a["max-height"] || 4000,

package/native/host.cjs

@@ -996,7 +996,9 @@ function processInput() {
                 }
               }
               sendToolResponse(socket, originalId, { 
-                message: `Saved to ${savePath} (${finalDims})`
+                message: `Saved to ${savePath} (${finalDims})`,
+                path: savePath,
+                screenshotId: msg.screenshotId,  // Preserve for upload_image workflow
               }, null);
             } catch (e) {
               sendToolResponse(socket, originalId, null, `Failed to save: ${e.message}`);

package/native/stream-viewer.html

@@ -0,0 +1,415 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Surf Stream Viewer</title>
+  <style>
+    * {
+      margin: 0;
+      padding: 0;
+      box-sizing: border-box;
+    }
+
+    body {
+      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+      background: #1a1a1a;
+      color: #e0e0e0;
+      display: flex;
+      flex-direction: column;
+      height: 100vh;
+      overflow: hidden;
+    }
+
+    #status {
+      background: #2d2d2d;
+      padding: 12px 20px;
+      border-bottom: 1px solid #404040;
+      display: flex;
+      align-items: center;
+      gap: 20px;
+      font-size: 14px;
+      flex-shrink: 0;
+    }
+
+    #status.connected {
+      border-bottom-color: #4caf50;
+    }
+
+    #status.disconnected {
+      border-bottom-color: #f44336;
+    }
+
+    .status-indicator {
+      width: 10px;
+      height: 10px;
+      border-radius: 50%;
+      background: #666;
+    }
+
+    .status-indicator.connected {
+      background: #4caf50;
+      box-shadow: 0 0 8px #4caf50;
+    }
+
+    .status-indicator.disconnected {
+      background: #f44336;
+    }
+
+    #viewport-container {
+      flex: 1;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      overflow: auto;
+      background: #0a0a0a;
+      position: relative;
+    }
+
+    #viewport {
+      display: block;
+      max-width: 100%;
+      max-height: 100%;
+      cursor: crosshair;
+      image-rendering: crisp-edges;
+      image-rendering: pixelated;
+    }
+
+    #viewport:not(.ready) {
+      opacity: 0.5;
+    }
+
+    #loading {
+      position: absolute;
+      top: 50%;
+      left: 50%;
+      transform: translate(-50%, -50%);
+      text-align: center;
+      color: #888;
+    }
+
+    .spinner {
+      border: 3px solid #333;
+      border-top: 3px solid #4caf50;
+      border-radius: 50%;
+      width: 40px;
+      height: 40px;
+      animation: spin 1s linear infinite;
+      margin: 0 auto 10px;
+    }
+
+    @keyframes spin {
+      0% { transform: rotate(0deg); }
+      100% { transform: rotate(360deg); }
+    }
+  </style>
+</head>
+<body>
+  <div id="status" class="disconnected">
+    <div class="status-indicator disconnected"></div>
+    <span id="status-text">Connecting...</span>
+    <span id="viewport-info" style="margin-left: auto;"></span>
+  </div>
+  <div id="viewport-container">
+    <div id="loading">
+      <div class="spinner"></div>
+      <div>Waiting for stream...</div>
+    </div>
+    <canvas id="viewport"></canvas>
+  </div>
+
+  <script>
+    const canvas = document.getElementById('viewport');
+    const ctx = canvas.getContext('2d');
+    const statusEl = document.getElementById('status');
+    const statusText = document.getElementById('status-text');
+    const viewportInfo = document.getElementById('viewport-info');
+    const loadingEl = document.getElementById('loading');
+
+    let ws = null;
+    let frameCount = 0;
+    let lastFrameTime = Date.now();
+    let fps = 0;
+    let isMouseDown = false;
+    let activeButton = 'left';
+    let activeTouches = new Map();
+
+    function connect() {
+      const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+      const wsUrl = `${protocol}//${window.location.host}`;
+      
+      ws = new WebSocket(wsUrl);
+
+      ws.onopen = () => {
+        statusEl.className = 'connected';
+        statusEl.querySelector('.status-indicator').className = 'status-indicator connected';
+        statusText.textContent = 'Connected';
+        loadingEl.style.display = 'none';
+      };
+
+      ws.onmessage = (event) => {
+        try {
+          const message = JSON.parse(event.data);
+          
+          if (message.type === 'frame') {
+            drawFrame(message);
+          } else if (message.type === 'status') {
+            updateStatus(message);
+          } else if (message.type === 'error') {
+            statusText.textContent = `Error: ${message.message}`;
+            statusEl.className = 'disconnected';
+            statusEl.querySelector('.status-indicator').className = 'status-indicator disconnected';
+          }
+        } catch (e) {
+          console.error('Error parsing message:', e);
+        }
+      };
+
+      ws.onerror = (error) => {
+        console.error('WebSocket error:', error);
+        statusText.textContent = 'Connection error';
+        statusEl.className = 'disconnected';
+        statusEl.querySelector('.status-indicator').className = 'status-indicator disconnected';
+      };
+
+      ws.onclose = () => {
+        statusText.textContent = 'Disconnected';
+        statusEl.className = 'disconnected';
+        statusEl.querySelector('.status-indicator').className = 'status-indicator disconnected';
+        loadingEl.style.display = 'block';
+        
+        setTimeout(connect, 2000);
+      };
+    }
+
+    function drawFrame(message) {
+      const img = new Image();
+      img.onload = () => {
+        canvas.width = img.width;
+        canvas.height = img.height;
+        ctx.drawImage(img, 0, 0);
+        canvas.classList.add('ready');
+        
+        frameCount++;
+        const now = Date.now();
+        if (now - lastFrameTime >= 1000) {
+          fps = frameCount;
+          frameCount = 0;
+          lastFrameTime = now;
+        }
+        
+        updateViewportInfo(img.width, img.height, fps);
+      };
+      const mimeType = message.data.startsWith('iVBORw0K') ? 'image/png' : 'image/jpeg';
+      img.src = `data:${mimeType};base64,${message.data}`;
+    }
+
+    function updateStatus(message) {
+      if (message.viewportWidth && message.viewportHeight) {
+        updateViewportInfo(message.viewportWidth, message.viewportHeight, fps);
+      }
+    }
+
+    function updateViewportInfo(width, height, fps) {
+      viewportInfo.textContent = `${width}×${height} | ${fps} fps`;
+    }
+
+    function getCanvasCoordinates(e) {
+      const rect = canvas.getBoundingClientRect();
+      const scaleX = canvas.width / rect.width;
+      const scaleY = canvas.height / rect.height;
+      
+      const x = Math.round((e.clientX - rect.left) * scaleX);
+      const y = Math.round((e.clientY - rect.top) * scaleY);
+      
+      // Check if click is within canvas bounds
+      if (x < 0 || x > canvas.width || y < 0 || y > canvas.height) {
+        return null;
+      }
+      
+      return { x, y };
+    }
+
+    function getModifiers(e) {
+      return (e.altKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.metaKey ? 4 : 0) | (e.shiftKey ? 8 : 0);
+    }
+
+    function sendMouseEvent(eventType, e, button = 'left') {
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      
+      const coords = getCanvasCoordinates(e);
+      if (!coords) return; // Click outside canvas bounds
+      
+      ws.send(JSON.stringify({
+        type: 'input_mouse',
+        eventType,
+        x: coords.x,
+        y: coords.y,
+        button,
+        clickCount: 1,
+        modifiers: getModifiers(e),
+      }));
+    }
+
+    canvas.addEventListener('mousedown', (e) => {
+      e.preventDefault();
+      isMouseDown = true;
+      activeButton = e.button === 2 ? 'right' : e.button === 1 ? 'middle' : 'left';
+      // Send mouseMoved first to ensure element has hover state
+      sendMouseEvent('mouseMoved', e, 'none');
+      sendMouseEvent('mousePressed', e, activeButton);
+    });
+
+    canvas.addEventListener('mouseup', (e) => {
+      e.preventDefault();
+      const button = e.button === 2 ? 'right' : e.button === 1 ? 'middle' : 'left';
+      sendMouseEvent('mouseReleased', e, button);
+      isMouseDown = false;
+      activeButton = 'left';
+    });
+
+    canvas.addEventListener('mousemove', (e) => {
+      e.preventDefault();
+      if (isMouseDown) {
+        sendMouseEvent('mouseMoved', e, activeButton);
+      }
+    });
+
+    canvas.addEventListener('mouseleave', (e) => {
+      if (isMouseDown) {
+        sendMouseEvent('mouseReleased', e, activeButton);
+        isMouseDown = false;
+        activeButton = 'left';
+      }
+    });
+
+    canvas.addEventListener('contextmenu', (e) => {
+      e.preventDefault();
+    });
+
+    canvas.addEventListener('wheel', (e) => {
+      e.preventDefault();
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      
+      const coords = getCanvasCoordinates(e);
+      let deltaX = e.deltaX;
+      let deltaY = e.deltaY;
+      if (e.deltaMode === 1) {
+        deltaX *= 40;
+        deltaY *= 40;
+      } else if (e.deltaMode === 2) {
+        deltaX *= window.innerHeight;
+        deltaY *= window.innerHeight;
+      }
+      ws.send(JSON.stringify({
+        type: 'input_mouse',
+        eventType: 'mouseWheel',
+        x: coords.x,
+        y: coords.y,
+        deltaX,
+        deltaY,
+        modifiers: getModifiers(e),
+      }));
+    });
+
+    function getTouchCoordinates(touch) {
+      const rect = canvas.getBoundingClientRect();
+      const scaleX = canvas.width / rect.width;
+      const scaleY = canvas.height / rect.height;
+      return {
+        x: Math.round((touch.clientX - rect.left) * scaleX),
+        y: Math.round((touch.clientY - rect.top) * scaleY),
+        id: touch.identifier,
+      };
+    }
+
+    function sendTouchEvent(eventType, touches) {
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      ws.send(JSON.stringify({
+        type: 'input_touch',
+        eventType,
+        touchPoints: touches,
+        modifiers: 0,
+      }));
+    }
+
+    canvas.addEventListener('touchstart', (e) => {
+      e.preventDefault();
+      for (const touch of e.changedTouches) {
+        const coords = getTouchCoordinates(touch);
+        activeTouches.set(touch.identifier, coords);
+      }
+      sendTouchEvent('touchStart', Array.from(activeTouches.values()));
+    });
+
+    canvas.addEventListener('touchend', (e) => {
+      e.preventDefault();
+      for (const touch of e.changedTouches) {
+        activeTouches.delete(touch.identifier);
+      }
+      sendTouchEvent('touchEnd', Array.from(activeTouches.values()));
+    });
+
+    canvas.addEventListener('touchmove', (e) => {
+      e.preventDefault();
+      for (const touch of e.changedTouches) {
+        const coords = getTouchCoordinates(touch);
+        activeTouches.set(touch.identifier, coords);
+      }
+      sendTouchEvent('touchMove', Array.from(activeTouches.values()));
+    });
+
+    canvas.addEventListener('touchcancel', (e) => {
+      e.preventDefault();
+      for (const touch of e.changedTouches) {
+        activeTouches.delete(touch.identifier);
+      }
+      sendTouchEvent('touchCancel', Array.from(activeTouches.values()));
+    });
+
+    document.addEventListener('keydown', (e) => {
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return;
+      
+      e.preventDefault();
+      const modifiers = (e.altKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.metaKey ? 4 : 0) | (e.shiftKey ? 8 : 0);
+      
+      // Send keyDown event
+      ws.send(JSON.stringify({
+        type: 'input_keyboard',
+        eventType: 'keyDown',
+        key: e.key,
+        code: e.code,
+        modifiers,
+      }));
+      
+      // For printable characters, also send a char event
+      if (e.key.length === 1 && !e.ctrlKey && !e.metaKey) {
+        ws.send(JSON.stringify({
+          type: 'input_keyboard',
+          eventType: 'char',
+          text: e.key,
+          modifiers,
+        }));
+      }
+    });
+
+    document.addEventListener('keyup', (e) => {
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return;
+      
+      e.preventDefault();
+      
+      ws.send(JSON.stringify({
+        type: 'input_keyboard',
+        eventType: 'keyUp',
+        key: e.key,
+        code: e.code,
+        modifiers: (e.altKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.metaKey ? 4 : 0) | (e.shiftKey ? 8 : 0),
+      }));
+    });
+
+    connect();
+  </script>
+</body>
+</html>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "surf-cli",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "CLI for AI agents to control Chrome. Zero config, agent-agnostic, battle-tested.",
   "keywords": [
     "chrome",
@@ -49,14 +49,14 @@
     "uninstall:native": "node scripts/uninstall-native-host.cjs"
   },
   "dependencies": {
-    "@google/generative-ai": "^0.21.0",
+    "@google/generative-ai": "^0.24.1",
     "@modelcontextprotocol/sdk": "^1.7.0",
     "buffer": "^6.0.3",
     "crypto-browserify": "^3.12.1",
     "events": "^3.3.0",
     "stream-browserify": "^3.0.0",
     "vite-plugin-node-polyfills": "^0.24.0",
-    "zod": "^3.24.0"
+    "zod": "^4.3.5"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.3.11",
@@ -64,7 +64,7 @@
     "@vitest/coverage-v8": "^4.0.16",
     "@vitest/ui": "^4.0.16",
     "typescript": "^5.7.2",
-    "vite": "^6.0.0",
+    "vite": "^7.3.1",
     "vitest": "^4.0.16"
   }
 }