pi-browser-harness 0.1.0

package/src/index.ts

@@ -0,0 +1,223 @@
+/**
+ * pi-browser-harness — browser control extension for pi.
+ *
+ * Gives pi agents full control of a real Chrome browser via CDP.
+ * Spawns the browser-harness daemon, registers browser_* tools,
+ * and injects browser control guidance into the system prompt.
+ *
+ * Install:
+ *   pi install npm:pi-browser-harness
+ *   # or copy to .pi/extensions/pi-browser-harness/
+ *
+ * Commands:
+ *   /browser-setup          — guided setup wizard
+ *   /browser-status         — show daemon status and current page
+ *   /browser-reload-daemon  — restart the daemon
+ *
+ * Flags:
+ *   --browser-namespace <name>  — override BU_NAME (default: auto)
+ *   --browser-debug-clicks       — enable debug click overlay
+ */
+
+import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
+import { BrowserDaemon } from "./daemon";
+import { getBrowserSystemPrompt } from "./prompt";
+import { registerRenderers } from "./renderers";
+import { registerSetupCommand } from "./setup";
+import { type BrowserState, defaultState, persistState, restoreState } from "./state";
+import { cleanupTempDirs, registerTools } from "./tools";
+
+export default function browserHarnessExtension(pi: ExtensionAPI) {
+  // ── Resolve namespace ──────────────────────────────────────────────────────
+  const flagNs = pi.getFlag("browser-namespace") as string | undefined;
+  const namespace = flagNs || `pi-${Math.random().toString(36).slice(2, 10)}`;
+
+  // ── State ──────────────────────────────────────────────────────────────────
+  let state: BrowserState = defaultState(namespace);
+  let daemon: BrowserDaemon | null = null;
+
+  // ── Tool initialization flag ───────────────────────────────────────────────
+  let toolsRegistered = false;
+
+  // ═══════════════════════════════════════════════════════════════════════════
+  // Flags
+  // ═══════════════════════════════════════════════════════════════════════════
+  pi.registerFlag("browser-namespace", {
+    description: "Browser daemon namespace (BU_NAME). Default: auto-generated",
+    type: "string",
+  });
+  pi.registerFlag("browser-debug-clicks", {
+    description: "Enable debug click overlay (saves annotated screenshots to /tmp)",
+    type: "boolean",
+    default: false,
+  });
+
+  // ═══════════════════════════════════════════════════════════════════════════
+  // Commands
+  // ═══════════════════════════════════════════════════════════════════════════
+  pi.registerCommand("browser-status", {
+    description: "Show browser connection status and current page",
+    handler: async (_args, ctx) => {
+      if (!daemon) {
+        ctx.ui.notify("Browser daemon not started. Run /browser-setup first.", "warning");
+        return;
+      }
+
+      const status = daemon.getStatus();
+      const lines = [
+        `Browser: ${status.alive ? "🟢 Connected" : "🔴 Disconnected"}`,
+        `Session: ${status.sessionId || "none"}`,
+      ];
+
+      if (status.remoteBrowserId) {
+        lines.push(`Browser ID: ${status.remoteBrowserId}`);
+      }
+
+      // Try to get current page info
+      if (status.alive) {
+        try {
+          const info = await daemon.getPageInfo();
+          if ("dialog" in info) {
+            lines.push(`\n⚠️  Dialog open: ${info.dialog.type} — "${info.dialog.message}"`);
+          } else {
+            lines.push(`\nCurrent Page:`);
+            lines.push(`  URL: ${info.url}`);
+            lines.push(`  Title: ${info.title}`);
+            lines.push(`  Viewport: ${info.width}x${info.height}`);
+          }
+        } catch {
+          lines.push("\n(Could not read page info)");
+        }
+      }
+
+      ctx.ui.notify(lines.join("\n"), "info");
+    },
+  });
+
+  pi.registerCommand("browser-reload-daemon", {
+    description: "Restart the browser daemon",
+    handler: async (_args, ctx) => {
+      if (!daemon) {
+        ctx.ui.notify("Browser daemon not started.", "warning");
+        return;
+      }
+
+      ctx.ui.notify("Restarting browser daemon...", "info");
+      try {
+        await daemon.stop();
+        await daemon.start();
+        ctx.ui.notify("Browser daemon restarted ✓", "info");
+      } catch (err) {
+        ctx.ui.notify(
+          `Restart failed: ${err instanceof Error ? err.message : String(err)}`,
+          "error",
+        );
+      }
+    },
+  });
+
+  // ═══════════════════════════════════════════════════════════════════════════
+  // Session lifecycle
+  // ═══════════════════════════════════════════════════════════════════════════
+  pi.on("session_start", async (_event, ctx) => {
+    // Restore persisted state (preserving the current namespace from flag)
+    state = restoreState(ctx, state.namespace);
+
+    // Create daemon instance
+    daemon = new BrowserDaemon(state.namespace);
+
+    // Attempt to start daemon (will fail gracefully if Chrome not connected)
+    try {
+      await daemon.start();
+    } catch {
+      // Don't block — user can run /browser-setup later
+    }
+
+    // Register tools (once)
+    if (!toolsRegistered && daemon) {
+      registerTools(pi, daemon);
+      toolsRegistered = true;
+    }
+
+    // Register renderers
+    registerRenderers(pi);
+
+    // Setup command (always available)
+    if (daemon) {
+      registerSetupCommand(pi, daemon);
+    }
+
+    // Update status
+    if (daemon.getStatus().alive) {
+      ctx.ui.setStatus("browser", "🟢 Browser connected");
+    } else {
+      ctx.ui.setStatus("browser", "🔴 Browser — run /browser-setup");
+    }
+  });
+
+  pi.on("session_shutdown", async (_event) => {
+    // Persist current state
+    if (state) {
+      persistState(pi, state);
+    }
+
+    // Stop daemon
+    if (daemon) {
+      try {
+        await daemon.stop();
+      } catch {
+        // best-effort
+      }
+      daemon = null;
+    }
+
+    toolsRegistered = false;
+
+    // Clean up temp files from truncated tool outputs
+    await cleanupTempDirs();
+  });
+
+  // Restore state on tree navigation
+  pi.on("session_tree", async (_event, ctx) => {
+    state = restoreState(ctx, daemon?.namespace);
+    if (daemon) {
+      persistState(pi, state);
+    }
+  });
+
+  // ═══════════════════════════════════════════════════════════════════════════
+  // System prompt injection
+  // ═══════════════════════════════════════════════════════════════════════════
+  pi.on("before_agent_start", async (event) => {
+    if (!daemon || !daemon.getStatus().alive) {
+      // Browser not connected — inject minimal note
+      return {
+        systemPrompt:
+          event.systemPrompt +
+          `\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser daemon is not connected. ` +
+          `Run /browser-setup to connect to Chrome, or /browser-status to check.`,
+      };
+    }
+
+    return {
+      systemPrompt: event.systemPrompt + getBrowserSystemPrompt(),
+    };
+  });
+
+  // ═══════════════════════════════════════════════════════════════════════════
+  // Tool result hook: track tab history
+  // ═══════════════════════════════════════════════════════════════════════════
+  pi.on("tool_result", async (event) => {
+    if (event.toolName === "browser_new_tab" || event.toolName === "browser_navigate") {
+      // Track new tabs in history
+      const details = event.details as { targetId?: string } | undefined;
+      if (details?.targetId) {
+        state = {
+          ...state,
+          tabHistory: [details.targetId, ...state.tabHistory.filter((id) => id !== details.targetId)].slice(0, 20),
+        };
+        persistState(pi, state);
+      }
+    }
+  });
+}

package/src/prompt.ts

@@ -0,0 +1,106 @@
+/**
+ * System prompt guidance for browser usage.
+ *
+ * Injects a compact browser control reference into the pi system prompt.
+ * Per-tool guidelines are injected via promptGuidelines on each tool registration.
+ * This prompt provides only the cross-cutting patterns and tool reference table.
+ */
+
+export function getBrowserSystemPrompt(): string {
+  return `
+## Browser Control
+
+You have full control of a real Chrome browser via \`browser_*\` tools.
+The browser connects to the user's running Chrome (not a headless instance).
+
+**These tools coexist with all other standard pi tools** — read, bash, edit,
+write, and any other built-in tools are still available. Use browser_* tools
+to interact with pages visually, and other tools alongside them.
+
+### Browser-Specific Tools
+
+| Tool | Purpose |
+|------|---------|
+| browser_screenshot | Capture a PNG screenshot of the current page |
+| browser_click | Click at viewport coordinates |
+| browser_type | Type text into the focused element |
+| browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) |
+| browser_scroll | Scroll the page at coordinates |
+| browser_navigate | Navigate to a URL |
+| browser_new_tab | Open a new tab, optionally navigate |
+| browser_open_urls | Open multiple URLs in new tabs (parallel) |
+| browser_go_back / browser_go_forward / browser_reload | History navigation |
+| browser_page_info | Get page URL, title, viewport, scroll position, or dialog info |
+| browser_list_tabs | List all open browser tabs |
+| browser_current_tab | Get current tab info |
+| browser_switch_tab | Switch to a different tab by targetId |
+| browser_execute_js | Execute JavaScript and return the result |
+| browser_http_get | Direct HTTP GET (outside browser, for APIs) |
+| browser_wait | Wait N seconds |
+| browser_wait_for_load | Wait for document.readyState === 'complete' |
+| browser_handle_dialog | Accept or dismiss a JS dialog |
+| browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) |
+
+### Temporary Scripts
+
+When the built-in tools aren't enough for a multi-step workflow, write a temporary
+script to disk and execute it with browser_run_script. The script runs in the
+harness process with direct access to the browser daemon and Node.js APIs.
+
+\`\`\`
+write("/tmp/scrape-pages.js", \`
+  const results = [];
+  for (const url of params.urls) {
+    await daemon.cdp("Page.navigate", { url });
+    await new Promise(r => setTimeout(r, 2000));
+    const data = await daemon.evaluateJS("document.title");
+    results.push({ url, title: data });
+  }
+  return { content: [{ type: "text", text: JSON.stringify(results, null, 2) }] };
+\`)
+browser_run_script("/tmp/scrape-pages.js", { urls: [...] })
+\`\`\`
+
+Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout.
+
+### Common Patterns
+
+**Navigation:**
+\`\`\`
+browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot()
+\`\`\`
+
+**Form filling:**
+\`\`\`
+browser_screenshot() → find input coordinates → browser_click(x, y)
+→ browser_type("text") → browser_press_key("Tab") → browser_screenshot()
+\`\`\`
+
+**Data extraction:**
+\`\`\`
+browser_execute_js("document.querySelector('.price').innerText")
+// or for APIs:
+browser_http_get("https://api.example.com/data")
+\`\`\`
+
+**Scrolling:**
+\`\`\`
+browser_screenshot() → browser_scroll({ deltaY: -500 }) → browser_screenshot()
+\`\`\`
+
+**Research Workflow (search + browser):**
+\`\`\`
+browser_navigate("https://google.com/search?q=...") → search engine in a tab
+browser_open_urls(urls: ["url1", "url2", ...]) → open result pages in parallel tabs
+browser_list_tabs() → see all open tabs with targetIds
+browser_switch_tab(targetId: "...") → switch to a tab
+browser_screenshot() → visually inspect the page
+browser_execute_js("document.querySelector('.main').innerText") → extract content
+\`\`\`
+
+**Temporary Scripts (extending the harness):**
+\`\`\`
+write("/tmp/extract.js", "...script with daemon access...") → browser_run_script("/tmp/extract.js")
+\`\`\`
+`;
+}

package/src/protocol.ts

@@ -0,0 +1,77 @@
+/**
+ * Protocol types for pi-browser-harness.
+ *
+ * Defines the shape of CDP communication and browser state.
+ * No socket transport — the daemon talks CDP directly over WebSocket.
+ */
+
+// ── CDP Event ────────────────────────────────────────────────────────────────
+
+export interface CDPEvent {
+  method: string;
+  params: unknown;
+  session_id?: string;
+}
+
+// ── Dialog ───────────────────────────────────────────────────────────────────
+
+export interface DialogInfo {
+  type: "alert" | "confirm" | "prompt" | "beforeunload";
+  message: string;
+  defaultPrompt?: string;
+}
+
+// ── Tab ──────────────────────────────────────────────────────────────────────
+
+export interface TabInfo {
+  targetId: string;
+  title: string;
+  url: string;
+}
+
+// ── Page State ───────────────────────────────────────────────────────────────
+
+export interface PageInfo {
+  url: string;
+  title: string;
+  width: number;
+  height: number;
+  scrollX: number;
+  scrollY: number;
+  pageWidth: number;
+  pageHeight: number;
+}
+
+export type PageInfoResult = PageInfo | { dialog: DialogInfo };
+
+// ── Daemon Status ────────────────────────────────────────────────────────────
+
+export interface DaemonStatus {
+  alive: boolean;
+  sessionId: string | null;
+  pid: number | null; // deprecated — always null (no subprocess)
+  namespace: string;
+  socketPath: string; // deprecated — always "" (no socket)
+  remoteBrowserId?: string;
+}
+
+// ── Remote Config ────────────────────────────────────────────────────────────
+
+export interface RemoteConfig {
+  cdpUrl: string;
+  browserId: string;
+}
+
+// ── Internal URL prefixes to filter ──────────────────────────────────────────
+
+const INTERNAL_PREFIXES = [
+  "chrome://",
+  "chrome-untrusted://",
+  "devtools://",
+  "chrome-extension://",
+  "about:",
+];
+
+export function isInternalUrl(url: string): boolean {
+  return INTERNAL_PREFIXES.some((p) => url.startsWith(p));
+}

package/src/renderers.ts

@@ -0,0 +1,21 @@
+/**
+ * TUI renderers for pi-browser-harness.
+ *
+ * Registers custom message renderers and tool renderCall/renderResult
+ * for browser tools.
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { Text } from "@mariozechner/pi-tui";
+
+export function registerRenderers(pi: ExtensionAPI): void {
+  // Register a message renderer for screenshot notifications
+  pi.registerMessageRenderer("browser-screenshot", (message, _options, theme) => {
+    const path = (message as { details?: { path?: string } }).details?.path || "unknown";
+    return new Text(
+      theme.fg("accent", `📸 Screenshot: ${path}`),
+      0,
+      0,
+    );
+  });
+}

package/src/setup.ts

@@ -0,0 +1,97 @@
+/**
+ * Setup command — verifies Chrome is running with remote debugging
+ * and connects the pi agent to it.
+ */
+
+import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
+import { execSync } from "node:child_process";
+import type { BrowserDaemon } from "./daemon";
+
+export function registerSetupCommand(pi: ExtensionAPI, daemon: BrowserDaemon): void {
+  pi.registerCommand("browser-setup", {
+    description: "Connect pi to your Chrome browser",
+    handler: async (_args, ctx) => {
+      await runSetup(ctx, daemon);
+    },
+  });
+}
+
+async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<void> {
+  ctx.ui.notify("browser setup: checking Chrome...", "info");
+
+  // Step 1: Check Chrome is running
+  const chromeRunning = checkChromeRunning();
+  if (!chromeRunning) {
+    ctx.ui.notify(
+      "Chrome/Chromium/Edge not detected. Please start your browser and retry /browser-setup.",
+      "error",
+    );
+    return;
+  }
+  ctx.ui.notify("Chrome is running ✓", "info");
+
+  // Step 2: Try to connect to Chrome DevTools
+  ctx.ui.notify("Connecting to Chrome DevTools...", "info");
+  try {
+    await daemon.start();
+    ctx.ui.notify("Connected to Chrome ✓", "info");
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    const lower = msg.toLowerCase();
+
+    if (
+      lower.includes("devtoolsactiveport") ||
+      lower.includes("remote debugging") ||
+      lower.includes("econnrefused") ||
+      lower.includes("cannot reach chrome devtools")
+    ) {
+      ctx.ui.notify(
+        "Chrome remote debugging needs to be enabled.\n\n" +
+          "Open chrome://inspect/#remote-debugging in your browser, tick the\n" +
+          "\"Discover network targets\" / Allow checkbox, then run /browser-setup again.\n\n" +
+          "Or set BU_CDP_WS to a remote browser WebSocket URL.",
+        "warning",
+      );
+      return;
+    }
+
+    ctx.ui.notify(`Connection failed: ${msg}`, "error");
+    return;
+  }
+
+  // Step 3: Verify with test navigation
+  ctx.ui.notify("Testing browser control...", "info");
+  try {
+    await daemon.newTab("https://github.com");
+    const info = await daemon.getPageInfo();
+    if ("dialog" in info) {
+      await daemon.cdp("Page.handleJavaScriptDialog", { accept: true });
+    }
+    ctx.ui.notify(
+      `Browser control verified ✓\nNavigated to: ${"url" in info ? info.url : "github.com"}`,
+      "info",
+    );
+  } catch (err) {
+    ctx.ui.notify(
+      `Browser connected but test navigation failed: ${err instanceof Error ? err.message : String(err)}`,
+      "warning",
+    );
+  }
+}
+
+function checkChromeRunning(): boolean {
+  try {
+    if (process.platform === "darwin" || process.platform === "linux") {
+      const out = execSync("ps -A -o comm=", { timeout: 5000 }).toString().toLowerCase();
+      return ["google chrome", "chrome", "chromium", "microsoft edge", "msedge"].some(
+        (n) => out.includes(n),
+      );
+    } else if (process.platform === "win32") {
+      const out = execSync("tasklist", { timeout: 5000 }).toString().toLowerCase();
+      return ["chrome.exe", "msedge.exe"].some((n) => out.includes(n));
+    }
+  } catch {
+    // best-effort
+  }
+  return false;
+}

package/src/state.ts

@@ -0,0 +1,77 @@
+/**
+ * Session persistence for pi-browser-harness.
+ *
+ * Persists daemon namespace, tab history, and preferences across
+ * session reloads and branch navigation.
+ */
+
+import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
+
+// ── Types ────────────────────────────────────────────────────────────────────
+
+export interface BrowserState {
+  /** Daemon namespace (BU_NAME) for this session */
+  namespace: string;
+  /** Recently active tab target IDs */
+  tabHistory: string[];
+  /** Cloud browser ID (for remote sessions) */
+  remoteBrowserId?: string;
+  /** Screenshot output directory */
+  screenshotDir: string;
+  /** Whether debug click overlay is enabled */
+  debugClicks: boolean;
+}
+
+// ── Defaults ─────────────────────────────────────────────────────────────────
+
+export function defaultState(namespace = "default"): BrowserState {
+  return {
+    namespace,
+    tabHistory: [],
+    screenshotDir: "",
+    debugClicks: false,
+  };
+}
+
+// ── Persist ──────────────────────────────────────────────────────────────────
+
+export function persistState(pi: ExtensionAPI, state: BrowserState): void {
+  pi.appendEntry<BrowserState>("browser-harness-state", state);
+}
+
+// ── Restore ──────────────────────────────────────────────────────────────────
+
+/**
+ * Find the last browser-harness-state entry in the current branch
+ * and return the restored state, merged with defaults.
+ */
+export function restoreState(ctx: ExtensionContext, currentNamespace?: string): BrowserState {
+  const branchEntries = ctx.sessionManager.getBranch();
+  const defaults = defaultState(currentNamespace);
+
+  // Walk branch from newest to oldest to find last persisted state
+  for (let i = branchEntries.length - 1; i >= 0; i--) {
+    const entry = branchEntries[i];
+    if (entry.type === "custom" && entry.customType === "browser-harness-state") {
+      const data = entry.data as Partial<BrowserState> | undefined;
+      if (data) {
+        // Merge persisted data onto defaults, but always keep the current
+        // namespace (which comes from the --browser-namespace flag or auto-gen).
+        return { ...defaults, ...data, namespace: currentNamespace || data.namespace || defaults.namespace };
+      }
+    }
+  }
+
+  return defaults;
+}
+
+// ── Merge helpers ────────────────────────────────────────────────────────────
+
+export function addToTabHistory(state: BrowserState, targetId: string): BrowserState {
+  const filtered = state.tabHistory.filter((id) => id !== targetId);
+  return { ...state, tabHistory: [targetId, ...filtered].slice(0, 20) };
+}
+
+export function setRemoteBrowser(state: BrowserState, browserId: string): BrowserState {
+  return { ...state, remoteBrowserId: browserId };
+}