verybot 0.1.3

package/dist/computer/browser/snapshot.js

@@ -0,0 +1,242 @@
+/**
+ * Parses Playwright's ariaSnapshot() output,
+ * assigns refs (e1, e2…) to interactive/content elements, and returns an
+ * annotated snapshot string plus a RoleRefMap for resolving refs back to locators.
+ */
+const INTERACTIVE_ROLES = new Set([
+    "button",
+    "link",
+    "textbox",
+    "checkbox",
+    "radio",
+    "combobox",
+    "listbox",
+    "menuitem",
+    "menuitemcheckbox",
+    "menuitemradio",
+    "option",
+    "searchbox",
+    "slider",
+    "spinbutton",
+    "switch",
+    "tab",
+    "treeitem",
+]);
+const CONTENT_ROLES = new Set([
+    "heading",
+    "cell",
+    "gridcell",
+    "columnheader",
+    "rowheader",
+    "listitem",
+    "article",
+    "region",
+    "main",
+    "navigation",
+]);
+const STRUCTURAL_ROLES = new Set([
+    "generic",
+    "group",
+    "list",
+    "table",
+    "row",
+    "rowgroup",
+    "grid",
+    "treegrid",
+    "menu",
+    "menubar",
+    "toolbar",
+    "tablist",
+    "tree",
+    "directory",
+    "document",
+    "application",
+    "presentation",
+    "none",
+]);
+function getIndentLevel(line) {
+    const match = line.match(/^(\s*)/);
+    return match ? Math.floor(match[1].length / 2) : 0;
+}
+function createRoleNameTracker() {
+    const counts = new Map();
+    const refsByKey = new Map();
+    return {
+        counts,
+        refsByKey,
+        getKey(role, name) {
+            return `${role}:${name ?? ""}`;
+        },
+        getNextIndex(role, name) {
+            const key = this.getKey(role, name);
+            const current = counts.get(key) ?? 0;
+            counts.set(key, current + 1);
+            return current;
+        },
+        trackRef(role, name, ref) {
+            const key = this.getKey(role, name);
+            const list = refsByKey.get(key) ?? [];
+            list.push(ref);
+            refsByKey.set(key, list);
+        },
+        getDuplicateKeys() {
+            const out = new Set();
+            for (const [key, refs] of refsByKey) {
+                if (refs.length > 1) {
+                    out.add(key);
+                }
+            }
+            return out;
+        },
+    };
+}
+function removeNthFromNonDuplicates(refs, tracker) {
+    const duplicates = tracker.getDuplicateKeys();
+    for (const [ref, data] of Object.entries(refs)) {
+        const key = tracker.getKey(data.role, data.name);
+        if (!duplicates.has(key)) {
+            delete refs[ref]?.nth;
+        }
+    }
+}
+function compactTree(tree) {
+    const lines = tree.split("\n");
+    const result = [];
+    for (let i = 0; i < lines.length; i += 1) {
+        const line = lines[i];
+        if (line.includes("[ref=")) {
+            result.push(line);
+            continue;
+        }
+        if (line.includes(":") && !line.trimEnd().endsWith(":")) {
+            result.push(line);
+            continue;
+        }
+        const currentIndent = getIndentLevel(line);
+        let hasRelevantChildren = false;
+        for (let j = i + 1; j < lines.length; j += 1) {
+            const childIndent = getIndentLevel(lines[j]);
+            if (childIndent <= currentIndent)
+                break;
+            if (lines[j]?.includes("[ref=")) {
+                hasRelevantChildren = true;
+                break;
+            }
+        }
+        if (hasRelevantChildren) {
+            result.push(line);
+        }
+    }
+    return result.join("\n");
+}
+function processLine(line, refs, options, tracker, nextRef) {
+    const depth = getIndentLevel(line);
+    if (options.maxDepth !== undefined && depth > options.maxDepth) {
+        return null;
+    }
+    const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
+    if (!match) {
+        return options.interactive ? null : line;
+    }
+    const [, prefix, roleRaw, name, suffix] = match;
+    if (roleRaw.startsWith("/")) {
+        return options.interactive ? null : line;
+    }
+    const role = roleRaw.toLowerCase();
+    const isInteractive = INTERACTIVE_ROLES.has(role);
+    const isContent = CONTENT_ROLES.has(role);
+    const isStructural = STRUCTURAL_ROLES.has(role);
+    if (options.interactive && !isInteractive)
+        return null;
+    if (options.compact && isStructural && !name)
+        return null;
+    const shouldHaveRef = isInteractive || (isContent && name);
+    if (!shouldHaveRef)
+        return line;
+    const ref = nextRef();
+    const nth = tracker.getNextIndex(role, name);
+    tracker.trackRef(role, name, ref);
+    refs[ref] = { role, name, nth };
+    let enhanced = `${prefix}${roleRaw}`;
+    if (name)
+        enhanced += ` "${name}"`;
+    enhanced += ` [ref=${ref}]`;
+    if (nth > 0)
+        enhanced += ` [nth=${nth}]`;
+    if (suffix)
+        enhanced += suffix;
+    return enhanced;
+}
+/** Validate and normalize a ref string like "e5", "@e5", or "ref=e5". Returns null if invalid. */
+export function parseRoleRef(raw) {
+    const trimmed = raw.trim();
+    if (!trimmed)
+        return null;
+    const normalized = trimmed.startsWith("@")
+        ? trimmed.slice(1)
+        : trimmed.startsWith("ref=")
+            ? trimmed.slice(4)
+            : trimmed;
+    return /^e\d+$/.test(normalized) ? normalized : null;
+}
+/**
+ * Parse Playwright's `ariaSnapshot()` text, assign refs to interactive/content
+ * elements, and return the annotated snapshot + a map from ref → role info.
+ */
+export function buildRoleSnapshotFromAriaSnapshot(ariaSnapshot, options = {}) {
+    const lines = ariaSnapshot.split("\n");
+    const refs = {};
+    const tracker = createRoleNameTracker();
+    let counter = 0;
+    const nextRef = () => {
+        counter += 1;
+        return `e${counter}`;
+    };
+    if (options.interactive) {
+        const result = [];
+        for (const line of lines) {
+            const depth = getIndentLevel(line);
+            if (options.maxDepth !== undefined && depth > options.maxDepth)
+                continue;
+            const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
+            if (!match)
+                continue;
+            const [, , roleRaw, name, suffix] = match;
+            if (roleRaw.startsWith("/"))
+                continue;
+            const role = roleRaw.toLowerCase();
+            if (!INTERACTIVE_ROLES.has(role))
+                continue;
+            const ref = nextRef();
+            const nth = tracker.getNextIndex(role, name);
+            tracker.trackRef(role, name, ref);
+            refs[ref] = { role, name, nth };
+            let enhanced = `- ${roleRaw}`;
+            if (name)
+                enhanced += ` "${name}"`;
+            enhanced += ` [ref=${ref}]`;
+            if (nth > 0)
+                enhanced += ` [nth=${nth}]`;
+            if (suffix.includes("["))
+                enhanced += suffix;
+            result.push(enhanced);
+        }
+        removeNthFromNonDuplicates(refs, tracker);
+        return {
+            snapshot: result.join("\n") || "(no interactive elements)",
+            refs,
+        };
+    }
+    const result = [];
+    for (const line of lines) {
+        const processed = processLine(line, refs, options, tracker, nextRef);
+        if (processed !== null)
+            result.push(processed);
+    }
+    removeNthFromNonDuplicates(refs, tracker);
+    const tree = result.join("\n") || "(empty)";
+    return {
+        snapshot: options.compact ? compactTree(tree) : tree,
+        refs,
+    };
+}

package/dist/computer/browser/tools.d.ts

@@ -0,0 +1,5 @@
+import { type ToolSet } from "ai";
+import type { BrowserManager } from "./manager.js";
+/** Tool names exported for delegate.ts to strip/replace per-worker instances. */
+export declare const BROWSER_TOOL_NAMES: string[];
+export declare function createBrowserTools(browser: BrowserManager): ToolSet;

package/dist/computer/browser/tools.js

@@ -0,0 +1,167 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { parseRoleRef } from "./snapshot.js";
+import * as actions from "./actions.js";
+/** Tool names exported for delegate.ts to strip/replace per-worker instances. */
+export const BROWSER_TOOL_NAMES = [
+    "browser_navigate",
+    "browser_snapshot",
+    "browser_click",
+    "browser_type",
+    "browser_press_key",
+    "browser_screenshot",
+    "browser_close",
+    "browser_switch_profile",
+];
+export function createBrowserTools(browser) {
+    return {
+        browser_navigate: tool({
+            description: "Navigate the browser to a URL. Launches the browser if not already open. " +
+                "Returns the page title, URL, and an accessibility snapshot with element refs (e1, e2…).",
+            inputSchema: z.object({
+                url: z.string().url().describe("The URL to navigate to"),
+            }),
+            execute: async ({ url }) => {
+                try {
+                    return await actions.navigate(browser, url);
+                }
+                catch (err) {
+                    return actions.toAIFriendlyError(err);
+                }
+            },
+        }),
+        browser_snapshot: tool({
+            description: "Take an accessibility snapshot of the current page. Returns the page structure " +
+                "with interactive elements labeled with refs (e1, e2…). Use these refs with " +
+                "browser_click and browser_type. Always take a fresh snapshot after page changes.",
+            inputSchema: z.object({}),
+            execute: async () => {
+                try {
+                    const snapshot = await actions.takeSnapshot(browser);
+                    return actions.withProfilePrefix(browser, snapshot);
+                }
+                catch (err) {
+                    return actions.toAIFriendlyError(err);
+                }
+            },
+        }),
+        browser_click: tool({
+            description: "Click an element by its ref from the latest snapshot (e.g. 'e5'). " +
+                "Automatically takes a new snapshot after clicking so you can see the result.",
+            inputSchema: z.object({
+                ref: z.string().describe("Element ref from snapshot, e.g. 'e5'"),
+            }),
+            execute: async ({ ref }) => {
+                const parsed = parseRoleRef(ref);
+                if (!parsed)
+                    return `Invalid ref "${ref}". Use a ref like "e5" from the latest snapshot.`;
+                try {
+                    return await actions.click(browser, parsed);
+                }
+                catch (err) {
+                    return actions.toAIFriendlyError(err);
+                }
+            },
+        }),
+        browser_type: tool({
+            description: "Type text into an input field by its ref from the latest snapshot. " +
+                "Clears existing content first (like a user selecting all and typing). " +
+                "Set submit=true to press Enter after typing (e.g. for search forms).",
+            inputSchema: z.object({
+                ref: z.string().describe("Element ref from snapshot, e.g. 'e3'"),
+                text: z.string().describe("Text to type into the field"),
+                submit: z.boolean().optional().describe("Press Enter after typing (default: false)"),
+            }),
+            execute: async ({ ref, text, submit }) => {
+                const parsed = parseRoleRef(ref);
+                if (!parsed)
+                    return `Invalid ref "${ref}". Use a ref like "e3" from the latest snapshot.`;
+                try {
+                    return await actions.type(browser, parsed, text, submit);
+                }
+                catch (err) {
+                    return actions.toAIFriendlyError(err);
+                }
+            },
+        }),
+        browser_press_key: tool({
+            description: "Press a keyboard key or combination. Examples: 'Enter', 'Escape', " +
+                "'ArrowDown', 'Control+A', 'Meta+C'. Use for keyboard shortcuts and navigation.",
+            inputSchema: z.object({
+                key: z.string().describe("Key or combination to press, e.g. 'Enter' or 'Control+A'"),
+            }),
+            execute: async ({ key }) => {
+                try {
+                    return await actions.pressKey(browser, key);
+                }
+                catch (err) {
+                    return actions.toAIFriendlyError(err);
+                }
+            },
+        }),
+        browser_screenshot: tool({
+            description: "Take a screenshot of the current page for visual inspection. " +
+                "Returns an image. Use when you need to see the visual layout, " +
+                "verify visual changes, or debug rendering issues.",
+            inputSchema: z.object({
+                fullPage: z.boolean().optional().describe("Capture the full scrollable page (default: false, viewport only)"),
+            }),
+            execute: async ({ fullPage }) => {
+                try {
+                    const { base64, mediaType } = await actions.takeScreenshot(browser, fullPage);
+                    return { base64, mediaType };
+                }
+                catch (err) {
+                    return actions.toAIFriendlyError(err);
+                }
+            },
+            toModelOutput({ output }) {
+                if (typeof output === "object" && output !== null && "base64" in output) {
+                    const { base64, mediaType } = output;
+                    return {
+                        type: "content",
+                        value: [
+                            {
+                                type: "image-data",
+                                data: base64,
+                                mediaType,
+                            },
+                        ],
+                    };
+                }
+                return { type: "text", value: String(output) };
+            },
+        }),
+        browser_switch_profile: tool({
+            description: "Switch the browser to a different named profile. Each profile has its own " +
+                "cookies, login state, and extensions. Closes the current browser — it will " +
+                "relaunch with the new profile on the next navigation. " +
+                "Profile names: letters, digits, and hyphens only (e.g. 'work', 'personal').",
+            inputSchema: z.object({
+                profile: z.string().describe("Profile name to switch to, e.g. 'work' or 'personal'"),
+            }),
+            execute: async ({ profile }) => {
+                try {
+                    return await actions.switchProfile(browser, profile);
+                }
+                catch (err) {
+                    return actions.toAIFriendlyError(err);
+                }
+            },
+        }),
+        browser_close: tool({
+            description: "Close the browser. Use when done with browser tasks.",
+            inputSchema: z.object({}),
+            execute: async () => {
+                try {
+                    const profile = browser.getActiveProfile();
+                    await browser.close();
+                    return `[profile: ${profile}] Browser closed.`;
+                }
+                catch (err) {
+                    return actions.toAIFriendlyError(err);
+                }
+            },
+        }),
+    };
+}

package/dist/computer/desktop/adapter.d.ts

@@ -0,0 +1,25 @@
+export interface ScreenInfo {
+    /** Screenshot pixel width (retina resolution). */
+    widthPx: number;
+    /** Screenshot pixel height (retina resolution). */
+    heightPx: number;
+    /** Retina scale factor (1 for non-retina, 2 for retina). */
+    scaleFactor: number;
+}
+export interface DesktopAdapter {
+    getScreenInfo(): ScreenInfo;
+    screenshot(): Promise<Buffer>;
+    click(x: number, y: number): Promise<void>;
+    doubleClick(x: number, y: number): Promise<void>;
+    rightClick(x: number, y: number): Promise<void>;
+    middleClick(x: number, y: number): Promise<void>;
+    tripleClick(x: number, y: number): Promise<void>;
+    mouseMove(x: number, y: number): Promise<void>;
+    type(text: string): Promise<void>;
+    /** Press a key or key combination (xdotool syntax, e.g. "Return", "ctrl+s"). */
+    key(combo: string): Promise<void>;
+    scroll(x: number, y: number, direction: "up" | "down" | "left" | "right", amount: number): Promise<void>;
+    getCursorPosition(): Promise<[number, number]>;
+    dragTo(startX: number, startY: number, endX: number, endY: number): Promise<void>;
+}
+export declare function createDesktopAdapter(): Promise<DesktopAdapter>;

package/dist/computer/desktop/adapter.js

@@ -0,0 +1,11 @@
+export async function createDesktopAdapter() {
+    switch (process.platform) {
+        case "darwin": {
+            const { MacOSAdapter } = await import("./macos.js");
+            return new MacOSAdapter();
+        }
+        default:
+            throw new Error(`Desktop control is not yet supported on ${process.platform}. ` +
+                `Currently supported: macOS (darwin).`);
+    }
+}

package/dist/computer/desktop/macos.d.ts

@@ -0,0 +1,24 @@
+import type { DesktopAdapter, ScreenInfo } from "./adapter.js";
+export declare class MacOSAdapter implements DesktopAdapter {
+    private screenInfo;
+    constructor();
+    getScreenInfo(): ScreenInfo;
+    screenshot(): Promise<Buffer>;
+    click(x: number, y: number): Promise<void>;
+    doubleClick(x: number, y: number): Promise<void>;
+    rightClick(x: number, y: number): Promise<void>;
+    middleClick(x: number, y: number): Promise<void>;
+    tripleClick(x: number, y: number): Promise<void>;
+    mouseMove(x: number, y: number): Promise<void>;
+    type(text: string): Promise<void>;
+    key(combo: string): Promise<void>;
+    scroll(x: number, y: number, direction: "up" | "down" | "left" | "right", amount: number): Promise<void>;
+    getCursorPosition(): Promise<[number, number]>;
+    dragTo(startX: number, startY: number, endX: number, endY: number): Promise<void>;
+    private checkDependencies;
+    private detectScreenInfo;
+    /** Convert pixel coordinates to logical (point) coordinates for cliclick. */
+    private toLogical;
+    /** Run cliclick with the given action arguments. */
+    private cliclick;
+}