speqs 0.5.1 → 0.7.0

package/dist/lib/local-sim/actions.js

@@ -0,0 +1,379 @@
+/**
+ * Action executor — resolves elements and executes Playwright actions.
+ *
+ * Resolution strategy:
+ * 1. CDP node resolution (using node_id from tree data)
+ * 2. Playwright locator fallback (using element_name + element_type)
+ * 3. Coordinate fallback (if returned by backend)
+ */
+import { resolveNodeToBoundingBox } from "./browser.js";
+import { isDebugEnabled } from "./debug.js";
+// --- ARIA role → Playwright role mapping ---
+const ELEMENT_TYPE_TO_ROLE = {
+    BUTTON: "button",
+    TEXT_INPUT: "input",
+    SELECT: "combobox",
+    LINK: "link",
+    CHECKBOX: "checkbox",
+    RADIO: "radio",
+    HEADING: "heading",
+    TAB: "tab",
+    MENU_ITEM: "menuitem",
+    SWITCH: "switch",
+    SLIDER: "slider",
+    IMAGE: "img",
+};
+/**
+ * Execute a single action on the page.
+ */
+export async function executeAction(page, action, treeData, contextValues) {
+    try {
+        // Intercept "back button" taps — the LLM often tries to tap the browser
+        // back button which doesn't exist in the viewport. Convert to page.goBack().
+        const isBackAttempt = action.element_name?.toLowerCase().includes("back") &&
+            (action.element_description?.toLowerCase().includes("browser") ||
+                action.element_description?.toLowerCase().includes("navigate") ||
+                action.element_name?.toLowerCase().includes("browser back"));
+        if (isBackAttempt || action.type === "navigate_back") {
+            await page.goBack({ timeout: 10_000 }).catch(() => { });
+            return { success: true, elementName: action.element_name, coordinates: null };
+        }
+        let coordinates = null;
+        switch (action.type) {
+            case "tap":
+                coordinates = await executeTap(page, action, treeData);
+                break;
+            case "text_input":
+                coordinates = await executeTextInput(page, action, treeData, contextValues);
+                break;
+            case "scroll":
+                await executeScroll(page, action, treeData);
+                break;
+            case "swipe":
+            case "pull_to_refresh":
+                await executeSwipe(page, action.direction ?? "down");
+                break;
+            case "wait":
+                await page.waitForTimeout(action.duration_ms ?? 1000);
+                break;
+            case "navigate_back":
+                await page.goBack({ timeout: 10_000 }).catch(() => { });
+                break;
+            case "long_press":
+                coordinates = await executeLongPress(page, action, treeData);
+                break;
+            case "double_tap":
+                coordinates = await executeDoubleTap(page, action, treeData);
+                break;
+            case "drag":
+                // Drag requires coordinates — skip if unavailable
+                break;
+            case "think":
+                // No-op: model is reasoning without acting
+                break;
+            case "pinch_zoom":
+            case "rotate_device":
+                // Not supported in desktop browser
+                break;
+        }
+        return { success: true, elementName: action.element_name, coordinates };
+    }
+    catch (err) {
+        if (!isRecoverableError(err))
+            throw err;
+        return { success: false, elementName: action.element_name, coordinates: null };
+    }
+}
+// --- Element Resolution ---
+/**
+ * Resolve an element: try CDP node_id first, then Playwright locator.
+ * Returns click coordinates { x, y } or null.
+ */
+async function resolveElement(page, action, treeData) {
+    // Strategy 1: CDP node resolution
+    if (action.node_id) {
+        const box = await resolveNodeToBoundingBox(page, action.node_id, treeData);
+        if (box) {
+            if (isDebugEnabled())
+                console.error(`      [resolve] CDP node ${action.node_id} → (${Math.round(box.x)}, ${Math.round(box.y)})`);
+            return { x: box.x, y: box.y };
+        }
+        if (isDebugEnabled())
+            console.error(`      [resolve] CDP node ${action.node_id} → FAILED, trying Playwright fallback`);
+    }
+    // Strategy 2: Playwright locator (tries multiple strategies)
+    const locator = await findElement(page, action);
+    if (locator) {
+        const box = await locator.boundingBox();
+        if (box) {
+            const coords = { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+            if (isDebugEnabled())
+                console.error(`      [resolve] Playwright locator → (${Math.round(coords.x)}, ${Math.round(coords.y)})`);
+            return coords;
+        }
+    }
+    if (isDebugEnabled())
+        console.error(`      [resolve] ALL strategies FAILED for "${action.element_name}"`);
+    return null;
+}
+/**
+ * Resolve to a Playwright Locator (for fill/type operations that need a Locator).
+ */
+async function resolveLocator(page, action, treeData) {
+    return findElement(page, action);
+}
+/**
+ * Build a Playwright locator, trying multiple strategies in order:
+ * 1. getByRole with element_name
+ * 2. getByRole with element_description (often more accurate)
+ * 3. getByText with element_name
+ * 4. getByText with element_description
+ */
+async function findElement(page, action) {
+    const role = action.element_type ? ELEMENT_TYPE_TO_ROLE[action.element_type] : null;
+    const name = action.element_name;
+    const desc = action.element_description;
+    const roleArg = role;
+    const candidates = [];
+    // Strategy 1: role + name
+    if (role && name) {
+        candidates.push(page.getByRole(roleArg, { name, exact: false }).first());
+    }
+    // Strategy 2: role + description
+    if (role && desc) {
+        candidates.push(page.getByRole(roleArg, { name: desc, exact: false }).first());
+    }
+    // Strategy 3: role only (if there's just one of that role)
+    if (role) {
+        candidates.push(page.getByRole(roleArg).first());
+    }
+    // Strategy 4: text search on name
+    if (name) {
+        candidates.push(page.getByText(name, { exact: false }).first());
+    }
+    // Strategy 5: text search on description
+    if (desc) {
+        candidates.push(page.getByText(desc, { exact: false }).first());
+    }
+    // Strategy 6: link by name (common case — LLM often calls links "buttons")
+    if (name) {
+        candidates.push(page.getByRole("link", { name, exact: false }).first());
+    }
+    if (desc) {
+        candidates.push(page.getByRole("link", { name: desc, exact: false }).first());
+    }
+    for (const locator of candidates) {
+        try {
+            await locator.waitFor({ state: "visible", timeout: 1500 });
+            return locator;
+        }
+        catch {
+            // Try next strategy
+        }
+    }
+    return null;
+}
+// --- Action Implementations ---
+async function executeTap(page, action, treeData) {
+    const count = action.count ?? 1;
+    const coords = await resolveElement(page, action, treeData);
+    if (coords) {
+        for (let i = 0; i < count; i++) {
+            await page.mouse.click(coords.x, coords.y);
+        }
+        return coords;
+    }
+    else {
+        throw new Error(`Cannot locate element for tap: ${action.element_name ?? "unknown"}`);
+    }
+}
+async function executeTextInput(page, action, treeData, contextValues) {
+    // Resolve the actual text to type
+    const text = resolveTextValue(action, contextValues);
+    // Try to get a Playwright locator for fill operations
+    const locator = await resolveLocator(page, action, treeData);
+    if (locator) {
+        if (action.mode === "click_type") {
+            await locator.click({ timeout: 5000 });
+            await locator.fill("");
+            await locator.pressSequentially(text, { delay: 30 });
+        }
+        else {
+            await locator.fill(text);
+        }
+        if (action.submit) {
+            await locator.press("Enter");
+        }
+        // Extract coordinates from the locator for recording
+        const box = await locator.boundingBox().catch(() => null);
+        return box ? { x: box.x + box.width / 2, y: box.y + box.height / 2 } : null;
+    }
+    else {
+        // Coordinate-based fallback
+        const coords = await resolveElement(page, action, treeData);
+        if (coords) {
+            await page.mouse.click(coords.x, coords.y);
+            await page.waitForTimeout(200);
+            const selectAll = process.platform === "darwin" ? "Meta+a" : "Control+a";
+            await page.keyboard.press(selectAll);
+            await page.keyboard.type(text, { delay: 30 });
+            if (action.submit) {
+                await page.keyboard.press("Enter");
+            }
+            return coords;
+        }
+        else {
+            throw new Error(`Cannot locate element for text input: ${action.element_name ?? "unknown"}`);
+        }
+    }
+}
+async function executeScroll(page, action, treeData) {
+    const viewport = page.viewportSize() ?? { width: 1440, height: 900 };
+    const amountMap = {
+        small: 0.5, medium: 0.8, large: 1.5, extra_large: 3.0,
+    };
+    const fraction = amountMap[action.amount ?? "medium"] ?? 0.8;
+    const pixels = Math.round(viewport.height * fraction);
+    switch (action.direction) {
+        case "up":
+            await page.evaluate((px) => window.scrollBy(0, -px), pixels);
+            break;
+        case "down":
+            await page.evaluate((px) => window.scrollBy(0, px), pixels);
+            break;
+        case "to_top":
+            await page.evaluate(() => window.scrollTo(0, 0));
+            break;
+        case "to_bottom":
+            await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight));
+            break;
+        case "to_element": {
+            const locator = await findElement(page, action);
+            if (locator) {
+                await locator.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => { });
+            }
+            else {
+                await page.evaluate((px) => window.scrollBy(0, px), pixels);
+            }
+            break;
+        }
+        default:
+            await page.evaluate((px) => window.scrollBy(0, px), pixels);
+    }
+    await page.waitForTimeout(300);
+}
+async function executeSwipe(page, direction) {
+    const viewport = page.viewportSize() ?? { width: 1440, height: 900 };
+    const cx = viewport.width / 2;
+    const cy = viewport.height / 2;
+    const d = viewport.height * 0.4;
+    let sx = cx, sy = cy, ex = cx, ey = cy;
+    switch (direction) {
+        case "up":
+            sy = cy + d / 2;
+            ey = cy - d / 2;
+            break;
+        case "down":
+            sy = cy - d / 2;
+            ey = cy + d / 2;
+            break;
+        case "left":
+            sx = cx + d / 2;
+            ex = cx - d / 2;
+            break;
+        case "right":
+            sx = cx - d / 2;
+            ex = cx + d / 2;
+            break;
+    }
+    await page.mouse.move(sx, sy);
+    await page.mouse.down();
+    await page.mouse.move(ex, ey, { steps: 10 });
+    await page.mouse.up();
+}
+async function executeLongPress(page, action, treeData) {
+    const coords = await resolveElement(page, action, treeData);
+    if (!coords)
+        throw new Error(`Cannot locate element for long press: ${action.element_name ?? "unknown"}`);
+    await page.mouse.move(coords.x, coords.y);
+    await page.mouse.down();
+    await page.waitForTimeout(action.duration_ms ?? 500);
+    await page.mouse.up();
+    return coords;
+}
+async function executeDoubleTap(page, action, treeData) {
+    const coords = await resolveElement(page, action, treeData);
+    if (coords) {
+        await page.mouse.dblclick(coords.x, coords.y);
+        return coords;
+    }
+    else {
+        throw new Error(`Cannot locate element for double tap: ${action.element_name ?? "unknown"}`);
+    }
+}
+// --- Helpers ---
+/**
+ * Resolve the actual text to type from an action, handling var/secret value types.
+ */
+function resolveTextValue(action, contextValues) {
+    if (action.value_type === "var" || action.value_type === "secret") {
+        const cv = contextValues.find(v => v.name === action.value);
+        if (cv?.value)
+            return cv.value;
+        // Fallback to the key name if resolution fails
+        return action.value ?? "";
+    }
+    return action.value ?? "";
+}
+function isRecoverableError(err) {
+    if (err instanceof Error && err.constructor.name === "TargetClosedError")
+        return false;
+    const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+    const fatal = ["target page", "browser has been closed", "target closed", "browser disconnected"];
+    return !fatal.some((f) => msg.includes(f));
+}
+/**
+ * Compare two base64 screenshots to detect visible change.
+ */
+export function detectNoVisibleChange(before, after) {
+    return before === after;
+}
+/**
+ * Build a human-readable action description matching backend's format_action_detail().
+ */
+export function describeAction(action) {
+    const element = action.element_name || "element";
+    switch (action.type) {
+        case "tap":
+            return action.count && action.count > 1
+                ? `tap on '${element}' x${action.count}`
+                : `tap on '${element}'`;
+        case "text_input": {
+            const val = action.value_type === "secret" ? "***" : `"${(action.value ?? "").slice(0, 30)}"`;
+            const modeStr = action.mode ? ` (${action.mode}${action.submit ? ", submit" : ""})` : "";
+            return `text_input on '${element}' → ${val}${modeStr}`;
+        }
+        case "scroll":
+            return action.direction === "to_element"
+                ? `scroll to '${element}'`
+                : `scroll ${action.direction ?? "down"} (${action.amount ?? "medium"})`;
+        case "swipe":
+            return `swipe ${action.direction ?? "up"} on '${element}'`;
+        case "wait":
+            return `wait ${action.duration_ms ?? 1000}ms`;
+        case "navigate_back":
+            return "navigate back";
+        case "long_press":
+            return `long_press on '${element}'`;
+        case "double_tap":
+            return `double_tap on '${element}'`;
+        case "drag":
+            return `drag '${element}'`;
+        case "think":
+            return `think: "${(action.thoughts ?? "").slice(0, 50)}"`;
+        case "pull_to_refresh":
+            return "pull_to_refresh";
+        default:
+            return `${action.type} on '${element}'`;
+    }
+}

package/dist/lib/local-sim/browser.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Playwright browser lifecycle, CDP tree extraction, and node resolution.
+ *
+ * Tree format matches backend's get_accessibility_tree():
+ *   [frame_index:node_id] role "name"
+ * with single-space indentation per level.
+ */
+import { type Browser, type BrowserContext, type Page } from "playwright-core";
+import type { LocalSimBrowserOptions, TreeData } from "./types.js";
+import "./install.js";
+export interface BrowserSession {
+    browser: Browser;
+    context: BrowserContext;
+    page: Page;
+}
+/**
+ * Launch a shared browser process with a pre-configured context.
+ * Create tabs with createTab() — they appear as tabs in one window.
+ */
+export declare function launchSharedBrowser(opts: LocalSimBrowserOptions): Promise<Browser>;
+/**
+ * Create a new tab in the shared browser's default context.
+ * Tabs share cookies/storage — appears as tabs in one window in headed mode.
+ */
+export declare function createTab(browser: Browser, opts: LocalSimBrowserOptions): Promise<BrowserSession>;
+/**
+ * Launch a standalone browser session (single tester, owns the browser).
+ */
+export declare function launchBrowser(opts: LocalSimBrowserOptions): Promise<BrowserSession>;
+export interface ObservationData {
+    screenshot: string;
+    treeData: TreeData;
+    url: string;
+    viewportWidth: number;
+    viewportHeight: number;
+    scrollPosition: number;
+    documentHeight: number;
+}
+/**
+ * Capture a full observation from the current page state.
+ */
+export declare function captureObservation(page: Page): Promise<ObservationData>;
+/**
+ * Extract accessibility tree via CDP, matching backend's [nodeId] role "name" format.
+ */
+export declare function extractAccessibilityTree(page: Page): Promise<TreeData>;
+/**
+ * Resolve a composite node_id (e.g., "0:42") to bounding box coordinates.
+ */
+export declare function resolveNodeToBoundingBox(page: Page, nodeId: string, treeData: TreeData): Promise<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+} | null>;
+/**
+ * Resolve a composite node_id to an XPath selector.
+ */
+export declare function resolveNodeToXPath(page: Page, nodeId: string, treeData: TreeData): Promise<string | null>;
+export declare function takeScreenshot(page: Page): Promise<string>;
+export declare function takeScreenshotJpeg(page: Page, quality?: number): Promise<Buffer>;
+export declare function navigateWithRetry(page: Page, url: string, maxRetries?: number): Promise<void>;
+export declare function closeBrowser(session: BrowserSession): Promise<void>;