speqs 0.6.0 → 0.7.0

package/dist/commands/simulation.js

@@ -16,8 +16,16 @@ function parseMaxInteractions(value) {
         throw new Error(`Invalid --max-interactions value: ${value}`);
     return n;
 }
+function parseSlowMo(value) {
+    const n = parseInt(value, 10);
+    if (isNaN(n) || n < 0)
+        throw new Error(`Invalid --slow-mo value: ${value}`);
+    return n;
+}
 import { MEDIA_MODALITIES } from "../lib/types.js";
 import { resolveContentUrl, resolveContentUrls, resolveTextContent } from "../lib/upload.js";
+import { runLocalSimulations } from "../lib/local-sim/loop.js";
+import { ensureBrowser } from "../lib/local-sim/install.js";
 function isMediaModality(modality) {
     return !!modality && MEDIA_MODALITIES.includes(modality);
 }
@@ -164,6 +172,13 @@ export function registerSimulationCommands(program) {
         .option("--language <lang>", "Language code (e.g. en, sv)")
         .option("--locale <locale>", "Locale code (e.g. en-US)")
         .option("-y, --yes", "Skip confirmation prompt")
+        // Local simulation options
+        .option("--local", "Run simulation with local browser (Playwright) instead of remote")
+        .option("--headed", "Show browser window (local mode only)")
+        .option("--slow-mo <ms>", "Slow down actions by ms (local mode only)")
+        .option("--devtools", "Open Chrome DevTools (local mode only)")
+        .option("--debug", "Enable detailed debug logging to stderr and ~/.speqs/local-sim.log")
+        .option("--parallel <n>", "Run N testers in parallel (local mode only, default: all)")
         .addHelpText("after", `
 Note: --workspace and --study are optional if you have set active context
   via \`speqs workspace use <alias>\` and \`speqs study use <alias>\`.
@@ -196,7 +211,11 @@ Examples:
   $ speqs sim run --image-urls ./post.png --copy-text @./caption.txt --social-platform instagram --config c-c3c
 
   # Re-run existing iteration:
-  $ speqs sim run --iteration i-d4e`)
+  $ speqs sim run --iteration i-d4e
+
+  # Local browser simulation (no remote Browserbase):
+  $ speqs sim run --local --url http://localhost:3000
+  $ speqs sim run --local --url http://localhost:3000 --headed --slow-mo 500`)
         .action(async (opts, cmd) => {
         await withClient(cmd, async (client, globals) => {
             const log = (msg) => { if (!globals.quiet)
@@ -373,6 +392,10 @@ Examples:
                 }
                 log("");
             }
+            // Ensure browser is ready before creating server-side state
+            if (opts.local) {
+                await ensureBrowser({ quiet: globals.quiet, skipPrompt: globals.json });
+            }
             // Step 1: Create or use existing iteration
             if (!iterationId) {
                 const iterName = resolvedOpts.iterationName || `CLI ${new Date().toISOString().slice(0, 16)}`;
@@ -385,6 +408,11 @@ Examples:
                 iterationId = iter.id;
                 log(`Created iteration "${iterName}"`);
             }
+            else if (!opts.iteration) {
+                // Auto-reused iteration — update its details to reflect current run
+                const newDetails = buildIterationDetails(modality, resolvedOpts);
+                await client.put(`/iterations/${iterationId}`, { details: newDetails });
+            }
             // Step 2: Create testers from profiles (or reuse from explicit iteration)
             let createdTesters;
             if (opts.iteration && !opts.profiles) {
@@ -412,6 +440,43 @@ Examples:
                 log(`Created ${createdTesters.length} tester${createdTesters.length > 1 ? "s" : ""}`);
             }
             // Step 3: Start simulations
+            // Local mode: run simulations with local browser
+            if (opts.local) {
+                if (isMedia) {
+                    throw new Error("Local mode is only supported for interactive simulations.");
+                }
+                const testerNameMap = new Map();
+                for (const t of createdTesters) {
+                    testerNameMap.set(t.id, t.tester_profile?.name ?? "Unknown");
+                }
+                await runLocalSimulations(client, {
+                    workspaceId: resolvedWorkspace,
+                    studyId: resolvedStudy,
+                    iterationId: iterationId,
+                    testerIds: createdTesters.map((t) => t.id),
+                    testerNames: testerNameMap,
+                    url: resolvedOpts.url,
+                    screenFormat: resolvedOpts.screenFormat,
+                    locale: opts.locale,
+                    maxInteractions: opts.maxInteractions ? parseMaxInteractions(opts.maxInteractions) : undefined,
+                    headed: !!opts.headed,
+                    slowMo: opts.slowMo ? parseSlowMo(opts.slowMo) : undefined,
+                    devtools: opts.devtools,
+                    debug: opts.debug,
+                    parallel: opts.parallel ? parseInt(opts.parallel, 10) : undefined,
+                    quiet: globals.quiet,
+                    json: globals.json,
+                });
+                if (globals.json) {
+                    output({
+                        iteration_id: iterationId,
+                        testers: createdTesters.map((t) => ({ id: t.id, profile_name: t.tester_profile?.name })),
+                        mode: "local",
+                    }, true);
+                }
+                return;
+            }
+            // Remote mode: delegate to backend
             log(`Starting ${createdTesters.length} simulation${createdTesters.length > 1 ? "s" : ""}...`);
             let simResults;
             if (isMedia) {

package/dist/lib/api-client.d.ts

@@ -24,5 +24,35 @@ export declare class ApiClient {
     }): Promise<T>;
     put<T = unknown>(path: string, body?: unknown): Promise<T>;
     del(path: string): Promise<void>;
+    localSimInit(body: {
+        tester_id: string;
+        study_id: string;
+        product_id: string;
+        iteration_id: string;
+    }): Promise<import("./local-sim/types.js").LocalSimInitResponse>;
+    localSimStep(body: import("./local-sim/types.js").LocalSimStepRequest): Promise<import("./local-sim/types.js").LocalSimStepResponseRaw>;
+    localSimRecord(body: import("./local-sim/types.js").LocalSimRecordRequest): Promise<import("./local-sim/types.js").LocalSimRecordResponse>;
+    localSimMatchFrame(body: {
+        product_id: string;
+        study_id: string;
+        screenshot_base64: string;
+        screenshot_url?: string;
+        location_name: string;
+        screen_format?: string;
+    }): Promise<{
+        frame_version_id: string;
+    }>;
+    localSimScreenshotUpload(body: {
+        product_id: string;
+        screenshot_id: string;
+        content_type: string;
+    }): Promise<{
+        upload_info: {
+            signed_upload_url: string;
+            file_path: string;
+            expires_in_seconds: number;
+        };
+        screenshot_url: string;
+    }>;
     private handleResponse;
 }

package/dist/lib/api-client.js

@@ -143,6 +143,22 @@ export class ApiClient {
             throw new ApiError(res.status, res.statusText, body);
         }
     }
+    // --- Local simulation endpoints ---
+    async localSimInit(body) {
+        return this.post("/simulation/local/init", body, { timeout: 30_000 });
+    }
+    async localSimStep(body) {
+        return this.post("/simulation/local/step", body, { timeout: 60_000 });
+    }
+    async localSimRecord(body) {
+        return this.post("/simulation/local/record", body, { timeout: 60_000 });
+    }
+    async localSimMatchFrame(body) {
+        return this.post("/simulation/local/match-frame", body, { timeout: 30_000 });
+    }
+    async localSimScreenshotUpload(body) {
+        return this.post("/simulation/local/screenshot/upload", body);
+    }
     async handleResponse(resp) {
         if (!resp.ok) {
             let body;

package/dist/lib/local-sim/actions.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Action executor — resolves elements and executes Playwright actions.
+ *
+ * Resolution strategy:
+ * 1. CDP node resolution (using node_id from tree data)
+ * 2. Playwright locator fallback (using element_name + element_type)
+ * 3. Coordinate fallback (if returned by backend)
+ */
+import type { Page } from "playwright-core";
+import type { LocalStepAction, ActionResult, ContextValue, TreeData } from "./types.js";
+/**
+ * Execute a single action on the page.
+ */
+export declare function executeAction(page: Page, action: LocalStepAction, treeData: TreeData, contextValues: ContextValue[]): Promise<ActionResult>;
+/**
+ * Compare two base64 screenshots to detect visible change.
+ */
+export declare function detectNoVisibleChange(before: string, after: string): boolean;
+/**
+ * Build a human-readable action description matching backend's format_action_detail().
+ */
+export declare function describeAction(action: LocalStepAction): string;

package/dist/lib/local-sim/actions.js

@@ -0,0 +1,379 @@
+/**
+ * Action executor — resolves elements and executes Playwright actions.
+ *
+ * Resolution strategy:
+ * 1. CDP node resolution (using node_id from tree data)
+ * 2. Playwright locator fallback (using element_name + element_type)
+ * 3. Coordinate fallback (if returned by backend)
+ */
+import { resolveNodeToBoundingBox } from "./browser.js";
+import { isDebugEnabled } from "./debug.js";
+// --- ARIA role → Playwright role mapping ---
+const ELEMENT_TYPE_TO_ROLE = {
+    BUTTON: "button",
+    TEXT_INPUT: "input",
+    SELECT: "combobox",
+    LINK: "link",
+    CHECKBOX: "checkbox",
+    RADIO: "radio",
+    HEADING: "heading",
+    TAB: "tab",
+    MENU_ITEM: "menuitem",
+    SWITCH: "switch",
+    SLIDER: "slider",
+    IMAGE: "img",
+};
+/**
+ * Execute a single action on the page.
+ */
+export async function executeAction(page, action, treeData, contextValues) {
+    try {
+        // Intercept "back button" taps — the LLM often tries to tap the browser
+        // back button which doesn't exist in the viewport. Convert to page.goBack().
+        const isBackAttempt = action.element_name?.toLowerCase().includes("back") &&
+            (action.element_description?.toLowerCase().includes("browser") ||
+                action.element_description?.toLowerCase().includes("navigate") ||
+                action.element_name?.toLowerCase().includes("browser back"));
+        if (isBackAttempt || action.type === "navigate_back") {
+            await page.goBack({ timeout: 10_000 }).catch(() => { });
+            return { success: true, elementName: action.element_name, coordinates: null };
+        }
+        let coordinates = null;
+        switch (action.type) {
+            case "tap":
+                coordinates = await executeTap(page, action, treeData);
+                break;
+            case "text_input":
+                coordinates = await executeTextInput(page, action, treeData, contextValues);
+                break;
+            case "scroll":
+                await executeScroll(page, action, treeData);
+                break;
+            case "swipe":
+            case "pull_to_refresh":
+                await executeSwipe(page, action.direction ?? "down");
+                break;
+            case "wait":
+                await page.waitForTimeout(action.duration_ms ?? 1000);
+                break;
+            case "navigate_back":
+                await page.goBack({ timeout: 10_000 }).catch(() => { });
+                break;
+            case "long_press":
+                coordinates = await executeLongPress(page, action, treeData);
+                break;
+            case "double_tap":
+                coordinates = await executeDoubleTap(page, action, treeData);
+                break;
+            case "drag":
+                // Drag requires coordinates — skip if unavailable
+                break;
+            case "think":
+                // No-op: model is reasoning without acting
+                break;
+            case "pinch_zoom":
+            case "rotate_device":
+                // Not supported in desktop browser
+                break;
+        }
+        return { success: true, elementName: action.element_name, coordinates };
+    }
+    catch (err) {
+        if (!isRecoverableError(err))
+            throw err;
+        return { success: false, elementName: action.element_name, coordinates: null };
+    }
+}
+// --- Element Resolution ---
+/**
+ * Resolve an element: try CDP node_id first, then Playwright locator.
+ * Returns click coordinates { x, y } or null.
+ */
+async function resolveElement(page, action, treeData) {
+    // Strategy 1: CDP node resolution
+    if (action.node_id) {
+        const box = await resolveNodeToBoundingBox(page, action.node_id, treeData);
+        if (box) {
+            if (isDebugEnabled())
+                console.error(`      [resolve] CDP node ${action.node_id} → (${Math.round(box.x)}, ${Math.round(box.y)})`);
+            return { x: box.x, y: box.y };
+        }
+        if (isDebugEnabled())
+            console.error(`      [resolve] CDP node ${action.node_id} → FAILED, trying Playwright fallback`);
+    }
+    // Strategy 2: Playwright locator (tries multiple strategies)
+    const locator = await findElement(page, action);
+    if (locator) {
+        const box = await locator.boundingBox();
+        if (box) {
+            const coords = { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+            if (isDebugEnabled())
+                console.error(`      [resolve] Playwright locator → (${Math.round(coords.x)}, ${Math.round(coords.y)})`);
+            return coords;
+        }
+    }
+    if (isDebugEnabled())
+        console.error(`      [resolve] ALL strategies FAILED for "${action.element_name}"`);
+    return null;
+}
+/**
+ * Resolve to a Playwright Locator (for fill/type operations that need a Locator).
+ */
+async function resolveLocator(page, action, treeData) {
+    return findElement(page, action);
+}
+/**
+ * Build a Playwright locator, trying multiple strategies in order:
+ * 1. getByRole with element_name
+ * 2. getByRole with element_description (often more accurate)
+ * 3. getByText with element_name
+ * 4. getByText with element_description
+ */
+async function findElement(page, action) {
+    const role = action.element_type ? ELEMENT_TYPE_TO_ROLE[action.element_type] : null;
+    const name = action.element_name;
+    const desc = action.element_description;
+    const roleArg = role;
+    const candidates = [];
+    // Strategy 1: role + name
+    if (role && name) {
+        candidates.push(page.getByRole(roleArg, { name, exact: false }).first());
+    }
+    // Strategy 2: role + description
+    if (role && desc) {
+        candidates.push(page.getByRole(roleArg, { name: desc, exact: false }).first());
+    }
+    // Strategy 3: role only (if there's just one of that role)
+    if (role) {
+        candidates.push(page.getByRole(roleArg).first());
+    }
+    // Strategy 4: text search on name
+    if (name) {
+        candidates.push(page.getByText(name, { exact: false }).first());
+    }
+    // Strategy 5: text search on description
+    if (desc) {
+        candidates.push(page.getByText(desc, { exact: false }).first());
+    }
+    // Strategy 6: link by name (common case — LLM often calls links "buttons")
+    if (name) {
+        candidates.push(page.getByRole("link", { name, exact: false }).first());
+    }
+    if (desc) {
+        candidates.push(page.getByRole("link", { name: desc, exact: false }).first());
+    }
+    for (const locator of candidates) {
+        try {
+            await locator.waitFor({ state: "visible", timeout: 1500 });
+            return locator;
+        }
+        catch {
+            // Try next strategy
+        }
+    }
+    return null;
+}
+// --- Action Implementations ---
+async function executeTap(page, action, treeData) {
+    const count = action.count ?? 1;
+    const coords = await resolveElement(page, action, treeData);
+    if (coords) {
+        for (let i = 0; i < count; i++) {
+            await page.mouse.click(coords.x, coords.y);
+        }
+        return coords;
+    }
+    else {
+        throw new Error(`Cannot locate element for tap: ${action.element_name ?? "unknown"}`);
+    }
+}
+async function executeTextInput(page, action, treeData, contextValues) {
+    // Resolve the actual text to type
+    const text = resolveTextValue(action, contextValues);
+    // Try to get a Playwright locator for fill operations
+    const locator = await resolveLocator(page, action, treeData);
+    if (locator) {
+        if (action.mode === "click_type") {
+            await locator.click({ timeout: 5000 });
+            await locator.fill("");
+            await locator.pressSequentially(text, { delay: 30 });
+        }
+        else {
+            await locator.fill(text);
+        }
+        if (action.submit) {
+            await locator.press("Enter");
+        }
+        // Extract coordinates from the locator for recording
+        const box = await locator.boundingBox().catch(() => null);
+        return box ? { x: box.x + box.width / 2, y: box.y + box.height / 2 } : null;
+    }
+    else {
+        // Coordinate-based fallback
+        const coords = await resolveElement(page, action, treeData);
+        if (coords) {
+            await page.mouse.click(coords.x, coords.y);
+            await page.waitForTimeout(200);
+            const selectAll = process.platform === "darwin" ? "Meta+a" : "Control+a";
+            await page.keyboard.press(selectAll);
+            await page.keyboard.type(text, { delay: 30 });
+            if (action.submit) {
+                await page.keyboard.press("Enter");
+            }
+            return coords;
+        }
+        else {
+            throw new Error(`Cannot locate element for text input: ${action.element_name ?? "unknown"}`);
+        }
+    }
+}
+async function executeScroll(page, action, treeData) {
+    const viewport = page.viewportSize() ?? { width: 1440, height: 900 };
+    const amountMap = {
+        small: 0.5, medium: 0.8, large: 1.5, extra_large: 3.0,
+    };
+    const fraction = amountMap[action.amount ?? "medium"] ?? 0.8;
+    const pixels = Math.round(viewport.height * fraction);
+    switch (action.direction) {
+        case "up":
+            await page.evaluate((px) => window.scrollBy(0, -px), pixels);
+            break;
+        case "down":
+            await page.evaluate((px) => window.scrollBy(0, px), pixels);
+            break;
+        case "to_top":
+            await page.evaluate(() => window.scrollTo(0, 0));
+            break;
+        case "to_bottom":
+            await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight));
+            break;
+        case "to_element": {
+            const locator = await findElement(page, action);
+            if (locator) {
+                await locator.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => { });
+            }
+            else {
+                await page.evaluate((px) => window.scrollBy(0, px), pixels);
+            }
+            break;
+        }
+        default:
+            await page.evaluate((px) => window.scrollBy(0, px), pixels);
+    }
+    await page.waitForTimeout(300);
+}
+async function executeSwipe(page, direction) {
+    const viewport = page.viewportSize() ?? { width: 1440, height: 900 };
+    const cx = viewport.width / 2;
+    const cy = viewport.height / 2;
+    const d = viewport.height * 0.4;
+    let sx = cx, sy = cy, ex = cx, ey = cy;
+    switch (direction) {
+        case "up":
+            sy = cy + d / 2;
+            ey = cy - d / 2;
+            break;
+        case "down":
+            sy = cy - d / 2;
+            ey = cy + d / 2;
+            break;
+        case "left":
+            sx = cx + d / 2;
+            ex = cx - d / 2;
+            break;
+        case "right":
+            sx = cx - d / 2;
+            ex = cx + d / 2;
+            break;
+    }
+    await page.mouse.move(sx, sy);
+    await page.mouse.down();
+    await page.mouse.move(ex, ey, { steps: 10 });
+    await page.mouse.up();
+}
+async function executeLongPress(page, action, treeData) {
+    const coords = await resolveElement(page, action, treeData);
+    if (!coords)
+        throw new Error(`Cannot locate element for long press: ${action.element_name ?? "unknown"}`);
+    await page.mouse.move(coords.x, coords.y);
+    await page.mouse.down();
+    await page.waitForTimeout(action.duration_ms ?? 500);
+    await page.mouse.up();
+    return coords;
+}
+async function executeDoubleTap(page, action, treeData) {
+    const coords = await resolveElement(page, action, treeData);
+    if (coords) {
+        await page.mouse.dblclick(coords.x, coords.y);
+        return coords;
+    }
+    else {
+        throw new Error(`Cannot locate element for double tap: ${action.element_name ?? "unknown"}`);
+    }
+}
+// --- Helpers ---
+/**
+ * Resolve the actual text to type from an action, handling var/secret value types.
+ */
+function resolveTextValue(action, contextValues) {
+    if (action.value_type === "var" || action.value_type === "secret") {
+        const cv = contextValues.find(v => v.name === action.value);
+        if (cv?.value)
+            return cv.value;
+        // Fallback to the key name if resolution fails
+        return action.value ?? "";
+    }
+    return action.value ?? "";
+}
+function isRecoverableError(err) {
+    if (err instanceof Error && err.constructor.name === "TargetClosedError")
+        return false;
+    const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+    const fatal = ["target page", "browser has been closed", "target closed", "browser disconnected"];
+    return !fatal.some((f) => msg.includes(f));
+}
+/**
+ * Compare two base64 screenshots to detect visible change.
+ */
+export function detectNoVisibleChange(before, after) {
+    return before === after;
+}
+/**
+ * Build a human-readable action description matching backend's format_action_detail().
+ */
+export function describeAction(action) {
+    const element = action.element_name || "element";
+    switch (action.type) {
+        case "tap":
+            return action.count && action.count > 1
+                ? `tap on '${element}' x${action.count}`
+                : `tap on '${element}'`;
+        case "text_input": {
+            const val = action.value_type === "secret" ? "***" : `"${(action.value ?? "").slice(0, 30)}"`;
+            const modeStr = action.mode ? ` (${action.mode}${action.submit ? ", submit" : ""})` : "";
+            return `text_input on '${element}' → ${val}${modeStr}`;
+        }
+        case "scroll":
+            return action.direction === "to_element"
+                ? `scroll to '${element}'`
+                : `scroll ${action.direction ?? "down"} (${action.amount ?? "medium"})`;
+        case "swipe":
+            return `swipe ${action.direction ?? "up"} on '${element}'`;
+        case "wait":
+            return `wait ${action.duration_ms ?? 1000}ms`;
+        case "navigate_back":
+            return "navigate back";
+        case "long_press":
+            return `long_press on '${element}'`;
+        case "double_tap":
+            return `double_tap on '${element}'`;
+        case "drag":
+            return `drag '${element}'`;
+        case "think":
+            return `think: "${(action.thoughts ?? "").slice(0, 50)}"`;
+        case "pull_to_refresh":
+            return "pull_to_refresh";
+        default:
+            return `${action.type} on '${element}'`;
+    }
+}

package/dist/lib/local-sim/browser.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Playwright browser lifecycle, CDP tree extraction, and node resolution.
+ *
+ * Tree format matches backend's get_accessibility_tree():
+ *   [frame_index:node_id] role "name"
+ * with single-space indentation per level.
+ */
+import { type Browser, type BrowserContext, type Page } from "playwright-core";
+import type { LocalSimBrowserOptions, TreeData } from "./types.js";
+import "./install.js";
+export interface BrowserSession {
+    browser: Browser;
+    context: BrowserContext;
+    page: Page;
+}
+/**
+ * Launch a shared browser process with a pre-configured context.
+ * Create tabs with createTab() — they appear as tabs in one window.
+ */
+export declare function launchSharedBrowser(opts: LocalSimBrowserOptions): Promise<Browser>;
+/**
+ * Create a new tab in the shared browser's default context.
+ * Tabs share cookies/storage — appears as tabs in one window in headed mode.
+ */
+export declare function createTab(browser: Browser, opts: LocalSimBrowserOptions): Promise<BrowserSession>;
+/**
+ * Launch a standalone browser session (single tester, owns the browser).
+ */
+export declare function launchBrowser(opts: LocalSimBrowserOptions): Promise<BrowserSession>;
+export interface ObservationData {
+    screenshot: string;
+    treeData: TreeData;
+    url: string;
+    viewportWidth: number;
+    viewportHeight: number;
+    scrollPosition: number;
+    documentHeight: number;
+}
+/**
+ * Capture a full observation from the current page state.
+ */
+export declare function captureObservation(page: Page): Promise<ObservationData>;
+/**
+ * Extract accessibility tree via CDP, matching backend's [nodeId] role "name" format.
+ */
+export declare function extractAccessibilityTree(page: Page): Promise<TreeData>;
+/**
+ * Resolve a composite node_id (e.g., "0:42") to bounding box coordinates.
+ */
+export declare function resolveNodeToBoundingBox(page: Page, nodeId: string, treeData: TreeData): Promise<{
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+} | null>;
+/**
+ * Resolve a composite node_id to an XPath selector.
+ */
+export declare function resolveNodeToXPath(page: Page, nodeId: string, treeData: TreeData): Promise<string | null>;
+export declare function takeScreenshot(page: Page): Promise<string>;
+export declare function takeScreenshotJpeg(page: Page, quality?: number): Promise<Buffer>;
+export declare function navigateWithRetry(page: Page, url: string, maxRetries?: number): Promise<void>;
+export declare function closeBrowser(session: BrowserSession): Promise<void>;