teamshare-bridge 0.26.13 → 0.26.15

package/dist/lib/orchestrator/handlers/browser.js

@@ -36,6 +36,32 @@ async function screenOf(session) {
         return null;
     }
 }
+/**
+ * Emit an action result, attaching an inline screenshot when the session is in
+ * QA vision mode (2026-10).
+ *
+ * `browser_open({ qaVision: true })` makes EVERY interaction result carry the
+ * page image, so the model sees what its click/type/hover actually did instead
+ * of spending a second round-trip on `screenshot` after each step. Failure
+ * payloads get one too: the toast or validation error that explains a failure
+ * is exactly what the agent needs to see. The capture is BEST-EFFORT - a
+ * screenshot problem must never turn a successful action into a failed one.
+ */
+async function emitWithVision(emit, session, cmdId, payload) {
+    if (session?.connected && session.qaVision === true) {
+        try {
+            const shot = await session.screenshot('jpeg', 62);
+            if (shot) {
+                payload.screenshot = shot;
+                payload.format = 'jpeg';
+            }
+        }
+        catch {
+            /* vision is best-effort */
+        }
+    }
+    emit('orchestrator:browser-result', cmdId, payload);
+}
 /**
  * The session an identity used most recently (2026-10).
  *
@@ -122,21 +148,36 @@ async function handleBrowserCommand(data, emit) {
                 // OPT-IN credential entry (default false). A QA task signs in with
                 // supplied credentials; the auto-answer browser is a separate path.
                 const allowCredentials = data.allowCredentials === true;
+                // OPT-IN QA vision mode (default false, 2026-10): every mutating action
+                // returns a screenshot. Only the QA brief turns it on.
+                const qaVision = data.qaVision === true;
                 // REUSE only a LIVE session. A disconnected one may have lost its port
                 // and its profile to a concurrent browser while it was down, so it is
                 // re-allocated below instead of silently reconnecting onto them.
                 const existing = manager.get(sessionId);
                 if (existing?.connected) {
                     existing.setAllowCredentials(allowCredentials);
-                    const pageState = data.url
-                        ? await existing.navigate(String(data.url))
-                        : await existing.pageInfo();
+                    existing.setQaVision(qaVision);
+                    // reset:true drops a crashed run's leftovers (open modal, corrupted
+                    // field values, stale validation errors) before the new run starts.
+                    // A resumed run omits it and keeps its page.
+                    let pageState;
+                    if (data.url) {
+                        pageState = await existing.navigate(String(data.url));
+                    }
+                    else if (data.reset === true) {
+                        pageState = await existing.navigate('about:blank');
+                    }
+                    else {
+                        pageState = await existing.pageInfo();
+                    }
                     const screen = await screenOf(existing);
                     emit('orchestrator:browser-result', data.cmdId, {
                         success: true,
                         alreadyConnected: true,
                         sessionId,
                         port: existing.port,
+                        qaVision: existing.qaVision,
                         ...pageState,
                         ...(screen ? { screen } : {}),
                     });
@@ -178,10 +219,12 @@ async function handleBrowserCommand(data, emit) {
                     headless,
                     profileDir: profile.dir,
                     allowCredentials,
+                    qaVision,
                 });
                 // A reused session keeps its flag from a previous connect; honour the
                 // explicit value on every connect so the caller stays in control.
                 session.setAllowCredentials(allowCredentials);
+                session.setQaVision(qaVision);
                 const info = await session.connect(port);
                 const actualPort = session.port;
                 log.info(`Browser session "${sessionId}" on port ${actualPort} ` +
@@ -195,6 +238,7 @@ async function handleBrowserCommand(data, emit) {
                     sessionId,
                     port: actualPort,
                     profile: profile.shared ? 'shared' : 'isolated',
+                    qaVision: session.qaVision,
                     ...pageState,
                     ...(screen ? { screen } : {}),
                 });
@@ -238,7 +282,7 @@ async function handleBrowserCommand(data, emit) {
                 }
                 const info = await session.navigate(url);
                 const screen = await screenOf(session);
-                emit('orchestrator:browser-result', data.cmdId, {
+                await emitWithVision(emit, session, data.cmdId, {
                     success: true,
                     ...info,
                     ...(screen ? { screen } : {}),
@@ -254,15 +298,92 @@ async function handleBrowserCommand(data, emit) {
                     });
                     return;
                 }
-                const x = typeof data.x === 'number' ? data.x : 0;
-                const y = typeof data.y === 'number' ? data.y : 0;
-                await session.click(x, y);
+                // Explicit pointer-sequence click (2026-10): the rescue for an
+                // activator that ignores a normal click. ASKED FOR, never automatic.
+                if (data.pointer === true) {
+                    if (typeof data.ref !== 'string' || !data.ref) {
+                        emit('orchestrator:browser-result', data.cmdId, {
+                            success: false,
+                            error: 'POINTER_CLICK_REF_REQUIRED: pointer:true needs a ref (getRefs / snapshot refs:true).',
+                        });
+                        return;
+                    }
+                    try {
+                        const pt = await session.clickPointerRef(data.ref);
+                        const screen = await screenOf(session);
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: true,
+                            clicked: true,
+                            pointer: true,
+                            ref: data.ref,
+                            x: pt.x,
+                            y: pt.y,
+                            ...(screen ? { screen } : {}),
+                        });
+                    }
+                    catch (err) {
+                        const screen = await screenOf(session);
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: false,
+                            error: err instanceof Error ? err.message : String(err),
+                            ...(screen ? { screen } : {}),
+                        });
+                    }
+                    break;
+                }
+                // Coordinates are CSS pixels (getElements/snapshot) — never screenshot
+                // pixels. Prefer elementIndex/selector: resolved bridge-side against a
+                // fresh DOM read, so a stale or scaled coordinate cannot misfire.
+                // Out-of-viewport coordinates are refused with CLICK_OUT_OF_VIEWPORT.
+                let clicked;
+                if (typeof data.ref === 'string' ||
+                    typeof data.elementIndex === 'number' ||
+                    typeof data.selector === 'string') {
+                    try {
+                        clicked = await session.clickRef({
+                            ...(typeof data.ref === 'string' ? { ref: data.ref } : {}),
+                            ...(typeof data.elementIndex === 'number'
+                                ? { elementIndex: data.elementIndex }
+                                : {}),
+                            ...(typeof data.selector === 'string'
+                                ? { selector: data.selector }
+                                : {}),
+                        });
+                    }
+                    catch (err) {
+                        const screen = await screenOf(session);
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: false,
+                            error: err instanceof Error ? err.message : String(err),
+                            ...(screen ? { screen } : {}),
+                        });
+                        return;
+                    }
+                }
+                else {
+                    const x = typeof data.x === 'number' ? data.x : 0;
+                    const y = typeof data.y === 'number' ? data.y : 0;
+                    try {
+                        await session.click(x, y);
+                    }
+                    catch (err) {
+                        const screen = await screenOf(session);
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: false,
+                            error: err instanceof Error ? err.message : String(err),
+                            ...(screen ? { screen } : {}),
+                        });
+                        return;
+                    }
+                    clicked = { x, y };
+                }
                 const screen = await screenOf(session);
-                emit('orchestrator:browser-result', data.cmdId, {
+                await emitWithVision(emit, session, data.cmdId, {
                     success: true,
                     clicked: true,
-                    x,
-                    y,
+                    ...(typeof data.ref === 'string' ? { ref: data.ref } : {}),
+                    x: clicked.x,
+                    y: clicked.y,
                     ...(screen ? { screen } : {}),
                 });
                 break;
@@ -281,14 +402,53 @@ async function handleBrowserCommand(data, emit) {
                 if (data.allowCredentials === true)
                     session.setAllowCredentials(true);
                 const text = String(data.text ?? '');
-                await session.type(text, { clearFirst: data.clearFirst === true });
-                const screen = await screenOf(session);
-                emit('orchestrator:browser-result', data.cmdId, {
-                    success: true,
-                    typed: true,
-                    length: text.length,
-                    ...(screen ? { screen } : {}),
-                });
+                // With a reference the field is clicked first (same as the batch
+                // type step), so one call fills a form field without a prior click.
+                if (typeof data.ref === 'string' ||
+                    typeof data.elementIndex === 'number' ||
+                    typeof data.selector === 'string') {
+                    try {
+                        await session.clickRef({
+                            ...(typeof data.ref === 'string' ? { ref: data.ref } : {}),
+                            ...(typeof data.elementIndex === 'number'
+                                ? { elementIndex: data.elementIndex }
+                                : {}),
+                            ...(typeof data.selector === 'string'
+                                ? { selector: data.selector }
+                                : {}),
+                        });
+                    }
+                    catch (err) {
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: false,
+                            error: err instanceof Error ? err.message : String(err),
+                        });
+                        return;
+                    }
+                }
+                // Verified typing: TYPE_NOT_VERIFIED (with the observed field state)
+                // when the insert does not land — never a silent typed:true.
+                try {
+                    const observed = await session.type(text, {
+                        clearFirst: data.clearFirst === true,
+                    });
+                    const screen = await screenOf(session);
+                    await emitWithVision(emit, session, data.cmdId, {
+                        success: true,
+                        typed: true,
+                        length: text.length,
+                        observed,
+                        ...(screen ? { screen } : {}),
+                    });
+                }
+                catch (err) {
+                    const screen = await screenOf(session);
+                    await emitWithVision(emit, session, data.cmdId, {
+                        success: false,
+                        error: err instanceof Error ? err.message : String(err),
+                        ...(screen ? { screen } : {}),
+                    });
+                }
                 break;
             }
             case 'login': {
@@ -320,7 +480,7 @@ async function handleBrowserCommand(data, emit) {
                 });
                 // Never echo the password: the payload carries booleans + an error code.
                 const screen = await screenOf(session);
-                emit('orchestrator:browser-result', data.cmdId, {
+                await emitWithVision(emit, session, data.cmdId, {
                     success: true,
                     ...result,
                     ...(screen ? { screen } : {}),
@@ -339,13 +499,185 @@ async function handleBrowserCommand(data, emit) {
                 const key = String(data.key ?? '');
                 await session.press(key);
                 const screen = await screenOf(session);
-                emit('orchestrator:browser-result', data.cmdId, {
+                await emitWithVision(emit, session, data.cmdId, {
                     success: true,
                     pressed: key,
                     ...(screen ? { screen } : {}),
                 });
                 break;
             }
+            // 2026-10: pointer hover without a click. Radix menus/tooltips open on
+            // hover, and a click on a hover-only trigger dismisses them again - this
+            // is the action that makes those reachable at all.
+            case 'hover': {
+                const session = manager.get(sessionId);
+                if (!session?.connected) {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'No active browser session.',
+                    });
+                    return;
+                }
+                try {
+                    if (typeof data.ref === 'string' ||
+                        typeof data.selector === 'string') {
+                        const pt = await session.hoverRef({
+                            ...(typeof data.ref === 'string' ? { ref: data.ref } : {}),
+                            ...(typeof data.selector === 'string'
+                                ? { selector: data.selector }
+                                : {}),
+                        });
+                        const screen = await screenOf(session);
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: true,
+                            hovered: true,
+                            ...(typeof data.ref === 'string' ? { ref: data.ref } : {}),
+                            x: pt.x,
+                            y: pt.y,
+                            ...(screen ? { screen } : {}),
+                        });
+                    }
+                    else {
+                        const x = typeof data.x === 'number' ? data.x : 0;
+                        const y = typeof data.y === 'number' ? data.y : 0;
+                        await session.hover(x, y);
+                        const screen = await screenOf(session);
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: true,
+                            hovered: true,
+                            x,
+                            y,
+                            ...(screen ? { screen } : {}),
+                        });
+                    }
+                }
+                catch (err) {
+                    const screen = await screenOf(session);
+                    await emitWithVision(emit, session, data.cmdId, {
+                        success: false,
+                        error: err instanceof Error ? err.message : String(err),
+                        ...(screen ? { screen } : {}),
+                    });
+                }
+                break;
+            }
+            // 2026-10: focus a control by ref. Reports whether focus actually moved;
+            // a non-focusable div says ok:false instead of pretending.
+            case 'focus': {
+                const session = manager.get(sessionId);
+                if (!session?.connected) {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'No active browser session.',
+                    });
+                    return;
+                }
+                try {
+                    if (typeof data.ref === 'string' && data.ref) {
+                        const result = await session.focusRef(data.ref);
+                        const screen = await screenOf(session);
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: result.ok === true,
+                            ...result,
+                            ...(result.ok
+                                ? {}
+                                : {
+                                    error: `FOCUS_NOT_TAKEN: ${data.ref} did not take focus (it may not be focusable).`,
+                                }),
+                            ...(screen ? { screen } : {}),
+                        });
+                    }
+                    else if (typeof data.selector === 'string' && data.selector) {
+                        const ok = await session.focus(data.selector);
+                        const screen = await screenOf(session);
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: ok,
+                            focused: ok,
+                            ...(ok
+                                ? {}
+                                : {
+                                    error: 'FOCUS_NOT_TAKEN: the selector matched no visible element, or it cannot take focus.',
+                                }),
+                            ...(screen ? { screen } : {}),
+                        });
+                    }
+                    else {
+                        await emitWithVision(emit, session, data.cmdId, {
+                            success: false,
+                            error: 'focus requires ref or selector.',
+                        });
+                    }
+                }
+                catch (err) {
+                    const screen = await screenOf(session);
+                    await emitWithVision(emit, session, data.cmdId, {
+                        success: false,
+                        error: err instanceof Error ? err.message : String(err),
+                        ...(screen ? { screen } : {}),
+                    });
+                }
+                break;
+            }
+            // 2026-10: dismiss whatever menu/popover layer is open.
+            case 'closePopup': {
+                const session = manager.get(sessionId);
+                if (!session?.connected) {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'No active browser session.',
+                    });
+                    return;
+                }
+                const result = await session.closePopup();
+                const screen = await screenOf(session);
+                await emitWithVision(emit, session, data.cmdId, {
+                    success: true,
+                    ...result,
+                    ...(screen ? { screen } : {}),
+                });
+                break;
+            }
+            // 2026-10: the interactive-element ref list (`e1 … eN`), including
+            // portalled menus/popovers/dialogs. Actions accept `ref`.
+            case 'getRefs': {
+                const session = manager.get(sessionId);
+                if (!session?.connected) {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'No active browser session. Call browser_open first.',
+                    });
+                    return;
+                }
+                const refs = await session.getRefs();
+                emit('orchestrator:browser-result', data.cmdId, {
+                    success: true,
+                    ...refs,
+                });
+                break;
+            }
+            // 2026-11: clear cookies/storage between test accounts. It existed on the
+            // session and in the batch path, but the SINGLE-action switch had no case,
+            // so browser_action({action:"clearSession"}) answered "Unknown browser
+            // action" - a silent dead end for invitation flows.
+            case 'clearSession': {
+                const session = manager.get(sessionId);
+                if (!session?.connected) {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'No active browser session. Call browser_open first.',
+                    });
+                    return;
+                }
+                const result = await session.clearSession();
+                await emitWithVision(emit, session, data.cmdId, {
+                    success: result.ok === true,
+                    cleared: result.ok === true,
+                    ...(result.ok
+                        ? {}
+                        : { error: result.error ?? 'the session could not be cleared' }),
+                });
+                break;
+            }
             case 'scroll': {
                 const session = manager.get(sessionId);
                 if (!session?.connected) {
@@ -357,7 +689,7 @@ async function handleBrowserCommand(data, emit) {
                 }
                 const delta = typeof data.delta === 'number' ? data.delta : 3;
                 await session.scroll(delta);
-                emit('orchestrator:browser-result', data.cmdId, {
+                await emitWithVision(emit, session, data.cmdId, {
                     success: true,
                     scrolled: true,
                     delta,
@@ -431,15 +763,24 @@ async function handleBrowserCommand(data, emit) {
                     ...(typeof data.maxChars === 'number'
                         ? { maxTextChars: data.maxChars }
                         : {}),
+                    ...(data.refs === true ? { refs: true } : {}),
                 });
                 const payload = {
                     success: true,
                     ...state,
                 };
-                if (data.screenshot === true && !session.nativeDialog) {
+                const wantsShot = data.screenshot === true ||
+                    (session.qaVision && data.screenshot !== false);
+                if (wantsShot && !session.nativeDialog) {
                     const format = data.format === 'png' ? 'png' : 'jpeg';
                     const quality = typeof data.quality === 'number' ? data.quality : 70;
-                    payload.screenshot = await session.screenshot(format, quality);
+                    if (data.annotate === true) {
+                        payload.screenshot = await session.annotatedScreenshot(format, quality);
+                        payload.annotated = true;
+                    }
+                    else {
+                        payload.screenshot = await session.screenshot(format, quality);
+                    }
                     payload.format = format;
                 }
                 emit('orchestrator:browser-result', data.cmdId, payload);
@@ -462,6 +803,7 @@ async function handleBrowserCommand(data, emit) {
                     'selector',
                     'text',
                     'dialog',
+                    'popover',
                     'toast',
                     'url',
                     'networkIdle',
@@ -543,13 +885,113 @@ async function handleBrowserCommand(data, emit) {
                         : {}),
                 });
                 const screen = await screenOf(session);
-                emit('orchestrator:browser-result', data.cmdId, {
+                await emitWithVision(emit, session, data.cmdId, {
+                    success: result.ok,
+                    ...result,
+                    ...(screen ? { screen } : {}),
+                });
+                break;
+            }
+            // 2026-10: pagination. `nextPage`/`prevPage` click the real control and
+            // wait for the page signature to change; `collectTable` walks every page
+            // and returns deduplicated rows in one call.
+            case 'nextPage':
+            case 'prevPage': {
+                const session = manager.get(sessionId);
+                if (!session?.connected) {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'No active browser session. Call browser_open first.',
+                    });
+                    return;
+                }
+                const timeout = typeof data.timeoutMs === 'number'
+                    ? data.timeoutMs
+                    : typeof data.stepTimeoutMs === 'number'
+                        ? data.stepTimeoutMs
+                        : 8000;
+                const result = action === 'nextPage'
+                    ? await session.nextPage(timeout)
+                    : await session.prevPage(timeout);
+                const screen = await screenOf(session);
+                await emitWithVision(emit, session, data.cmdId, {
+                    success: true,
+                    ...result,
+                    ...(screen ? { screen } : {}),
+                });
+                break;
+            }
+            case 'gotoPage': {
+                const session = manager.get(sessionId);
+                if (!session?.connected) {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'No active browser session. Call browser_open first.',
+                    });
+                    return;
+                }
+                if (typeof data.page !== 'number') {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'page parameter required for gotoPage',
+                    });
+                    return;
+                }
+                const result = await session.gotoPage(data.page, typeof data.timeoutMs === 'number' ? data.timeoutMs : undefined);
+                const screen = await screenOf(session);
+                await emitWithVision(emit, session, data.cmdId, {
+                    success: true,
+                    ...result,
+                    ...(screen ? { screen } : {}),
+                });
+                break;
+            }
+            case 'setPageSize': {
+                const session = manager.get(sessionId);
+                if (!session?.connected) {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'No active browser session. Call browser_open first.',
+                    });
+                    return;
+                }
+                if (typeof data.size !== 'number') {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'size parameter required for setPageSize',
+                    });
+                    return;
+                }
+                const result = await session.setPageSize(data.size);
+                const screen = await screenOf(session);
+                await emitWithVision(emit, session, data.cmdId, {
                     success: result.ok,
                     ...result,
                     ...(screen ? { screen } : {}),
                 });
                 break;
             }
+            case 'collectTable': {
+                const session = manager.get(sessionId);
+                if (!session?.connected) {
+                    emit('orchestrator:browser-result', data.cmdId, {
+                        success: false,
+                        error: 'No active browser session. Call browser_open first.',
+                    });
+                    return;
+                }
+                const result = await session.collectTable({
+                    selector: typeof data.selector === 'string' ? data.selector : undefined,
+                    maxPages: typeof data.maxPages === 'number' ? data.maxPages : undefined,
+                    maxRows: typeof data.maxRows === 'number' ? data.maxRows : undefined,
+                    pageSize: typeof data.size === 'number' ? data.size : undefined,
+                });
+                emit('orchestrator:browser-result', data.cmdId, {
+                    success: true,
+                    ...result,
+                });
+                break;
+            }
             case 'networkLog': {
                 const session = manager.get(sessionId);
                 if (!session?.connected) {
@@ -831,7 +1273,7 @@ async function handleBrowserCommand(data, emit) {
                     return;
                 }
                 const result = await session.submit(typeof data.selector === 'string' ? data.selector : undefined);
-                emit('orchestrator:browser-result', data.cmdId, {
+                await emitWithVision(emit, session, data.cmdId, {
                     success: result?.ok === true,
                     submitted: result?.ok === true,
                     ...(result?.ok
@@ -856,7 +1298,7 @@ async function handleBrowserCommand(data, emit) {
                 const steps = data.actions;
                 const summary = await (0, batch_1.executeBatch)(session, steps ?? []);
                 const screen = await screenOf(session);
-                emit('orchestrator:browser-result', data.cmdId, {
+                await emitWithVision(emit, session, data.cmdId, {
                     success: true,
                     batch: true,
                     ...summary,
@@ -901,7 +1343,10 @@ async function handleBrowserCommand(data, emit) {
     catch (err) {
         const message = err instanceof Error ? err.message : String(err);
         log.info(`Browser command FAILED (${action}): ${message}`);
-        emit('orchestrator:browser-result', data.cmdId, {
+        // In QA vision mode a thrown failure still shows the page: the error text
+        // alone ("TYPE_NOT_VERIFIED", "SELECTOR_NOT_FOUND") leaves the model
+        // guessing what the screen actually looked like.
+        await emitWithVision(emit, manager.get(sessionId), data.cmdId, {
             success: false,
             error: message,
         });