zan-browser 1.3.37 → 2.0.0

package/dist/session.js

@@ -5,56 +5,43 @@ const interceptor_1 = require("./interceptor");
 const observer_1 = require("./observer");
 class Session {
     page;
-    browser;
+    browserHandle;
     interceptor;
     observer;
     ai;
     defaultTimeout;
-    lastInteractiveElements = new Map();
-    viewerUrl; // Browserbase live viewer URL
-    constructor(page, browser, ai, timeout = 30_000, viewerUrl) {
+    viewerUrl;
+    constructor(page, browserHandle, ai, timeout = 30_000, viewerUrl) {
         this.page = page;
-        this.browser = browser;
+        this.browserHandle = browserHandle;
         this.ai = ai;
         this.defaultTimeout = timeout;
         this.viewerUrl = viewerUrl;
         this.interceptor = new interceptor_1.Interceptor(page);
         this.observer = new observer_1.Observer(page);
         // Start capturing traffic immediately — t=0
-        // This catches requests the site fires on load before we do anything
         this.interceptor.start();
     }
+    // ─── Accessors for SmartNavigator ───────────────────────────────────────────
+    getPage() { return this.page; }
+    getInterceptor() { return this.interceptor; }
+    getObserver() { return this.observer; }
+    getAI() { return this.ai; }
     // ─── Navigation ───────────────────────────────────────────────────────────────
     async goto(url, options) {
         const timeout = options?.timeout ?? this.defaultTimeout;
-        try {
-            await this.page.goto(url, {
-                waitUntil: "domcontentloaded",
-                timeout,
-            });
-        }
-        catch (err) {
-            // On timeout, don't destroy the session — wait for network idle on the
-            // partially loaded page, then continue if the URL actually changed.
-            try {
-                await this.page.waitForLoadState("networkidle", { timeout: 5_000 });
-            }
-            catch {
-                // networkidle also timed out — that's fine, continue with what we have
-            }
-            const currentUrl = this.page.url();
-            if (currentUrl === "about:blank" || currentUrl === "") {
-                throw new Error(`Navigation to ${url} failed: page is still on about:blank — ${err.message}`);
-            }
-            // Page URL changed — partial load is usable, continue
-        }
+        await this.page.goto(url, {
+            waitUntil: "domcontentloaded",
+            timeout,
+        });
         // Wait for network to go idle — catches XHR that SPAs fire after DOM is parsed.
         try {
-            await this.page.waitForLoadState("networkidle", { timeout: 8_000 });
+            await this.page.waitForNetworkIdle({ timeout: 8_000 });
         }
         catch {
             // networkidle timed out — page likely has background polling, continue anyway
         }
+        // Minimum 800ms after networkidle (or timeout) — some SPAs fire XHR slightly after idle
         await this.page.waitForTimeout(800);
     }
     async wait(ms) {
@@ -65,21 +52,28 @@ class Session {
     }
     // ─── Observe — always first, before any action ────────────────────────────────
     async observe() {
-        this.lastInteractiveElements.clear();
         return this.observer.observe();
     }
-    // ─── Screenshot — only when observe() is not enough ──────────────────────────
+    // ─── Form and autocomplete detection ────────────────────────────────────────
+    async detectForms() {
+        return this.observer.detectForms();
+    }
+    async detectAutocomplete() {
+        return this.observer.detectAutocomplete();
+    }
+    async detectErrors() {
+        return this.observer.detectErrors();
+    }
+    // ─── Screenshot ────────────────────────────────────────────────────────────────
     async screenshot() {
-        const buffer = await this.page.screenshot({ type: "jpeg", quality: 80 });
+        const result = await this.page.screenshot({ type: "jpeg", quality: 80, encoding: "base64" });
         return {
-            base64: buffer.toString("base64"),
+            base64: typeof result === "string" ? result : result.toString("base64"),
             mimeType: "image/jpeg",
             timestamp: Date.now(),
         };
     }
     // ─── DOM text extraction — fallback when no useful XHR is captured ────────────
-    // Strips chrome (nav, footer, scripts) and returns visible text capped at 15k chars.
-    // Used by web-agent when the page renders data server-side with no API calls.
     async extractPageContent() {
         return await this.page.evaluate(() => {
             const clone = document.body.cloneNode(true);
@@ -93,344 +87,99 @@ class Session {
                 .slice(0, 15000);
         });
     }
-    // ─── DOM text + links extraction — for link-aware intelligent navigation ─────
-    // Returns visible text (same as extractPageContent) plus all <a> links on the page.
-    // Used by the caller to feed real links into AI so it picks from actual URLs, not guesses.
-    async extractPageWithLinks() {
-        return await this.page.evaluate(() => {
-            const clone = document.body.cloneNode(true);
-            clone.querySelectorAll("script,style,nav,footer,header")
-                .forEach((el) => el.remove());
-            const content = clone.innerText
-                .split("\n")
-                .map((l) => l.trim())
-                .filter((l) => l.length > 0)
-                .join("\n")
-                .slice(0, 12000);
-            const links = [];
-            const seen = new Set();
-            document.querySelectorAll("a[href]").forEach((a) => {
-                const href = a.href;
-                const text = a.innerText.trim().slice(0, 100);
-                if (href && text && !seen.has(href) && href.startsWith("http")) {
-                    seen.add(href);
-                    links.push({ href, text });
-                }
-            });
-            return { content, links: links.slice(0, 80) };
-        });
-    }
-    // ─── Interactive DOM observation — full interactive element scan ─────────────
-    // Replaces extractPageWithLinks() for agent navigation: returns ALL interactive
-    // elements (buttons, menus, dropdowns, inputs) not just anchor links.
-    // Sets data-rid attributes on matched elements so clickById() can target them.
-    async observeInteractiveDom() {
-        const INTERACTIVE_SEL = [
-            "button", "[role='button']", "[role='switch']",
-            "input:not([type='hidden'])",
-            "a[href]", "select", "textarea",
-            "[role='menuitem']", "[role='option']", "[role='tab']", "[role='combobox']",
-            "[onclick]", "[tabindex]:not([tabindex='-1'])",
-        ].join(", ");
-        const result = await this.page.evaluate((interactiveSel) => {
-            // Clean previous markers
-            document.querySelectorAll("[data-rid]").forEach((el) => el.removeAttribute("data-rid"));
-            const isVisible = (el) => {
-                const htmlEl = el;
-                // offsetParent is null when the element or any ancestor has display:none.
-                // Exception: <body>, <html>, and position:fixed/sticky elements legitimately have offsetParent === null.
-                if (htmlEl.offsetParent === null) {
-                    const style = window.getComputedStyle(el);
-                    const pos = style.position;
-                    if (pos !== "fixed" && pos !== "sticky" && el.tagName !== "BODY" && el.tagName !== "HTML") {
-                        return false;
-                    }
-                }
-                const style = window.getComputedStyle(el);
-                if (style.visibility === "hidden" || style.opacity === "0")
-                    return false;
-                const rect = el.getBoundingClientRect();
-                if (rect.width === 0 || rect.height === 0)
-                    return false;
-                // Check if element is within viewport bounds
-                const vh = window.innerHeight;
-                const vw = window.innerWidth;
-                return rect.bottom > 0 && rect.top < vh && rect.right > 0 && rect.left < vw;
-            };
-            const truncate = (v, max = 60) => v.length > max ? v.slice(0, max) : v;
-            const resolveUrl = (href) => {
-                if (!href)
-                    return href;
-                try {
-                    return new URL(href, window.location.href).href;
-                }
-                catch {
-                    return href;
-                }
-            };
-            const ATTR_KEYS = [
-                "title", "type", "name", "role", "aria-label",
-                "placeholder", "value", "alt", "href", "aria-expanded",
-            ];
-            const ATTR_RENDER_ORDER = [
-                "title", "type", "name", "role", "aria_label",
-                "placeholder", "value", "alt", "href", "aria_expanded",
-            ];
-            const getAttrs = (el) => {
-                const out = {};
-                for (const attr of ATTR_KEYS) {
-                    let val = el.getAttribute(attr);
-                    if (val == null || val === "")
-                        continue;
-                    if (attr === "href")
-                        val = resolveUrl(val);
-                    const key = attr.replace(/-/g, "_");
-                    out[key] = truncate(val);
-                }
-                if ((el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement)
-                    && el.value) {
-                    out["value"] = truncate(el.value);
-                }
-                return out;
-            };
-            const getInnerText = (el) => (el.textContent ?? "").replace(/\s+/g, " ").trim().slice(0, 100);
-            // B — buttons, checkboxes, radios, switches, role="button"
-            // L — anchor links
-            // I — inputs, textareas, selects
-            // M — menuitem, option, tab, combobox, other interactables
-            const categorize = (el) => {
-                const tag = el.tagName.toLowerCase();
-                const role = el.getAttribute("role") ?? "";
-                const type = el.type ?? "";
-                if (tag === "a" && el.hasAttribute("href"))
-                    return "L";
-                if (tag === "textarea" || tag === "select")
-                    return "I";
-                if (tag === "input" && !["submit", "button", "checkbox", "radio", "hidden"].includes(type))
-                    return "I";
-                if (tag === "button")
-                    return "B";
-                if (tag === "input" && ["submit", "button", "checkbox", "radio"].includes(type))
-                    return "B";
-                if (role === "button" || role === "switch")
-                    return "B";
-                if (["menuitem", "option", "tab", "combobox"].includes(role))
-                    return "M";
-                return "M";
-            };
-            const interactiveSet = new Set();
-            document.querySelectorAll(interactiveSel).forEach((el) => interactiveSet.add(el));
-            // Only include text near interactive elements — headings, labels, or
-            // nodes whose parent/grandparent contains an interactive child.
-            const isNearInteractive = (parent) => {
-                if (!parent)
-                    return false;
-                const tag = parent.tagName.toLowerCase();
-                if (["h1", "h2", "h3", "h4", "h5", "h6", "label", "legend", "figcaption", "caption", "th", "li"].includes(tag))
-                    return true;
-                try {
-                    if (parent.querySelector(interactiveSel))
-                        return true;
-                    if (parent.parentElement?.querySelector(interactiveSel))
-                        return true;
-                }
-                catch { /* selector might fail on exotic DOMs */ }
-                return false;
-            };
-            const SELF_CLOSING = new Set(["input", "img", "br", "hr"]);
-            const lines = [];
-            const elements = [];
-            const counters = { B: 0, L: 0, I: 0, M: 0 };
-            const walk = (node) => {
-                if (node.nodeType === Node.TEXT_NODE) {
-                    const text = (node.textContent ?? "").replace(/\s+/g, " ").trim();
-                    if (text.length < 2)
-                        return;
-                    const parent = node.parentElement;
-                    if (!parent)
-                        return;
-                    if (parent.tagName === "SCRIPT" || parent.tagName === "STYLE" || parent.tagName === "NOSCRIPT")
-                        return;
-                    if (interactiveSet.has(parent))
-                        return; // text goes inside the tag
-                    if (!isVisible(parent))
-                        return;
-                    if (!isNearInteractive(parent))
-                        return;
-                    lines.push(`_[:] ${text.slice(0, 150)}`);
-                    return;
-                }
-                if (node.nodeType !== Node.ELEMENT_NODE)
-                    return;
-                const el = node;
-                const tag = el.tagName.toLowerCase();
-                if (tag === "script" || tag === "style" || tag === "noscript")
-                    return;
-                if (interactiveSet.has(el)) {
-                    if (!isVisible(el))
-                        return;
-                    const cat = categorize(el);
-                    counters[cat]++;
-                    const id = `${cat}${counters[cat]}`;
-                    const attrs = getAttrs(el);
-                    const text = getInnerText(el);
-                    const htmlTag = tag === "a" ? "a" : tag;
-                    // Mark element for Playwright selector lookup
-                    el.setAttribute("data-rid", id);
-                    const attrParts = [];
-                    for (const key of ATTR_RENDER_ORDER) {
-                        if (attrs[key])
-                            attrParts.push(`${key}="${attrs[key]}"`);
-                    }
-                    const attrStr = attrParts.length > 0 ? " " + attrParts.join(" ") : "";
-                    if (SELF_CLOSING.has(htmlTag)) {
-                        lines.push(`${id}[:] <${htmlTag}${attrStr}/>`);
-                    }
-                    else {
-                        lines.push(`${id}[:] <${htmlTag}${attrStr}>${text}</${htmlTag}>`);
-                    }
-                    elements.push({ id, selector: `[data-rid="${id}"]` });
-                    return; // don't recurse into interactive elements
-                }
-                for (const child of Array.from(el.childNodes)) {
-                    walk(child);
-                }
-            };
-            walk(document.body);
-            return { lines, elements };
-        }, INTERACTIVE_SEL);
-        this.lastInteractiveElements = new Map(result.elements.map((e) => [e.id, e.selector]));
-        return {
-            dom: result.lines.join("\n"),
-            elements: result.elements,
-        };
-    }
-    // ─── Intelligent DOM scrape — Notte-style content understanding ──────────────
-    // Passes visible page text to Claude Sonnet and asks whether the goal data is present.
-    // If not found, follows nextUrl suggestions up to maxHops times before giving up.
-    // Caller must supply initialContent (from extractPageContent()) to avoid a double call.
-    async scrapeWithGoal(goal, initialContent, seedValues, maxHops = 3, initialLinks) {
-        let content = initialContent;
-        let links = initialLinks;
-        for (let hop = 0; hop <= maxHops; hop++) {
-            const result = await this.ai.scrapePageContent(goal, content, seedValues, links);
-            if (result.found || !result.nextUrl || hop === maxHops) {
-                return result;
-            }
-            // Follow the suggested link and re-scrape
+    // ─── Interaction ──────────────────────────────────────────────────────────────
+    async clickById(elementId) {
+        // First try cached CSS selector (reliable)
+        const selector = this.observer.getSelector(elementId);
+        if (selector) {
             try {
-                await this.goto(result.nextUrl);
-                const page = await this.extractPageWithLinks();
-                content = page.content;
-                links = page.links;
+                await this.page.click(selector);
+                return;
             }
             catch {
-                return result;
+                // Fall through to coordinate-based click
             }
         }
-        return { found: false, data: null, nextUrl: null, reasoning: "max hops reached without finding data" };
+        const coords = await this.observer.resolveElementId(elementId);
+        if (!coords)
+            throw new Error(`Element ${elementId} not found in DOM`);
+        await this.page.mouse.click(coords.x, coords.y);
     }
-    // ─── Interaction ──────────────────────────────────────────────────────────────
-    // Click by element ID — checks stored selectors from observeInteractiveDom()
-    // first, then falls back to coordinate-based resolution from observe().
-    async clickById(elementId) {
-        const pagesBefore = this.page.context().pages().length;
-        const selector = this.lastInteractiveElements.get(elementId);
+    async fillById(elementId, value) {
+        // First try cached CSS selector (reliable)
+        const selector = this.observer.getSelector(elementId);
         if (selector) {
-            await this.page.locator(selector).click();
-        }
-        else {
-            const coords = await this.observer.resolveElementId(elementId);
-            if (!coords)
-                throw new Error(`Element ${elementId} not found in DOM`);
-            await this.page.mouse.click(coords.x, coords.y);
+            try {
+                await this.page.click(selector, { clickCount: 3 }); // select all
+                await this.page.keyboard.press("Backspace");
+                await this.page.type(selector, value, { delay: 80 });
+                return;
+            }
+            catch {
+                // Fall through to coordinate-based fill
+            }
         }
-        await this.switchToNewTabIfOpened(pagesBefore);
-    }
-    // Fill input by element ID from observe()
-    async fillById(elementId, value) {
         const coords = await this.observer.resolveElementId(elementId);
         if (!coords)
             throw new Error(`Element ${elementId} not found in DOM`);
         await this.page.mouse.click(coords.x, coords.y);
         await this.page.keyboard.press("Control+a");
-        await this.page.keyboard.type(value);
+        await this.page.keyboard.type(value, { delay: 80 });
+    }
+    // ─── Direct selector interaction (for autocomplete clicks) ──────────────────
+    async clickSelector(selector) {
+        const el = await this.page.waitForSelector(selector, { timeout: 3_000, visible: true });
+        if (el) {
+            await el.click();
+        }
+        else {
+            throw new Error(`Selector ${selector} not found`);
+        }
     }
     // ─── Captured requests ────────────────────────────────────────────────────────
-    // All captured requests sorted by relevance score
     getCapturedRequests() {
         return this.interceptor.getAll();
     }
-    // Only useful XHR/fetch requests above score threshold
     getUsefulRequests(minScore = 1) {
         return this.interceptor.getUseful(minScore);
     }
-    // Only requests that fired as a direct result of a user action
     getActionTriggeredRequests() {
         return this.interceptor.getActionTriggered();
     }
-    // Listen to requests in real-time as they come in
     onRequest(handler) {
         this.interceptor.on("request", handler);
     }
+    hasSeedValueInResponses(seedValues) {
+        return this.interceptor.hasSeedValueInResponses(seedValues);
+    }
+    markActionTimestamp() {
+        this.interceptor.markActionTimestamp();
+    }
     // ─── act() — achieve a goal through multi-step navigation ────────────────────
     async act(goal, maxSteps = 10, seedValues) {
         const steps = [];
-        const trajectory = [];
-        let consecutiveFailures = 0;
-        const MAX_CONSECUTIVE_FAILURES = 3;
+        const history = [];
         for (let i = 0; i < maxSteps; i++) {
-            // ALWAYS observe first — no screenshot by default
             const observation = await this.observe();
-            let completion;
+            let action;
             let usedScreenshot = false;
-            // If observe() gives us enough elements, use text-based decision
             if (observation.elements.filter((e) => e.visible).length > 0) {
-                completion = await this.ai.decideFromObservation(goal, observation, trajectory, i + 1, maxSteps, seedValues);
+                action = await this.ai.decideFromObservation(goal, observation, history, seedValues);
             }
             else {
-                // Fallback: page has no readable elements (canvas, iframe, etc.) → screenshot
                 const shot = await this.screenshot();
-                completion = await this.ai.decideFromScreenshot(goal, shot.base64, trajectory, i + 1, maxSteps, seedValues);
+                action = await this.ai.decideFromScreenshot(goal, shot.base64, history, seedValues);
                 usedScreenshot = true;
             }
-            const { action, state } = completion;
             steps.push({ action, observeBefore: observation, usedScreenshot });
-            // Track trajectory for multi-turn conversation (assistant response, then next user observation)
-            trajectory.push({
-                role: "assistant",
-                content: JSON.stringify(completion),
-            });
-            // Track consecutive failures
-            if (state.previous_goal_status === "failure") {
-                consecutiveFailures++;
-                if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
-                    return {
-                        success: false,
-                        steps,
-                        finalUrl: this.url,
-                        reason: `${MAX_CONSECUTIVE_FAILURES} consecutive failures: ${state.previous_goal_eval}`,
-                    };
-                }
-            }
-            else if (state.previous_goal_status === "success") {
-                consecutiveFailures = 0;
-            }
-            // Terminal states
+            history.push(`Step ${i + 1}: ${action.type} — ${action.reason}`);
             if (action.type === "done") {
                 return { success: true, steps, finalUrl: this.url, reason: action.reason };
             }
             if (action.type === "impossible") {
                 return { success: false, steps, finalUrl: this.url, reason: action.reason };
             }
-            // Execute action
-            const actionSuccess = await this.executeAction(action);
-            // Add execution result to trajectory
-            const resultMsg = actionSuccess
-                ? `Action '${action.type}' succeeded`
-                : `Action '${action.type}' failed`;
-            trajectory.push({ role: "user", content: resultMsg });
-            // Short wait after interactions for page to settle
+            await this.executeAction(action);
             await this.page.waitForTimeout(500);
         }
         return {
@@ -441,10 +190,7 @@ class Session {
         };
     }
     // ─── findData() — navigate + capture requests until goal achieved ─────────────
-    // seedValues: concrete example inputs to use during discovery (e.g. what to type in a search box)
     async findData(goal, maxSteps = 15, seedValues) {
-        // Do NOT clear — requests captured during goto() (t=0) are valuable.
-        // goto() already waits for networkidle + 800ms minimum, so no additional delay needed here.
         const actResult = await this.act(goal, maxSteps, seedValues);
         const capturedRequests = this.interceptor.getUseful();
         return {
@@ -455,84 +201,49 @@ class Session {
     // ─── Cleanup ──────────────────────────────────────────────────────────────────
     async close() {
         this.interceptor.stop();
-        await this.page.close();
+        if (!this.page.isClosed()) {
+            await this.page.close();
+        }
     }
     async closeAll() {
         this.interceptor.stop();
-        await this.browser.close();
-    }
-    // ─── New tab detection ─────────────────────────────────────────────────────────
-    // After a click, check if a new tab was opened and switch the session to it.
-    // Rebuilds interceptor/observer so they reference the new page.
-    async switchToNewTabIfOpened(pagesBeforeCount) {
-        await this.page.waitForTimeout(1000);
-        const pages = this.page.context().pages();
-        if (pages.length <= pagesBeforeCount)
-            return;
-        // Switch to the newest tab
-        const newPage = pages[pages.length - 1];
-        if (newPage === this.page)
-            return;
-        // Tear down interceptor on old page, rebuild on new page
-        this.interceptor.stop();
-        this.page = newPage;
-        this.interceptor = new interceptor_1.Interceptor(this.page);
-        this.observer = new observer_1.Observer(this.page);
-        this.interceptor.start();
-        try {
-            await this.page.waitForLoadState("domcontentloaded", { timeout: this.defaultTimeout });
-        }
-        catch {
-            // Partial load — continue with what we have
-        }
+        await this.browserHandle.close();
     }
     // ─── Execute action ───────────────────────────────────────────────────────────
     async executeAction(action) {
-        try {
-            switch (action.type) {
-                case "click":
-                    this.interceptor.markActionTimestamp();
-                    await this.clickById(action.elementId).catch(async () => {
-                        await this.page.getByText(action.elementId).first().click();
-                    });
-                    break;
-                case "fill":
-                    this.interceptor.markActionTimestamp();
-                    await this.fillById(action.elementId, action.value).catch(async () => {
-                        await this.page.getByRole("textbox").first().fill(action.value);
-                    });
-                    break;
-                case "scroll":
-                    this.interceptor.markActionTimestamp();
-                    await this.page.evaluate(({ direction, amount }) => {
-                        window.scrollBy(0, direction === "down" ? amount : -amount);
-                    }, { direction: action.direction, amount: action.amount ?? 300 }).catch(() => {
-                        // Execution context destroyed = page navigated mid-scroll. Not a crash — continue.
-                    });
-                    break;
-                case "navigate":
-                    this.interceptor.markActionTimestamp();
-                    await this.goto(action.url);
-                    break;
-                case "press_key":
-                    await this.page.keyboard.press(action.key);
-                    break;
-                case "wait":
-                    await this.page.waitForTimeout(action.ms);
-                    break;
-                case "scrape":
-                    // Scrape is handled by the caller — no-op here
-                    break;
-                case "screenshot":
-                    // Already handled in act() loop
-                    break;
-                default:
-                    break;
-            }
-            return true;
-        }
-        catch {
-            return false;
+        switch (action.type) {
+            case "click":
+                this.interceptor.markActionTimestamp();
+                await this.clickById(action.elementId).catch(() => {
+                    // If all resolution fails, log and continue
+                    console.warn(`[Session] Click failed for ${action.elementId}`);
+                });
+                break;
+            case "fill":
+                this.interceptor.markActionTimestamp();
+                await this.fillById(action.elementId, action.value).catch(() => {
+                    console.warn(`[Session] Fill failed for ${action.elementId}`);
+                });
+                break;
+            case "scroll":
+                this.interceptor.markActionTimestamp();
+                await this.page.evaluate(({ direction, amount }) => {
+                    window.scrollBy(0, direction === "down" ? amount : -amount);
+                }, { direction: action.direction, amount: action.amount ?? 300 }).catch(() => {
+                    // Execution context destroyed = page navigated mid-scroll
+                });
+                break;
+            case "navigate":
+                this.interceptor.markActionTimestamp();
+                await this.page.goto(action.url, { waitUntil: "domcontentloaded", timeout: this.defaultTimeout });
+                break;
+            case "wait":
+                await this.page.waitForTimeout(action.ms);
+                break;
+            case "screenshot":
+                break;
+            default:
+                break;
         }
     }
 }