sentienceapi 0.90.1

package/dist/actions.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Actions v1 - click, type, press
+ */
+import { SentienceBrowser } from './browser';
+import { ActionResult, BBox } from './types';
+export interface ClickRect {
+    x: number;
+    y: number;
+    w?: number;
+    width?: number;
+    h?: number;
+    height?: number;
+}
+export declare function click(browser: SentienceBrowser, elementId: number, useMouse?: boolean, takeSnapshot?: boolean): Promise<ActionResult>;
+export declare function typeText(browser: SentienceBrowser, elementId: number, text: string, takeSnapshot?: boolean): Promise<ActionResult>;
+export declare function press(browser: SentienceBrowser, key: string, takeSnapshot?: boolean): Promise<ActionResult>;
+/**
+ * Click at the center of a rectangle using Playwright's native mouse simulation.
+ * This uses a hybrid approach: calculates center coordinates and uses mouse.click()
+ * for realistic event simulation (triggers hover, focus, mousedown, mouseup).
+ *
+ * @param browser - SentienceBrowser instance
+ * @param rect - Rectangle with x, y, w (or width), h (or height) keys, or BBox object
+ * @param highlight - Whether to show a red border highlight when clicking (default: true)
+ * @param highlightDuration - How long to show the highlight in seconds (default: 2.0)
+ * @param takeSnapshot - Whether to take snapshot after action
+ * @returns ActionResult
+ *
+ * @example
+ * ```typescript
+ * // Click using rect object
+ * await clickRect(browser, { x: 100, y: 200, w: 50, h: 30 });
+ *
+ * // Click using BBox from element
+ * const snap = await snapshot(browser);
+ * const element = find(snap, "role=button");
+ * if (element) {
+ *   await clickRect(browser, {
+ *     x: element.bbox.x,
+ *     y: element.bbox.y,
+ *     w: element.bbox.width,
+ *     h: element.bbox.height
+ *   });
+ * }
+ *
+ * // Without highlight
+ * await clickRect(browser, { x: 100, y: 200, w: 50, h: 30 }, false);
+ *
+ * // Custom highlight duration
+ * await clickRect(browser, { x: 100, y: 200, w: 50, h: 30 }, true, 3.0);
+ * ```
+ */
+export declare function clickRect(browser: SentienceBrowser, rect: ClickRect | BBox, highlight?: boolean, highlightDuration?: number, takeSnapshot?: boolean): Promise<ActionResult>;
+//# sourceMappingURL=actions.d.ts.map

package/dist/actions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAY,IAAI,EAAE,MAAM,SAAS,CAAC;AAGvD,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA4DD,wBAAsB,KAAK,CACzB,OAAO,EAAE,gBAAgB,EACzB,SAAS,EAAE,MAAM,EACjB,QAAQ,GAAE,OAAc,EACxB,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CAyGvB;AAED,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,gBAAgB,EACzB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CA6CvB;AAED,wBAAsB,KAAK,CACzB,OAAO,EAAE,gBAAgB,EACzB,GAAG,EAAE,MAAM,EACX,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CA6BvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,wBAAsB,SAAS,CAC7B,OAAO,EAAE,gBAAgB,EACzB,IAAI,EAAE,SAAS,GAAG,IAAI,EACtB,SAAS,GAAE,OAAc,EACzB,iBAAiB,GAAE,MAAY,EAC/B,YAAY,GAAE,OAAe,GAC5B,OAAO,CAAC,YAAY,CAAC,CA2FvB"}

package/dist/actions.js

@@ -0,0 +1,349 @@
+"use strict";
+/**
+ * Actions v1 - click, type, press
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.click = click;
+exports.typeText = typeText;
+exports.press = press;
+exports.clickRect = clickRect;
+const snapshot_1 = require("./snapshot");
+/**
+ * Highlight a rectangle with a red border overlay
+ */
+async function highlightRect(browser, rect, durationSec = 2.0) {
+    const page = browser.getPage();
+    const highlightId = `sentience_highlight_${Date.now()}`;
+    // Combine all arguments into a single object for Playwright
+    const args = {
+        rect: {
+            x: rect.x,
+            y: rect.y,
+            w: rect.w || rect.width || 0,
+            h: rect.h || rect.height || 0,
+        },
+        highlightId,
+        durationSec,
+    };
+    await page.evaluate((args) => {
+        const { rect, highlightId, durationSec } = args;
+        // Create overlay div
+        const overlay = document.createElement('div');
+        overlay.id = highlightId;
+        overlay.style.position = 'fixed';
+        overlay.style.left = `${rect.x}px`;
+        overlay.style.top = `${rect.y}px`;
+        overlay.style.width = `${rect.w}px`;
+        overlay.style.height = `${rect.h}px`;
+        overlay.style.border = '3px solid red';
+        overlay.style.borderRadius = '2px';
+        overlay.style.boxSizing = 'border-box';
+        overlay.style.pointerEvents = 'none';
+        overlay.style.zIndex = '999999';
+        overlay.style.backgroundColor = 'rgba(255, 0, 0, 0.1)';
+        overlay.style.transition = 'opacity 0.3s ease-out';
+        document.body.appendChild(overlay);
+        // Remove after duration
+        setTimeout(() => {
+            overlay.style.opacity = '0';
+            setTimeout(() => {
+                if (overlay.parentNode) {
+                    overlay.parentNode.removeChild(overlay);
+                }
+            }, 300); // Wait for fade-out transition
+        }, durationSec * 1000);
+    }, args);
+}
+async function click(browser, elementId, useMouse = true, takeSnapshot = false) {
+    const page = browser.getPage();
+    const startTime = Date.now();
+    const urlBefore = page.url();
+    let success;
+    if (useMouse) {
+        // Hybrid approach: Get element bbox from snapshot, calculate center, use mouse.click()
+        try {
+            const snap = await (0, snapshot_1.snapshot)(browser);
+            const element = snap.elements.find((el) => el.id === elementId);
+            if (element) {
+                // Calculate center of element bbox
+                const centerX = element.bbox.x + element.bbox.width / 2;
+                const centerY = element.bbox.y + element.bbox.height / 2;
+                // Use Playwright's native mouse click for realistic simulation
+                await page.mouse.click(centerX, centerY);
+                success = true;
+            }
+            else {
+                // Fallback to JS click if element not found in snapshot
+                try {
+                    success = await page.evaluate((id) => {
+                        return window.sentience.click(id);
+                    }, elementId);
+                }
+                catch (error) {
+                    // Navigation might have destroyed context, assume success if URL changed
+                    success = true;
+                }
+            }
+        }
+        catch (error) {
+            // Fallback to JS click on error
+            try {
+                success = await page.evaluate((id) => {
+                    return window.sentience.click(id);
+                }, elementId);
+            }
+            catch (evalError) {
+                // Navigation might have destroyed context, assume success
+                success = true;
+            }
+        }
+    }
+    else {
+        // Legacy JS-based click
+        try {
+            success = await page.evaluate((id) => {
+                return window.sentience.click(id);
+            }, elementId);
+        }
+        catch (error) {
+            // Navigation might have destroyed context, assume success
+            success = true;
+        }
+    }
+    // Wait a bit for navigation/DOM updates
+    try {
+        await page.waitForTimeout(500);
+    }
+    catch (error) {
+        // Navigation might have happened, context destroyed
+    }
+    const durationMs = Date.now() - startTime;
+    // Check if URL changed (handle navigation gracefully)
+    let urlAfter;
+    let urlChanged;
+    try {
+        urlAfter = page.url();
+        urlChanged = urlBefore !== urlAfter;
+    }
+    catch (error) {
+        // Context destroyed due to navigation - assume URL changed
+        urlAfter = urlBefore;
+        urlChanged = true;
+    }
+    // Determine outcome
+    let outcome;
+    if (urlChanged) {
+        outcome = 'navigated';
+    }
+    else if (success) {
+        outcome = 'dom_updated';
+    }
+    else {
+        outcome = 'error';
+    }
+    // Optional snapshot after
+    let snapshotAfter;
+    if (takeSnapshot) {
+        try {
+            snapshotAfter = await (0, snapshot_1.snapshot)(browser);
+        }
+        catch (error) {
+            // Navigation might have destroyed context
+        }
+    }
+    return {
+        success,
+        duration_ms: durationMs,
+        outcome,
+        url_changed: urlChanged,
+        snapshot_after: snapshotAfter,
+        error: success
+            ? undefined
+            : { code: 'click_failed', reason: 'Element not found or not clickable' },
+    };
+}
+async function typeText(browser, elementId, text, takeSnapshot = false) {
+    const page = browser.getPage();
+    const startTime = Date.now();
+    const urlBefore = page.url();
+    // Focus element first
+    const focused = await page.evaluate((id) => {
+        const el = window.sentience_registry[id];
+        if (el) {
+            el.focus();
+            return true;
+        }
+        return false;
+    }, elementId);
+    if (!focused) {
+        return {
+            success: false,
+            duration_ms: Date.now() - startTime,
+            outcome: 'error',
+            error: { code: 'focus_failed', reason: 'Element not found' },
+        };
+    }
+    // Type using Playwright keyboard
+    await page.keyboard.type(text);
+    const durationMs = Date.now() - startTime;
+    const urlAfter = page.url();
+    const urlChanged = urlBefore !== urlAfter;
+    const outcome = urlChanged ? 'navigated' : 'dom_updated';
+    let snapshotAfter;
+    if (takeSnapshot) {
+        snapshotAfter = await (0, snapshot_1.snapshot)(browser);
+    }
+    return {
+        success: true,
+        duration_ms: durationMs,
+        outcome,
+        url_changed: urlChanged,
+        snapshot_after: snapshotAfter,
+    };
+}
+async function press(browser, key, takeSnapshot = false) {
+    const page = browser.getPage();
+    const startTime = Date.now();
+    const urlBefore = page.url();
+    // Press key using Playwright
+    await page.keyboard.press(key);
+    // Wait a bit for navigation/DOM updates
+    await page.waitForTimeout(500);
+    const durationMs = Date.now() - startTime;
+    const urlAfter = page.url();
+    const urlChanged = urlBefore !== urlAfter;
+    const outcome = urlChanged ? 'navigated' : 'dom_updated';
+    let snapshotAfter;
+    if (takeSnapshot) {
+        snapshotAfter = await (0, snapshot_1.snapshot)(browser);
+    }
+    return {
+        success: true,
+        duration_ms: durationMs,
+        outcome,
+        url_changed: urlChanged,
+        snapshot_after: snapshotAfter,
+    };
+}
+/**
+ * Click at the center of a rectangle using Playwright's native mouse simulation.
+ * This uses a hybrid approach: calculates center coordinates and uses mouse.click()
+ * for realistic event simulation (triggers hover, focus, mousedown, mouseup).
+ *
+ * @param browser - SentienceBrowser instance
+ * @param rect - Rectangle with x, y, w (or width), h (or height) keys, or BBox object
+ * @param highlight - Whether to show a red border highlight when clicking (default: true)
+ * @param highlightDuration - How long to show the highlight in seconds (default: 2.0)
+ * @param takeSnapshot - Whether to take snapshot after action
+ * @returns ActionResult
+ *
+ * @example
+ * ```typescript
+ * // Click using rect object
+ * await clickRect(browser, { x: 100, y: 200, w: 50, h: 30 });
+ *
+ * // Click using BBox from element
+ * const snap = await snapshot(browser);
+ * const element = find(snap, "role=button");
+ * if (element) {
+ *   await clickRect(browser, {
+ *     x: element.bbox.x,
+ *     y: element.bbox.y,
+ *     w: element.bbox.width,
+ *     h: element.bbox.height
+ *   });
+ * }
+ *
+ * // Without highlight
+ * await clickRect(browser, { x: 100, y: 200, w: 50, h: 30 }, false);
+ *
+ * // Custom highlight duration
+ * await clickRect(browser, { x: 100, y: 200, w: 50, h: 30 }, true, 3.0);
+ * ```
+ */
+async function clickRect(browser, rect, highlight = true, highlightDuration = 2.0, takeSnapshot = false) {
+    const page = browser.getPage();
+    const startTime = Date.now();
+    const urlBefore = page.url();
+    // Handle BBox object or ClickRect dict
+    let x, y, w, h;
+    if ('width' in rect && 'height' in rect && !('w' in rect) && !('h' in rect)) {
+        // BBox object (width and height are required in BBox)
+        const bbox = rect;
+        x = bbox.x;
+        y = bbox.y;
+        w = bbox.width;
+        h = bbox.height;
+    }
+    else {
+        // ClickRect dict
+        const clickRect = rect;
+        x = clickRect.x;
+        y = clickRect.y;
+        w = clickRect.w || clickRect.width || 0;
+        h = clickRect.h || clickRect.height || 0;
+    }
+    if (w <= 0 || h <= 0) {
+        return {
+            success: false,
+            duration_ms: 0,
+            outcome: 'error',
+            error: {
+                code: 'invalid_rect',
+                reason: 'Rectangle width and height must be positive',
+            },
+        };
+    }
+    // Calculate center of rectangle
+    const centerX = x + w / 2;
+    const centerY = y + h / 2;
+    // Show highlight before clicking (if enabled)
+    if (highlight) {
+        await highlightRect(browser, { x, y, w, h }, highlightDuration);
+        // Small delay to ensure highlight is visible
+        await page.waitForTimeout(50);
+    }
+    // Use Playwright's native mouse click for realistic simulation
+    let success;
+    let errorMsg;
+    try {
+        await page.mouse.click(centerX, centerY);
+        success = true;
+    }
+    catch (error) {
+        success = false;
+        errorMsg = error instanceof Error ? error.message : String(error);
+    }
+    // Wait a bit for navigation/DOM updates
+    await page.waitForTimeout(500);
+    const durationMs = Date.now() - startTime;
+    const urlAfter = page.url();
+    const urlChanged = urlBefore !== urlAfter;
+    // Determine outcome
+    let outcome;
+    if (urlChanged) {
+        outcome = 'navigated';
+    }
+    else if (success) {
+        outcome = 'dom_updated';
+    }
+    else {
+        outcome = 'error';
+    }
+    // Optional snapshot after
+    let snapshotAfter;
+    if (takeSnapshot) {
+        snapshotAfter = await (0, snapshot_1.snapshot)(browser);
+    }
+    return {
+        success,
+        duration_ms: durationMs,
+        outcome,
+        url_changed: urlChanged,
+        snapshot_after: snapshotAfter,
+        error: success
+            ? undefined
+            : { code: 'click_failed', reason: errorMsg || 'Click failed' },
+    };
+}
+//# sourceMappingURL=actions.js.map

package/dist/actions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actions.js","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAyEH,sBA8GC;AAED,4BAkDC;AAED,sBAiCC;AAsCD,8BAiGC;AAjZD,yCAAsC;AAWtC;;GAEG;AACH,KAAK,UAAU,aAAa,CAC1B,OAAyB,EACzB,IAAe,EACf,cAAsB,GAAG;IAEzB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,WAAW,GAAG,uBAAuB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAExD,4DAA4D;IAC5D,MAAM,IAAI,GAAG;QACX,IAAI,EAAE;YACJ,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC;YAC5B,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC;SAC9B;QACD,WAAW;QACX,WAAW;KACZ,CAAC;IAEF,MAAM,IAAI,CAAC,QAAQ,CACjB,CAAC,IAAwG,EAAE,EAAE;QAC3G,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;QAChD,qBAAqB;QACrB,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC9C,OAAO,CAAC,EAAE,GAAG,WAAW,CAAC;QACzB,OAAO,CAAC,KAAK,CAAC,QAAQ,GAAG,OAAO,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC;QACnC,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC;QAClC,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC;QACpC,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC;QACrC,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,eAAe,CAAC;QACvC,OAAO,CAAC,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC;QACnC,OAAO,CAAC,KAAK,CAAC,SAAS,GAAG,YAAY,CAAC;QACvC,OAAO,CAAC,KAAK,CAAC,aAAa,GAAG,MAAM,CAAC;QACrC,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,eAAe,GAAG,sBAAsB,CAAC;QACvD,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,uBAAuB,CAAC;QAEnD,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAEnC,wBAAwB;QACxB,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC;YAC5B,UAAU,CAAC,GAAG,EAAE;gBACd,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;oBACvB,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,+BAA+B;QAC1C,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC,CAAC;IACzB,CAAC,EACD,IAAI,CACL,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,KAAK,CACzB,OAAyB,EACzB,SAAiB,EACjB,WAAoB,IAAI,EACxB,eAAwB,KAAK;IAE7B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,OAAgB,CAAC;IAErB,IAAI,QAAQ,EAAE,CAAC;QACb,uFAAuF;QACvF,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAA,mBAAQ,EAAC,OAAO,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;YAEhE,IAAI,OAAO,EAAE,CAAC;gBACZ,mCAAmC;gBACnC,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;gBACxD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;gBACzD,+DAA+D;gBAC/D,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBACzC,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,wDAAwD;gBACxD,IAAI,CAAC;oBACH,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE;wBACnC,OAAQ,MAAc,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC7C,CAAC,EAAE,SAAS,CAAC,CAAC;gBAChB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,yEAAyE;oBACzE,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gCAAgC;YAChC,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE;oBACnC,OAAQ,MAAc,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7C,CAAC,EAAE,SAAS,CAAC,CAAC;YAChB,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBACnB,0DAA0D;gBAC1D,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,wBAAwB;QACxB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE;gBACnC,OAAQ,MAAc,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7C,CAAC,EAAE,SAAS,CAAC,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,0DAA0D;YAC1D,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,oDAAoD;IACtD,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAE1C,sDAAsD;IACtD,IAAI,QAAgB,CAAC;IACrB,IAAI,UAAmB,CAAC;IACxB,IAAI,CAAC;QACH,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,UAAU,GAAG,SAAS,KAAK,QAAQ,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,2DAA2D;QAC3D,QAAQ,GAAG,SAAS,CAAC;QACrB,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,oBAAoB;IACpB,IAAI,OAAwE,CAAC;IAC7E,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,GAAG,WAAW,CAAC;IACxB,CAAC;SAAM,IAAI,OAAO,EAAE,CAAC;QACnB,OAAO,GAAG,aAAa,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,OAAO,CAAC;IACpB,CAAC;IAED,0BAA0B;IAC1B,IAAI,aAAmC,CAAC;IACxC,IAAI,YAAY,EAAE,CAAC;QACjB,IAAI,CAAC;YACH,aAAa,GAAG,MAAM,IAAA,mBAAQ,EAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,0CAA0C;QAC5C,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO;QACP,WAAW,EAAE,UAAU;QACvB,OAAO;QACP,WAAW,EAAE,UAAU;QACvB,cAAc,EAAE,aAAa;QAC7B,KAAK,EAAE,OAAO;YACZ,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,oCAAoC,EAAE;KAC3E,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,QAAQ,CAC5B,OAAyB,EACzB,SAAiB,EACjB,IAAY,EACZ,eAAwB,KAAK;IAE7B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,sBAAsB;IACtB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE;QACzC,MAAM,EAAE,GAAI,MAAc,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;QAClD,IAAI,EAAE,EAAE,CAAC;YACP,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,EAAE,SAAS,CAAC,CAAC;IAEd,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YACnC,OAAO,EAAE,OAAO;YAChB,KAAK,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,mBAAmB,EAAE;SAC7D,CAAC;IACJ,CAAC;IAED,iCAAiC;IACjC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,UAAU,GAAG,SAAS,KAAK,QAAQ,CAAC;IAE1C,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC;IAEzD,IAAI,aAAmC,CAAC;IACxC,IAAI,YAAY,EAAE,CAAC;QACjB,aAAa,GAAG,MAAM,IAAA,mBAAQ,EAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO;QACL,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,UAAU;QACvB,OAAO;QACP,WAAW,EAAE,UAAU;QACvB,cAAc,EAAE,aAAa;KAC9B,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,KAAK,CACzB,OAAyB,EACzB,GAAW,EACX,eAAwB,KAAK;IAE7B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,6BAA6B;IAC7B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAE/B,wCAAwC;IACxC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAE/B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,UAAU,GAAG,SAAS,KAAK,QAAQ,CAAC;IAE1C,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC;IAEzD,IAAI,aAAmC,CAAC;IACxC,IAAI,YAAY,EAAE,CAAC;QACjB,aAAa,GAAG,MAAM,IAAA,mBAAQ,EAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO;QACL,OAAO,EAAE,IAAI;QACb,WAAW,EAAE,UAAU;QACvB,OAAO;QACP,WAAW,EAAE,UAAU;QACvB,cAAc,EAAE,aAAa;KAC9B,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACI,KAAK,UAAU,SAAS,CAC7B,OAAyB,EACzB,IAAsB,EACtB,YAAqB,IAAI,EACzB,oBAA4B,GAAG,EAC/B,eAAwB,KAAK;IAE7B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,uCAAuC;IACvC,IAAI,CAAS,EAAE,CAAS,EAAE,CAAS,EAAE,CAAS,CAAC;IAE/C,IAAI,OAAO,IAAI,IAAI,IAAI,QAAQ,IAAI,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;QAC5E,sDAAsD;QACtD,MAAM,IAAI,GAAG,IAAY,CAAC;QAC1B,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACX,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QACX,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,iBAAiB;QACjB,MAAM,SAAS,GAAG,IAAiB,CAAC;QACpC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAChB,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAChB,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,SAAS,CAAC,KAAK,IAAI,CAAC,CAAC;QACxC,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO;YACL,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,CAAC;YACd,OAAO,EAAE,OAAO;YAChB,KAAK,EAAE;gBACL,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,6CAA6C;aACtD;SACF,CAAC;IACJ,CAAC;IAED,gCAAgC;IAChC,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1B,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAE1B,8CAA8C;IAC9C,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAC;QAChE,6CAA6C;QAC7C,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,+DAA+D;IAC/D,IAAI,OAAgB,CAAC;IACrB,IAAI,QAA4B,CAAC;IACjC,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,GAAG,IAAI,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,GAAG,KAAK,CAAC;QAChB,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,wCAAwC;IACxC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAE/B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,UAAU,GAAG,SAAS,KAAK,QAAQ,CAAC;IAE1C,oBAAoB;IACpB,IAAI,OAAwE,CAAC;IAC7E,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,GAAG,WAAW,CAAC;IACxB,CAAC;SAAM,IAAI,OAAO,EAAE,CAAC;QACnB,OAAO,GAAG,aAAa,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,OAAO,CAAC;IACpB,CAAC;IAED,0BAA0B;IAC1B,IAAI,aAAmC,CAAC;IACxC,IAAI,YAAY,EAAE,CAAC;QACjB,aAAa,GAAG,MAAM,IAAA,mBAAQ,EAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO;QACL,OAAO;QACP,WAAW,EAAE,UAAU;QACvB,OAAO;QACP,WAAW,EAAE,UAAU;QACvB,cAAc,EAAE,aAAa;QAC7B,KAAK,EAAE,OAAO;YACZ,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,IAAI,cAAc,EAAE;KACjE,CAAC;AACJ,CAAC"}

package/dist/agent.d.ts

@@ -0,0 +1,157 @@
+/**
+ * Sentience Agent: High-level automation agent using LLM + SDK
+ * Implements observe-think-act loop for natural language commands
+ */
+import { SentienceBrowser } from './browser';
+import { SnapshotOptions } from './snapshot';
+import { LLMProvider } from './llm-provider';
+import { Tracer } from './tracing/tracer';
+/**
+ * Execution result from agent.act()
+ */
+export interface AgentActResult {
+    success: boolean;
+    action?: string;
+    elementId?: number;
+    text?: string;
+    key?: string;
+    outcome?: string;
+    urlChanged?: boolean;
+    durationMs: number;
+    attempt: number;
+    goal: string;
+    error?: string;
+    message?: string;
+}
+/**
+ * History entry for executed action
+ */
+export interface HistoryEntry {
+    goal: string;
+    action: string;
+    result: AgentActResult;
+    success: boolean;
+    attempt: number;
+    durationMs: number;
+}
+/**
+ * Token usage statistics
+ */
+export interface TokenStats {
+    totalPromptTokens: number;
+    totalCompletionTokens: number;
+    totalTokens: number;
+    byAction: Array<{
+        goal: string;
+        promptTokens?: number;
+        completionTokens?: number;
+        totalTokens?: number;
+        model?: string;
+    }>;
+}
+/**
+ * High-level agent that combines Sentience SDK with any LLM provider.
+ *
+ * Uses observe-think-act loop to execute natural language commands:
+ * 1. OBSERVE: Get snapshot of current page state
+ * 2. THINK: Query LLM to decide next action
+ * 3. ACT: Execute action using SDK
+ *
+ * Example:
+ * ```typescript
+ * import { SentienceBrowser, SentienceAgent, OpenAIProvider } from 'sentience-ts';
+ *
+ * const browser = await SentienceBrowser.create({ apiKey: 'sentience_key' });
+ * const llm = new OpenAIProvider('openai_key', 'gpt-4o');
+ * const agent = new SentienceAgent(browser, llm);
+ *
+ * await browser.getPage().goto('https://google.com');
+ * await agent.act('Click the search box');
+ * await agent.act("Type 'magic mouse' into the search field");
+ * await agent.act('Press Enter key');
+ * ```
+ */
+export declare class SentienceAgent {
+    private browser;
+    private llm;
+    private snapshotLimit;
+    private verbose;
+    private tracer?;
+    private stepCount;
+    private history;
+    private tokenUsage;
+    /**
+     * Initialize Sentience Agent
+     * @param browser - SentienceBrowser instance
+     * @param llm - LLM provider (OpenAIProvider, AnthropicProvider, etc.)
+     * @param snapshotLimit - Maximum elements to include in context (default: 50)
+     * @param verbose - Print execution logs (default: true)
+     * @param tracer - Optional tracer for recording execution (default: undefined)
+     */
+    constructor(browser: SentienceBrowser, llm: LLMProvider, snapshotLimit?: number, verbose?: boolean, tracer?: Tracer);
+    /**
+     * Execute a high-level goal using observe → think → act loop
+     * @param goal - Natural language instruction (e.g., "Click the Sign In button")
+     * @param maxRetries - Number of retries on failure (default: 2)
+     * @param snapshotOptions - Optional snapshot parameters (limit, filter, etc.)
+     * @returns Result dict with status, action_taken, reasoning, and execution data
+     *
+     * Example:
+     * ```typescript
+     * const result = await agent.act('Click the search box');
+     * console.log(result);
+     * // { success: true, action: 'click', elementId: 42, ... }
+     * ```
+     */
+    act(goal: string, maxRetries?: number, snapshotOptions?: SnapshotOptions): Promise<AgentActResult>;
+    /**
+     * Filter elements from snapshot based on goal context.
+     * Applies goal-based keyword matching to boost relevant elements and filters out irrelevant ones.
+     */
+    private filterElements;
+    /**
+     * Extract meaningful keywords from goal text
+     */
+    private extractKeywords;
+    /**
+     * Convert snapshot elements to token-efficient prompt string
+     * Format: [ID] <role> "text" {cues} @ (x,y) (Imp:score)
+     * Note: elements are already filtered by filterElements() in act()
+     */
+    private buildContext;
+    /**
+     * Query LLM with standardized prompt template
+     */
+    private queryLLM;
+    /**
+     * Parse action string and execute SDK call
+     */
+    private executeAction;
+    /**
+     * Track token usage for analytics
+     */
+    private trackTokens;
+    /**
+     * Get token usage statistics
+     * @returns Dictionary with token usage breakdown
+     */
+    getTokenStats(): TokenStats;
+    /**
+     * Get execution history
+     * @returns List of all actions taken with results
+     */
+    getHistory(): HistoryEntry[];
+    /**
+     * Clear execution history and reset token counters
+     */
+    clearHistory(): void;
+    /**
+     * Close the tracer and flush events to disk
+     */
+    closeTracer(): Promise<void>;
+    /**
+     * Get the tracer instance (if any)
+     */
+    getTracer(): Tracer | undefined;
+}
+//# sourceMappingURL=agent.d.ts.map

package/dist/agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAY,eAAe,EAAE,MAAM,YAAY,CAAC;AAGvD,OAAO,EAAE,WAAW,EAAe,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG1C;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,OAAO,CAAmB;IAClC,OAAO,CAAC,GAAG,CAAc;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,UAAU,CAAa;IAE/B;;;;;;;OAOG;gBAED,OAAO,EAAE,gBAAgB,EACzB,GAAG,EAAE,WAAW,EAChB,aAAa,GAAE,MAAW,EAC1B,OAAO,GAAE,OAAc,EACvB,MAAM,CAAC,EAAE,MAAM;IAiBjB;;;;;;;;;;;;;OAaG;IACG,GAAG,CACP,IAAI,EAAE,MAAM,EACZ,UAAU,GAAE,MAAU,EACtB,eAAe,CAAC,EAAE,eAAe,GAChC,OAAO,CAAC,cAAc,CAAC;IAmJ1B;;;OAGG;IACH,OAAO,CAAC,cAAc;IA+CtB;;OAEG;IACH,OAAO,CAAC,eAAe;IASvB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IA0BpB;;OAEG;YACW,QAAQ;IAkCtB;;OAEG;YACW,aAAa;IAsE3B;;OAEG;IACH,OAAO,CAAC,WAAW;IAoBnB;;;OAGG;IACH,aAAa,IAAI,UAAU;IAI3B;;;OAGG;IACH,UAAU,IAAI,YAAY,EAAE;IAI5B;;OAEG;IACH,YAAY,IAAI,IAAI;IAWpB;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAMlC;;OAEG;IACH,SAAS,IAAI,MAAM,GAAG,SAAS;CAGhC"}