screenhand 0.1.0

package/dist/runtime/composite-adapter.js

@@ -0,0 +1,205 @@
+import { AccessibilityAdapter } from "./accessibility-adapter.js";
+import { AppleScriptAdapter } from "./applescript-adapter.js";
+import { CdpChromeAdapter } from "./cdp-chrome-adapter.js";
+import { VisionAdapter } from "./vision-adapter.js";
+/** macOS bundle IDs routed to CDP. */
+const BROWSER_BUNDLES = new Set([
+    "com.google.Chrome",
+    "com.google.Chrome.canary",
+    "com.brave.Browser",
+    "com.microsoft.edgemac",
+    "com.vivaldi.Vivaldi",
+    "org.chromium.Chromium",
+]);
+/** Windows process names routed to CDP. */
+const BROWSER_PROCESS_NAMES = new Set([
+    "chrome",
+    "chrome.exe",
+    "brave",
+    "brave.exe",
+    "msedge",
+    "msedge.exe",
+    "vivaldi",
+    "vivaldi.exe",
+    "chromium",
+    "chromium.exe",
+]);
+const isWindows = process.platform === "win32";
+/**
+ * Composite adapter that auto-selects the best adapter per app:
+ *   - Chromium browsers → CDP
+ *   - Scriptable apps → AppleScript (with AX fallback)
+ *   - Default → Accessibility
+ *   - Fallback → Vision (if AX locate fails)
+ */
+export class CompositeAdapter {
+    bridge;
+    cdp;
+    accessibility;
+    applescript;
+    vision;
+    sessionRouting = new Map();
+    constructor(bridge, cdpOptions) {
+        this.bridge = bridge;
+        this.cdp = new CdpChromeAdapter(cdpOptions);
+        this.accessibility = new AccessibilityAdapter(bridge);
+        this.applescript = new AppleScriptAdapter();
+        this.vision = new VisionAdapter(bridge);
+    }
+    async attach(profile) {
+        // Default to accessibility adapter; routing is set per-session when app is known
+        const info = await this.accessibility.attach(profile);
+        this.sessionRouting.set(info.sessionId, {
+            adapter: this.accessibility,
+            adapterName: "accessibility",
+        });
+        // Override adapterType
+        return { ...info, adapterType: "composite" };
+    }
+    async getAppContext(sessionId) {
+        return this.getAdapter(sessionId).getAppContext(sessionId);
+    }
+    async getPageMeta(sessionId) {
+        return this.getAdapter(sessionId).getPageMeta(sessionId);
+    }
+    async navigate(sessionId, url, timeoutMs) {
+        return this.getAdapter(sessionId).navigate(sessionId, url, timeoutMs);
+    }
+    async locate(sessionId, target, timeoutMs) {
+        const primary = this.getAdapter(sessionId);
+        const result = await primary.locate(sessionId, target, timeoutMs);
+        if (result)
+            return result;
+        // Fallback to vision if primary (accessibility/applescript) fails
+        const routing = this.sessionRouting.get(sessionId);
+        if (routing && routing.adapterName !== "vision" && routing.adapterName !== "cdp") {
+            try {
+                return await this.vision.locate(sessionId, target, Math.min(timeoutMs, 2000));
+            }
+            catch {
+                // Vision also failed
+            }
+        }
+        return null;
+    }
+    async click(sessionId, element) {
+        // If the element was found by vision (coordinates-based), use vision adapter for click
+        if (element.locatorUsed.startsWith("vision:") && element.coordinates) {
+            return this.vision.click(sessionId, element);
+        }
+        return this.getAdapter(sessionId).click(sessionId, element);
+    }
+    async setValue(sessionId, element, text, clear) {
+        return this.getAdapter(sessionId).setValue(sessionId, element, text, clear);
+    }
+    async getValue(sessionId, element) {
+        return this.getAdapter(sessionId).getValue(sessionId, element);
+    }
+    async waitFor(sessionId, condition, timeoutMs) {
+        return this.getAdapter(sessionId).waitFor(sessionId, condition, timeoutMs);
+    }
+    async extract(sessionId, target, format) {
+        return this.getAdapter(sessionId).extract(sessionId, target, format);
+    }
+    async screenshot(sessionId, region) {
+        return this.getAdapter(sessionId).screenshot(sessionId, region);
+    }
+    // ── Desktop methods (delegate to the best adapter that supports them) ──
+    async launchApp(sessionId, bundleId) {
+        // Route to the appropriate adapter based on the app being launched
+        this.routeSession(sessionId, bundleId);
+        const adapter = this.getAdapter(sessionId);
+        if (adapter.launchApp) {
+            return adapter.launchApp(sessionId, bundleId);
+        }
+        // Fallback to accessibility
+        return this.accessibility.launchApp(sessionId, bundleId);
+    }
+    async focusApp(sessionId, bundleId) {
+        this.routeSession(sessionId, bundleId);
+        const adapter = this.getAdapter(sessionId);
+        if (adapter.focusApp) {
+            return adapter.focusApp(sessionId, bundleId);
+        }
+        return this.accessibility.focusApp(sessionId, bundleId);
+    }
+    async listApps(sessionId) {
+        return this.accessibility.listApps(sessionId);
+    }
+    async listWindows(sessionId) {
+        return this.accessibility.listWindows(sessionId);
+    }
+    async menuClick(sessionId, menuPath) {
+        const adapter = this.getAdapter(sessionId);
+        if (adapter.menuClick) {
+            return adapter.menuClick(sessionId, menuPath);
+        }
+        return this.accessibility.menuClick(sessionId, menuPath);
+    }
+    async keyCombo(sessionId, keys) {
+        const adapter = this.getAdapter(sessionId);
+        if (adapter.keyCombo) {
+            return adapter.keyCombo(sessionId, keys);
+        }
+        return this.accessibility.keyCombo(sessionId, keys);
+    }
+    async elementTree(sessionId, maxDepth, root) {
+        const adapter = this.getAdapter(sessionId);
+        if (adapter.elementTree) {
+            return adapter.elementTree(sessionId, maxDepth, root);
+        }
+        return this.accessibility.elementTree(sessionId, maxDepth, root);
+    }
+    async drag(sessionId, from, to) {
+        const adapter = this.getAdapter(sessionId);
+        if (adapter.drag) {
+            return adapter.drag(sessionId, from, to);
+        }
+        return this.accessibility.drag(sessionId, from, to);
+    }
+    async scroll(sessionId, direction, amount, element) {
+        const adapter = this.getAdapter(sessionId);
+        if (adapter.scroll) {
+            return adapter.scroll(sessionId, direction, amount, element);
+        }
+        return this.accessibility.scroll(sessionId, direction, amount, element);
+    }
+    // ── Routing logic ──
+    routeSession(sessionId, bundleId) {
+        let adapter;
+        let adapterName;
+        if (isWindows) {
+            // On Windows: route by process name
+            const processName = bundleId.toLowerCase().replace(/\.exe$/, "");
+            if (BROWSER_PROCESS_NAMES.has(processName) || BROWSER_PROCESS_NAMES.has(bundleId.toLowerCase())) {
+                adapter = this.cdp;
+                adapterName = "cdp";
+            }
+            else {
+                // No AppleScript on Windows — always use accessibility (UI Automation)
+                adapter = this.accessibility;
+                adapterName = "accessibility";
+            }
+        }
+        else {
+            // On macOS: route by bundle ID
+            if (BROWSER_BUNDLES.has(bundleId)) {
+                adapter = this.cdp;
+                adapterName = "cdp";
+            }
+            else if (AppleScriptAdapter.isScriptable(bundleId)) {
+                adapter = this.applescript;
+                adapterName = "applescript";
+            }
+            else {
+                adapter = this.accessibility;
+                adapterName = "accessibility";
+            }
+        }
+        this.sessionRouting.set(sessionId, { adapter, adapterName });
+    }
+    getAdapter(sessionId) {
+        const routing = this.sessionRouting.get(sessionId);
+        return routing?.adapter ?? this.accessibility;
+    }
+}

package/dist/runtime/executor.js

@@ -0,0 +1,250 @@
+import { DEFAULT_ACTION_BUDGET } from "../config.js";
+export class Executor {
+    adapter;
+    cache;
+    logger;
+    constructor(adapter, cache, logger) {
+        this.adapter = adapter;
+        this.cache = cache;
+        this.logger = logger;
+    }
+    async press(input) {
+        const telemetry = this.logger.start("press", input.sessionId);
+        const budget = this.resolveBudget(input.budget);
+        const attempts = [];
+        let lastError;
+        for (let retry = 0; retry <= budget.maxRetries; retry += 1) {
+            telemetry.retries = retry;
+            try {
+                const siteKey = await this.currentSiteKey(input.sessionId);
+                const actionKey = this.targetToKey(input.target);
+                const locateResult = await this.locateWithBudget(input.sessionId, siteKey, actionKey, input.target, budget.locateMs, retry > 0);
+                attempts.push(...locateResult.attempts);
+                telemetry.locateMs += locateResult.attempts.reduce((sum, attempt) => sum + attempt.timeoutMs, 0);
+                await this.timed(budget.actMs, async () => {
+                    await this.adapter.click(input.sessionId, locateResult.element);
+                }, "ACTION_FAILED");
+                telemetry.actMs += budget.actMs;
+                if (input.verify) {
+                    const verified = await this.timed(budget.verifyMs, () => this.adapter.waitFor(input.sessionId, input.verify, budget.verifyMs), "VERIFY_FAILED");
+                    telemetry.verifyMs += budget.verifyMs;
+                    if (!verified) {
+                        throw this.runtimeError("VERIFY_FAILED", "Verification condition not met.");
+                    }
+                }
+                const page = await this.adapter.getPageMeta(input.sessionId);
+                return this.success(page, telemetry);
+            }
+            catch (error) {
+                lastError = this.asRuntimeError(error, attempts);
+            }
+        }
+        return this.failure(lastError ??
+            this.runtimeError("ACTION_FAILED", "Press failed with unknown runtime error."), telemetry);
+    }
+    async typeInto(input) {
+        const telemetry = this.logger.start("type_into", input.sessionId);
+        const budget = this.resolveBudget(input.budget);
+        const attempts = [];
+        try {
+            const siteKey = await this.currentSiteKey(input.sessionId);
+            const actionKey = `type:${this.targetToKey(input.target)}`;
+            const locateResult = await this.locateWithBudget(input.sessionId, siteKey, actionKey, input.target, budget.locateMs, false);
+            attempts.push(...locateResult.attempts);
+            telemetry.locateMs += budget.locateMs;
+            await this.timed(budget.actMs, async () => {
+                await this.adapter.setValue(input.sessionId, locateResult.element, input.text, input.clear ?? true);
+            }, "ACTION_FAILED");
+            telemetry.actMs += budget.actMs;
+            if (input.verifyValue ?? true) {
+                const read = await this.adapter.getValue(input.sessionId, locateResult.element);
+                if (read !== input.text) {
+                    throw this.runtimeError("VERIFY_FAILED", `Field value mismatch. Expected "${input.text}", got "${read}".`);
+                }
+            }
+            if (input.verify) {
+                const verified = await this.timed(budget.verifyMs, () => this.adapter.waitFor(input.sessionId, input.verify, budget.verifyMs), "VERIFY_FAILED");
+                telemetry.verifyMs += budget.verifyMs;
+                if (!verified) {
+                    throw this.runtimeError("VERIFY_FAILED", "Verification condition not met.");
+                }
+            }
+            const page = await this.adapter.getPageMeta(input.sessionId);
+            return this.success(page, telemetry);
+        }
+        catch (error) {
+            return this.failure(this.asRuntimeError(error, attempts), telemetry);
+        }
+    }
+    async locateWithBudget(sessionId, siteKey, actionKey, target, locateBudgetMs, skipCache) {
+        const attempts = [];
+        const strategyBudget = Math.max(50, Math.floor(locateBudgetMs / 3));
+        if (!skipCache) {
+            const cachedLocator = this.cache.get(siteKey, actionKey);
+            if (cachedLocator) {
+                const cachedTarget = { type: "selector", value: cachedLocator };
+                const match = await this.tryLocate(sessionId, "cache", cachedTarget, strategyBudget, attempts);
+                if (match) {
+                    return { element: match, attempts };
+                }
+            }
+        }
+        const strategies = this.expandTargetStrategies(target);
+        for (const strategy of strategies) {
+            const match = await this.tryLocate(sessionId, strategy.strategy, strategy.target, strategyBudget, attempts);
+            if (match) {
+                if (strategy.target.type === "selector") {
+                    this.cache.set(siteKey, actionKey, strategy.target.value);
+                }
+                return { element: match, attempts };
+            }
+        }
+        throw this.runtimeError("LOCATE_FAILED", "Could not locate target.", attempts);
+    }
+    async tryLocate(sessionId, strategyName, target, timeoutMs, attempts) {
+        try {
+            const found = await this.timed(timeoutMs, () => this.adapter.locate(sessionId, target, timeoutMs), "LOCATE_FAILED");
+            attempts.push({
+                strategy: strategyName,
+                target: this.targetToKey(target),
+                timeoutMs,
+                matched: Boolean(found),
+            });
+            return found;
+        }
+        catch (error) {
+            attempts.push({
+                strategy: strategyName,
+                target: this.targetToKey(target),
+                timeoutMs,
+                matched: false,
+                reason: error instanceof Error ? error.message : "Unknown locate error",
+            });
+            return null;
+        }
+    }
+    expandTargetStrategies(target) {
+        if (target.type === "selector") {
+            return [{ strategy: "selector", target }];
+        }
+        if (target.type === "text") {
+            return [
+                { strategy: "text_exact", target: { type: "text", value: target.value, exact: true } },
+                { strategy: "text_fuzzy", target: { type: "text", value: target.value, exact: false } },
+            ];
+        }
+        if (target.type === "role") {
+            return [
+                { strategy: "role_name_exact", target: { type: "role", role: target.role, name: target.name, exact: true } },
+                { strategy: "role_name_fuzzy", target: { type: "role", role: target.role, name: target.name, exact: false } },
+                { strategy: "fallback_text", target: { type: "text", value: target.name } },
+            ];
+        }
+        // For new target types (ax_path, ax_attribute, coordinates, image), pass through directly
+        return [{ strategy: target.type, target }];
+    }
+    async currentSiteKey(sessionId) {
+        // Try app context first for desktop apps, fall back to page URL for browsers
+        try {
+            const ctx = await this.adapter.getAppContext(sessionId);
+            if (ctx.url) {
+                try {
+                    return new URL(ctx.url).host || ctx.bundleId;
+                }
+                catch {
+                    // URL parsing failed, use bundleId + windowTitle
+                }
+            }
+            return `${ctx.bundleId}::${ctx.windowTitle}`;
+        }
+        catch {
+            // Fallback to page meta
+            try {
+                const page = await this.adapter.getPageMeta(sessionId);
+                return new URL(page.url).host || "unknown-site";
+            }
+            catch {
+                return "unknown-site";
+            }
+        }
+    }
+    resolveBudget(input) {
+        return {
+            ...DEFAULT_ACTION_BUDGET,
+            ...input,
+        };
+    }
+    async timed(timeoutMs, operation, errorCode) {
+        const timeout = new Promise((_, reject) => {
+            setTimeout(() => {
+                reject(this.runtimeError("TIMEOUT", `Timed out after ${timeoutMs}ms.`));
+            }, timeoutMs);
+        });
+        try {
+            return await Promise.race([operation(), timeout]);
+        }
+        catch (error) {
+            if (this.isRuntimeError(error)) {
+                throw error;
+            }
+            throw this.runtimeError(errorCode, error instanceof Error ? error.message : "Unexpected runtime error");
+        }
+    }
+    targetToKey(target) {
+        switch (target.type) {
+            case "selector":
+                return `selector:${target.value}`;
+            case "text":
+                return `text:${target.value}`;
+            case "role":
+                return `role:${target.role}|name:${target.name}`;
+            case "ax_path":
+                return `ax_path:${target.path.join("/")}`;
+            case "ax_attribute":
+                return `ax_attr:${target.attribute}=${target.value}`;
+            case "coordinates":
+                return `coords:${target.x},${target.y}`;
+            case "image":
+                return `image:${target.base64.slice(0, 20)}`;
+        }
+    }
+    success(data, telemetry) {
+        return {
+            ok: true,
+            data,
+            telemetry: this.logger.finish(telemetry, "success"),
+        };
+    }
+    failure(error, telemetry) {
+        return {
+            ok: false,
+            error,
+            telemetry: this.logger.finish(telemetry, "failed"),
+        };
+    }
+    runtimeError(code, message, attempts) {
+        const error = { code, message };
+        if (attempts && attempts.length > 0) {
+            error.attempts = attempts;
+        }
+        return error;
+    }
+    isRuntimeError(error) {
+        if (typeof error !== "object" || error === null) {
+            return false;
+        }
+        return "code" in error && "message" in error;
+    }
+    asRuntimeError(error, attempts) {
+        if (this.isRuntimeError(error)) {
+            if (error.attempts || !attempts || attempts.length === 0) {
+                return error;
+            }
+            return {
+                ...error,
+                attempts,
+            };
+        }
+        return this.runtimeError("ACTION_FAILED", error instanceof Error ? error.message : "Unexpected runtime error", attempts);
+    }
+}

package/dist/runtime/locator-cache.js

@@ -0,0 +1,12 @@
+export class LocatorCache {
+    store = new Map();
+    get(siteKey, actionKey) {
+        return this.store.get(this.key(siteKey, actionKey));
+    }
+    set(siteKey, actionKey, locator) {
+        this.store.set(this.key(siteKey, actionKey), locator);
+    }
+    key(siteKey, actionKey) {
+        return `${siteKey}::${actionKey}`;
+    }
+}

package/dist/runtime/planning-loop.js

@@ -0,0 +1,47 @@
+/**
+ * Bidirectional planning loop that buffers UI events between LLM actions
+ * and provides state snapshots for the LLM to react to.
+ */
+export class PlanningLoop {
+    observer;
+    adapter;
+    lastActionResults = new Map();
+    constructor(observer, adapter) {
+        this.observer = observer;
+        this.adapter = adapter;
+    }
+    /** Get a state snapshot for the LLM after an action. */
+    async getStateSnapshot(sessionId) {
+        const recentEvents = this.observer.drainEvents();
+        let appContext = null;
+        try {
+            appContext = await this.adapter.getAppContext(sessionId);
+        }
+        catch {
+            // May not have an active session
+        }
+        return {
+            recentEvents,
+            appContext,
+            lastActionResult: this.lastActionResults.get(sessionId) ?? null,
+            observing: this.observer.isObserving,
+            timestamp: new Date().toISOString(),
+        };
+    }
+    /** Record the result of the last action for a session. */
+    recordActionResult(sessionId, result) {
+        this.lastActionResults.set(sessionId, result);
+    }
+    /** Start observing a process for state changes. */
+    async startObserving(sessionId, pid) {
+        await this.observer.startObserving(pid);
+    }
+    /** Stop observing a process. */
+    async stopObserving(sessionId, pid) {
+        await this.observer.stopObserving(pid);
+    }
+    /** Peek at recent events without draining. */
+    peekEvents(limit = 50) {
+        return this.observer.peekEvents(limit);
+    }
+}