screenhand 0.1.1 → 0.3.0

package/src/runtime/composite-adapter.ts

@@ -1,274 +0,0 @@
-// Copyright (C) 2025 Clazro Technology Private Limited
-// SPDX-License-Identifier: AGPL-3.0-only
-//
-// This file is part of ScreenHand.
-//
-// ScreenHand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, version 3.
-//
-// ScreenHand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
-
-import type {
-  AXNode,
-  AppContext,
-  ExtractFormat,
-  LocatedElement,
-  PageMeta,
-  RunningApp,
-  SessionInfo,
-  Target,
-  WaitCondition,
-  WindowInfo,
-} from "../types.js";
-import type { AppAdapter } from "./app-adapter.js";
-import type { BridgeClient } from "../native/bridge-client.js";
-// Legacy alias for backward compatibility
-type MacOSBridgeClient = BridgeClient;
-import { AccessibilityAdapter } from "./accessibility-adapter.js";
-import { AppleScriptAdapter } from "./applescript-adapter.js";
-import { CdpChromeAdapter, type CdpChromeAdapterOptions } from "./cdp-chrome-adapter.js";
-import { VisionAdapter } from "./vision-adapter.js";
-
-/** macOS bundle IDs routed to CDP. */
-const BROWSER_BUNDLES = new Set([
-  "com.google.Chrome",
-  "com.google.Chrome.canary",
-  "com.brave.Browser",
-  "com.microsoft.edgemac",
-  "com.vivaldi.Vivaldi",
-  "org.chromium.Chromium",
-]);
-
-/** Windows process names routed to CDP. */
-const BROWSER_PROCESS_NAMES = new Set([
-  "chrome",
-  "chrome.exe",
-  "brave",
-  "brave.exe",
-  "msedge",
-  "msedge.exe",
-  "vivaldi",
-  "vivaldi.exe",
-  "chromium",
-  "chromium.exe",
-]);
-
-const isWindows = process.platform === "win32";
-
-interface SessionRouting {
-  adapter: AppAdapter;
-  adapterName: string;
-}
-
-/**
- * Composite adapter that auto-selects the best adapter per app:
- *   - Chromium browsers → CDP
- *   - Scriptable apps → AppleScript (with AX fallback)
- *   - Default → Accessibility
- *   - Fallback → Vision (if AX locate fails)
- */
-export class CompositeAdapter implements AppAdapter {
-  private readonly cdp: CdpChromeAdapter;
-  private readonly accessibility: AccessibilityAdapter;
-  private readonly applescript: AppleScriptAdapter;
-  private readonly vision: VisionAdapter;
-
-  private readonly sessionRouting = new Map<string, SessionRouting>();
-
-  constructor(
-    private readonly bridge: MacOSBridgeClient,
-    cdpOptions?: CdpChromeAdapterOptions,
-  ) {
-    this.cdp = new CdpChromeAdapter(cdpOptions);
-    this.accessibility = new AccessibilityAdapter(bridge);
-    this.applescript = new AppleScriptAdapter();
-    this.vision = new VisionAdapter(bridge);
-  }
-
-  async attach(profile: string): Promise<SessionInfo> {
-    // Default to accessibility adapter; routing is set per-session when app is known
-    const info = await this.accessibility.attach(profile);
-    this.sessionRouting.set(info.sessionId, {
-      adapter: this.accessibility,
-      adapterName: "accessibility",
-    });
-
-    // Override adapterType
-    return { ...info, adapterType: "composite" };
-  }
-
-  async getAppContext(sessionId: string): Promise<AppContext> {
-    return this.getAdapter(sessionId).getAppContext(sessionId);
-  }
-
-  async getPageMeta(sessionId: string): Promise<PageMeta> {
-    return this.getAdapter(sessionId).getPageMeta(sessionId);
-  }
-
-  async navigate(sessionId: string, url: string, timeoutMs: number): Promise<PageMeta> {
-    return this.getAdapter(sessionId).navigate(sessionId, url, timeoutMs);
-  }
-
-  async locate(sessionId: string, target: Target, timeoutMs: number): Promise<LocatedElement | null> {
-    const primary = this.getAdapter(sessionId);
-    const result = await primary.locate(sessionId, target, timeoutMs);
-    if (result) return result;
-
-    // Fallback to vision if primary (accessibility/applescript) fails
-    const routing = this.sessionRouting.get(sessionId);
-    if (routing && routing.adapterName !== "vision" && routing.adapterName !== "cdp") {
-      try {
-        return await this.vision.locate(sessionId, target, Math.min(timeoutMs, 2000));
-      } catch {
-        // Vision also failed
-      }
-    }
-
-    return null;
-  }
-
-  async click(sessionId: string, element: LocatedElement): Promise<void> {
-    // If the element was found by vision (coordinates-based), use vision adapter for click
-    if (element.locatorUsed.startsWith("vision:") && element.coordinates) {
-      return this.vision.click(sessionId, element);
-    }
-    return this.getAdapter(sessionId).click(sessionId, element);
-  }
-
-  async setValue(sessionId: string, element: LocatedElement, text: string, clear: boolean): Promise<void> {
-    return this.getAdapter(sessionId).setValue(sessionId, element, text, clear);
-  }
-
-  async getValue(sessionId: string, element: LocatedElement): Promise<string> {
-    return this.getAdapter(sessionId).getValue(sessionId, element);
-  }
-
-  async waitFor(sessionId: string, condition: WaitCondition, timeoutMs: number): Promise<boolean> {
-    return this.getAdapter(sessionId).waitFor(sessionId, condition, timeoutMs);
-  }
-
-  async extract(sessionId: string, target: Target, format: ExtractFormat): Promise<unknown> {
-    return this.getAdapter(sessionId).extract(sessionId, target, format);
-  }
-
-  async screenshot(sessionId: string, region?: { x: number; y: number; width: number; height: number }): Promise<string> {
-    return this.getAdapter(sessionId).screenshot(sessionId, region);
-  }
-
-  // ── Desktop methods (delegate to the best adapter that supports them) ──
-
-  async launchApp(sessionId: string, bundleId: string): Promise<AppContext> {
-    // Route to the appropriate adapter based on the app being launched
-    this.routeSession(sessionId, bundleId);
-
-    const adapter = this.getAdapter(sessionId);
-    if (adapter.launchApp) {
-      return adapter.launchApp(sessionId, bundleId);
-    }
-    // Fallback to accessibility
-    return this.accessibility.launchApp(sessionId, bundleId);
-  }
-
-  async focusApp(sessionId: string, bundleId: string): Promise<void> {
-    this.routeSession(sessionId, bundleId);
-    const adapter = this.getAdapter(sessionId);
-    if (adapter.focusApp) {
-      return adapter.focusApp(sessionId, bundleId);
-    }
-    return this.accessibility.focusApp(sessionId, bundleId);
-  }
-
-  async listApps(sessionId: string): Promise<RunningApp[]> {
-    return this.accessibility.listApps(sessionId);
-  }
-
-  async listWindows(sessionId: string): Promise<WindowInfo[]> {
-    return this.accessibility.listWindows(sessionId);
-  }
-
-  async menuClick(sessionId: string, menuPath: string[]): Promise<void> {
-    const adapter = this.getAdapter(sessionId);
-    if (adapter.menuClick) {
-      return adapter.menuClick(sessionId, menuPath);
-    }
-    return this.accessibility.menuClick(sessionId, menuPath);
-  }
-
-  async keyCombo(sessionId: string, keys: string[]): Promise<void> {
-    const adapter = this.getAdapter(sessionId);
-    if (adapter.keyCombo) {
-      return adapter.keyCombo(sessionId, keys);
-    }
-    return this.accessibility.keyCombo(sessionId, keys);
-  }
-
-  async elementTree(sessionId: string, maxDepth?: number, root?: Target): Promise<AXNode> {
-    const adapter = this.getAdapter(sessionId);
-    if (adapter.elementTree) {
-      return adapter.elementTree(sessionId, maxDepth, root);
-    }
-    return this.accessibility.elementTree(sessionId, maxDepth, root);
-  }
-
-  async drag(sessionId: string, from: LocatedElement, to: LocatedElement): Promise<void> {
-    const adapter = this.getAdapter(sessionId);
-    if (adapter.drag) {
-      return adapter.drag(sessionId, from, to);
-    }
-    return this.accessibility.drag(sessionId, from, to);
-  }
-
-  async scroll(sessionId: string, direction: "up" | "down" | "left" | "right", amount: number, element?: LocatedElement): Promise<void> {
-    const adapter = this.getAdapter(sessionId);
-    if (adapter.scroll) {
-      return adapter.scroll(sessionId, direction, amount, element);
-    }
-    return this.accessibility.scroll(sessionId, direction, amount, element);
-  }
-
-  // ── Routing logic ──
-
-  private routeSession(sessionId: string, bundleId: string): void {
-    let adapter: AppAdapter;
-    let adapterName: string;
-
-    if (isWindows) {
-      // On Windows: route by process name
-      const processName = bundleId.toLowerCase().replace(/\.exe$/, "");
-      if (BROWSER_PROCESS_NAMES.has(processName) || BROWSER_PROCESS_NAMES.has(bundleId.toLowerCase())) {
-        adapter = this.cdp;
-        adapterName = "cdp";
-      } else {
-        // No AppleScript on Windows — always use accessibility (UI Automation)
-        adapter = this.accessibility;
-        adapterName = "accessibility";
-      }
-    } else {
-      // On macOS: route by bundle ID
-      if (BROWSER_BUNDLES.has(bundleId)) {
-        adapter = this.cdp;
-        adapterName = "cdp";
-      } else if (AppleScriptAdapter.isScriptable(bundleId)) {
-        adapter = this.applescript;
-        adapterName = "applescript";
-      } else {
-        adapter = this.accessibility;
-        adapterName = "accessibility";
-      }
-    }
-
-    this.sessionRouting.set(sessionId, { adapter, adapterName });
-  }
-
-  private getAdapter(sessionId: string): AppAdapter {
-    const routing = this.sessionRouting.get(sessionId);
-    return routing?.adapter ?? this.accessibility;
-  }
-}

package/src/runtime/executor.ts

@@ -1,396 +0,0 @@
-// Copyright (C) 2025 Clazro Technology Private Limited
-// SPDX-License-Identifier: AGPL-3.0-only
-//
-// This file is part of ScreenHand.
-//
-// ScreenHand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, version 3.
-//
-// ScreenHand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
-
-import { DEFAULT_ACTION_BUDGET } from "../config.js";
-import type { TimelineLogger } from "../logging/timeline-logger.js";
-import type {
-  ActionBudget,
-  ActionTelemetry,
-  LocatedElement,
-  LocatorAttempt,
-  PageMeta,
-  PressInput,
-  RuntimeError,
-  Target,
-  ToolResult,
-  TypeIntoInput,
-} from "../types.js";
-import type { AppAdapter } from "./app-adapter.js";
-import type { LocatorCache } from "./locator-cache.js";
-
-interface LocateResult {
-  element: LocatedElement;
-  attempts: LocatorAttempt[];
-}
-
-export class Executor {
-  constructor(
-    private readonly adapter: AppAdapter,
-    private readonly cache: LocatorCache,
-    private readonly logger: TimelineLogger,
-  ) {}
-
-  async press(input: PressInput): Promise<ToolResult<PageMeta>> {
-    const telemetry = this.logger.start("press", input.sessionId);
-    const budget = this.resolveBudget(input.budget);
-    const attempts: LocatorAttempt[] = [];
-    let lastError: RuntimeError | undefined;
-
-    for (let retry = 0; retry <= budget.maxRetries; retry += 1) {
-      telemetry.retries = retry;
-      try {
-        const siteKey = await this.currentSiteKey(input.sessionId);
-        const actionKey = this.targetToKey(input.target);
-        const locateResult = await this.locateWithBudget(
-          input.sessionId,
-          siteKey,
-          actionKey,
-          input.target,
-          budget.locateMs,
-          retry > 0,
-        );
-        attempts.push(...locateResult.attempts);
-        telemetry.locateMs += locateResult.attempts.reduce(
-          (sum, attempt) => sum + attempt.timeoutMs,
-          0,
-        );
-
-        await this.timed(
-          budget.actMs,
-          async () => {
-            await this.adapter.click(input.sessionId, locateResult.element);
-          },
-          "ACTION_FAILED",
-        );
-        telemetry.actMs += budget.actMs;
-
-        if (input.verify) {
-          const verified = await this.timed(
-            budget.verifyMs,
-            () => this.adapter.waitFor(input.sessionId, input.verify!, budget.verifyMs),
-            "VERIFY_FAILED",
-          );
-          telemetry.verifyMs += budget.verifyMs;
-          if (!verified) {
-            throw this.runtimeError("VERIFY_FAILED", "Verification condition not met.");
-          }
-        }
-
-        const page = await this.adapter.getPageMeta(input.sessionId);
-        return this.success(page, telemetry);
-      } catch (error) {
-        lastError = this.asRuntimeError(error, attempts);
-      }
-    }
-
-    return this.failure(
-      lastError ??
-        this.runtimeError("ACTION_FAILED", "Press failed with unknown runtime error."),
-      telemetry,
-    );
-  }
-
-  async typeInto(input: TypeIntoInput): Promise<ToolResult<PageMeta>> {
-    const telemetry = this.logger.start("type_into", input.sessionId);
-    const budget = this.resolveBudget(input.budget);
-    const attempts: LocatorAttempt[] = [];
-
-    try {
-      const siteKey = await this.currentSiteKey(input.sessionId);
-      const actionKey = `type:${this.targetToKey(input.target)}`;
-      const locateResult = await this.locateWithBudget(
-        input.sessionId,
-        siteKey,
-        actionKey,
-        input.target,
-        budget.locateMs,
-        false,
-      );
-      attempts.push(...locateResult.attempts);
-      telemetry.locateMs += budget.locateMs;
-
-      await this.timed(
-        budget.actMs,
-        async () => {
-          await this.adapter.setValue(
-            input.sessionId,
-            locateResult.element,
-            input.text,
-            input.clear ?? true,
-          );
-        },
-        "ACTION_FAILED",
-      );
-      telemetry.actMs += budget.actMs;
-
-      if (input.verifyValue ?? true) {
-        const read = await this.adapter.getValue(input.sessionId, locateResult.element);
-        if (read !== input.text) {
-          throw this.runtimeError(
-            "VERIFY_FAILED",
-            `Field value mismatch. Expected "${input.text}", got "${read}".`,
-          );
-        }
-      }
-
-      if (input.verify) {
-        const verified = await this.timed(
-          budget.verifyMs,
-          () => this.adapter.waitFor(input.sessionId, input.verify!, budget.verifyMs),
-          "VERIFY_FAILED",
-        );
-        telemetry.verifyMs += budget.verifyMs;
-        if (!verified) {
-          throw this.runtimeError("VERIFY_FAILED", "Verification condition not met.");
-        }
-      }
-
-      const page = await this.adapter.getPageMeta(input.sessionId);
-      return this.success(page, telemetry);
-    } catch (error) {
-      return this.failure(this.asRuntimeError(error, attempts), telemetry);
-    }
-  }
-
-  private async locateWithBudget(
-    sessionId: string,
-    siteKey: string,
-    actionKey: string,
-    target: Target,
-    locateBudgetMs: number,
-    skipCache: boolean,
-  ): Promise<LocateResult> {
-    const attempts: LocatorAttempt[] = [];
-    const strategyBudget = Math.max(50, Math.floor(locateBudgetMs / 3));
-
-    if (!skipCache) {
-      const cachedLocator = this.cache.get(siteKey, actionKey);
-      if (cachedLocator) {
-        const cachedTarget: Target = { type: "selector", value: cachedLocator };
-        const match = await this.tryLocate(
-          sessionId,
-          "cache",
-          cachedTarget,
-          strategyBudget,
-          attempts,
-        );
-        if (match) {
-          return { element: match, attempts };
-        }
-      }
-    }
-
-    const strategies = this.expandTargetStrategies(target);
-    for (const strategy of strategies) {
-      const match = await this.tryLocate(
-        sessionId,
-        strategy.strategy,
-        strategy.target,
-        strategyBudget,
-        attempts,
-      );
-      if (match) {
-        if (strategy.target.type === "selector") {
-          this.cache.set(siteKey, actionKey, strategy.target.value);
-        }
-        return { element: match, attempts };
-      }
-    }
-
-    throw this.runtimeError("LOCATE_FAILED", "Could not locate target.", attempts);
-  }
-
-  private async tryLocate(
-    sessionId: string,
-    strategyName: string,
-    target: Target,
-    timeoutMs: number,
-    attempts: LocatorAttempt[],
-  ): Promise<LocatedElement | null> {
-    try {
-      const found = await this.timed(
-        timeoutMs,
-        () => this.adapter.locate(sessionId, target, timeoutMs),
-        "LOCATE_FAILED",
-      );
-      attempts.push({
-        strategy: strategyName,
-        target: this.targetToKey(target),
-        timeoutMs,
-        matched: Boolean(found),
-      });
-      return found;
-    } catch (error) {
-      attempts.push({
-        strategy: strategyName,
-        target: this.targetToKey(target),
-        timeoutMs,
-        matched: false,
-        reason: error instanceof Error ? error.message : "Unknown locate error",
-      });
-      return null;
-    }
-  }
-
-  private expandTargetStrategies(
-    target: Target,
-  ): Array<{ strategy: string; target: Target }> {
-    if (target.type === "selector") {
-      return [{ strategy: "selector", target }];
-    }
-    if (target.type === "text") {
-      return [
-        { strategy: "text_exact", target: { type: "text", value: target.value, exact: true } },
-        { strategy: "text_fuzzy", target: { type: "text", value: target.value, exact: false } },
-      ];
-    }
-    if (target.type === "role") {
-      return [
-        { strategy: "role_name_exact", target: { type: "role", role: target.role, name: target.name, exact: true } },
-        { strategy: "role_name_fuzzy", target: { type: "role", role: target.role, name: target.name, exact: false } },
-        { strategy: "fallback_text", target: { type: "text", value: target.name } },
-      ];
-    }
-    // For new target types (ax_path, ax_attribute, coordinates, image), pass through directly
-    return [{ strategy: target.type, target }];
-  }
-
-  private async currentSiteKey(sessionId: string): Promise<string> {
-    // Try app context first for desktop apps, fall back to page URL for browsers
-    try {
-      const ctx = await this.adapter.getAppContext(sessionId);
-      if (ctx.url) {
-        try {
-          return new URL(ctx.url).host || ctx.bundleId;
-        } catch {
-          // URL parsing failed, use bundleId + windowTitle
-        }
-      }
-      return `${ctx.bundleId}::${ctx.windowTitle}`;
-    } catch {
-      // Fallback to page meta
-      try {
-        const page = await this.adapter.getPageMeta(sessionId);
-        return new URL(page.url).host || "unknown-site";
-      } catch {
-        return "unknown-site";
-      }
-    }
-  }
-
-  private resolveBudget(input?: Partial<ActionBudget>): ActionBudget {
-    return {
-      ...DEFAULT_ACTION_BUDGET,
-      ...input,
-    };
-  }
-
-  private async timed<T>(
-    timeoutMs: number,
-    operation: () => Promise<T>,
-    errorCode: RuntimeError["code"],
-  ): Promise<T> {
-    const timeout = new Promise<never>((_, reject) => {
-      setTimeout(() => {
-        reject(this.runtimeError("TIMEOUT", `Timed out after ${timeoutMs}ms.`));
-      }, timeoutMs);
-    });
-
-    try {
-      return await Promise.race([operation(), timeout]);
-    } catch (error) {
-      if (this.isRuntimeError(error)) {
-        throw error;
-      }
-      throw this.runtimeError(
-        errorCode,
-        error instanceof Error ? error.message : "Unexpected runtime error",
-      );
-    }
-  }
-
-  private targetToKey(target: Target): string {
-    switch (target.type) {
-      case "selector":
-        return `selector:${target.value}`;
-      case "text":
-        return `text:${target.value}`;
-      case "role":
-        return `role:${target.role}|name:${target.name}`;
-      case "ax_path":
-        return `ax_path:${target.path.join("/")}`;
-      case "ax_attribute":
-        return `ax_attr:${target.attribute}=${target.value}`;
-      case "coordinates":
-        return `coords:${target.x},${target.y}`;
-      case "image":
-        return `image:${target.base64.slice(0, 20)}`;
-    }
-  }
-
-  private success<T>(data: T, telemetry: ActionTelemetry): ToolResult<T> {
-    return {
-      ok: true,
-      data,
-      telemetry: this.logger.finish(telemetry, "success"),
-    };
-  }
-
-  private failure<T>(error: RuntimeError, telemetry: ActionTelemetry): ToolResult<T> {
-    return {
-      ok: false,
-      error,
-      telemetry: this.logger.finish(telemetry, "failed"),
-    };
-  }
-
-  private runtimeError(
-    code: RuntimeError["code"],
-    message: string,
-    attempts?: LocatorAttempt[],
-  ): RuntimeError {
-    const error: RuntimeError = { code, message };
-    if (attempts && attempts.length > 0) {
-      error.attempts = attempts;
-    }
-    return error;
-  }
-
-  private isRuntimeError(error: unknown): error is RuntimeError {
-    if (typeof error !== "object" || error === null) {
-      return false;
-    }
-    return "code" in error && "message" in error;
-  }
-
-  private asRuntimeError(error: unknown, attempts?: LocatorAttempt[]): RuntimeError {
-    if (this.isRuntimeError(error)) {
-      if (error.attempts || !attempts || attempts.length === 0) {
-        return error;
-      }
-      return {
-        ...error,
-        attempts,
-      };
-    }
-    return this.runtimeError(
-      "ACTION_FAILED",
-      error instanceof Error ? error.message : "Unexpected runtime error",
-      attempts,
-    );
-  }
-}