screenhand 0.1.1 → 0.2.0

package/src/runtime/state-observer.ts

@@ -1,136 +0,0 @@
-// Copyright (C) 2025 Clazro Technology Private Limited
-// SPDX-License-Identifier: AGPL-3.0-only
-//
-// This file is part of ScreenHand.
-//
-// ScreenHand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, version 3.
-//
-// ScreenHand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
-
-import { EventEmitter } from "node:events";
-import type { MacOSBridgeClient } from "../native/macos-bridge-client.js";
-import type { UIEvent, UIEventType } from "../types.js";
-
-/**
- * Wraps the native bridge's AX observer events into typed UIEvent objects.
- * Buffers events for consumption by the planning loop.
- */
-export class StateObserver extends EventEmitter {
-  private observedPids = new Set<number>();
-  private eventBuffer: UIEvent[] = [];
-  private readonly maxBufferSize: number;
-
-  constructor(
-    private readonly bridge: MacOSBridgeClient,
-    maxBufferSize = 200,
-  ) {
-    super();
-    this.maxBufferSize = maxBufferSize;
-
-    // Listen for AX events from the bridge
-    this.bridge.on("ax-event", (raw: Record<string, unknown>) => {
-      const event = this.parseEvent(raw);
-      if (event) {
-        this.eventBuffer.push(event);
-        if (this.eventBuffer.length > this.maxBufferSize) {
-          this.eventBuffer.shift();
-        }
-        this.emit("event", event);
-      }
-    });
-  }
-
-  async startObserving(pid: number, eventTypes?: UIEventType[]): Promise<void> {
-    if (this.observedPids.has(pid)) return;
-
-    const notifications = eventTypes
-      ? this.mapEventTypesToNotifications(eventTypes)
-      : undefined;
-
-    await this.bridge.call("observer.start", {
-      pid,
-      notifications,
-    });
-
-    this.observedPids.add(pid);
-  }
-
-  async stopObserving(pid: number): Promise<void> {
-    if (!this.observedPids.has(pid)) return;
-
-    await this.bridge.call("observer.stop", { pid });
-    this.observedPids.delete(pid);
-  }
-
-  /** Get and clear the event buffer. */
-  drainEvents(): UIEvent[] {
-    const events = [...this.eventBuffer];
-    this.eventBuffer = [];
-    return events;
-  }
-
-  /** Get recent events without clearing. */
-  peekEvents(limit = 50): UIEvent[] {
-    return this.eventBuffer.slice(-limit);
-  }
-
-  /** Clear the event buffer. */
-  clearEvents(): void {
-    this.eventBuffer = [];
-  }
-
-  get isObserving(): boolean {
-    return this.observedPids.size > 0;
-  }
-
-  get observedProcesses(): number[] {
-    return [...this.observedPids];
-  }
-
-  private parseEvent(raw: Record<string, unknown>): UIEvent | null {
-    const type = raw.type as UIEventType | undefined;
-    if (!type) return null;
-
-    const event: UIEvent = {
-      type,
-      timestamp: (raw.timestamp as string) ?? new Date().toISOString(),
-      pid: (raw.pid as number) ?? 0,
-    };
-
-    if (typeof raw.bundleId === "string") event.bundleId = raw.bundleId;
-    if (typeof raw.elementRole === "string") event.elementRole = raw.elementRole;
-    if (typeof raw.elementLabel === "string") event.elementLabel = raw.elementLabel;
-    if (typeof raw.oldValue === "string") event.oldValue = raw.oldValue;
-    if (typeof raw.newValue === "string") event.newValue = raw.newValue;
-    if (typeof raw.windowTitle === "string") event.windowTitle = raw.windowTitle;
-
-    return event;
-  }
-
-  private mapEventTypesToNotifications(types: UIEventType[]): string[] {
-    const map: Record<string, string> = {
-      value_changed: "AXValueChanged",
-      focus_changed: "AXFocusedUIElementChanged",
-      window_created: "AXWindowCreated",
-      window_closed: "AXUIElementDestroyed",
-      title_changed: "AXTitleChanged",
-      menu_opened: "AXMenuOpened",
-      layout_changed: "AXLayoutChanged",
-      dialog_appeared: "AXSheetCreated",
-      app_activated: "AXApplicationActivated",
-      app_deactivated: "AXApplicationDeactivated",
-    };
-
-    return types
-      .map((t) => map[t])
-      .filter((n): n is string => n !== undefined);
-  }
-}

package/src/runtime/vision-adapter.ts

@@ -1,297 +0,0 @@
-// Copyright (C) 2025 Clazro Technology Private Limited
-// SPDX-License-Identifier: AGPL-3.0-only
-//
-// This file is part of ScreenHand.
-//
-// ScreenHand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, version 3.
-//
-// ScreenHand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
-
-import type {
-  AXNode,
-  AppContext,
-  ExtractFormat,
-  LocatedElement,
-  PageMeta,
-  SessionInfo,
-  Target,
-  WaitCondition,
-} from "../types.js";
-import type { AppAdapter } from "./app-adapter.js";
-import type { MacOSBridgeClient } from "../native/macos-bridge-client.js";
-
-const POLL_INTERVAL_MS = 200;
-
-interface VisionSessionState {
-  info: SessionInfo;
-  pid: number;
-  bundleId: string;
-  appName: string;
-  lastScreenshotPath?: string;
-}
-
-interface OCRResult {
-  text: string;
-  confidence: number;
-  bounds: { x: number; y: number; width: number; height: number };
-}
-
-/**
- * Vision-based adapter for apps with poor/no accessibility support.
- * Uses screenshots + OCR to locate elements and CG events to interact.
- */
-export class VisionAdapter implements AppAdapter {
-  private readonly sessions = new Map<string, VisionSessionState>();
-  private readonly sessionsByProfile = new Map<string, VisionSessionState>();
-
-  constructor(private readonly bridge: MacOSBridgeClient) {}
-
-  async attach(profile: string): Promise<SessionInfo> {
-    const existing = this.sessionsByProfile.get(profile);
-    if (existing) return existing.info;
-
-    await this.bridge.start();
-
-    const frontmost = await this.bridge.call<{ bundleId: string; name: string; pid: number }>(
-      "app.frontmost",
-    );
-
-    const info: SessionInfo = {
-      sessionId: `vision_session_${profile}_${Date.now()}`,
-      profile,
-      createdAt: new Date().toISOString(),
-      adapterType: "vision",
-    };
-
-    const state: VisionSessionState = {
-      info,
-      pid: frontmost.pid,
-      bundleId: frontmost.bundleId,
-      appName: frontmost.name,
-    };
-
-    this.sessions.set(info.sessionId, state);
-    this.sessionsByProfile.set(profile, state);
-    return info;
-  }
-
-  async getAppContext(sessionId: string): Promise<AppContext> {
-    const state = this.requireSession(sessionId);
-    return {
-      bundleId: state.bundleId,
-      appName: state.appName,
-      pid: state.pid,
-      windowTitle: state.appName,
-    };
-  }
-
-  async getPageMeta(sessionId: string): Promise<PageMeta> {
-    const ctx = await this.getAppContext(sessionId);
-    return {
-      url: `app://${ctx.bundleId}`,
-      title: ctx.appName,
-    };
-  }
-
-  async navigate(sessionId: string, url: string, _timeoutMs: number): Promise<PageMeta> {
-    if (url.startsWith("app://")) {
-      const bundleId = url.slice(6);
-      const result = await this.bridge.call<{ bundleId: string; appName: string; pid: number }>(
-        "app.launch",
-        { bundleId },
-      );
-      const state = this.requireSession(sessionId);
-      state.pid = result.pid;
-      state.bundleId = result.bundleId;
-      state.appName = result.appName;
-    }
-    return this.getPageMeta(sessionId);
-  }
-
-  async locate(sessionId: string, target: Target, timeoutMs: number): Promise<LocatedElement | null> {
-    const state = this.requireSession(sessionId);
-    const deadline = Date.now() + timeoutMs;
-
-    while (Date.now() < deadline) {
-      // Take a screenshot
-      const screenshotResult = await this.bridge.call<{ path: string }>(
-        "cg.captureScreen",
-        {},
-      );
-      state.lastScreenshotPath = screenshotResult.path;
-
-      const searchText = this.getSearchText(target);
-      if (!searchText) {
-        // For coordinate targets, just return coordinates directly
-        if (target.type === "coordinates") {
-          return {
-            handleId: `vision_coords_${target.x}_${target.y}`,
-            locatorUsed: "vision:coordinates",
-            coordinates: { x: target.x, y: target.y, width: 1, height: 1 },
-          };
-        }
-        return null;
-      }
-
-      // OCR the screenshot
-      const matches = await this.bridge.call<OCRResult[]>("vision.findText", {
-        imagePath: screenshotResult.path,
-        searchText,
-      });
-
-      if (matches.length > 0) {
-        const best = matches.reduce((a, b) => (a.confidence > b.confidence ? a : b));
-        return {
-          handleId: `vision_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
-          locatorUsed: `vision:text:${searchText}`,
-          label: best.text,
-          coordinates: best.bounds,
-        };
-      }
-
-      await sleep(POLL_INTERVAL_MS);
-    }
-
-    return null;
-  }
-
-  async click(_sessionId: string, element: LocatedElement): Promise<void> {
-    if (!element.coordinates) {
-      throw new Error("Vision adapter requires coordinates to click");
-    }
-    const cx = element.coordinates.x + element.coordinates.width / 2;
-    const cy = element.coordinates.y + element.coordinates.height / 2;
-    await this.bridge.call("cg.mouseClick", { x: cx, y: cy });
-  }
-
-  async setValue(_sessionId: string, element: LocatedElement, text: string, clear: boolean): Promise<void> {
-    // Click to focus
-    await this.click(_sessionId, element);
-    await sleep(100);
-
-    if (clear) {
-      await this.bridge.call("cg.keyCombo", { keys: ["cmd", "a"] });
-      await sleep(50);
-    }
-
-    await this.bridge.call("cg.typeText", { text });
-  }
-
-  async getValue(_sessionId: string, element: LocatedElement): Promise<string> {
-    // Vision can't reliably read values; return label if available
-    return element.label ?? "";
-  }
-
-  async waitFor(sessionId: string, condition: WaitCondition, timeoutMs: number): Promise<boolean> {
-    const deadline = Date.now() + timeoutMs;
-
-    while (Date.now() < deadline) {
-      if (condition.type === "text_appears") {
-        const found = await this.locate(
-          sessionId,
-          { type: "text", value: condition.text },
-          200,
-        );
-        if (found) return true;
-      } else if (condition.type === "element_exists") {
-        const found = await this.locate(sessionId, condition.target, 200);
-        if (found) return true;
-      } else if (condition.type === "element_gone") {
-        const found = await this.locate(sessionId, condition.target, 200);
-        if (!found) return true;
-      } else {
-        // Unsupported condition types
-        return false;
-      }
-      await sleep(POLL_INTERVAL_MS);
-    }
-    return false;
-  }
-
-  async extract(sessionId: string, _target: Target, format: ExtractFormat): Promise<unknown> {
-    const state = this.requireSession(sessionId);
-
-    // Take a fresh screenshot and OCR it
-    const screenshotResult = await this.bridge.call<{ path: string }>("cg.captureScreen", {});
-    state.lastScreenshotPath = screenshotResult.path;
-
-    const ocrResult = await this.bridge.call<{ text: string; regions: OCRResult[] }>(
-      "vision.ocr",
-      { imagePath: screenshotResult.path },
-    );
-
-    if (format === "text") {
-      return ocrResult.text;
-    }
-
-    if (format === "json") {
-      return ocrResult;
-    }
-
-    // table format
-    return {
-      headers: ["text", "confidence", "x", "y", "width", "height"],
-      rows: ocrResult.regions.map((r) => [
-        r.text,
-        r.confidence,
-        r.bounds.x,
-        r.bounds.y,
-        r.bounds.width,
-        r.bounds.height,
-      ]),
-    };
-  }
-
-  async screenshot(_sessionId: string, region?: { x: number; y: number; width: number; height: number }): Promise<string> {
-    const result = await this.bridge.call<{ path: string }>(
-      "cg.captureScreen",
-      region ? { region } : {},
-    );
-    return result.path;
-  }
-
-  async keyCombo(_sessionId: string, keys: string[]): Promise<void> {
-    await this.bridge.call("cg.keyCombo", { keys });
-  }
-
-  async elementTree(_sessionId: string, _maxDepth?: number, _root?: Target): Promise<AXNode> {
-    throw new Error("Vision adapter does not support elementTree — use accessibility adapter");
-  }
-
-  // ── Private ──
-
-  private requireSession(sessionId: string): VisionSessionState {
-    const state = this.sessions.get(sessionId);
-    if (!state) throw new Error(`Session not found: ${sessionId}`);
-    return state;
-  }
-
-  private getSearchText(target: Target): string | null {
-    switch (target.type) {
-      case "text":
-        return target.value;
-      case "role":
-        return target.name;
-      case "selector":
-        return target.value;
-      case "ax_attribute":
-        return target.value;
-      case "image":
-      case "coordinates":
-      case "ax_path":
-        return null;
-    }
-  }
-}
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}