screenhand 0.1.1 → 0.3.0

package/src/types.ts

@@ -1,297 +0,0 @@
-// Copyright (C) 2025 Clazro Technology Private Limited
-// SPDX-License-Identifier: AGPL-3.0-only
-//
-// This file is part of ScreenHand.
-//
-// ScreenHand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, version 3.
-//
-// ScreenHand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
-
-export type ToolName =
-  | "session_start"
-  | "navigate"
-  | "press"
-  | "type_into"
-  | "wait_for"
-  | "extract"
-  | "screenshot"
-  | "app_launch"
-  | "app_focus"
-  | "app_list"
-  | "window_list"
-  | "menu_click"
-  | "key_combo"
-  | "element_tree"
-  | "observe_start"
-  | "observe_stop"
-  | "drag"
-  | "scroll";
-
-export type Target =
-  | { type: "selector"; value: string }
-  | { type: "text"; value: string; exact?: boolean }
-  | { type: "role"; role: string; name: string; exact?: boolean }
-  | { type: "ax_path"; path: string[] }
-  | { type: "ax_attribute"; attribute: string; value: string }
-  | { type: "coordinates"; x: number; y: number }
-  | { type: "image"; base64: string; confidence?: number };
-
-export type WaitCondition =
-  | { type: "selector_visible"; selector: string }
-  | { type: "selector_hidden"; selector: string }
-  | { type: "url_matches"; regex: string }
-  | { type: "text_appears"; text: string }
-  | { type: "spinner_disappears"; selector: string }
-  | { type: "element_exists"; target: Target }
-  | { type: "element_gone"; target: Target }
-  | { type: "window_title_matches"; regex: string }
-  | { type: "app_idle"; bundleId: string; timeoutMs?: number };
-
-export type ExtractFormat = "text" | "table" | "json";
-
-export type ActionStatus = "success" | "failed";
-
-export interface ActionBudget {
-  locateMs: number;
-  actMs: number;
-  verifyMs: number;
-  maxRetries: number;
-}
-
-export interface SessionInfo {
-  sessionId: string;
-  profile: string;
-  createdAt: string;
-  adapterType?: "cdp" | "accessibility" | "applescript" | "vision" | "composite";
-}
-
-export interface PageMeta {
-  url: string;
-  title: string;
-}
-
-export interface AppContext {
-  bundleId: string;
-  appName: string;
-  pid: number;
-  windowTitle: string;
-  windowId?: number;
-  url?: string;
-}
-
-export interface LocatedElement {
-  handleId: string;
-  locatorUsed: string;
-  role?: string;
-  label?: string;
-  coordinates?: { x: number; y: number; width: number; height: number };
-}
-
-export interface LocatorAttempt {
-  strategy: string;
-  target: string;
-  timeoutMs: number;
-  matched: boolean;
-  reason?: string;
-}
-
-export interface RuntimeError {
-  code:
-    | "SESSION_NOT_FOUND"
-    | "LOCATE_FAILED"
-    | "ACTION_FAILED"
-    | "VERIFY_FAILED"
-    | "TIMEOUT"
-    | "NOT_IMPLEMENTED"
-    | "PERMISSION_DENIED"
-    | "APP_NOT_FOUND"
-    | "BRIDGE_ERROR";
-  message: string;
-  page?: PageMeta;
-  appContext?: AppContext;
-  attempts?: LocatorAttempt[];
-  cause?: string;
-}
-
-export interface ActionTelemetry {
-  action: string;
-  sessionId: string;
-  startedAt: string;
-  finishedAt?: string;
-  totalMs?: number;
-  locateMs: number;
-  actMs: number;
-  verifyMs: number;
-  retries: number;
-  status?: ActionStatus;
-}
-
-export interface ToolSuccess<T> {
-  ok: true;
-  data: T;
-  telemetry: ActionTelemetry;
-}
-
-export interface ToolFailure {
-  ok: false;
-  error: RuntimeError;
-  telemetry: ActionTelemetry;
-}
-
-export type ToolResult<T> = ToolSuccess<T> | ToolFailure;
-
-export interface PressInput {
-  sessionId: string;
-  target: Target;
-  verify?: WaitCondition;
-  budget?: Partial<ActionBudget>;
-}
-
-export interface TypeIntoInput {
-  sessionId: string;
-  target: Target;
-  text: string;
-  clear?: boolean;
-  verifyValue?: boolean;
-  verify?: WaitCondition;
-  budget?: Partial<ActionBudget>;
-}
-
-export interface NavigateInput {
-  sessionId: string;
-  url: string;
-  timeoutMs?: number;
-}
-
-export interface WaitForInput {
-  sessionId: string;
-  condition: WaitCondition;
-  timeoutMs?: number;
-}
-
-export interface ExtractInput {
-  sessionId: string;
-  target: Target;
-  format: ExtractFormat;
-}
-
-export interface ScreenshotInput {
-  sessionId: string;
-  region?: {
-    x: number;
-    y: number;
-    width: number;
-    height: number;
-  };
-}
-
-export interface AppLaunchInput {
-  sessionId: string;
-  bundleId: string;
-  waitForReady?: boolean;
-}
-
-export interface AppFocusInput {
-  sessionId: string;
-  bundleId: string;
-}
-
-export interface MenuClickInput {
-  sessionId: string;
-  menuPath: string[];
-}
-
-export interface KeyComboInput {
-  sessionId: string;
-  keys: string[];
-}
-
-export interface ElementTreeInput {
-  sessionId: string;
-  maxDepth?: number;
-  root?: Target;
-}
-
-export interface DragInput {
-  sessionId: string;
-  from: Target;
-  to: Target;
-  budget?: Partial<ActionBudget>;
-}
-
-export interface ScrollInput {
-  sessionId: string;
-  target?: Target;
-  direction: "up" | "down" | "left" | "right";
-  amount?: number;
-}
-
-export interface ObserveStartInput {
-  sessionId: string;
-  events?: UIEventType[];
-}
-
-export interface ObserveStopInput {
-  sessionId: string;
-}
-
-export type UIEventType =
-  | "value_changed"
-  | "focus_changed"
-  | "window_created"
-  | "window_closed"
-  | "dialog_appeared"
-  | "menu_opened"
-  | "title_changed"
-  | "layout_changed"
-  | "app_activated"
-  | "app_deactivated";
-
-export interface UIEvent {
-  type: UIEventType;
-  timestamp: string;
-  pid: number;
-  bundleId?: string;
-  elementRole?: string;
-  elementLabel?: string;
-  oldValue?: string;
-  newValue?: string;
-  windowTitle?: string;
-}
-
-export interface RunningApp {
-  bundleId: string;
-  name: string;
-  pid: number;
-  isActive: boolean;
-}
-
-export interface WindowInfo {
-  windowId: number;
-  title: string;
-  bundleId: string;
-  pid: number;
-  bounds: { x: number; y: number; width: number; height: number };
-  isOnScreen: boolean;
-}
-
-export interface AXNode {
-  role: string;
-  title?: string;
-  value?: string;
-  description?: string;
-  identifier?: string;
-  enabled?: boolean;
-  focused?: boolean;
-  position?: { x: number; y: number };
-  size?: { width: number; height: number };
-  children?: AXNode[];
-}

package/tests/bridge-client.test.ts

@@ -1,176 +0,0 @@
-import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-import { BridgeClient } from "../src/native/bridge-client.js";
-import { EventEmitter } from "node:events";
-import { ChildProcess } from "node:child_process";
-import path from "node:path";
-
-// ── Platform detection ──
-
-describe("BridgeClient platform detection", () => {
-  it("selects macOS binary path when platform is darwin", () => {
-    const originalPlatform = process.platform;
-    Object.defineProperty(process, "platform", { value: "darwin", writable: true });
-
-    const client = new BridgeClient();
-    // Access the private binaryPath via any cast
-    const binaryPath = (client as any).binaryPath as string;
-    expect(binaryPath).toContain("macos-bridge");
-    expect(binaryPath).not.toContain("windows-bridge");
-
-    Object.defineProperty(process, "platform", { value: originalPlatform, writable: true });
-  });
-
-  it("selects Windows binary path when platform is win32", () => {
-    const originalPlatform = process.platform;
-    Object.defineProperty(process, "platform", { value: "win32", writable: true });
-
-    const client = new BridgeClient();
-    const binaryPath = (client as any).binaryPath as string;
-    expect(binaryPath).toContain("windows-bridge");
-    expect(binaryPath).toContain(".exe");
-
-    Object.defineProperty(process, "platform", { value: originalPlatform, writable: true });
-  });
-
-  it("accepts a custom binary path", () => {
-    const customPath = "/custom/path/to/bridge";
-    const client = new BridgeClient(customPath);
-    expect((client as any).binaryPath).toBe(customPath);
-  });
-});
-
-// ── JSON-RPC line parsing ──
-
-describe("BridgeClient JSON-RPC parsing", () => {
-  let client: BridgeClient;
-
-  beforeEach(() => {
-    client = new BridgeClient("/fake/path");
-  });
-
-  afterEach(async () => {
-    await client.stop();
-  });
-
-  it("resolves pending request on valid response", async () => {
-    // Manually set up a pending request and feed it a line
-    const pending = (client as any).pending as Map<number, any>;
-
-    const resultPromise = new Promise((resolve, reject) => {
-      pending.set(1, {
-        resolve,
-        reject,
-        timer: setTimeout(() => reject(new Error("timeout")), 5000),
-      });
-    });
-
-    // Simulate receiving a response line
-    (client as any).handleLine(JSON.stringify({ id: 1, result: { pong: true } }));
-
-    const result = await resultPromise;
-    expect(result).toEqual({ pong: true });
-    expect(pending.size).toBe(0);
-  });
-
-  it("rejects pending request on error response", async () => {
-    const pending = (client as any).pending as Map<number, any>;
-
-    const resultPromise = new Promise((resolve, reject) => {
-      pending.set(2, {
-        resolve,
-        reject,
-        timer: setTimeout(() => reject(new Error("timeout")), 5000),
-      });
-    });
-
-    (client as any).handleLine(JSON.stringify({
-      id: 2,
-      result: null,
-      error: { code: -1, message: "Not found: element" },
-    }));
-
-    await expect(resultPromise).rejects.toThrow("Not found: element");
-    expect(pending.size).toBe(0);
-  });
-
-  it("emits ax-event for event messages", async () => {
-    const eventPromise = new Promise<Record<string, unknown>>((resolve) => {
-      client.on("ax-event", resolve);
-    });
-
-    (client as any).handleLine(JSON.stringify({
-      id: 0,
-      event: { type: "value_changed", pid: 123 },
-    }));
-
-    const event = await eventPromise;
-    expect(event).toEqual({ type: "value_changed", pid: 123 });
-  });
-
-  it("ignores malformed JSON lines", () => {
-    // Should not throw
-    (client as any).handleLine("not json at all");
-    (client as any).handleLine("{incomplete");
-    (client as any).handleLine("");
-  });
-
-  it("ignores responses for unknown request IDs", () => {
-    // Should not throw
-    (client as any).handleLine(JSON.stringify({ id: 999, result: "orphan" }));
-  });
-});
-
-// ── Lifecycle ──
-
-describe("BridgeClient lifecycle", () => {
-  it("rejects all pending requests on stop", async () => {
-    const client = new BridgeClient("/fake/path");
-    const pending = (client as any).pending as Map<number, any>;
-
-    const p1 = new Promise((resolve, reject) => {
-      pending.set(10, {
-        resolve,
-        reject,
-        timer: setTimeout(() => reject(new Error("timeout")), 5000),
-      });
-    });
-
-    const p2 = new Promise((resolve, reject) => {
-      pending.set(11, {
-        resolve,
-        reject,
-        timer: setTimeout(() => reject(new Error("timeout")), 5000),
-      });
-    });
-
-    await client.stop();
-
-    await expect(p1).rejects.toThrow("Bridge stopped");
-    await expect(p2).rejects.toThrow("Bridge stopped");
-    expect(pending.size).toBe(0);
-  });
-
-  it("start is idempotent after first call", async () => {
-    const client = new BridgeClient("/fake/path");
-    (client as any).started = true;
-
-    // Second start() should return immediately without spawning
-    await client.start(); // should not throw or spawn
-    expect((client as any).process).toBeNull();
-  });
-});
-
-// ── MacOSBridgeClient backward compat ──
-
-describe("MacOSBridgeClient backward compatibility", () => {
-  it("MacOSBridgeClient is the same as BridgeClient", async () => {
-    const { MacOSBridgeClient } = await import("../src/native/bridge-client.js");
-    expect(MacOSBridgeClient).toBe(BridgeClient);
-  });
-
-  it("re-export from macos-bridge-client.ts works", async () => {
-    const mod = await import("../src/native/macos-bridge-client.js");
-    expect(mod.MacOSBridgeClient).toBe(BridgeClient);
-    expect(mod.BridgeClient).toBe(BridgeClient);
-  });
-});

package/tests/browser-stealth.test.ts

@@ -1,210 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
-
-/**
- * Tests for browser stealth / anti-detection features.
- * These test the stealth script validity, CDP Input event sequences,
- * and human-like timing without requiring a real Chrome instance.
- */
-
-// ── Stealth script validation ──
-
-// Extract the stealth script source inline (mirrors mcp-desktop.ts STEALTH_SCRIPT)
-const STEALTH_SCRIPT = `
-Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
-for (const key of Object.keys(window)) {
-  if (key.match(/^cdc_/)) delete (window)[key];
-}
-Object.defineProperty(navigator, 'plugins', {
-  get: () => [
-    { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
-    { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
-    { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
-  ],
-});
-Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
-`;
-
-describe("browser_stealth", () => {
-  it("stealth patches are valid JavaScript (no syntax errors)", () => {
-    // If this throws, the script has syntax errors
-    expect(() => new Function(STEALTH_SCRIPT)).not.toThrow();
-  });
-
-  it("stealth script hides webdriver property", () => {
-    // Simulate in a minimal environment
-    const nav: any = { webdriver: true };
-    const fn = new Function("navigator", `
-      Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
-    `);
-    fn(nav);
-    expect(nav.webdriver).toBeUndefined();
-  });
-
-  it("stealth script sets realistic plugins", () => {
-    const nav: any = { plugins: [] };
-    const fn = new Function("navigator", `
-      Object.defineProperty(navigator, 'plugins', {
-        get: () => [
-          { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
-          { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
-          { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
-        ],
-      });
-    `);
-    fn(nav);
-    expect(nav.plugins).toHaveLength(3);
-    expect(nav.plugins[0].name).toBe("Chrome PDF Plugin");
-  });
-
-  it("stealth script sets realistic languages", () => {
-    const nav: any = { languages: [] };
-    const fn = new Function("navigator", `
-      Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
-    `);
-    fn(nav);
-    expect(nav.languages).toEqual(["en-US", "en"]);
-  });
-});
-
-// ── Mock CDP client for Input domain tests ──
-
-function createMockCDPClient() {
-  const calls: { method: string; params: any }[] = [];
-  return {
-    calls,
-    Runtime: {
-      enable: vi.fn(),
-      evaluate: vi.fn().mockResolvedValue({
-        result: { value: { ok: true, x: 100, y: 200, text: "Submit" } },
-      }),
-    },
-    Input: {
-      dispatchKeyEvent: vi.fn().mockImplementation(async (params: any) => {
-        calls.push({ method: "Input.dispatchKeyEvent", params });
-      }),
-      dispatchMouseEvent: vi.fn().mockImplementation(async (params: any) => {
-        calls.push({ method: "Input.dispatchMouseEvent", params });
-      }),
-    },
-    Page: {
-      enable: vi.fn(),
-      addScriptToEvaluateOnNewDocument: vi.fn(),
-    },
-    close: vi.fn(),
-  };
-}
-
-describe("browser_fill_form (mock CDP)", () => {
-  it("types character by character using Input.dispatchKeyEvent", async () => {
-    const client = createMockCDPClient();
-    const text = "hello";
-
-    // Simulate what browser_fill_form does internally
-    // Clear: select all + backspace
-    await client.Input.dispatchKeyEvent({ type: "keyDown", key: "a", code: "KeyA", modifiers: 4 });
-    await client.Input.dispatchKeyEvent({ type: "keyUp", key: "a", code: "KeyA", modifiers: 4 });
-    await client.Input.dispatchKeyEvent({ type: "keyDown", key: "Backspace", code: "Backspace" });
-    await client.Input.dispatchKeyEvent({ type: "keyUp", key: "Backspace", code: "Backspace" });
-
-    // Type each char
-    for (const char of text) {
-      await client.Input.dispatchKeyEvent({ type: "keyDown", text: char, key: char, unmodifiedText: char });
-      await client.Input.dispatchKeyEvent({ type: "keyUp", text: char, key: char, unmodifiedText: char });
-    }
-
-    // 4 clear events + 2 per char = 4 + 10 = 14
-    const keyEvents = client.calls.filter(c => c.method === "Input.dispatchKeyEvent");
-    expect(keyEvents).toHaveLength(14);
-
-    // Verify keyDown/keyUp pairs for typed chars
-    const typedEvents = keyEvents.slice(4); // skip clear events
-    for (let i = 0; i < text.length; i++) {
-      const down = typedEvents[i * 2];
-      const up = typedEvents[i * 2 + 1];
-      expect(down.params.type).toBe("keyDown");
-      expect(down.params.text).toBe(text[i]);
-      expect(up.params.type).toBe("keyUp");
-      expect(up.params.text).toBe(text[i]);
-    }
-  });
-
-  it("does not use el.value assignment (no Runtime.evaluate with el.value =)", () => {
-    // The implementation should use Input.dispatchKeyEvent, NOT el.value = text
-    // This test verifies the approach by checking that the mock client
-    // receives dispatchKeyEvent calls instead of evaluate calls with el.value
-    const client = createMockCDPClient();
-
-    // The evaluate call should only be for focusing, not for setting value
-    // In the real implementation, evaluate is called once for focus only
-    expect(client.Runtime.evaluate).not.toHaveBeenCalledWith(
-      expect.objectContaining({
-        expression: expect.stringContaining("el.value ="),
-      })
-    );
-  });
-});
-
-describe("browser_human_click (mock CDP)", () => {
-  it("dispatches mouseMoved → mousePressed → mouseReleased sequence", async () => {
-    const client = createMockCDPClient();
-    const x = 100;
-    const y = 200;
-
-    await client.Input.dispatchMouseEvent({ type: "mouseMoved", x, y });
-    await client.Input.dispatchMouseEvent({ type: "mousePressed", x, y, button: "left", clickCount: 1 });
-    await client.Input.dispatchMouseEvent({ type: "mouseReleased", x, y, button: "left", clickCount: 1 });
-
-    const mouseEvents = client.calls.filter(c => c.method === "Input.dispatchMouseEvent");
-    expect(mouseEvents).toHaveLength(3);
-    expect(mouseEvents[0].params.type).toBe("mouseMoved");
-    expect(mouseEvents[1].params.type).toBe("mousePressed");
-    expect(mouseEvents[1].params.button).toBe("left");
-    expect(mouseEvents[2].params.type).toBe("mouseReleased");
-    expect(mouseEvents[2].params.button).toBe("left");
-  });
-
-  it("uses correct coordinates from element bounding rect", async () => {
-    const client = createMockCDPClient();
-    // Simulate element at (50, 100) with size 200x40
-    const rect = { x: 50, y: 100, width: 200, height: 40 };
-    const centerX = rect.x + rect.width / 2; // 150
-    const centerY = rect.y + rect.height / 2; // 120
-
-    await client.Input.dispatchMouseEvent({ type: "mouseMoved", x: centerX, y: centerY });
-    await client.Input.dispatchMouseEvent({ type: "mousePressed", x: centerX, y: centerY, button: "left", clickCount: 1 });
-    await client.Input.dispatchMouseEvent({ type: "mouseReleased", x: centerX, y: centerY, button: "left", clickCount: 1 });
-
-    const mouseEvents = client.calls.filter(c => c.method === "Input.dispatchMouseEvent");
-    for (const evt of mouseEvents) {
-      expect(evt.params.x).toBe(150);
-      expect(evt.params.y).toBe(120);
-    }
-  });
-});
-
-describe("random delay timing", () => {
-  it("randomDelay produces values within expected range", async () => {
-    // Test the delay logic directly
-    const min = 30;
-    const max = 80;
-    const samples: number[] = [];
-
-    for (let i = 0; i < 50; i++) {
-      const start = Date.now();
-      await new Promise(r => setTimeout(r, min + Math.random() * (max - min)));
-      const elapsed = Date.now() - start;
-      samples.push(elapsed);
-    }
-
-    // All delays should be roughly in the expected range
-    // Allow some tolerance for timer imprecision
-    for (const s of samples) {
-      expect(s).toBeGreaterThanOrEqual(min - 5);
-      expect(s).toBeLessThan(max + 50); // generous upper bound for CI
-    }
-
-    // Should not all be the same (randomness check)
-    const unique = new Set(samples);
-    expect(unique.size).toBeGreaterThan(1);
-  });
-});