screenhand 0.1.0 → 0.1.1

package/tests/types.test.ts

@@ -0,0 +1,92 @@
+import { describe, it, expect, expectTypeOf } from "vitest";
+import type {
+  Target,
+  WaitCondition,
+  ToolName,
+  UIEventType,
+  SessionInfo,
+  AppContext,
+  LocatedElement,
+  WindowInfo,
+  RunningApp,
+} from "../src/types.js";
+
+/**
+ * Type-level tests to ensure the type definitions compile correctly
+ * and cover expected shapes. These catch regressions in the public API.
+ */
+
+describe("Type definitions", () => {
+  it("Target union covers all locator strategies", () => {
+    const selectorTarget: Target = { type: "selector", value: "button.submit" };
+    const textTarget: Target = { type: "text", value: "Click me", exact: true };
+    const roleTarget: Target = { type: "role", role: "button", name: "Submit" };
+    const axPathTarget: Target = { type: "ax_path", path: ["0", "1", "3"] };
+    const axAttrTarget: Target = { type: "ax_attribute", attribute: "AXIdentifier", value: "btn1" };
+    const coordTarget: Target = { type: "coordinates", x: 100, y: 200 };
+    const imageTarget: Target = { type: "image", base64: "abc123", confidence: 0.9 };
+
+    // All should compile and be assignable
+    const targets: Target[] = [
+      selectorTarget, textTarget, roleTarget,
+      axPathTarget, axAttrTarget, coordTarget, imageTarget,
+    ];
+    expect(targets).toHaveLength(7);
+  });
+
+  it("WaitCondition covers all condition types", () => {
+    const conditions: WaitCondition[] = [
+      { type: "selector_visible", selector: ".loaded" },
+      { type: "selector_hidden", selector: ".spinner" },
+      { type: "url_matches", regex: "https://.*" },
+      { type: "text_appears", text: "Success" },
+      { type: "element_exists", target: { type: "text", value: "OK" } },
+      { type: "element_gone", target: { type: "text", value: "Loading" } },
+      { type: "window_title_matches", regex: "Untitled" },
+    ];
+    expect(conditions.length).toBeGreaterThan(0);
+  });
+
+  it("SessionInfo has required fields", () => {
+    const session: SessionInfo = {
+      sessionId: "test-123",
+      profile: "default",
+      createdAt: Date.now(),
+      adapterType: "composite",
+    };
+    expect(session.sessionId).toBe("test-123");
+  });
+
+  it("AppContext has required fields", () => {
+    const ctx: AppContext = {
+      bundleId: "com.apple.Notes",
+      appName: "Notes",
+      pid: 1234,
+      windowTitle: "My Note",
+    };
+    expect(ctx.pid).toBe(1234);
+  });
+
+  it("WindowInfo has required fields", () => {
+    const win: WindowInfo = {
+      windowId: 42,
+      title: "Test Window",
+      appName: "TestApp",
+      bundleId: "com.test.app",
+      pid: 5678,
+      bounds: { x: 0, y: 0, width: 800, height: 600 },
+      isOnScreen: true,
+    };
+    expect(win.windowId).toBe(42);
+  });
+
+  it("RunningApp has required fields", () => {
+    const app: RunningApp = {
+      name: "Finder",
+      bundleId: "com.apple.finder",
+      pid: 100,
+      isActive: true,
+    };
+    expect(app.name).toBe("Finder");
+  });
+});

package/tsconfig.check.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true,
+    "esModuleInterop": true,
+    "noEmit": true
+  },
+  "include": ["src/**/*.ts", "mcp-desktop.ts", "mcp-bridge.ts"],
+  "exclude": ["dist", "node_modules"]
+}

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+    "skipLibCheck": true,
+    "rootDir": "src",
+    "outDir": "dist",
+    "resolveJsonModule": true,
+    "esModuleInterop": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["dist", "node_modules"]
+}
+

package/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    include: ["tests/**/*.test.ts"],
+    testTimeout: 15_000,
+  },
+});

package/dist/config.js

@@ -1,9 +0,0 @@
-export const DEFAULT_ACTION_BUDGET = {
-    locateMs: 800,
-    actMs: 200,
-    verifyMs: 2000,
-    maxRetries: 1,
-};
-export const DEFAULT_NAVIGATE_TIMEOUT_MS = 10_000;
-export const DEFAULT_WAIT_TIMEOUT_MS = 2_000;
-export const DEFAULT_PROFILE = "automation";

package/dist/index.js

@@ -1,55 +0,0 @@
-import { TimelineLogger } from "./logging/timeline-logger.js";
-import { MvpMcpServer } from "./mcp/server.js";
-import { PlaceholderAppAdapter, } from "./runtime/app-adapter.js";
-import { CdpChromeAdapter } from "./runtime/cdp-chrome-adapter.js";
-import { AutomationRuntimeService } from "./runtime/service.js";
-export { PlaceholderAppAdapter } from "./runtime/app-adapter.js";
-export { CdpChromeAdapter } from "./runtime/cdp-chrome-adapter.js";
-export { AccessibilityAdapter } from "./runtime/accessibility-adapter.js";
-export { AppleScriptAdapter } from "./runtime/applescript-adapter.js";
-export { VisionAdapter } from "./runtime/vision-adapter.js";
-export { CompositeAdapter } from "./runtime/composite-adapter.js";
-export { BridgeClient, BridgeClient as MacOSBridgeClient } from "./native/bridge-client.js";
-export { StateObserver } from "./runtime/state-observer.js";
-export { PlanningLoop } from "./runtime/planning-loop.js";
-export { AutomationRuntimeService } from "./runtime/service.js";
-export { MvpMcpServer } from "./mcp/server.js";
-export { createMcpStdioServer, startMcpStdioServer } from "./mcp/mcp-stdio-server.js";
-export function createRuntimeApp(adapter) {
-    const logger = new TimelineLogger();
-    const runtime = new AutomationRuntimeService(adapter, logger);
-    const mcp = new MvpMcpServer(runtime);
-    return { runtime, mcp };
-}
-async function createDefaultAdapter() {
-    if (process.env.AUTOMATOR_ADAPTER === "placeholder") {
-        return new PlaceholderAppAdapter();
-    }
-    if (process.env.AUTOMATOR_ADAPTER === "composite") {
-        // Lazy import to avoid requiring Swift bridge for CDP-only usage
-        const { MacOSBridgeClient } = await import("./native/macos-bridge-client.js");
-        const { CompositeAdapter } = await import("./runtime/composite-adapter.js");
-        const bridge = new MacOSBridgeClient();
-        return new CompositeAdapter(bridge, {
-            headless: process.env.AUTOMATOR_HEADLESS === "1",
-        });
-    }
-    if (process.env.AUTOMATOR_ADAPTER === "accessibility") {
-        const { MacOSBridgeClient } = await import("./native/macos-bridge-client.js");
-        const { AccessibilityAdapter } = await import("./runtime/accessibility-adapter.js");
-        const bridge = new MacOSBridgeClient();
-        return new AccessibilityAdapter(bridge);
-    }
-    return new CdpChromeAdapter({
-        headless: process.env.AUTOMATOR_HEADLESS === "1",
-    });
-}
-const app = createRuntimeApp(await createDefaultAdapter());
-if (process.argv.includes("--healthcheck")) {
-    const session = await app.runtime.sessionStart("automation");
-    console.log(JSON.stringify({
-        status: "ok",
-        session,
-        note: "Runtime loaded with universal adapter support.",
-    }, null, 2));
-}

package/dist/logging/timeline-logger.js

@@ -1,29 +0,0 @@
-export class TimelineLogger {
-    timeline = [];
-    start(action, sessionId) {
-        return {
-            action,
-            sessionId,
-            startedAt: new Date().toISOString(),
-            locateMs: 0,
-            actMs: 0,
-            verifyMs: 0,
-            retries: 0,
-        };
-    }
-    finish(telemetry, status) {
-        const finishedAt = new Date().toISOString();
-        const totalMs = new Date(finishedAt).getTime() - new Date(telemetry.startedAt).getTime();
-        const finalized = {
-            ...telemetry,
-            finishedAt,
-            totalMs,
-            status,
-        };
-        this.timeline.push(finalized);
-        return finalized;
-    }
-    getRecent(limit = 50) {
-        return this.timeline.slice(-limit);
-    }
-}

package/dist/mcp/mcp-stdio-server.js

@@ -1,284 +0,0 @@
-import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
-import { z } from "zod";
-// ── Schema building blocks ──
-const TargetSchema = z.union([
-    z.string().describe("Shorthand: text to find, or 'css=...' / 'text=...' / 'ax_id=...' prefix"),
-    z.object({
-        text: z.string(),
-        exact: z.boolean().optional(),
-    }).describe("Find by visible text"),
-    z.object({
-        role: z.string(),
-        name: z.string(),
-        exact: z.boolean().optional(),
-    }).describe("Find by ARIA/AX role and accessible name"),
-    z.object({
-        selector: z.string(),
-    }).describe("Find by CSS selector (browser) or AX identifier (desktop)"),
-    z.object({
-        x: z.number(),
-        y: z.number(),
-    }).describe("Click at screen coordinates"),
-    z.object({
-        attribute: z.string(),
-        value: z.string(),
-    }).describe("Find by accessibility attribute"),
-]);
-const WaitConditionSchema = z.object({
-    type: z.enum([
-        "selector_visible",
-        "selector_hidden",
-        "url_matches",
-        "text_appears",
-        "spinner_disappears",
-        "element_exists",
-        "element_gone",
-        "window_title_matches",
-        "app_idle",
-    ]),
-    selector: z.string().optional(),
-    regex: z.string().optional(),
-    text: z.string().optional(),
-    target: TargetSchema.optional(),
-    bundleId: z.string().optional(),
-    timeoutMs: z.number().optional(),
-}).describe("Condition to wait for");
-const RegionSchema = z.object({
-    x: z.number(),
-    y: z.number(),
-    width: z.number(),
-    height: z.number(),
-});
-// ── Target parser ──
-function parseTarget(input) {
-    if (typeof input === "string") {
-        if (input.startsWith("css="))
-            return { type: "selector", value: input.slice(4) };
-        if (input.startsWith("text="))
-            return { type: "text", value: input.slice(5), exact: true };
-        if (input.startsWith("ax_id="))
-            return { type: "ax_attribute", attribute: "identifier", value: input.slice(6) };
-        return { type: "text", value: input };
-    }
-    const obj = input;
-    if (typeof obj.selector === "string")
-        return { type: "selector", value: obj.selector };
-    if (typeof obj.text === "string")
-        return { type: "text", value: obj.text, exact: obj.exact === true };
-    if (typeof obj.role === "string" && typeof obj.name === "string")
-        return { type: "role", role: obj.role, name: obj.name, exact: obj.exact === true };
-    if (typeof obj.x === "number" && typeof obj.y === "number")
-        return { type: "coordinates", x: obj.x, y: obj.y };
-    if (typeof obj.attribute === "string" && typeof obj.value === "string")
-        return { type: "ax_attribute", attribute: obj.attribute, value: obj.value };
-    throw new Error("Invalid target");
-}
-function parseWaitCondition(input) {
-    const obj = input;
-    const type = obj.type;
-    switch (type) {
-        case "selector_visible": return { type: "selector_visible", selector: obj.selector };
-        case "selector_hidden": return { type: "selector_hidden", selector: obj.selector };
-        case "url_matches": return { type: "url_matches", regex: obj.regex };
-        case "text_appears": return { type: "text_appears", text: obj.text };
-        case "spinner_disappears": return { type: "spinner_disappears", selector: obj.selector };
-        case "element_exists": return { type: "element_exists", target: parseTarget(obj.target) };
-        case "element_gone": return { type: "element_gone", target: parseTarget(obj.target) };
-        case "window_title_matches": return { type: "window_title_matches", regex: obj.regex };
-        case "app_idle": {
-            const cond = { type: "app_idle", bundleId: obj.bundleId };
-            if (typeof obj.timeoutMs === "number")
-                cond.timeoutMs = obj.timeoutMs;
-            return cond;
-        }
-        default: throw new Error(`Unknown wait condition type: ${type}`);
-    }
-}
-// ── Helpers ──
-function ok(data) {
-    return { content: [{ type: "text", text: JSON.stringify(data, null, 2) }] };
-}
-function err(message) {
-    return { content: [{ type: "text", text: message }], isError: true };
-}
-// ── Server builder ──
-export function createMcpStdioServer(runtime) {
-    const mcp = new McpServer({ name: "screenhand", version: "0.1.0" }, {
-        capabilities: { tools: {} },
-        instructions: "ScreenHand gives AI agents eyes and hands on the desktop. Use session_start to begin, then call tools to control apps.",
-    });
-    // ── session_start ──
-    mcp.tool("session_start", "Start a new automation session. Returns a sessionId needed by all other tools. Automatically attaches to the frontmost app.", { profile: z.string().optional().describe("Session profile name (default: 'automation')") }, async ({ profile }) => {
-        try {
-            const session = await runtime.sessionStart(profile);
-            return ok(session);
-        }
-        catch (e) {
-            return err(`Failed to start session: ${e instanceof Error ? e.message : String(e)}`);
-        }
-    });
-    // ── press ──
-    mcp.tool("press", "Click/press a UI element. Finds the element by text, role, selector, or coordinates, then clicks it.", {
-        sessionId: z.string().describe("Session ID from session_start"),
-        target: TargetSchema.describe("What to click — text string, {role, name}, {selector}, or {x, y}"),
-        verify: WaitConditionSchema.optional().describe("Optional condition to verify after clicking"),
-    }, async ({ sessionId, target, verify }) => {
-        const input = { sessionId, target: parseTarget(target) };
-        if (verify)
-            input.verify = parseWaitCondition(verify);
-        const result = await runtime.press(input);
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── type_into ──
-    mcp.tool("type_into", "Type text into a UI element (text field, search box, etc). Locates the field, optionally clears it, then types.", {
-        sessionId: z.string(),
-        target: TargetSchema.describe("The input field to type into"),
-        text: z.string().describe("Text to type"),
-        clear: z.boolean().optional().describe("Clear the field first (default: true)"),
-        verify: WaitConditionSchema.optional(),
-    }, async ({ sessionId, target, text, clear, verify }) => {
-        const input = { sessionId, target: parseTarget(target), text };
-        if (typeof clear === "boolean")
-            input.clear = clear;
-        if (verify)
-            input.verify = parseWaitCondition(verify);
-        const result = await runtime.typeInto(input);
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── navigate ──
-    mcp.tool("navigate", "Navigate a browser to a URL, or open an app via 'app://com.bundle.id'.", {
-        sessionId: z.string(),
-        url: z.string().describe("URL to navigate to, or 'app://bundleId' to launch an app"),
-        timeoutMs: z.number().optional().describe("Navigation timeout in ms (default: 10000)"),
-    }, async ({ sessionId, url, timeoutMs }) => {
-        const input = { sessionId, url };
-        if (typeof timeoutMs === "number")
-            input.timeoutMs = timeoutMs;
-        const result = await runtime.navigate(input);
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── wait_for ──
-    mcp.tool("wait_for", "Wait for a condition: element appears/disappears, text appears, URL changes, window title matches, etc.", {
-        sessionId: z.string(),
-        condition: WaitConditionSchema,
-        timeoutMs: z.number().optional().describe("Timeout in ms (default: 2000)"),
-    }, async ({ sessionId, condition, timeoutMs }) => {
-        const input = { sessionId, condition: parseWaitCondition(condition) };
-        if (typeof timeoutMs === "number")
-            input.timeoutMs = timeoutMs;
-        const result = await runtime.waitFor(input);
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── extract ──
-    mcp.tool("extract", "Extract data from a UI element. Returns text content, table data, or structured JSON from the element.", {
-        sessionId: z.string(),
-        target: TargetSchema,
-        format: z.enum(["text", "table", "json"]).describe("Output format"),
-    }, async ({ sessionId, target, format }) => {
-        const result = await runtime.extract({
-            sessionId,
-            target: parseTarget(target),
-            format: format,
-        });
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── screenshot ──
-    mcp.tool("screenshot", "Capture a screenshot of the current app window or a specific screen region. Returns the file path.", {
-        sessionId: z.string(),
-        region: RegionSchema.optional().describe("Optional screen region to capture"),
-    }, async ({ sessionId, region }) => {
-        const input = { sessionId };
-        if (region)
-            input.region = region;
-        const result = await runtime.screenshot(input);
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── app_launch ──
-    mcp.tool("app_launch", "Launch a macOS/Windows application by bundle ID (e.g., 'com.apple.Safari', 'com.google.Chrome').", {
-        sessionId: z.string(),
-        bundleId: z.string().describe("macOS bundle ID or Windows process name"),
-    }, async ({ sessionId, bundleId }) => {
-        const result = await runtime.appLaunch({ sessionId, bundleId });
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── app_focus ──
-    mcp.tool("app_focus", "Bring a running application to the foreground.", {
-        sessionId: z.string(),
-        bundleId: z.string(),
-    }, async ({ sessionId, bundleId }) => {
-        const result = await runtime.appFocus({ sessionId, bundleId });
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── app_list ──
-    mcp.tool("app_list", "List all running applications with their bundle IDs, names, and PIDs.", { sessionId: z.string() }, async ({ sessionId }) => {
-        const result = await runtime.appList(sessionId);
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── window_list ──
-    mcp.tool("window_list", "List all visible windows with their titles, positions, and sizes.", { sessionId: z.string() }, async ({ sessionId }) => {
-        const result = await runtime.windowList(sessionId);
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── menu_click ──
-    mcp.tool("menu_click", "Click a menu item by path. For example ['File', 'Save As...'] clicks File → Save As.", {
-        sessionId: z.string(),
-        menuPath: z.array(z.string()).describe("Menu path, e.g. ['File', 'New Window']"),
-    }, async ({ sessionId, menuPath }) => {
-        const result = await runtime.menuClick({ sessionId, menuPath });
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── key_combo ──
-    mcp.tool("key_combo", "Send a keyboard shortcut. Keys: 'cmd', 'ctrl', 'alt', 'shift', plus any character. E.g. ['cmd', 'c'] for copy.", {
-        sessionId: z.string(),
-        keys: z.array(z.string()).describe("Key combination, e.g. ['cmd', 's']"),
-    }, async ({ sessionId, keys }) => {
-        const result = await runtime.keyCombo({ sessionId, keys });
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── element_tree ──
-    mcp.tool("element_tree", "Get the accessibility element tree of the current app. Useful for understanding the UI structure and finding elements to interact with.", {
-        sessionId: z.string(),
-        maxDepth: z.number().optional().describe("Max tree depth (default: 5)"),
-    }, async ({ sessionId, maxDepth }) => {
-        const input = { sessionId };
-        if (typeof maxDepth === "number")
-            input.maxDepth = maxDepth;
-        const result = await runtime.elementTree(input);
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── drag ──
-    mcp.tool("drag", "Drag from one UI element to another.", {
-        sessionId: z.string(),
-        from: TargetSchema.describe("Element to drag from"),
-        to: TargetSchema.describe("Element to drag to"),
-    }, async ({ sessionId, from, to }) => {
-        const result = await runtime.drag({
-            sessionId,
-            from: parseTarget(from),
-            to: parseTarget(to),
-        });
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    // ── scroll ──
-    mcp.tool("scroll", "Scroll in a direction, optionally targeting a specific element.", {
-        sessionId: z.string(),
-        direction: z.enum(["up", "down", "left", "right"]),
-        amount: z.number().optional().describe("Scroll amount (default: 3)"),
-        target: TargetSchema.optional().describe("Element to scroll within"),
-    }, async ({ sessionId, direction, amount, target }) => {
-        const input = { sessionId, direction };
-        if (typeof amount === "number")
-            input.amount = amount;
-        if (target)
-            input.target = parseTarget(target);
-        const result = await runtime.scroll(input);
-        return result.ok ? ok(result) : err(result.error.message);
-    });
-    return mcp;
-}
-export async function startMcpStdioServer(runtime) {
-    const mcp = createMcpStdioServer(runtime);
-    const transport = new StdioServerTransport();
-    await mcp.connect(transport);
-}