screenhand 0.1.1 → 0.3.0

package/dist/src/runtime/state-observer.js

@@ -0,0 +1,121 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { EventEmitter } from "node:events";
+/**
+ * Wraps the native bridge's AX observer events into typed UIEvent objects.
+ * Buffers events for consumption by the planning loop.
+ */
+export class StateObserver extends EventEmitter {
+    bridge;
+    observedPids = new Set();
+    eventBuffer = [];
+    maxBufferSize;
+    constructor(bridge, maxBufferSize = 200) {
+        super();
+        this.bridge = bridge;
+        this.maxBufferSize = maxBufferSize;
+        // Listen for AX events from the bridge
+        this.bridge.on("ax-event", (raw) => {
+            const event = this.parseEvent(raw);
+            if (event) {
+                this.eventBuffer.push(event);
+                if (this.eventBuffer.length > this.maxBufferSize) {
+                    this.eventBuffer.shift();
+                }
+                this.emit("event", event);
+            }
+        });
+    }
+    async startObserving(pid, eventTypes) {
+        if (this.observedPids.has(pid))
+            return;
+        const notifications = eventTypes
+            ? this.mapEventTypesToNotifications(eventTypes)
+            : undefined;
+        await this.bridge.call("observer.start", {
+            pid,
+            notifications,
+        });
+        this.observedPids.add(pid);
+    }
+    async stopObserving(pid) {
+        if (!this.observedPids.has(pid))
+            return;
+        await this.bridge.call("observer.stop", { pid });
+        this.observedPids.delete(pid);
+    }
+    /** Get and clear the event buffer. */
+    drainEvents() {
+        const events = [...this.eventBuffer];
+        this.eventBuffer = [];
+        return events;
+    }
+    /** Get recent events without clearing. */
+    peekEvents(limit = 50) {
+        return this.eventBuffer.slice(-limit);
+    }
+    /** Clear the event buffer. */
+    clearEvents() {
+        this.eventBuffer = [];
+    }
+    get isObserving() {
+        return this.observedPids.size > 0;
+    }
+    get observedProcesses() {
+        return [...this.observedPids];
+    }
+    parseEvent(raw) {
+        const type = raw.type;
+        if (!type)
+            return null;
+        const event = {
+            type,
+            timestamp: raw.timestamp ?? new Date().toISOString(),
+            pid: raw.pid ?? 0,
+        };
+        if (typeof raw.bundleId === "string")
+            event.bundleId = raw.bundleId;
+        if (typeof raw.elementRole === "string")
+            event.elementRole = raw.elementRole;
+        if (typeof raw.elementLabel === "string")
+            event.elementLabel = raw.elementLabel;
+        if (typeof raw.oldValue === "string")
+            event.oldValue = raw.oldValue;
+        if (typeof raw.newValue === "string")
+            event.newValue = raw.newValue;
+        if (typeof raw.windowTitle === "string")
+            event.windowTitle = raw.windowTitle;
+        return event;
+    }
+    mapEventTypesToNotifications(types) {
+        const map = {
+            value_changed: "AXValueChanged",
+            focus_changed: "AXFocusedUIElementChanged",
+            window_created: "AXWindowCreated",
+            window_closed: "AXUIElementDestroyed",
+            title_changed: "AXTitleChanged",
+            menu_opened: "AXMenuOpened",
+            layout_changed: "AXLayoutChanged",
+            dialog_appeared: "AXSheetCreated",
+            app_activated: "AXApplicationActivated",
+            app_deactivated: "AXApplicationDeactivated",
+        };
+        return types
+            .map((t) => map[t])
+            .filter((n) => n !== undefined);
+    }
+}

package/dist/src/runtime/vision-adapter.js

@@ -0,0 +1,225 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { randomUUID } from "node:crypto";
+const POLL_INTERVAL_MS = 200;
+/**
+ * Vision-based adapter for apps with poor/no accessibility support.
+ * Uses screenshots + OCR to locate elements and CG events to interact.
+ */
+export class VisionAdapter {
+    bridge;
+    sessions = new Map();
+    sessionsByProfile = new Map();
+    constructor(bridge) {
+        this.bridge = bridge;
+    }
+    async attach(profile, reuseSessionId) {
+        const existing = this.sessionsByProfile.get(profile);
+        if (existing)
+            return existing.info;
+        await this.bridge.start();
+        const frontmost = await this.bridge.call("app.frontmost");
+        const info = {
+            sessionId: reuseSessionId ?? `vision_session_${profile}_${Date.now()}_${randomUUID().slice(0, 8)}`,
+            profile,
+            createdAt: new Date().toISOString(),
+            adapterType: "vision",
+        };
+        const state = {
+            info,
+            pid: frontmost.pid,
+            bundleId: frontmost.bundleId,
+            appName: frontmost.name,
+        };
+        this.sessions.set(info.sessionId, state);
+        this.sessionsByProfile.set(profile, state);
+        return info;
+    }
+    async getAppContext(sessionId) {
+        const state = this.requireSession(sessionId);
+        return {
+            bundleId: state.bundleId,
+            appName: state.appName,
+            pid: state.pid,
+            windowTitle: state.appName,
+        };
+    }
+    async getPageMeta(sessionId) {
+        const ctx = await this.getAppContext(sessionId);
+        return {
+            url: `app://${ctx.bundleId}`,
+            title: ctx.appName,
+        };
+    }
+    async navigate(sessionId, url, _timeoutMs) {
+        if (url.startsWith("app://")) {
+            const bundleId = url.slice(6);
+            const result = await this.bridge.call("app.launch", { bundleId });
+            const state = this.requireSession(sessionId);
+            state.pid = result.pid;
+            state.bundleId = result.bundleId;
+            state.appName = result.appName;
+        }
+        return this.getPageMeta(sessionId);
+    }
+    async locate(sessionId, target, timeoutMs) {
+        const state = this.requireSession(sessionId);
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            // Take a screenshot
+            const screenshotResult = await this.bridge.call("cg.captureScreen", {});
+            state.lastScreenshotPath = screenshotResult.path;
+            const searchText = this.getSearchText(target);
+            if (!searchText) {
+                // For coordinate targets, just return coordinates directly
+                if (target.type === "coordinates") {
+                    return {
+                        handleId: `vision_coords_${target.x}_${target.y}`,
+                        locatorUsed: "vision:coordinates",
+                        coordinates: { x: target.x, y: target.y, width: 1, height: 1 },
+                    };
+                }
+                return null;
+            }
+            // OCR the screenshot
+            const matches = await this.bridge.call("vision.findText", {
+                imagePath: screenshotResult.path,
+                searchText,
+            });
+            if (matches.length > 0) {
+                const best = matches.reduce((a, b) => (a.confidence > b.confidence ? a : b));
+                return {
+                    handleId: `vision_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+                    locatorUsed: `vision:text:${searchText}`,
+                    label: best.text,
+                    coordinates: best.bounds,
+                };
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return null;
+    }
+    async click(_sessionId, element) {
+        if (!element.coordinates) {
+            throw new Error("Vision adapter requires coordinates to click");
+        }
+        const cx = element.coordinates.x + element.coordinates.width / 2;
+        const cy = element.coordinates.y + element.coordinates.height / 2;
+        await this.bridge.call("cg.mouseClick", { x: cx, y: cy });
+    }
+    async setValue(_sessionId, element, text, clear) {
+        // Click to focus
+        await this.click(_sessionId, element);
+        await sleep(100);
+        if (clear) {
+            await this.bridge.call("cg.keyCombo", { keys: ["cmd", "a"] });
+            await sleep(50);
+        }
+        await this.bridge.call("cg.typeText", { text });
+    }
+    async getValue(_sessionId, element) {
+        // Vision can't reliably read values; return label if available
+        return element.label ?? "";
+    }
+    async waitFor(sessionId, condition, timeoutMs) {
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            if (condition.type === "text_appears") {
+                const found = await this.locate(sessionId, { type: "text", value: condition.text }, 200);
+                if (found)
+                    return true;
+            }
+            else if (condition.type === "element_exists") {
+                const found = await this.locate(sessionId, condition.target, 200);
+                if (found)
+                    return true;
+            }
+            else if (condition.type === "element_gone") {
+                const found = await this.locate(sessionId, condition.target, 200);
+                if (!found)
+                    return true;
+            }
+            else {
+                // Unsupported condition types
+                return false;
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        return false;
+    }
+    async extract(sessionId, _target, format) {
+        const state = this.requireSession(sessionId);
+        // Take a fresh screenshot and OCR it
+        const screenshotResult = await this.bridge.call("cg.captureScreen", {});
+        state.lastScreenshotPath = screenshotResult.path;
+        const ocrResult = await this.bridge.call("vision.ocr", { imagePath: screenshotResult.path });
+        if (format === "text") {
+            return ocrResult.text;
+        }
+        if (format === "json") {
+            return ocrResult;
+        }
+        // table format
+        return {
+            headers: ["text", "confidence", "x", "y", "width", "height"],
+            rows: ocrResult.regions.map((r) => [
+                r.text,
+                r.confidence,
+                r.bounds.x,
+                r.bounds.y,
+                r.bounds.width,
+                r.bounds.height,
+            ]),
+        };
+    }
+    async screenshot(_sessionId, region) {
+        const result = await this.bridge.call("cg.captureScreen", region ? { region } : {});
+        return result.path;
+    }
+    async keyCombo(_sessionId, keys) {
+        await this.bridge.call("cg.keyCombo", { keys });
+    }
+    async elementTree(_sessionId, _maxDepth, _root) {
+        throw new Error("Vision adapter does not support elementTree — use accessibility adapter");
+    }
+    // ── Private ──
+    requireSession(sessionId) {
+        const state = this.sessions.get(sessionId);
+        if (!state)
+            throw new Error(`Session not found: ${sessionId}`);
+        return state;
+    }
+    getSearchText(target) {
+        switch (target.type) {
+            case "text":
+                return target.value;
+            case "role":
+                return target.name;
+            case "selector":
+                return target.value;
+            case "ax_attribute":
+                return target.value;
+            case "image":
+            case "coordinates":
+            case "ax_path":
+                return null;
+        }
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/src/state/app-map-types.js

@@ -0,0 +1,72 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+// ── Rating System (F → 0) ──────────────────────────────────────────
+//
+// Game-style rating: F E D C B A S SS SSS 0
+// Each grade has 3 sub-tiers: e.g. B1 (entry), B2 (mid), B3 (top)
+// Graded by 10 weighted factors scored 0-100
+//
+// Years-equivalent mapping:
+//   F   = just opened the app
+//   E   = ~1 week user
+//   D   = ~1-3 months
+//   C   = ~6-12 months
+//   B   = ~1-3 years (consistent daily user)
+//   A   = ~3-5 years (power user / team lead)
+//   S   = ~5-10 years (department architect)
+//   SS  = ~10-20 years (platform expert, builds systems)
+//   SSS = ~20+ years (framework builder, trains others)
+//   0   = Class Zero — transcendent mastery, all 10 factors maxed
+/** Rating grades from lowest to highest */
+export const RATING_GRADES = ["F", "E", "D", "C", "B", "A", "S", "SS", "SSS", "0"];
+/** Factor weights — hard-to-fake signals dominate (57%), session-gated evidence (43%) */
+export const RATING_FACTOR_WEIGHTS = {
+    consistency: 20, // THE core signal — can't fake showing up 50+ times
+    platformKnowledge: 15, // shortcuts, deep features — proves real knowledge
+    edgeCaseHandling: 12, // surviving unexpected states — proves resilience
+    teachingAbility: 10, // exporting playbooks — proves codifiable mastery
+    featureCoverage: 10, // breadth of features used (session-gated)
+    workflowDepth: 8, // multi-step workflows completed (session-gated)
+    outcomeVerification: 8, // verified outcomes (session-gated)
+    errorRecovery: 7, // healing from failures — honest if it happens
+    crossFeatureChains: 5, // combining features end-to-end
+    speedEfficiency: 5, // repeat mastery across sessions
+};
+export const DEFAULT_APP_MAP_CONFIG = {
+    mapsDir: "",
+    staleThresholdDays: 7,
+    versionDecayFactor: 0.5,
+    pruneSessionThreshold: 10,
+    maxZonesPerApp: 50,
+    maxElementsPerZone: 100,
+    maxEdges: 200,
+    maxHistoryEntries: 100,
+    maxHierarchyEntriesPerZone: 50,
+    maxContractsPerZone: 30,
+    maxOutcomesPerContract: 5,
+    maxStateDimensions: 30,
+    maxStateTransitions: 100,
+    maxVisibilityConditions: 200,
+    maxTimingProfiles: 100,
+    maxReadySignals: 50,
+};
+// ── Rating Utility ──────────────────────────────────────────────────
+/** Convert Rating to display string: "B2", "SS3", "0" */
+export function ratingToString(r) {
+    if (r.grade === "0")
+        return "0"; // Class Zero has no sub-tier display
+    return `${r.grade}${r.subTier}`;
+}
+/** Grade thresholds: weighted score needed for each grade (0-100 scale) */
+export const GRADE_THRESHOLDS = [
+    { grade: "0", minScore: 97 }, // Class Zero — near-perfect across all factors
+    { grade: "SSS", minScore: 90 }, // 20+ years equivalent
+    { grade: "SS", minScore: 82 }, // 10-20 years
+    { grade: "S", minScore: 73 }, // 5-10 years
+    { grade: "A", minScore: 62 }, // 3-5 years
+    { grade: "B", minScore: 50 }, // 1-3 years
+    { grade: "C", minScore: 38 }, // 6-12 months
+    { grade: "D", minScore: 25 }, // 1-3 months
+    { grade: "E", minScore: 12 }, // ~1 week
+    { grade: "F", minScore: 0 }, // just opened the app
+];