screenhand 0.1.1 → 0.3.0

package/dist/src/playbook/recorder.js

@@ -0,0 +1,536 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * Playbook Recorder v2 — event-driven + screenshot-based
+ *
+ * Two capture modes running in parallel:
+ *
+ * 1. AX Event Stream (real-time, ~0ms latency)
+ *    - Listens to macOS accessibility notifications via the native bridge
+ *    - Captures: focus changes, value changes, window creates, app switches
+ *    - This is how we know WHAT the user clicked/typed
+ *
+ * 2. Periodic Screenshots (every 2s)
+ *    - Captures visual state of the screen
+ *    - At stop time, AI analyzes the screenshot sequence + AX events
+ *    - This is how we handle things AX events miss (Chrome DOM, visual changes)
+ *
+ * On stop:
+ *    - All AX events + screenshots sent to AI
+ *    - AI produces clean PlaybookStep[] from the combined data
+ *    - Saved to disk as a replayable playbook
+ */
+import Anthropic from "@anthropic-ai/sdk";
+import fs from "node:fs";
+import { PlaybookStore } from "./store.js";
+const SCREENSHOT_INTERVAL_MS = 2500;
+const AX_POLL_INTERVAL_MS = 500;
+export class PlaybookRecorder {
+    runtime;
+    options;
+    recording = false;
+    events = [];
+    screenshots = [];
+    screenshotTimer = null;
+    axPollTimer = null;
+    sessionId = "";
+    // Track previous AX state for diff detection
+    prevFocused = "";
+    prevActiveApp = "";
+    prevWindowTitle = "";
+    prevUrl = "";
+    prevTextFields = new Map();
+    store;
+    ai;
+    model;
+    log;
+    onEvent;
+    captureScreenshots;
+    constructor(runtime, playbookDir, options = {}) {
+        this.runtime = runtime;
+        this.options = options;
+        this.store = new PlaybookStore(playbookDir);
+        this.store.load();
+        this.ai = new Anthropic();
+        this.model = options.model ?? "claude-sonnet-4-20250514";
+        this.log = options.onLog ?? ((msg) => console.error(`[Recorder] ${msg}`));
+        this.onEvent = options.onEvent;
+        this.captureScreenshots = options.screenshots !== false;
+    }
+    /**
+     * Start recording user actions.
+     */
+    async start(sessionId) {
+        if (this.recording) {
+            this.log("Already recording");
+            return;
+        }
+        this.recording = true;
+        this.sessionId = sessionId;
+        this.events = [];
+        this.screenshots = [];
+        this.prevFocused = "";
+        this.prevActiveApp = "";
+        this.prevWindowTitle = "";
+        this.prevUrl = "";
+        this.prevTextFields.clear();
+        // Take initial state snapshot
+        await this.captureState("initial");
+        this.log("Recording started — watching for AX events + taking screenshots");
+        // Start AX event polling (fast — every 500ms)
+        this.axPollTimer = setInterval(async () => {
+            if (!this.recording)
+                return;
+            try {
+                await this.pollAXState();
+            }
+            catch { /* non-fatal */ }
+        }, AX_POLL_INTERVAL_MS);
+        // Start screenshot capture (slower — every 2.5s)
+        if (this.captureScreenshots) {
+            this.screenshotTimer = setInterval(async () => {
+                if (!this.recording)
+                    return;
+                try {
+                    await this.takeScreenshot();
+                }
+                catch { /* non-fatal */ }
+            }, SCREENSHOT_INTERVAL_MS);
+        }
+    }
+    /**
+     * Stop recording and generate a playbook.
+     */
+    async stop(name, description, platform) {
+        this.recording = false;
+        this.clearTimers();
+        // Take final screenshot
+        if (this.captureScreenshots) {
+            try {
+                await this.takeScreenshot();
+            }
+            catch { /* ignore */ }
+        }
+        this.log(`Recording stopped. ${this.events.length} events, ${this.screenshots.length} screenshots captured.`);
+        // Convert raw events + screenshots to playbook steps via AI
+        const steps = await this.eventsToSteps(this.events, this.screenshots, name, platform);
+        // Save as playbook
+        const id = `rec_${platform}_${Date.now()}`;
+        const playbook = {
+            id,
+            name,
+            description,
+            platform,
+            steps,
+            version: "1.0.0",
+            tags: [
+                platform,
+                ...name.toLowerCase().split(/\W+/).filter((w) => w.length >= 3),
+            ],
+            successCount: 0,
+            failCount: 0,
+            lastRun: new Date().toISOString(),
+        };
+        this.store.save(playbook);
+        this.log(`Playbook saved: ${id} (${steps.length} steps)`);
+        return playbook;
+    }
+    /**
+     * Cancel recording without saving.
+     */
+    cancel() {
+        this.recording = false;
+        this.clearTimers();
+        this.events = [];
+        this.screenshots = [];
+        this.log("Recording cancelled");
+    }
+    get isRecording() {
+        return this.recording;
+    }
+    get eventCount() {
+        return this.events.length;
+    }
+    getEvents() {
+        return [...this.events];
+    }
+    // ── AX State Polling (fast, event-driven feel) ──
+    async pollAXState() {
+        // 1. Check which app is active
+        try {
+            const apps = await this.runtime.appList(this.sessionId);
+            if (apps.ok) {
+                const active = apps.data.find((a) => a.isActive);
+                if (active && active.bundleId !== this.prevActiveApp) {
+                    if (this.prevActiveApp) {
+                        this.addEvent({
+                            type: "app_activated",
+                            details: {
+                                from: this.prevActiveApp,
+                                to: active.bundleId,
+                                appName: active.name,
+                            },
+                        });
+                    }
+                    this.prevActiveApp = active.bundleId;
+                }
+            }
+        }
+        catch { /* ignore */ }
+        // 2. Get accessibility tree — find focused element and text field values
+        try {
+            const tree = await this.runtime.elementTree({ sessionId: this.sessionId, maxDepth: 4 });
+            if (!tree.ok)
+                return;
+            // Detect focus change
+            const focused = findFocused(tree.data);
+            if (focused && focused !== this.prevFocused) {
+                this.addEvent({
+                    type: "focus_changed",
+                    details: {
+                        from: this.prevFocused,
+                        to: focused,
+                        element: describeFocused(tree.data),
+                    },
+                });
+                this.prevFocused = focused;
+            }
+            // Detect text field value changes (typing detection)
+            const currentFields = collectTextFields(tree.data);
+            for (const [fieldId, value] of currentFields) {
+                const prev = this.prevTextFields.get(fieldId);
+                if (prev !== undefined && prev !== value) {
+                    this.addEvent({
+                        type: "value_changed",
+                        details: {
+                            field: fieldId,
+                            from: prev.slice(-50),
+                            to: value.slice(-50),
+                            typed: value.slice(prev.length),
+                        },
+                    });
+                }
+            }
+            this.prevTextFields = currentFields;
+            // Detect window title change (navigation in browser)
+            const title = tree.data.title ?? "";
+            if (title && title !== this.prevWindowTitle) {
+                if (this.prevWindowTitle) {
+                    this.addEvent({
+                        type: "title_changed",
+                        details: { from: this.prevWindowTitle, to: title },
+                    });
+                }
+                this.prevWindowTitle = title;
+            }
+        }
+        catch { /* ignore */ }
+    }
+    // ── Screenshot Capture ──
+    async takeScreenshot() {
+        try {
+            const result = await this.runtime.screenshot({ sessionId: this.sessionId });
+            if (result.ok) {
+                const record = {
+                    path: result.data.path,
+                    timestamp: new Date().toISOString(),
+                    index: this.screenshots.length,
+                };
+                this.screenshots.push(record);
+            }
+        }
+        catch { /* non-fatal */ }
+    }
+    // ── State Capture ──
+    async captureState(label) {
+        // Capture initial app state
+        try {
+            const apps = await this.runtime.appList(this.sessionId);
+            if (apps.ok) {
+                const active = apps.data.find((a) => a.isActive);
+                if (active) {
+                    this.prevActiveApp = active.bundleId;
+                    this.addEvent({
+                        type: "app_activated",
+                        details: { to: active.bundleId, appName: active.name, label },
+                    });
+                }
+            }
+        }
+        catch { /* ignore */ }
+        // Capture initial tree state
+        try {
+            const tree = await this.runtime.elementTree({ sessionId: this.sessionId, maxDepth: 4 });
+            if (tree.ok) {
+                this.prevFocused = findFocused(tree.data);
+                this.prevWindowTitle = tree.data.title ?? "";
+                this.prevTextFields = collectTextFields(tree.data);
+            }
+        }
+        catch { /* ignore */ }
+        // Take initial screenshot
+        if (this.captureScreenshots) {
+            await this.takeScreenshot();
+        }
+    }
+    // ── Event Management ──
+    addEvent(partial) {
+        const event = {
+            ...partial,
+            timestamp: new Date().toISOString(),
+        };
+        this.events.push(event);
+        this.log(`Event: ${event.type} — ${JSON.stringify(event.details).slice(0, 120)}`);
+        if (this.onEvent)
+            this.onEvent(event);
+    }
+    clearTimers() {
+        if (this.axPollTimer) {
+            clearInterval(this.axPollTimer);
+            this.axPollTimer = null;
+        }
+        if (this.screenshotTimer) {
+            clearInterval(this.screenshotTimer);
+            this.screenshotTimer = null;
+        }
+    }
+    // ── AI Conversion ──
+    /**
+     * Convert raw events + screenshots into clean playbook steps.
+     * Sends first + last screenshot as images so AI can see what happened visually.
+     */
+    async eventsToSteps(events, screenshots, taskName, platform) {
+        if (events.length === 0)
+            return [];
+        // Build the content array — text + optional images
+        const content = [];
+        // Add text prompt
+        content.push({
+            type: "text",
+            text: `Convert these recorded user events into a clean, replayable automation playbook.
+
+Task: ${taskName}
+Platform: ${platform}
+
+Raw events recorded (in chronological order):
+${events.map((e, i) => `${i + 1}. [${e.timestamp}] ${e.type}: ${JSON.stringify(e.details)}`).join("\n")}
+
+${screenshots.length > 0 ? `\n${screenshots.length} screenshots were taken during recording. The first and last are attached below for visual context.\n` : ""}
+Convert these into a JSON array of playbook steps. Each step:
+{
+  "action": "navigate" | "press" | "type_into" | "key" | "key_combo" | "menu_click" | "scroll" | "wait" | "screenshot",
+  "target": "CSS selector, text label, or {\"selector\": \"...\"}",
+  "url": "for navigate",
+  "text": "for type_into",
+  "keys": ["for", "key or key_combo"],
+  "menuPath": ["for", "menu_click"],
+  "ms": 1000,
+  "description": "human-readable description of what this step does",
+  "verify": "optional CSS selector or text to verify success",
+  "optional": false
+}
+
+Rules:
+- Infer the user's INTENT from events, not just mirror them mechanically
+- focus_changed events usually mean a click — convert to "press" with the element label
+- value_changed events mean typing — convert to "type_into" with the field and text
+- title_changed often means navigation — add appropriate navigate or wait steps
+- app_activated means switching apps — use app_focus or app_launch
+- Use stable selectors: data-testid, aria-label, role+name over fragile CSS
+- Merge rapid consecutive events into single meaningful steps
+- Add wait steps (500-2000ms) after navigation/page loads
+- Add verify conditions for critical steps (modal opened, page loaded, etc.)
+- Skip noise (duplicate events, layout thrash, irrelevant focus changes)
+
+Respond with ONLY a valid JSON array, no markdown fences, no explanation.`,
+        });
+        // Attach first and last screenshots as images (if available)
+        if (screenshots.length > 0) {
+            const toAttach = [screenshots[0]];
+            if (screenshots.length > 1) {
+                toAttach.push(screenshots[screenshots.length - 1]);
+            }
+            for (const shot of toAttach) {
+                try {
+                    const imageData = fs.readFileSync(shot.path);
+                    const base64 = imageData.toString("base64");
+                    content.push({
+                        type: "image",
+                        source: {
+                            type: "base64",
+                            media_type: "image/png",
+                            data: base64,
+                        },
+                    });
+                    content.push({
+                        type: "text",
+                        text: `Screenshot ${shot.index + 1} taken at ${shot.timestamp}`,
+                    });
+                }
+                catch {
+                    // Skip unreadable screenshots
+                }
+            }
+        }
+        try {
+            const resp = await this.ai.messages.create({
+                model: this.model,
+                max_tokens: 4096,
+                messages: [{ role: "user", content }],
+            });
+            const text = resp.content[0]?.type === "text" ? resp.content[0].text : "";
+            const jsonMatch = text.match(/\[[\s\S]*\]/);
+            if (jsonMatch) {
+                const steps = JSON.parse(jsonMatch[0]);
+                this.log(`AI generated ${steps.length} playbook steps`);
+                return steps;
+            }
+        }
+        catch (err) {
+            this.log(`AI conversion failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        // Fallback without AI
+        return this.eventsToStepsFallback(events);
+    }
+    eventsToStepsFallback(events) {
+        const steps = [];
+        for (const event of events) {
+            switch (event.type) {
+                case "app_activated":
+                    if (event.details.label === "initial")
+                        break;
+                    steps.push({
+                        action: "wait",
+                        ms: 500,
+                        description: `Switched to ${event.details.appName ?? event.details.to}`,
+                    });
+                    break;
+                case "focus_changed": {
+                    const target = String(event.details.to ?? "");
+                    if (!target || target === this.prevFocused)
+                        break;
+                    steps.push({
+                        action: "press",
+                        target,
+                        description: `Click on ${event.details.element ?? target}`,
+                    });
+                    break;
+                }
+                case "value_changed": {
+                    const typed = String(event.details.typed ?? "");
+                    const field = String(event.details.field ?? "");
+                    if (typed) {
+                        steps.push({
+                            action: "type_into",
+                            target: field,
+                            text: typed,
+                            description: `Type "${typed.slice(0, 30)}" into ${field}`,
+                        });
+                    }
+                    break;
+                }
+                case "title_changed":
+                    steps.push({
+                        action: "wait",
+                        ms: 1500,
+                        description: `Page changed to: ${event.details.to}`,
+                    });
+                    break;
+                case "url_changed":
+                    steps.push({
+                        action: "navigate",
+                        url: event.details.to,
+                        description: `Navigate to ${event.details.to}`,
+                    });
+                    break;
+                case "menu_opened":
+                case "dialog_appeared":
+                    steps.push({
+                        action: "wait",
+                        ms: 1000,
+                        description: `${event.type}: ${JSON.stringify(event.details).slice(0, 50)}`,
+                    });
+                    break;
+            }
+        }
+        return steps;
+    }
+}
+// ── AX Tree Helpers ──
+/** Find the focused element and return a stable identifier. */
+function findFocused(node, depth = 0) {
+    if (depth > 6)
+        return "";
+    if (node.focused === true) {
+        const role = node.role?.replace("AX", "") ?? "";
+        const label = node.title ?? node.description ?? node.identifier ?? "";
+        return `${role}:${label}`;
+    }
+    if (node.children) {
+        for (const child of node.children) {
+            const found = findFocused(child, depth + 1);
+            if (found)
+                return found;
+        }
+    }
+    return "";
+}
+/** Get a human-readable description of the focused element + context. */
+function describeFocused(node, depth = 0) {
+    if (depth > 6)
+        return "";
+    if (node.focused === true) {
+        const parts = [node.role?.replace("AX", "")];
+        if (node.title)
+            parts.push(`"${node.title}"`);
+        if (node.description)
+            parts.push(`desc="${node.description}"`);
+        if (node.value)
+            parts.push(`val="${node.value.slice(0, 30)}"`);
+        if (node.position)
+            parts.push(`@${Math.round(node.position.x)},${Math.round(node.position.y)}`);
+        return parts.filter(Boolean).join(" ");
+    }
+    if (node.children) {
+        for (const child of node.children) {
+            const found = describeFocused(child, depth + 1);
+            if (found)
+                return found;
+        }
+    }
+    return "";
+}
+/** Collect all text field values from the tree for typing detection. */
+function collectTextFields(node, depth = 0) {
+    const fields = new Map();
+    if (depth > 5)
+        return fields;
+    const role = node.role?.replace("AX", "").toLowerCase() ?? "";
+    const isTextField = role === "textfield" || role === "textarea" || role === "combobox" || role === "searchfield";
+    if (isTextField && node.value !== undefined) {
+        const id = node.identifier ?? node.title ?? node.description ?? `field_${depth}`;
+        fields.set(id, node.value);
+    }
+    if (node.children) {
+        for (const child of node.children) {
+            for (const [k, v] of collectTextFields(child, depth + 1)) {
+                fields.set(k, v);
+            }
+        }
+    }
+    return fields;
+}