screenhand 0.5.0 → 0.5.2

package/dist/src/planner/plan-refiner.js

@@ -0,0 +1,213 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * PlanRefiner — Self-improving plans.
+ *
+ * After a successful plan execution, diffs the original plan vs what actually
+ * happened (steps skipped, fallbacks used, extra retries). Produces a refined
+ * plan stripped of dead steps. After 3 successful refinements of the same goal
+ * pattern, graduates the plan to a playbook.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { writeFileAtomicSync } from "../util/atomic-write.js";
+const GRADUATION_THRESHOLD = 3;
+const MAX_REFINED_PLANS = 200;
+export class PlanRefiner {
+    stateDir;
+    cache = new Map();
+    filePath;
+    constructor(stateDir) {
+        this.stateDir = stateDir;
+        this.filePath = path.join(stateDir, "refined-plans.json");
+        this.load();
+    }
+    load() {
+        try {
+            if (fs.existsSync(this.filePath)) {
+                const data = JSON.parse(fs.readFileSync(this.filePath, "utf-8"));
+                if (Array.isArray(data)) {
+                    for (const entry of data) {
+                        if (entry.goalKey && typeof entry.goalKey === "string") {
+                            this.cache.set(entry.goalKey, entry);
+                        }
+                    }
+                }
+            }
+        }
+        catch {
+            // Corrupted file — start fresh
+            this.cache.clear();
+        }
+    }
+    save() {
+        try {
+            // Evict oldest graduated plans if over limit
+            if (this.cache.size > MAX_REFINED_PLANS) {
+                const sorted = [...this.cache.entries()]
+                    .sort((a, b) => new Date(a[1].lastRefinedAt).getTime() - new Date(b[1].lastRefinedAt).getTime());
+                while (this.cache.size > MAX_REFINED_PLANS && sorted.length > 0) {
+                    const oldest = sorted.shift();
+                    this.cache.delete(oldest[0]);
+                }
+            }
+            const data = [...this.cache.values()];
+            writeFileAtomicSync(this.filePath, JSON.stringify(data, null, 2));
+        }
+        catch (err) {
+            process.stderr.write(`[plan-refiner] Save failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        }
+    }
+    normalizeGoal(description) {
+        return description.toLowerCase().trim().replace(/\s+/g, " ");
+    }
+    /**
+     * Refine a plan after successful execution.
+     * Extracts only the steps that actually completed (strips skipped/failed).
+     * Returns the refinement count and whether the plan was graduated.
+     */
+    refine(goal, result) {
+        if (!result.success)
+            return { refinementCount: 0, graduated: false };
+        const goalKey = this.normalizeGoal(goal.description);
+        // Extract completed steps from all subgoals
+        const completedSteps = [];
+        for (const sg of goal.subgoals) {
+            if (sg.plan) {
+                for (const step of sg.plan.steps) {
+                    if (step.status === "completed") {
+                        completedSteps.push({ tool: step.tool, params: step.params });
+                    }
+                }
+            }
+        }
+        if (completedSteps.length === 0)
+            return { refinementCount: 0, graduated: false };
+        const existing = this.cache.get(goalKey);
+        const refinementCount = (existing?.refinementCount ?? 0) + 1;
+        const prevAvg = existing?.avgDurationMs ?? result.durationMs;
+        const avgDurationMs = Math.round((prevAvg * (refinementCount - 1) + result.durationMs) / refinementCount);
+        const refined = {
+            goalKey,
+            goalDescription: goal.description,
+            steps: completedSteps,
+            refinementCount,
+            avgDurationMs,
+            lastRefinedAt: new Date().toISOString(),
+            graduated: existing?.graduated ?? false,
+        };
+        this.cache.set(goalKey, refined);
+        this.save();
+        return { refinementCount, graduated: refined.graduated };
+    }
+    /**
+     * Check if a goal should graduate to a playbook.
+     * Returns the playbook if ready, null otherwise.
+     */
+    checkGraduation(goalDescription, bundleId, appName) {
+        const goalKey = this.normalizeGoal(goalDescription);
+        const refined = this.cache.get(goalKey);
+        if (!refined)
+            return null;
+        if (refined.graduated)
+            return null;
+        if (refined.refinementCount < GRADUATION_THRESHOLD)
+            return null;
+        // Graduate: create a playbook from the refined steps
+        const playbookSteps = [];
+        for (let i = 0; i < refined.steps.length; i++) {
+            const s = refined.steps[i];
+            const mapped = mapToolToPlaybookStep(s.tool, s.params, i);
+            if (mapped)
+                playbookSteps.push(mapped);
+        }
+        const playbook = {
+            id: `graduated_${goalKey.replace(/[^a-z0-9]/g, "_").slice(0, 40)}_${Date.now().toString(36)}`,
+            name: refined.goalDescription,
+            description: `Auto-graduated from ${refined.refinementCount} successful plan executions (avg ${refined.avgDurationMs}ms)`,
+            platform: appName.toLowerCase(),
+            bundleId,
+            version: "1.0.0",
+            steps: playbookSteps,
+            tags: [appName.toLowerCase(), "auto-graduated", "refined"],
+            successCount: refined.refinementCount,
+            failCount: 0,
+            selectors: {},
+            errors: [],
+        };
+        // Mark as graduated
+        refined.graduated = true;
+        this.save();
+        return playbook;
+    }
+    /** Get all refined plans (for debugging/status). */
+    getAll() {
+        return [...this.cache.values()];
+    }
+    /** Look up a refined plan for a goal. */
+    lookup(goalDescription) {
+        return this.cache.get(this.normalizeGoal(goalDescription)) ?? null;
+    }
+}
+/** Map MCP tool name + params to a PlaybookStep. Returns null if unmappable. */
+function mapToolToPlaybookStep(tool, params, index) {
+    const TOOL_TO_ACTION = {
+        ui_press: "press",
+        click_text: "press",
+        click: "press",
+        click_with_fallback: "press",
+        type_text: "type_into",
+        type_with_fallback: "type_into",
+        key: "key",
+        browser_navigate: "navigate",
+        browser_click: "browser_click",
+        browser_type: "browser_type",
+        browser_human_click: "browser_human_click",
+        browser_js: "browser_js",
+        menu_click: "menu_click",
+        scroll: "scroll",
+        scroll_with_fallback: "scroll",
+        screenshot: "screenshot",
+        applescript: "applescript",
+    };
+    const action = TOOL_TO_ACTION[tool];
+    if (!action)
+        return null; // focus, launch, etc. are not playbook-able
+    const step = {
+        action,
+        description: `Step ${index + 1}: ${tool}`,
+    };
+    // Map params to playbook step fields
+    const target = params.title ?? params.text ?? params.selector ?? params.target;
+    if (typeof target === "string")
+        step.target = target;
+    if (typeof params.text === "string")
+        step.text = params.text;
+    if (typeof params.url === "string")
+        step.url = params.url;
+    if (typeof params.combo === "string")
+        step.keys = [params.combo];
+    if (typeof params.key === "string")
+        step.keys = [params.key];
+    if (Array.isArray(params.menuPath))
+        step.menuPath = params.menuPath;
+    if (typeof params.direction === "string")
+        step.direction = params.direction;
+    if (typeof params.amount === "number")
+        step.amount = params.amount;
+    return step;
+}

package/dist/src/playbook/engine.js

@@ -23,9 +23,15 @@ export class PlaybookEngine {
     appleScriptRunner;
     /** Enable observer-based popup checks before each step */
     popupCheckEnabled = false;
+    /** Optional callback invoked after each step with outcome — wires into learning/memory. */
+    onOutcome;
     constructor(runtime) {
         this.runtime = runtime;
     }
+    /** Set callback for step outcomes so PlaybookEngine feeds learning. */
+    setOutcomeCallback(cb) {
+        this.onOutcome = cb;
+    }
     /** Enable/disable pre-step popup detection via observer daemon */
     setPopupCheck(enabled) {
         this.popupCheckEnabled = enabled;
@@ -61,6 +67,9 @@ export class PlaybookEngine {
                 }
                 const result = await this.executeStep(sessionId, step, playbook.cdpPort);
                 stepsCompleted++;
+                // Report success to learning pipeline
+                if (this.onOutcome)
+                    this.onOutcome(step, true, null);
                 if (options.onStep) {
                     options.onStep(i, step, result);
                 }
@@ -68,6 +77,8 @@ export class PlaybookEngine {
                 if (step.verify) {
                     const verified = await this.verifyStep(sessionId, step);
                     if (!verified && !step.optional) {
+                        if (this.onOutcome)
+                            this.onOutcome(step, false, `Verification failed at step ${i}`);
                         return {
                             playbook: playbook.id,
                             success: false,
@@ -83,10 +94,14 @@ export class PlaybookEngine {
                 await sleep(STEP_DELAY_MS);
             }
             catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                // Report failure to learning pipeline
+                if (this.onOutcome)
+                    this.onOutcome(step, false, errMsg);
                 if (step.optional) {
                     stepsCompleted++;
                     if (options.onStep) {
-                        options.onStep(i, step, `Skipped (optional): ${err instanceof Error ? err.message : String(err)}`);
+                        options.onStep(i, step, `Skipped (optional): ${errMsg}`);
                     }
                     continue;
                 }
@@ -96,7 +111,7 @@ export class PlaybookEngine {
                     stepsCompleted,
                     totalSteps: playbook.steps.length,
                     failedAtStep: i,
-                    error: err instanceof Error ? err.message : String(err),
+                    error: errMsg,
                     durationMs: Date.now() - start,
                 };
             }
@@ -115,7 +130,7 @@ export class PlaybookEngine {
      */
     /** Tools that the PlaybookEngine refuses to execute (defense in depth). */
     static BLOCKED_ACTIONS = new Set([
-        "applescript", "browser_stealth",
+        "browser_stealth",
         "memory_save", "memory_clear", "memory_snapshot",
         "supervisor_start", "supervisor_stop", "supervisor_install", "supervisor_uninstall",
         "job_create", "worker_start",

package/dist/src/playbook/recorder.js

@@ -98,7 +98,9 @@ export class PlaybookRecorder {
             try {
                 await this.pollAXState();
             }
-            catch { /* non-fatal */ }
+            catch (e) {
+                process.stderr.write(`[recorder] AX poll failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }, AX_POLL_INTERVAL_MS);
         // Start screenshot capture (slower — every 2.5s)
         if (this.captureScreenshots) {
@@ -108,7 +110,9 @@ export class PlaybookRecorder {
                 try {
                     await this.takeScreenshot();
                 }
-                catch { /* non-fatal */ }
+                catch (e) {
+                    process.stderr.write(`[recorder] screenshot capture failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }, SCREENSHOT_INTERVAL_MS);
         }
     }
@@ -123,7 +127,9 @@ export class PlaybookRecorder {
             try {
                 await this.takeScreenshot();
             }
-            catch { /* ignore */ }
+            catch (e) {
+                process.stderr.write(`[recorder] final screenshot failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }
         this.log(`Recording stopped. ${this.events.length} events, ${this.screenshots.length} screenshots captured.`);
         // Convert raw events + screenshots to playbook steps via AI
@@ -190,7 +196,9 @@ export class PlaybookRecorder {
                 }
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[recorder] app list poll failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // 2. Get accessibility tree — find focused element and text field values
         try {
             const tree = await this.runtime.elementTree({ sessionId: this.sessionId, maxDepth: 4 });
@@ -238,7 +246,9 @@ export class PlaybookRecorder {
                 this.prevWindowTitle = title;
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[recorder] window title poll failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     // ── Screenshot Capture ──
     async takeScreenshot() {
@@ -253,7 +263,9 @@ export class PlaybookRecorder {
                 this.screenshots.push(record);
             }
         }
-        catch { /* non-fatal */ }
+        catch (e) {
+            process.stderr.write(`[recorder] takeScreenshot failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     // ── State Capture ──
     async captureState(label) {
@@ -271,7 +283,9 @@ export class PlaybookRecorder {
                 }
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[recorder] initial app state capture failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // Capture initial tree state
         try {
             const tree = await this.runtime.elementTree({ sessionId: this.sessionId, maxDepth: 4 });
@@ -281,7 +295,9 @@ export class PlaybookRecorder {
                 this.prevTextFields = collectTextFields(tree.data);
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[recorder] initial tree state capture failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // Take initial screenshot
         if (this.captureScreenshots) {
             await this.takeScreenshot();

package/dist/src/playbook/runner.js

@@ -97,7 +97,9 @@ export class PlaybookRunner {
             if (shot.ok)
                 screenshotInfo = `Screenshot saved to: ${shot.data.path}`;
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[playbook-runner] screenshot for recovery failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // Get current page state
         let pageState = "";
         try {
@@ -106,7 +108,9 @@ export class PlaybookRunner {
                 pageState = JSON.stringify(tree.data).slice(0, 3000);
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[playbook-runner] elementTree for recovery failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // Build rich context from playbook metadata
         const playbookContext = buildPlaybookContext(playbook);
         const prompt = `A playbook automation failed. Help me recover.
@@ -221,7 +225,9 @@ Or if unrecoverable, respond with: { "unrecoverable": true, "reason": "..." }`;
                 if (tree.ok)
                     pageState = JSON.stringify(tree.data).slice(0, 4000);
             }
-            catch { /* ignore */ }
+            catch (e) {
+                process.stderr.write(`[playbook-runner] elementTree in AI loop failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
             const prompt = `Task: ${task}
 
 Steps taken so far:

package/dist/src/playbook/store.js

@@ -63,6 +63,10 @@ export class PlaybookStore {
             }
         }
     }
+    /** Reload all playbooks from disk (call after ingestion writes new data). */
+    reload() {
+        this.load();
+    }
     /** Get all loaded playbooks. */
     getAll() {
         return [...this.playbooks.values()];
@@ -274,6 +278,10 @@ export class PlaybookStore {
         if (raw.flows && typeof raw.flows === "object") {
             return this.convertLegacy(raw, filename);
         }
+        // Reference format: has selectors but no steps/flows (e.g. *-explore.json from platform_explore)
+        if (raw.selectors && typeof raw.selectors === "object") {
+            return this.convertLegacy(raw, filename);
+        }
         return null;
     }
     /**

package/dist/src/recovery/engine.js

@@ -109,7 +109,9 @@ export class RecoveryEngine {
                 if (undoStrategy)
                     candidates.push(undoStrategy);
             }
-            catch { /* best-effort */ }
+            catch (e) {
+                process.stderr.write(`[recovery] buildUndoStrategy failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }
         // Reference strategies second (app-specific)
         if (blocker.bundleId) {
@@ -355,7 +357,9 @@ export class RecoveryEngine {
                         break;
                     }
                 }
-                catch { /* skip malformed */ }
+                catch (e) {
+                    process.stderr.write(`[recovery] malformed reference file: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
         }
         catch { /* dir doesn't exist */ }
@@ -366,7 +370,9 @@ export class RecoveryEngine {
         try {
             this.memory.recordError(`recovery:${event.strategyId}`, event.error ?? "", event.success ? event.strategyLabel : null, event.blocker.bundleId ?? undefined);
         }
-        catch { /* best-effort */ }
+        catch (e) {
+            process.stderr.write(`[recovery] recordEvent failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
 }
 function sleep(ms) {

package/dist/src/state/app-map.js

@@ -339,7 +339,9 @@ export class AppMap {
             fs.mkdirSync(this.config.mapsDir, { recursive: true });
             writeFileAtomicSync(this.ladderFilePath(bundleId), JSON.stringify(data, null, 2));
         }
-        catch { /* non-critical */ }
+        catch (e) {
+            process.stderr.write(`[app-map] saveGeneratedLadder failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     // ── Create ────────────────────────────────────────────────────────
     createEmpty(bundleId, appName, version = "unknown") {
@@ -1591,7 +1593,9 @@ export class AppMap {
                 writeFileAtomicSync(this.filePath(data.app), JSON.stringify(data, null, 2) + "\n");
                 this.dirty.delete(data.app); // Only remove the one we just wrote
             }
-            catch { /* non-fatal — will be picked up by next debounced save */ }
+            catch (e) {
+                process.stderr.write(`[app-map] urgent mastery save failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }
     }
     refreshMastery(bundleId) {

package/dist/src/state/state-watcher.js

@@ -0,0 +1,144 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export class StateWatcher {
+    worldModel;
+    execute;
+    rules = new Map();
+    interval = null;
+    pollMs;
+    constructor(worldModel, execute, pollMs = 2_000) {
+        this.worldModel = worldModel;
+        this.execute = execute;
+        this.pollMs = pollMs;
+    }
+    static MAX_RULES = 50;
+    /**
+     * Register a watch rule. Returns the rule ID for later removal.
+     */
+    register(rule) {
+        if (this.rules.size >= StateWatcher.MAX_RULES && !this.rules.has(rule.id)) {
+            throw new Error(`Maximum watch rules (${StateWatcher.MAX_RULES}) reached. Remove existing rules first.`);
+        }
+        this.rules.set(rule.id, {
+            rule,
+            fireCount: 0,
+            lastFiredAt: 0,
+        });
+        return rule.id;
+    }
+    /** Register a convenience rule: fire action when a control with matching title appears. */
+    watchForElement(id, elementTitle, action, bundleId) {
+        const titleLower = elementTitle.toLowerCase();
+        return this.register({
+            id,
+            description: `Watch for element "${elementTitle}"`,
+            condition: (state) => {
+                for (const win of state.windows.values()) {
+                    for (const ctrl of win.controls.values()) {
+                        if (ctrl.label?.value?.toLowerCase().includes(titleLower)) {
+                            return true;
+                        }
+                    }
+                }
+                return false;
+            },
+            action,
+            maxFires: 1,
+            cooldownMs: 10_000,
+            ...(bundleId ? { bundleId } : {}),
+        });
+    }
+    /** Register: fire action when a dialog appears with matching text. */
+    watchForDialog(id, titlePattern, action) {
+        return this.register({
+            id,
+            description: `Watch for dialog matching ${titlePattern}`,
+            condition: (state) => state.activeDialogs.some((d) => titlePattern.test(d.title ?? "")),
+            action,
+            maxFires: 0, // unlimited — dialogs can recur
+            cooldownMs: 5_000,
+        });
+    }
+    /** Remove a watch rule by ID. */
+    unregister(id) {
+        return this.rules.delete(id);
+    }
+    /** Remove all rules. */
+    clear() {
+        this.rules.clear();
+    }
+    /** Get all registered rules. */
+    getRules() {
+        return [...this.rules.values()].map((rs) => ({
+            id: rs.rule.id,
+            description: rs.rule.description,
+            fireCount: rs.fireCount,
+        }));
+    }
+    /** Start the polling loop. */
+    start() {
+        if (this.interval)
+            return;
+        this.interval = setInterval(() => {
+            void this.tick();
+        }, this.pollMs);
+    }
+    /** Stop the polling loop. */
+    stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+    }
+    get isRunning() {
+        return this.interval !== null;
+    }
+    async tick() {
+        const state = this.worldModel.getState();
+        const now = Date.now();
+        const focusedBundleId = state.focusedApp?.bundleId;
+        for (const [id, rs] of this.rules) {
+            // Max fires check
+            if (rs.rule.maxFires > 0 && rs.fireCount >= rs.rule.maxFires)
+                continue;
+            // Cooldown check
+            if (now - rs.lastFiredAt < rs.rule.cooldownMs)
+                continue;
+            // BundleId filter
+            if (rs.rule.bundleId && rs.rule.bundleId !== focusedBundleId)
+                continue;
+            try {
+                if (rs.rule.condition(state)) {
+                    rs.fireCount++;
+                    rs.lastFiredAt = now;
+                    process.stderr.write(`[state-watcher] Rule "${id}" fired (${rs.fireCount}x): ${rs.rule.description}\n`);
+                    // Fire and forget — don't block the poll loop
+                    this.execute(rs.rule.action.tool, rs.rule.action.params).catch((err) => {
+                        process.stderr.write(`[state-watcher] Rule "${id}" action failed: ${err instanceof Error ? err.message : String(err)}\n`);
+                    });
+                    // Remove exhausted rules
+                    if (rs.rule.maxFires > 0 && rs.fireCount >= rs.rule.maxFires) {
+                        this.rules.delete(id);
+                    }
+                }
+            }
+            catch (err) {
+                process.stderr.write(`[state-watcher] Rule "${id}" condition threw: ${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        }
+    }
+}

package/dist/src/supervisor/supervisor.js

@@ -297,7 +297,7 @@ export class SessionSupervisor {
             this.log(`Poll error (${this.consecutiveErrors}/${this.config.maxConsecutiveErrors}): ${err instanceof Error ? err.message : String(err)}`);
             if (this.consecutiveErrors >= this.config.maxConsecutiveErrors) {
                 this.log("Max consecutive errors reached — stopping supervisor");
-                this.stop().catch(() => { });
+                this.stop().catch((e) => { process.stderr.write(`[supervisor] stop after max errors failed: ${e instanceof Error ? e.message : String(e)}\n`); });
             }
         }
     }