screenhand 0.5.0 → 0.5.3

package/dist/src/perception/manager.js

@@ -150,7 +150,9 @@ export class PerceptionManager extends EventEmitter {
                 windowId = (frontmost ?? matching[0])?.windowId;
             }
         }
-        catch { /* best-effort */ }
+        catch (e) {
+            process.stderr.write(`[perception-mgr] window ID lookup failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         const ctx = {
             bundleId: focusedApp.bundleId,
             appName: focusedApp.bundleId,
@@ -189,9 +191,44 @@ export class PerceptionManager extends EventEmitter {
     notifyToolCall() {
         this.coordinator?.notifyToolCall();
     }
+    // ── Focus/Crash Tracking ──
+    expectedBundleId = null;
+    lastUIChangeTs = Date.now();
+    stallCheckInterval = null;
+    /** Set the expected focused app — enables focus_lost and app_crash detection. */
+    setExpectedApp(bundleId) {
+        this.expectedBundleId = bundleId;
+        this.lastUIChangeTs = Date.now();
+    }
+    /** Start stall detection (fires stall_detected if no UI changes for stallMs). */
+    startStallDetection(stallMs = 30_000) {
+        this.stopStallDetection();
+        this.lastUIChangeTs = Date.now();
+        this.stallCheckInterval = setInterval(() => {
+            if (!this.expectedBundleId)
+                return;
+            const elapsed = Date.now() - this.lastUIChangeTs;
+            if (elapsed >= stallMs) {
+                this.emit("stall_detected", {
+                    bundleId: this.expectedBundleId,
+                    stallMs: elapsed,
+                });
+                // Reset so we don't fire every interval tick
+                this.lastUIChangeTs = Date.now();
+            }
+        }, 5_000);
+    }
+    stopStallDetection() {
+        if (this.stallCheckInterval) {
+            clearInterval(this.stallCheckInterval);
+            this.stallCheckInterval = null;
+        }
+    }
     handleReactiveEvent(event) {
         if (event.data?.type === "ax_events" && Array.isArray(event.data.events)) {
             for (const uiEvent of event.data.events) {
+                // Track any UI change for stall detection
+                this.lastUIChangeTs = Date.now();
                 if (uiEvent.type === "dialog_appeared") {
                     this.emit("dialog_detected", {
                         title: uiEvent.windowTitle ?? "",
@@ -205,6 +242,33 @@ export class PerceptionManager extends EventEmitter {
                         bundleId: uiEvent.bundleId,
                         pid: uiEvent.pid,
                     });
+                    // Focus loss detection: if we expected a specific app and a different one took focus
+                    if (this.expectedBundleId && uiEvent.bundleId !== this.expectedBundleId) {
+                        this.emit("focus_lost", {
+                            expectedBundleId: this.expectedBundleId,
+                            actualBundleId: uiEvent.bundleId,
+                            pid: uiEvent.pid,
+                        });
+                    }
+                }
+                // App crash detection: when expected app deactivates, check if PID is still alive.
+                // Only emit app_crash if the process is actually gone (not just lost focus).
+                if (uiEvent.type === "app_deactivated" &&
+                    uiEvent.bundleId &&
+                    uiEvent.bundleId === this.expectedBundleId &&
+                    uiEvent.pid) {
+                    try {
+                        // process.kill(pid, 0) throws if PID doesn't exist
+                        process.kill(uiEvent.pid, 0);
+                        // PID alive — just a focus switch, handled by focus_lost above
+                    }
+                    catch {
+                        // PID dead — app crashed
+                        this.emit("app_crash", {
+                            bundleId: uiEvent.bundleId,
+                            pid: uiEvent.pid,
+                        });
+                    }
                 }
             }
         }

package/dist/src/planner/executor.js

@@ -429,7 +429,9 @@ export class PlanExecutor {
                         }
                     }
                 }
-                catch { /* best-effort contract check */ }
+                catch (e) {
+                    process.stderr.write(`[planner] contract precondition check failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
         }
         // 4. Focus validation: for type_text, verify a text field is focused
@@ -785,7 +787,9 @@ export class PlanExecutor {
                         }
                     }
                 }
-                catch { /* best-effort */ }
+                catch (e) {
+                    process.stderr.write(`[planner] contract outcome inference failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
         }
         // Navigation or state-changing click → next step's target should be visible

package/dist/src/planner/plan-refiner.js

@@ -0,0 +1,213 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * PlanRefiner — Self-improving plans.
+ *
+ * After a successful plan execution, diffs the original plan vs what actually
+ * happened (steps skipped, fallbacks used, extra retries). Produces a refined
+ * plan stripped of dead steps. After 3 successful refinements of the same goal
+ * pattern, graduates the plan to a playbook.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { writeFileAtomicSync } from "../util/atomic-write.js";
+const GRADUATION_THRESHOLD = 3;
+const MAX_REFINED_PLANS = 200;
+export class PlanRefiner {
+    stateDir;
+    cache = new Map();
+    filePath;
+    constructor(stateDir) {
+        this.stateDir = stateDir;
+        this.filePath = path.join(stateDir, "refined-plans.json");
+        this.load();
+    }
+    load() {
+        try {
+            if (fs.existsSync(this.filePath)) {
+                const data = JSON.parse(fs.readFileSync(this.filePath, "utf-8"));
+                if (Array.isArray(data)) {
+                    for (const entry of data) {
+                        if (entry.goalKey && typeof entry.goalKey === "string") {
+                            this.cache.set(entry.goalKey, entry);
+                        }
+                    }
+                }
+            }
+        }
+        catch {
+            // Corrupted file — start fresh
+            this.cache.clear();
+        }
+    }
+    save() {
+        try {
+            // Evict oldest graduated plans if over limit
+            if (this.cache.size > MAX_REFINED_PLANS) {
+                const sorted = [...this.cache.entries()]
+                    .sort((a, b) => new Date(a[1].lastRefinedAt).getTime() - new Date(b[1].lastRefinedAt).getTime());
+                while (this.cache.size > MAX_REFINED_PLANS && sorted.length > 0) {
+                    const oldest = sorted.shift();
+                    this.cache.delete(oldest[0]);
+                }
+            }
+            const data = [...this.cache.values()];
+            writeFileAtomicSync(this.filePath, JSON.stringify(data, null, 2));
+        }
+        catch (err) {
+            process.stderr.write(`[plan-refiner] Save failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        }
+    }
+    normalizeGoal(description) {
+        return description.toLowerCase().trim().replace(/\s+/g, " ");
+    }
+    /**
+     * Refine a plan after successful execution.
+     * Extracts only the steps that actually completed (strips skipped/failed).
+     * Returns the refinement count and whether the plan was graduated.
+     */
+    refine(goal, result) {
+        if (!result.success)
+            return { refinementCount: 0, graduated: false };
+        const goalKey = this.normalizeGoal(goal.description);
+        // Extract completed steps from all subgoals
+        const completedSteps = [];
+        for (const sg of goal.subgoals) {
+            if (sg.plan) {
+                for (const step of sg.plan.steps) {
+                    if (step.status === "completed") {
+                        completedSteps.push({ tool: step.tool, params: step.params });
+                    }
+                }
+            }
+        }
+        if (completedSteps.length === 0)
+            return { refinementCount: 0, graduated: false };
+        const existing = this.cache.get(goalKey);
+        const refinementCount = (existing?.refinementCount ?? 0) + 1;
+        const prevAvg = existing?.avgDurationMs ?? result.durationMs;
+        const avgDurationMs = Math.round((prevAvg * (refinementCount - 1) + result.durationMs) / refinementCount);
+        const refined = {
+            goalKey,
+            goalDescription: goal.description,
+            steps: completedSteps,
+            refinementCount,
+            avgDurationMs,
+            lastRefinedAt: new Date().toISOString(),
+            graduated: existing?.graduated ?? false,
+        };
+        this.cache.set(goalKey, refined);
+        this.save();
+        return { refinementCount, graduated: refined.graduated };
+    }
+    /**
+     * Check if a goal should graduate to a playbook.
+     * Returns the playbook if ready, null otherwise.
+     */
+    checkGraduation(goalDescription, bundleId, appName) {
+        const goalKey = this.normalizeGoal(goalDescription);
+        const refined = this.cache.get(goalKey);
+        if (!refined)
+            return null;
+        if (refined.graduated)
+            return null;
+        if (refined.refinementCount < GRADUATION_THRESHOLD)
+            return null;
+        // Graduate: create a playbook from the refined steps
+        const playbookSteps = [];
+        for (let i = 0; i < refined.steps.length; i++) {
+            const s = refined.steps[i];
+            const mapped = mapToolToPlaybookStep(s.tool, s.params, i);
+            if (mapped)
+                playbookSteps.push(mapped);
+        }
+        const playbook = {
+            id: `graduated_${goalKey.replace(/[^a-z0-9]/g, "_").slice(0, 40)}_${Date.now().toString(36)}`,
+            name: refined.goalDescription,
+            description: `Auto-graduated from ${refined.refinementCount} successful plan executions (avg ${refined.avgDurationMs}ms)`,
+            platform: appName.toLowerCase(),
+            bundleId,
+            version: "1.0.0",
+            steps: playbookSteps,
+            tags: [appName.toLowerCase(), "auto-graduated", "refined"],
+            successCount: refined.refinementCount,
+            failCount: 0,
+            selectors: {},
+            errors: [],
+        };
+        // Mark as graduated
+        refined.graduated = true;
+        this.save();
+        return playbook;
+    }
+    /** Get all refined plans (for debugging/status). */
+    getAll() {
+        return [...this.cache.values()];
+    }
+    /** Look up a refined plan for a goal. */
+    lookup(goalDescription) {
+        return this.cache.get(this.normalizeGoal(goalDescription)) ?? null;
+    }
+}
+/** Map MCP tool name + params to a PlaybookStep. Returns null if unmappable. */
+function mapToolToPlaybookStep(tool, params, index) {
+    const TOOL_TO_ACTION = {
+        ui_press: "press",
+        click_text: "press",
+        click: "press",
+        click_with_fallback: "press",
+        type_text: "type_into",
+        type_with_fallback: "type_into",
+        key: "key",
+        browser_navigate: "navigate",
+        browser_click: "browser_click",
+        browser_type: "browser_type",
+        browser_human_click: "browser_human_click",
+        browser_js: "browser_js",
+        menu_click: "menu_click",
+        scroll: "scroll",
+        scroll_with_fallback: "scroll",
+        screenshot: "screenshot",
+        applescript: "applescript",
+    };
+    const action = TOOL_TO_ACTION[tool];
+    if (!action)
+        return null; // focus, launch, etc. are not playbook-able
+    const step = {
+        action,
+        description: `Step ${index + 1}: ${tool}`,
+    };
+    // Map params to playbook step fields
+    const target = params.title ?? params.text ?? params.selector ?? params.target;
+    if (typeof target === "string")
+        step.target = target;
+    if (typeof params.text === "string")
+        step.text = params.text;
+    if (typeof params.url === "string")
+        step.url = params.url;
+    if (typeof params.combo === "string")
+        step.keys = [params.combo];
+    if (typeof params.key === "string")
+        step.keys = [params.key];
+    if (Array.isArray(params.menuPath))
+        step.menuPath = params.menuPath;
+    if (typeof params.direction === "string")
+        step.direction = params.direction;
+    if (typeof params.amount === "number")
+        step.amount = params.amount;
+    return step;
+}

package/dist/src/playbook/engine.js

@@ -23,9 +23,15 @@ export class PlaybookEngine {
     appleScriptRunner;
     /** Enable observer-based popup checks before each step */
     popupCheckEnabled = false;
+    /** Optional callback invoked after each step with outcome — wires into learning/memory. */
+    onOutcome;
     constructor(runtime) {
         this.runtime = runtime;
     }
+    /** Set callback for step outcomes so PlaybookEngine feeds learning. */
+    setOutcomeCallback(cb) {
+        this.onOutcome = cb;
+    }
     /** Enable/disable pre-step popup detection via observer daemon */
     setPopupCheck(enabled) {
         this.popupCheckEnabled = enabled;
@@ -61,6 +67,9 @@ export class PlaybookEngine {
                 }
                 const result = await this.executeStep(sessionId, step, playbook.cdpPort);
                 stepsCompleted++;
+                // Report success to learning pipeline
+                if (this.onOutcome)
+                    this.onOutcome(step, true, null);
                 if (options.onStep) {
                     options.onStep(i, step, result);
                 }
@@ -68,6 +77,8 @@ export class PlaybookEngine {
                 if (step.verify) {
                     const verified = await this.verifyStep(sessionId, step);
                     if (!verified && !step.optional) {
+                        if (this.onOutcome)
+                            this.onOutcome(step, false, `Verification failed at step ${i}`);
                         return {
                             playbook: playbook.id,
                             success: false,
@@ -83,10 +94,14 @@ export class PlaybookEngine {
                 await sleep(STEP_DELAY_MS);
             }
             catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                // Report failure to learning pipeline
+                if (this.onOutcome)
+                    this.onOutcome(step, false, errMsg);
                 if (step.optional) {
                     stepsCompleted++;
                     if (options.onStep) {
-                        options.onStep(i, step, `Skipped (optional): ${err instanceof Error ? err.message : String(err)}`);
+                        options.onStep(i, step, `Skipped (optional): ${errMsg}`);
                     }
                     continue;
                 }
@@ -96,7 +111,7 @@ export class PlaybookEngine {
                     stepsCompleted,
                     totalSteps: playbook.steps.length,
                     failedAtStep: i,
-                    error: err instanceof Error ? err.message : String(err),
+                    error: errMsg,
                     durationMs: Date.now() - start,
                 };
             }
@@ -115,7 +130,7 @@ export class PlaybookEngine {
      */
     /** Tools that the PlaybookEngine refuses to execute (defense in depth). */
     static BLOCKED_ACTIONS = new Set([
-        "applescript", "browser_stealth",
+        "browser_stealth",
         "memory_save", "memory_clear", "memory_snapshot",
         "supervisor_start", "supervisor_stop", "supervisor_install", "supervisor_uninstall",
         "job_create", "worker_start",

package/dist/src/playbook/recorder.js

@@ -98,7 +98,9 @@ export class PlaybookRecorder {
             try {
                 await this.pollAXState();
             }
-            catch { /* non-fatal */ }
+            catch (e) {
+                process.stderr.write(`[recorder] AX poll failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }, AX_POLL_INTERVAL_MS);
         // Start screenshot capture (slower — every 2.5s)
         if (this.captureScreenshots) {
@@ -108,7 +110,9 @@ export class PlaybookRecorder {
                 try {
                     await this.takeScreenshot();
                 }
-                catch { /* non-fatal */ }
+                catch (e) {
+                    process.stderr.write(`[recorder] screenshot capture failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }, SCREENSHOT_INTERVAL_MS);
         }
     }
@@ -123,7 +127,9 @@ export class PlaybookRecorder {
             try {
                 await this.takeScreenshot();
             }
-            catch { /* ignore */ }
+            catch (e) {
+                process.stderr.write(`[recorder] final screenshot failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }
         this.log(`Recording stopped. ${this.events.length} events, ${this.screenshots.length} screenshots captured.`);
         // Convert raw events + screenshots to playbook steps via AI
@@ -190,7 +196,9 @@ export class PlaybookRecorder {
                 }
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[recorder] app list poll failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // 2. Get accessibility tree — find focused element and text field values
         try {
             const tree = await this.runtime.elementTree({ sessionId: this.sessionId, maxDepth: 4 });
@@ -238,7 +246,9 @@ export class PlaybookRecorder {
                 this.prevWindowTitle = title;
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[recorder] window title poll failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     // ── Screenshot Capture ──
     async takeScreenshot() {
@@ -253,7 +263,9 @@ export class PlaybookRecorder {
                 this.screenshots.push(record);
             }
         }
-        catch { /* non-fatal */ }
+        catch (e) {
+            process.stderr.write(`[recorder] takeScreenshot failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     // ── State Capture ──
     async captureState(label) {
@@ -271,7 +283,9 @@ export class PlaybookRecorder {
                 }
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[recorder] initial app state capture failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // Capture initial tree state
         try {
             const tree = await this.runtime.elementTree({ sessionId: this.sessionId, maxDepth: 4 });
@@ -281,7 +295,9 @@ export class PlaybookRecorder {
                 this.prevTextFields = collectTextFields(tree.data);
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[recorder] initial tree state capture failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // Take initial screenshot
         if (this.captureScreenshots) {
             await this.takeScreenshot();

package/dist/src/playbook/runner.js

@@ -97,7 +97,9 @@ export class PlaybookRunner {
             if (shot.ok)
                 screenshotInfo = `Screenshot saved to: ${shot.data.path}`;
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[playbook-runner] screenshot for recovery failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // Get current page state
         let pageState = "";
         try {
@@ -106,7 +108,9 @@ export class PlaybookRunner {
                 pageState = JSON.stringify(tree.data).slice(0, 3000);
             }
         }
-        catch { /* ignore */ }
+        catch (e) {
+            process.stderr.write(`[playbook-runner] elementTree for recovery failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         // Build rich context from playbook metadata
         const playbookContext = buildPlaybookContext(playbook);
         const prompt = `A playbook automation failed. Help me recover.
@@ -221,7 +225,9 @@ Or if unrecoverable, respond with: { "unrecoverable": true, "reason": "..." }`;
                 if (tree.ok)
                     pageState = JSON.stringify(tree.data).slice(0, 4000);
             }
-            catch { /* ignore */ }
+            catch (e) {
+                process.stderr.write(`[playbook-runner] elementTree in AI loop failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
             const prompt = `Task: ${task}
 
 Steps taken so far:

package/dist/src/playbook/store.js

@@ -63,6 +63,10 @@ export class PlaybookStore {
             }
         }
     }
+    /** Reload all playbooks from disk (call after ingestion writes new data). */
+    reload() {
+        this.load();
+    }
     /** Get all loaded playbooks. */
     getAll() {
         return [...this.playbooks.values()];
@@ -274,6 +278,10 @@ export class PlaybookStore {
         if (raw.flows && typeof raw.flows === "object") {
             return this.convertLegacy(raw, filename);
         }
+        // Reference format: has selectors but no steps/flows (e.g. *-explore.json from platform_explore)
+        if (raw.selectors && typeof raw.selectors === "object") {
+            return this.convertLegacy(raw, filename);
+        }
         return null;
     }
     /**

package/dist/src/recovery/engine.js

@@ -109,7 +109,9 @@ export class RecoveryEngine {
                 if (undoStrategy)
                     candidates.push(undoStrategy);
             }
-            catch { /* best-effort */ }
+            catch (e) {
+                process.stderr.write(`[recovery] buildUndoStrategy failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }
         // Reference strategies second (app-specific)
         if (blocker.bundleId) {
@@ -355,7 +357,9 @@ export class RecoveryEngine {
                         break;
                     }
                 }
-                catch { /* skip malformed */ }
+                catch (e) {
+                    process.stderr.write(`[recovery] malformed reference file: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
         }
         catch { /* dir doesn't exist */ }
@@ -366,7 +370,9 @@ export class RecoveryEngine {
         try {
             this.memory.recordError(`recovery:${event.strategyId}`, event.error ?? "", event.success ? event.strategyLabel : null, event.blocker.bundleId ?? undefined);
         }
-        catch { /* best-effort */ }
+        catch (e) {
+            process.stderr.write(`[recovery] recordEvent failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
 }
 function sleep(ms) {