screenhand 0.4.9 → 0.5.2

package/dist/mcp-desktop.js

@@ -52,8 +52,10 @@ import { PlaybookStore } from "./src/playbook/store.js";
 import { ContextTracker } from "./src/context-tracker.js";
 import { McpPlaybookRecorder } from "./src/playbook/mcp-recorder.js";
 import { WorldModel } from "./src/state/index.js";
+import { StateWatcher } from "./src/state/state-watcher.js";
 import { PerceptionManager } from "./src/perception/index.js";
 import { Planner, PlanExecutor, GoalStore, ToolRegistry } from "./src/planner/index.js";
+import { PlanRefiner } from "./src/planner/plan-refiner.js";
 import { RecoveryEngine } from "./src/recovery/index.js";
 import { LearningEngine, LocatorPolicy } from "./src/learning/index.js";
 import { discoverWebElements, testWebElement, compileReference, saveExploreResult, discoverNativeElements } from "./src/platform/explorer.js";
@@ -531,17 +533,24 @@ const leaseManager = new LeaseManager(LOCK_DIR);
 // Playbooks dir holds only executable step sequences for job_create
 // Resolution order: local dev paths → npm dist paths → ~/.screenhand/ user paths
 function resolveDataDir(name) {
-    // 1. Local dev path (when running from source)
+    const hasJson = (dir) => fs.existsSync(dir) && fs.readdirSync(dir).some(f => f.endsWith(".json"));
+    // 1. Local dev path (when running from source: references/, playbooks/)
     const local = path.resolve(__dirname, name);
-    if (fs.existsSync(local) && fs.readdirSync(local).some(f => f.endsWith(".json"))) {
+    if (hasJson(local))
         return local;
-    }
-    // 2. npm dist path (when installed via npx/npm)
+    // 2. npm dist path — same level (dist-references/ next to dist/)
     const dist = path.resolve(__dirname, `dist-${name}`);
-    if (fs.existsSync(dist) && fs.readdirSync(dist).some(f => f.endsWith(".json"))) {
+    if (hasJson(dist))
         return dist;
-    }
-    // 3. User home path (always available for user-generated content)
+    // 3. npm dist path — parent level (when __dirname is dist/, check ../dist-references/)
+    const parentDist = path.resolve(__dirname, "..", `dist-${name}`);
+    if (hasJson(parentDist))
+        return parentDist;
+    // 4. Parent level plain name (../references/)
+    const parentLocal = path.resolve(__dirname, "..", name);
+    if (hasJson(parentLocal))
+        return parentLocal;
+    // 5. User home path (always available for user-generated content)
     const userDir = path.join(os.homedir(), ".screenhand", name);
     if (!fs.existsSync(userDir)) {
         fs.mkdirSync(userDir, { recursive: true });
@@ -563,6 +572,9 @@ const seedAppMapsDir = (() => {
     const dist = path.resolve(__dirname, "dist-app-maps");
     if (fs.existsSync(dist))
         return dist;
+    const parentDist = path.resolve(__dirname, "..", "dist-app-maps");
+    if (fs.existsSync(parentDist))
+        return parentDist;
     const local = path.resolve(__dirname, "seed-app-maps");
     if (fs.existsSync(local))
         return local;
@@ -599,6 +611,7 @@ catch { /* dir may not exist */ }
 const planner = new Planner(_executablePlaybookStore, memory, contextTracker, worldModel, learningEngine);
 const goalStore = new GoalStore(path.join(os.homedir(), ".screenhand", "planner"));
 goalStore.init();
+const planRefiner = new PlanRefiner(path.join(os.homedir(), ".screenhand", "planner"));
 const toolRegistry = new ToolRegistry();
 const recoveryEngine = new RecoveryEngine(worldModel, toolRegistry.toExecutor(), memory);
 recoveryEngine.setLearningEngine(learningEngine);
@@ -606,6 +619,7 @@ recoveryEngine.setAppMap(appMap);
 planner.setToolRegistry(toolRegistry);
 planner.setAppMap(appMap);
 perceptionManager.setLearningEngine(learningEngine);
+const stateWatcher = new StateWatcher(worldModel, toolRegistry.toExecutor(), 2_000);
 // ── Reactive event loop: wire perception events to automatic responses ──
 // These fire at perception speed (100-300ms), not LLM speed (~2-3s).
 perceptionManager.on("dialog_detected", (event) => {
@@ -639,6 +653,28 @@ perceptionManager.on("app_switched", (event) => {
     // Log for observability
     console.error(`[reactive] App switched to ${event.bundleId} (pid=${event.pid})`);
 });
+// ── Perception-triggered recovery: focus loss, app crash, stall ──
+perceptionManager.on("focus_lost", (event) => {
+    console.error(`[reactive] Focus lost: expected ${event.expectedBundleId}, got ${event.actualBundleId} — auto-refocusing`);
+    // Auto-refocus the expected app
+    toolRegistry.toExecutor()("focus", { bundleId: event.expectedBundleId }).catch((err) => {
+        console.error(`[reactive] Auto-refocus failed: ${err instanceof Error ? err.message : err}`);
+    });
+});
+perceptionManager.on("app_crash", (event) => {
+    console.error(`[reactive] App crash detected: ${event.bundleId} (pid=${event.pid}) — auto-relaunching`);
+    // Auto-relaunch the crashed app
+    toolRegistry.toExecutor()("launch", { bundleId: event.bundleId }).catch((err) => {
+        console.error(`[reactive] Auto-relaunch failed: ${err instanceof Error ? err.message : err}`);
+    });
+});
+perceptionManager.on("stall_detected", (event) => {
+    console.error(`[reactive] UI stall detected: ${event.bundleId} — no changes for ${(event.stallMs / 1000).toFixed(0)}s — taking screenshot for diagnosis`);
+    // Take a screenshot so the next LLM call can see what's on screen
+    toolRegistry.toExecutor()("screenshot", {}).catch((err) => {
+        console.error(`[reactive] Stall screenshot failed: ${err instanceof Error ? err.message : err}`);
+    });
+});
 const mcpRecorder = new McpPlaybookRecorder(playbooksDir);
 const referenceMerger = new ReferenceMerger(referencesDir);
 const communityPublisher = new PlaybookPublisher();
@@ -671,6 +707,7 @@ const MEMORY_TOOLS = new Set([
 ]);
 // Track the strategy we're currently following (for feedback loop)
 let activeStrategyFingerprint = null;
+let autoExecutionInProgress = false; // guard against concurrent auto-execution
 let currentAdaptiveBudget = null;
 // Intercept all tool registrations to auto-log + auto-recall
 const _rawOriginalTool = server.tool.bind(server);
@@ -760,7 +797,7 @@ server.tool = (...args) => {
         if (!perceptionManager.isRunning && bridgeReady) {
             const focusApp = worldModel.getState().focusedApp;
             if (focusApp?.bundleId && focusApp?.pid) {
-                perceptionManager.tryAutoStart(focusApp, bridge).catch(() => { });
+                perceptionManager.tryAutoStart(focusApp, bridge).catch((e) => { process.stderr.write(`[screenhand] perception auto-start failed: ${e instanceof Error ? e.message : String(e)}\n`); });
                 installSafariEnricher(focusApp.bundleId);
             }
         }
@@ -811,7 +848,7 @@ server.tool = (...args) => {
             "type_with_fallback", "select_with_fallback", "scroll_with_fallback",
         ]);
         try {
-            const result = await originalHandler(params, extra);
+            let result = await originalHandler(params, extra);
             const durationMs = Date.now() - start;
             // ── POST-CALL: log action (async, non-blocking) ──
             const entry = {
@@ -863,7 +900,9 @@ server.tool = (...args) => {
                 try {
                     appMap.recordPageTransition(postBundleIdForCtx, pageTransition.from, pageTransition.to, toolName);
                 }
-                catch { /* non-critical — don't break tool execution for nav tracking */ }
+                catch (e) {
+                    process.stderr.write(`[screenhand] nav tracking failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
             // ── POST-CALL: detect focus drift ──
             const postBundleId = worldModel.getState().focusedApp?.bundleId ?? null;
@@ -937,7 +976,9 @@ server.tool = (...args) => {
                         }
                     }
                 }
-                catch { /* non-fatal */ }
+                catch (e) {
+                    process.stderr.write(`[screenhand] app map feature learning failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
             if (!resultIsError && learnBundleId !== "unknown") {
                 try {
@@ -1359,7 +1400,9 @@ server.tool = (...args) => {
                                     }
                                 }
                             }
-                            catch { /* hierarchy extraction non-fatal */ }
+                            catch (e) {
+                                process.stderr.write(`[screenhand] hierarchy extraction failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                            }
                         }
                     }
                     // ── Conditional UI visibility tracking (throttled) ──
@@ -1407,7 +1450,9 @@ server.tool = (...args) => {
                                     }
                                 }
                             }
-                            catch { /* visibility tracking non-fatal */ }
+                            catch (e) {
+                                process.stderr.write(`[screenhand] visibility tracking failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                            }
                         }
                     }
                     // ── Timing recording: track tool response times per element ──
@@ -1501,25 +1546,65 @@ server.tool = (...args) => {
             if (knownError) {
                 hints.push(`⚡ Memory: "${toolName}" has failed before: "${knownError.error}" (${knownError.occurrences}x). Fix: ${knownError.resolution}`);
             }
-            // Suggest next step if we're mid-strategy
+            // ── Strategy matching: auto-execute proven strategies OR hint unproven ones ──
             const recentTools = memory.getRecentToolNames();
-            const strategyHint = memory.quickStrategyHint(recentTools, worldModel.getState().focusedApp?.bundleId);
-            if (strategyHint) {
-                activeStrategyFingerprint = strategyHint.fingerprint;
-                const nextParams = Object.keys(strategyHint.nextStep.params).length > 0
-                    ? `(${JSON.stringify(strategyHint.nextStep.params)})`
-                    : "";
-                hints.push(`💡 Memory: This matches strategy "${strategyHint.strategy.task}" (${strategyHint.strategy.successCount} wins, ${strategyHint.strategy.failCount ?? 0} fails). Next step: ${strategyHint.nextStep.tool}${nextParams}`);
-                // If this was the last step of the strategy, record success
-                if (recentTools.length === strategyHint.strategy.steps.length - 1) {
-                    // Next call will be the final step — but this call completing means we're on track
+            const currentBundleForStrategy = worldModel.getState().focusedApp?.bundleId;
+            // Try auto-execution first (10+ successes, 0 failures)
+            // Guard: skip if another auto-execution is already in progress
+            const autoExec = autoExecutionInProgress ? null : memory.getAutoExecutableStrategy(recentTools, currentBundleForStrategy);
+            if (autoExec) {
+                autoExecutionInProgress = true;
+                activeStrategyFingerprint = autoExec.fingerprint;
+                const autoResults = [];
+                let allOk = true;
+                hints.push(`🚀 Auto-executing proven strategy "${autoExec.strategy.task}" (${autoExec.strategy.successCount} wins) — ${autoExec.remainingSteps.length} steps remaining`);
+                for (const step of autoExec.remainingSteps) {
+                    try {
+                        const stepResult = await toolRegistry.toExecutor()(step.tool, step.params);
+                        autoResults.push({ tool: step.tool, ...stepResult });
+                        // Record outcome for learning
+                        const target = typeof step.params.target === "string" ? step.params.target
+                            : typeof step.params.title === "string" ? step.params.title
+                                : typeof step.params.text === "string" ? step.params.text
+                                    : null;
+                        contextTracker.recordOutcome(step.tool, { target, text: typeof step.params.text === "string" ? step.params.text : null }, stepResult.ok, stepResult.ok ? null : (stepResult.error ?? null));
+                        if (!stepResult.ok) {
+                            allOk = false;
+                            hints.push(`  ✗ ${step.tool} failed: ${stepResult.error ?? "unknown"}`);
+                            break; // Stop auto-execution on first failure
+                        }
+                        hints.push(`  ✓ ${step.tool} — ok`);
+                    }
+                    catch (err) {
+                        allOk = false;
+                        hints.push(`  ✗ ${step.tool} threw: ${err instanceof Error ? err.message : String(err)}`);
+                        break;
+                    }
                 }
-            }
-            else if (activeStrategyFingerprint && recentTools.length > 0) {
-                // We were following a strategy but the sequence diverged — record success
-                // (the agent completed the strategy or went its own way after it)
-                memory.recordStrategyOutcome(activeStrategyFingerprint, true);
+                // Record strategy outcome
+                memory.recordStrategyOutcome(autoExec.fingerprint, allOk);
                 activeStrategyFingerprint = null;
+                autoExecutionInProgress = false;
+                // Append auto-execution results to the response
+                const autoSummary = autoResults.map((r) => `${r.tool}: ${r.ok ? "ok" : r.error}`).join("\n");
+                const resultContent = Array.isArray(result?.content) ? result.content : [];
+                resultContent.push({ type: "text", text: `\n── AUTO-EXECUTED (${autoResults.length} steps) ──\n${autoSummary}` });
+                result = { ...result, content: resultContent };
+            }
+            else {
+                // Fall back to strategy hint (suggest but don't execute)
+                const strategyHint = memory.quickStrategyHint(recentTools, currentBundleForStrategy);
+                if (strategyHint) {
+                    activeStrategyFingerprint = strategyHint.fingerprint;
+                    const nextParams = Object.keys(strategyHint.nextStep.params).length > 0
+                        ? `(${JSON.stringify(strategyHint.nextStep.params)})`
+                        : "";
+                    hints.push(`💡 Memory: This matches strategy "${strategyHint.strategy.task}" (${strategyHint.strategy.successCount} wins, ${strategyHint.strategy.failCount ?? 0} fails). Next step: ${strategyHint.nextStep.tool}${nextParams}`);
+                }
+                else if (activeStrategyFingerprint && recentTools.length > 0) {
+                    memory.recordStrategyOutcome(activeStrategyFingerprint, true);
+                    activeStrategyFingerprint = null;
+                }
             }
             // Attach hints in BOTH content (visible) and _meta (for programmatic access)
             if (hints.length > 0) {
@@ -1735,7 +1820,9 @@ server.tool("focus", "Focus/activate an application (or a specific window by win
                     targetApp = { bundleId, name: appWin.appName, pid: appWin.pid || appWin.ownerPid };
                 }
             }
-            catch { /* ignore */ }
+            catch (e) {
+                process.stderr.write(`[screenhand] focus window check for ${bundleId} failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
             if (!targetApp) {
                 return { content: [{ type: "text", text: `Error: ${bundleId} is not running. Use launch("${bundleId}") first.` }], isError: true };
             }
@@ -1806,10 +1893,14 @@ server.tool("focus", "Focus/activate an application (or a specific window by win
                     await perceptionManager.ensureStarted(ctx);
                     installSafariEnricher(bundleId);
                 }
-                catch { /* best-effort */ }
+                catch (e) {
+                    process.stderr.write(`[screenhand] perception ensureStarted in focus failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
         }
-        catch { /* app.list failed — world model update is best-effort */ }
+        catch (e) {
+            process.stderr.write(`[screenhand] focus world-model update failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         return { content: [{ type: "text", text: focusMsg }] };
     }
     finally {
@@ -1873,7 +1964,9 @@ server.tool("launch", "Launch an application. Chrome/Chromium browsers are launc
         await perceptionManager.ensureStarted({ bundleId, appName: r.appName ?? bundleId, pid: r.pid, windowTitle: "", ...(windowId != null ? { windowId } : {}) });
         installSafariEnricher(bundleId);
     }
-    catch { /* perception start is best-effort */ }
+    catch (e) {
+        process.stderr.write(`[screenhand] perception start after launch failed: ${e instanceof Error ? e.message : String(e)}\n`);
+    }
     let msg = `Launched ${r.appName} pid=${r.pid}`;
     if (chromeAppName) {
         const port = cdpPort ?? 9222;
@@ -2102,7 +2195,9 @@ server.tool("ui_press", "PREFERRED: Find and press/click a UI element by its tit
                     return { content: [{ type: "text", text: `Element "${title}" not found in PID ${pid}. A system dialog from "${front.name}" (${front.bundleId}, PID ${front.pid}) may be blocking. Dismiss it first, or use click(x, y) to interact with the dialog directly.` }], isError: true };
                 }
             }
-            catch { /* ignore frontmost check failure */ }
+            catch (e) {
+                process.stderr.write(`[screenhand] frontmost check in ui_press failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
             throw new Error(`Element "${title}" not found (searched title, value, and description)`);
         }
     }
@@ -2323,7 +2418,9 @@ server.tool("type_text", "Type text using the keyboard. Auto-detects Electron ap
                 catch { /* not available on this port */ }
             }
         }
-        catch { /* auto-detect is best-effort */ }
+        catch (e) {
+            process.stderr.write(`[screenhand] CDP auto-detect failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     if (electronCdpPort) {
         // CDP path: click editor to ensure focus, then type via key events
@@ -2386,7 +2483,9 @@ server.tool("key", "Press a key combination", {
             const front = await bridge.call("app.frontmost", {});
             targetPid = front.pid;
         }
-        catch { /* fallback to global posting */ }
+        catch (e) {
+            process.stderr.write(`[screenhand] key frontmost PID resolve failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     const keys = combo.split("+");
     const hasModifier = keys.some(k => ["cmd", "ctrl", "alt", "shift"].includes(k.toLowerCase()));
@@ -2456,7 +2555,9 @@ async function getCDPClient(tabId, overridePort) {
     try {
         perceptionManager.activateCDP(client);
     }
-    catch { /* best-effort */ }
+    catch (e) {
+        process.stderr.write(`[screenhand] perception CDP activate failed: ${e instanceof Error ? e.message : String(e)}\n`);
+    }
     return { client, targetId: targetId, CDP: cdp, port };
 }
 // ── Random delay helper ──
@@ -3393,6 +3494,12 @@ server.tool("platform_explore", "Autonomously explore an app or website. Maps al
         // Compile and save
         const result = compileReference(platform, "web", tested, url);
         const filePath = saveExploreResult(referencesDir, result);
+        // Auto-merge explore selectors into main reference so data isn't fragmented
+        if (result.selectors && Object.keys(result.selectors).length > 0) {
+            referenceMerger.mergeExploreSelectors(result.selectors, result.errors, "", platform);
+        }
+        // Hot-reload: make new data immediately available to context tracker
+        _playbookStoreForContext.reload();
         return { content: [{ type: "text", text: `Exploration complete: ${filePath}\n\nElements found: ${elements.length}\nTested: ${result.testedElements}\nWorking selectors: ${result.workingSelectors}\nErrors: ${result.errors.length}\n\nKey discoveries:\n${result.keyDiscoveries.map(d => `  - ${d}`).join("\n")}` }] };
     }
     else if (bundleId) {
@@ -3414,6 +3521,12 @@ server.tool("platform_explore", "Autonomously explore an app or website. Maps al
             ...el, clickWorked: true, result: "discovered_not_tested",
         })), undefined, bundleId);
         const filePath = saveExploreResult(referencesDir, result);
+        // Auto-merge explore selectors into main reference so data isn't fragmented
+        if (result.selectors && Object.keys(result.selectors).length > 0) {
+            referenceMerger.mergeExploreSelectors(result.selectors, result.errors, bundleId, platform);
+        }
+        // Hot-reload: make new data immediately available to context tracker
+        _playbookStoreForContext.reload();
         return { content: [{ type: "text", text: `Native app exploration complete: ${filePath}\n\nElements discovered: ${elements.length}\n(Native elements discovered but not auto-clicked for safety. Use playbook_record to test interactively.)` }] };
     }
     else {
@@ -5027,7 +5140,9 @@ server.tool("scroll_with_fallback", "Scroll within an element or the active wind
                     return { content: [{ type: "text", text: `"${target}" is visible after ${i} scroll(s).` }] };
                 }
             }
-            catch { /* OCR failed, keep scrolling */ }
+            catch (e) {
+                process.stderr.write(`[screenhand] OCR during scroll search failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
             // Scroll once
             const deltaX = direction === "left" ? -scrollAmount : direction === "right" ? scrollAmount : 0;
             const deltaY = direction === "up" ? -scrollAmount : direction === "down" ? scrollAmount : 0;
@@ -5154,7 +5269,9 @@ server.tool("wait_for_state", "Wait until a condition is met on screen: text app
                     await client.close();
                 }
             }
-            catch { /* CDP unavailable */ }
+            catch (e) {
+                process.stderr.write(`[screenhand] wait_for_state CDP check failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }
         const elapsed = Date.now() - (deadline - timeout);
         lastCheck = `${elapsed}ms`;
@@ -5440,6 +5557,18 @@ function getJobRunner() {
                 timeout: 15000,
             }).trim();
         });
+        // Wire learning feedback: PlaybookEngine reports step outcomes to context tracker + AppMap
+        playbookEngine.setOutcomeCallback((step, success, error) => {
+            const target = typeof step.target === "string" ? step.target : null;
+            contextTracker.recordOutcome(step.action, { target, text: step.text }, success, error);
+            const bid = worldModel.getState().focusedApp?.bundleId ?? lastKnownBundleId;
+            if (bid && target) {
+                try {
+                    appMap.recordElementOutcome(bid, "auto", target, success);
+                }
+                catch { /* non-critical */ }
+            }
+        });
         activeJobRunner = new JobRunner(bridge, jobManager, leaseManager, supervisor, (() => {
             const cfg = {
                 hasCDP: cdpPort !== null,
@@ -5616,10 +5745,22 @@ originalTool("plan_execute", "Run a plan automatically. Known steps (from playbo
     if (!goal) {
         return { content: [{ type: "text", text: `Goal not found: ${goalId}` }] };
     }
-    const adaptiveBudget = learningEngine.getAdaptiveBudget(worldModel.getState().focusedApp?.bundleId ?? "unknown");
+    const focusedBundleId = worldModel.getState().focusedApp?.bundleId ?? "unknown";
+    const adaptiveBudget = learningEngine.getAdaptiveBudget(focusedBundleId);
     const executor = new PlanExecutor(worldModel, planner, toolRegistry.toExecutor(), { postconditionWaitMs: adaptiveBudget.verifyMs, defaultStepTimeout: Math.max(30_000, adaptiveBudget.actMs * 2) }, recoveryEngine, learningEngine);
     executor.setAppMap(appMap);
-    const result = await executor.executeGoal(goal);
+    // Enable perception-triggered recovery during plan execution
+    perceptionManager.setExpectedApp(focusedBundleId);
+    perceptionManager.startStallDetection(30_000);
+    let result;
+    try {
+        result = await executor.executeGoal(goal);
+    }
+    finally {
+        // Disable reactive recovery after plan completes
+        perceptionManager.setExpectedApp(null);
+        perceptionManager.stopStallDetection();
+    }
     goalStore.update(goalId, goal);
     // Check if paused at an LLM step
     if ("paused" in result) {
@@ -5658,7 +5799,25 @@ originalTool("plan_execute", "Run a plan automatically. Known steps (from playbo
                 }
             }
         }
-        catch { /* strategy recording is best-effort */ }
+        catch (e) {
+            process.stderr.write(`[screenhand] strategy recording failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
+        // Self-improving plans: refine and check for graduation
+        try {
+            const refinement = planRefiner.refine(goal, result);
+            if (refinement.refinementCount > 0) {
+                process.stderr.write(`[plan-refiner] Refined plan for "${goal.description}" (${refinement.refinementCount}x)\n`);
+            }
+            // Check graduation to playbook (3+ refinements)
+            const playbook = planRefiner.checkGraduation(goal.description, focusedBundleId, worldModel.getState().focusedApp?.appName ?? focusedBundleId);
+            if (playbook) {
+                _playbookStoreForContext.save(playbook);
+                process.stderr.write(`[plan-refiner] Plan GRADUATED to playbook: ${playbook.id}\n`);
+            }
+        }
+        catch (e) {
+            process.stderr.write(`[plan-refiner] Refinement failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     const lines = [
         result.success ? "Goal completed successfully." : `Goal failed: ${result.error}`,
@@ -6419,6 +6578,88 @@ server.tool("observer_ocr_roi", "Submit a targeted ROI OCR command to the runnin
     return { content: [{ type: "text", text: `ROI OCR command submitted: ${id}\nRegion: (${x}, ${y}, ${width}×${height})\nThe daemon will process this on its next cycle. Call observer_ocr_roi with commandId="${id}" to poll the result.` }] };
 });
 // ═══════════════════════════════════════════════
+// STATE WATCHER — Continuous observation event bus
+// ═══════════════════════════════════════════════
+server.tool("watch_start", "Start the state watcher polling loop. Evaluates registered watch rules every 2s against the world model.", {}, async () => {
+    stateWatcher.start();
+    const rules = stateWatcher.getRules();
+    return { content: [{ type: "text", text: `State watcher started. ${rules.length} rules registered.` }] };
+});
+server.tool("watch_stop", "Stop the state watcher polling loop.", {}, async () => {
+    stateWatcher.stop();
+    return { content: [{ type: "text", text: "State watcher stopped." }] };
+});
+server.tool("watch_register", "Register a watch rule: when element with matching title appears, execute an action. Use for automated responses to known UI states.", {
+    id: z.string().describe("Unique rule ID"),
+    elementTitle: z.string().describe("UI element title/label to watch for (case-insensitive substring match)"),
+    actionTool: z.string().describe("Tool to execute when element appears (e.g. click_text, key)"),
+    actionParams: z.record(z.string(), z.unknown()).describe("Params for the action tool"),
+    bundleId: z.string().optional().describe("Only match when this app is focused"),
+    maxFires: z.number().optional().describe("Max times to fire (0=unlimited, default=1)"),
+}, async ({ id, elementTitle, actionTool, actionParams, bundleId, maxFires }) => {
+    // Validate tool exists and is safe for automated execution
+    const BLOCKED_WATCH_TOOLS = new Set(["applescript", "browser_js", "browser_stealth"]);
+    if (BLOCKED_WATCH_TOOLS.has(actionTool)) {
+        return { content: [{ type: "text", text: `Tool "${actionTool}" is not allowed in watch rules (security: prevents arbitrary code execution)` }], isError: true };
+    }
+    if (!toolRegistry.has(actionTool)) {
+        return { content: [{ type: "text", text: `Unknown tool: "${actionTool}"` }], isError: true };
+    }
+    stateWatcher.watchForElement(id, elementTitle, { tool: actionTool, params: actionParams }, bundleId);
+    if (maxFires !== undefined) {
+        const rules = stateWatcher.getRules();
+        const rule = rules.find((r) => r.id === id);
+        if (rule) {
+            // Update maxFires on the registered rule
+            const ruleState = stateWatcher.rules.get(id);
+            if (ruleState)
+                ruleState.rule.maxFires = maxFires;
+        }
+    }
+    return { content: [{ type: "text", text: `Watch rule "${id}" registered: when "${elementTitle}" appears → ${actionTool}(${JSON.stringify(actionParams)})` }] };
+});
+server.tool("watch_dialog", "Register a dialog watch rule: when a dialog matching the pattern appears, auto-execute an action.", {
+    id: z.string().describe("Unique rule ID"),
+    titlePattern: z.string().describe("Regex pattern to match dialog titles"),
+    actionTool: z.string().describe("Tool to execute (e.g. click_text, key)"),
+    actionParams: z.record(z.string(), z.unknown()).describe("Params for the action tool"),
+}, async ({ id, titlePattern, actionTool, actionParams }) => {
+    // Validate regex — reject patterns that could cause ReDoS
+    let regex;
+    try {
+        regex = new RegExp(titlePattern, "i");
+        // Quick sanity check — if it takes >50ms on a test string, reject
+        const testStr = "a".repeat(100);
+        const t0 = Date.now();
+        regex.test(testStr);
+        if (Date.now() - t0 > 50) {
+            return { content: [{ type: "text", text: `Rejected: regex pattern "${titlePattern}" is too expensive (potential ReDoS)` }], isError: true };
+        }
+    }
+    catch (e) {
+        return { content: [{ type: "text", text: `Invalid regex: ${e instanceof Error ? e.message : String(e)}` }], isError: true };
+    }
+    stateWatcher.watchForDialog(id, regex, { tool: actionTool, params: actionParams });
+    return { content: [{ type: "text", text: `Dialog watch "${id}" registered: /${titlePattern}/i → ${actionTool}(${JSON.stringify(actionParams)})` }] };
+});
+server.tool("watch_unregister", "Remove a watch rule by ID.", {
+    id: z.string().describe("Rule ID to remove"),
+}, async ({ id }) => {
+    const removed = stateWatcher.unregister(id);
+    return { content: [{ type: "text", text: removed ? `Rule "${id}" removed.` : `Rule "${id}" not found.` }] };
+});
+server.tool("watch_status", "Get all registered watch rules and their fire counts.", {}, async () => {
+    const rules = stateWatcher.getRules();
+    const running = stateWatcher.isRunning;
+    const lines = [
+        `State watcher: ${running ? "running" : "stopped"}`,
+        `Rules: ${rules.length}`,
+        "",
+        ...rules.map((r) => `  [${r.id}] ${r.description} (fired ${r.fireCount}x)`),
+    ];
+    return { content: [{ type: "text", text: lines.join("\n") }] };
+});
+// ═══════════════════════════════════════════════
 // PHASE 6: TOOL MASTERY — Ingestion + Community
 // ═══════════════════════════════════════════════
 server.tool("scan_menu_bar", "Scan an app's menu bar via AX tree. Extracts all menu paths, keyboard shortcuts, and enabled/disabled states. Automatically merges discovered shortcuts into the reference file.", {
@@ -6480,6 +6721,8 @@ server.tool("scan_menu_bar", "Scan an app's menu bar via AX tree. Extracts all m
             });
         }
     }
+    // Hot-reload: make new reference data immediately available to context tracker
+    _playbookStoreForContext.reload();
     let output = lines.join("\n") + bootstrapInfo;
     output = redactUsername(output);
     output = output.replace(/Log Out [^\n:]+/g, "Log Out [USER]");
@@ -6543,6 +6786,8 @@ server.tool("ingest_documentation", "Parse a documentation page (HTML, markdown,
             }
         }
     }
+    // Hot-reload: make new reference data immediately available to context tracker
+    _playbookStoreForContext.reload();
     return { content: [{ type: "text", text: lines.join("\n") }] };
 });
 server.tool("ingest_tutorial", "Extract structured playbook steps from a video transcript (e.g. YouTube captions). Converts tutorial narration into actionable automation steps with tool mappings.", {
@@ -6633,6 +6878,8 @@ server.tool("discover_features", "Extract features from an app's official websit
             lines.push(`  [${f.category}] ${f.name}: ${f.description}`);
         }
     }
+    // Hot-reload: make new reference data immediately available to context tracker
+    _playbookStoreForContext.reload();
     return { content: [{ type: "text", text: lines.join("\n") }] };
 });
 server.tool("coverage_report", "Check what ScreenHand knows about an app: shortcuts, selectors, flows, playbooks, error patterns, and stability %. Useful before complex workflows to decide strategy: learn first (if empty), go fast (if high coverage), or use fallback tools (if error patterns exist). Optional for quick actions.", {
@@ -6741,10 +6988,14 @@ originalTool("community_fetch", "Search community playbooks for a platform or wo
 // START
 // ═══════════════════════════════════════════════
 async function main() {
-    // Flush playbook learnings on graceful shutdown
-    process.on("SIGINT", () => { void perceptionManager.stop(); contextTracker.flush(); learningEngine.flush(); appMap.flush(); process.exit(0); });
-    process.on("SIGTERM", () => { void perceptionManager.stop(); contextTracker.flush(); learningEngine.flush(); appMap.flush(); process.exit(0); });
-    process.on("beforeExit", () => { void perceptionManager.stop(); contextTracker.flush(); learningEngine.flush(); appMap.flush(); });
+    // Flush all learned state on shutdown (signals, stdin EOF, or normal exit)
+    const flushAll = () => { void perceptionManager.stop(); perceptionManager.stopStallDetection(); stateWatcher.stop(); contextTracker.flush(); learningEngine.flush(); appMap.flush(); };
+    process.on("SIGINT", () => { flushAll(); process.exit(0); });
+    process.on("SIGTERM", () => { flushAll(); process.exit(0); });
+    process.on("beforeExit", flushAll);
+    // MCP clients often close stdin without sending a signal — flush on stdin end too
+    process.stdin.on("end", () => { flushAll(); process.exit(0); });
+    process.stdin.on("close", () => { flushAll(); process.exit(0); });
     const transport = new StdioServerTransport();
     await server.connect(transport);
 }

package/dist/src/community/publisher.js

@@ -75,9 +75,11 @@ export class PlaybookPublisher {
             return null;
         }
         writeFileAtomicSync(filePath, JSON.stringify(shared, null, 2) + "\n");
-        // Best-effort sync to remote API
+        // Best-effort sync to remote API — log failures so user knows data didn't leave machine
         if (this.remote) {
-            void this.remote.publish(shared).catch(() => { });
+            void this.remote.publish(shared).catch((err) => {
+                process.stderr.write(`[screenhand] Remote publish failed: ${err instanceof Error ? err.message : String(err)}\n`);
+            });
         }
         return shared;
     }