screenhand 0.5.0 → 0.5.3

package/dist/mcp-desktop.js

@@ -52,8 +52,10 @@ import { PlaybookStore } from "./src/playbook/store.js";
 import { ContextTracker } from "./src/context-tracker.js";
 import { McpPlaybookRecorder } from "./src/playbook/mcp-recorder.js";
 import { WorldModel } from "./src/state/index.js";
+import { StateWatcher } from "./src/state/state-watcher.js";
 import { PerceptionManager } from "./src/perception/index.js";
 import { Planner, PlanExecutor, GoalStore, ToolRegistry } from "./src/planner/index.js";
+import { PlanRefiner } from "./src/planner/plan-refiner.js";
 import { RecoveryEngine } from "./src/recovery/index.js";
 import { LearningEngine, LocatorPolicy } from "./src/learning/index.js";
 import { discoverWebElements, testWebElement, compileReference, saveExploreResult, discoverNativeElements } from "./src/platform/explorer.js";
@@ -70,6 +72,7 @@ import { MenuScanner } from "./src/ingestion/menu-scanner.js";
 import { DocParser } from "./src/ingestion/doc-parser.js";
 import { TutorialExtractor } from "./src/ingestion/tutorial-extractor.js";
 import { extractFeaturesFromHTML } from "./src/ingestion/feature-extractor.js";
+import { quickScan, llmEnrich, buildVisualMeta, isSensitiveApp } from "./src/state/visual-mapper.js";
 import { CoverageAuditor } from "./src/ingestion/coverage-auditor.js";
 import { ReferenceMerger } from "./src/ingestion/reference-merger.js";
 import { PlaybookPublisher } from "./src/community/publisher.js";
@@ -609,6 +612,7 @@ catch { /* dir may not exist */ }
 const planner = new Planner(_executablePlaybookStore, memory, contextTracker, worldModel, learningEngine);
 const goalStore = new GoalStore(path.join(os.homedir(), ".screenhand", "planner"));
 goalStore.init();
+const planRefiner = new PlanRefiner(path.join(os.homedir(), ".screenhand", "planner"));
 const toolRegistry = new ToolRegistry();
 const recoveryEngine = new RecoveryEngine(worldModel, toolRegistry.toExecutor(), memory);
 recoveryEngine.setLearningEngine(learningEngine);
@@ -616,6 +620,7 @@ recoveryEngine.setAppMap(appMap);
 planner.setToolRegistry(toolRegistry);
 planner.setAppMap(appMap);
 perceptionManager.setLearningEngine(learningEngine);
+const stateWatcher = new StateWatcher(worldModel, toolRegistry.toExecutor(), 2_000);
 // ── Reactive event loop: wire perception events to automatic responses ──
 // These fire at perception speed (100-300ms), not LLM speed (~2-3s).
 perceptionManager.on("dialog_detected", (event) => {
@@ -649,6 +654,28 @@ perceptionManager.on("app_switched", (event) => {
     // Log for observability
     console.error(`[reactive] App switched to ${event.bundleId} (pid=${event.pid})`);
 });
+// ── Perception-triggered recovery: focus loss, app crash, stall ──
+perceptionManager.on("focus_lost", (event) => {
+    console.error(`[reactive] Focus lost: expected ${event.expectedBundleId}, got ${event.actualBundleId} — auto-refocusing`);
+    // Auto-refocus the expected app
+    toolRegistry.toExecutor()("focus", { bundleId: event.expectedBundleId }).catch((err) => {
+        console.error(`[reactive] Auto-refocus failed: ${err instanceof Error ? err.message : err}`);
+    });
+});
+perceptionManager.on("app_crash", (event) => {
+    console.error(`[reactive] App crash detected: ${event.bundleId} (pid=${event.pid}) — auto-relaunching`);
+    // Auto-relaunch the crashed app
+    toolRegistry.toExecutor()("launch", { bundleId: event.bundleId }).catch((err) => {
+        console.error(`[reactive] Auto-relaunch failed: ${err instanceof Error ? err.message : err}`);
+    });
+});
+perceptionManager.on("stall_detected", (event) => {
+    console.error(`[reactive] UI stall detected: ${event.bundleId} — no changes for ${(event.stallMs / 1000).toFixed(0)}s — taking screenshot for diagnosis`);
+    // Take a screenshot so the next LLM call can see what's on screen
+    toolRegistry.toExecutor()("screenshot", {}).catch((err) => {
+        console.error(`[reactive] Stall screenshot failed: ${err instanceof Error ? err.message : err}`);
+    });
+});
 const mcpRecorder = new McpPlaybookRecorder(playbooksDir);
 const referenceMerger = new ReferenceMerger(referencesDir);
 const communityPublisher = new PlaybookPublisher();
@@ -681,6 +708,7 @@ const MEMORY_TOOLS = new Set([
 ]);
 // Track the strategy we're currently following (for feedback loop)
 let activeStrategyFingerprint = null;
+let autoExecutionInProgress = false; // guard against concurrent auto-execution
 let currentAdaptiveBudget = null;
 // Intercept all tool registrations to auto-log + auto-recall
 const _rawOriginalTool = server.tool.bind(server);
@@ -770,7 +798,7 @@ server.tool = (...args) => {
         if (!perceptionManager.isRunning && bridgeReady) {
             const focusApp = worldModel.getState().focusedApp;
             if (focusApp?.bundleId && focusApp?.pid) {
-                perceptionManager.tryAutoStart(focusApp, bridge).catch(() => { });
+                perceptionManager.tryAutoStart(focusApp, bridge).catch((e) => { process.stderr.write(`[screenhand] perception auto-start failed: ${e instanceof Error ? e.message : String(e)}\n`); });
                 installSafariEnricher(focusApp.bundleId);
             }
         }
@@ -821,7 +849,7 @@ server.tool = (...args) => {
             "type_with_fallback", "select_with_fallback", "scroll_with_fallback",
         ]);
         try {
-            const result = await originalHandler(params, extra);
+            let result = await originalHandler(params, extra);
             const durationMs = Date.now() - start;
             // ── POST-CALL: log action (async, non-blocking) ──
             const entry = {
@@ -873,7 +901,9 @@ server.tool = (...args) => {
                 try {
                     appMap.recordPageTransition(postBundleIdForCtx, pageTransition.from, pageTransition.to, toolName);
                 }
-                catch { /* non-critical — don't break tool execution for nav tracking */ }
+                catch (e) {
+                    process.stderr.write(`[screenhand] nav tracking failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
             // ── POST-CALL: detect focus drift ──
             const postBundleId = worldModel.getState().focusedApp?.bundleId ?? null;
@@ -947,7 +977,9 @@ server.tool = (...args) => {
                         }
                     }
                 }
-                catch { /* non-fatal */ }
+                catch (e) {
+                    process.stderr.write(`[screenhand] app map feature learning failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
             if (!resultIsError && learnBundleId !== "unknown") {
                 try {
@@ -1369,7 +1401,9 @@ server.tool = (...args) => {
                                     }
                                 }
                             }
-                            catch { /* hierarchy extraction non-fatal */ }
+                            catch (e) {
+                                process.stderr.write(`[screenhand] hierarchy extraction failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                            }
                         }
                     }
                     // ── Conditional UI visibility tracking (throttled) ──
@@ -1417,7 +1451,9 @@ server.tool = (...args) => {
                                     }
                                 }
                             }
-                            catch { /* visibility tracking non-fatal */ }
+                            catch (e) {
+                                process.stderr.write(`[screenhand] visibility tracking failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                            }
                         }
                     }
                     // ── Timing recording: track tool response times per element ──
@@ -1511,25 +1547,65 @@ server.tool = (...args) => {
             if (knownError) {
                 hints.push(`⚡ Memory: "${toolName}" has failed before: "${knownError.error}" (${knownError.occurrences}x). Fix: ${knownError.resolution}`);
             }
-            // Suggest next step if we're mid-strategy
+            // ── Strategy matching: auto-execute proven strategies OR hint unproven ones ──
             const recentTools = memory.getRecentToolNames();
-            const strategyHint = memory.quickStrategyHint(recentTools, worldModel.getState().focusedApp?.bundleId);
-            if (strategyHint) {
-                activeStrategyFingerprint = strategyHint.fingerprint;
-                const nextParams = Object.keys(strategyHint.nextStep.params).length > 0
-                    ? `(${JSON.stringify(strategyHint.nextStep.params)})`
-                    : "";
-                hints.push(`💡 Memory: This matches strategy "${strategyHint.strategy.task}" (${strategyHint.strategy.successCount} wins, ${strategyHint.strategy.failCount ?? 0} fails). Next step: ${strategyHint.nextStep.tool}${nextParams}`);
-                // If this was the last step of the strategy, record success
-                if (recentTools.length === strategyHint.strategy.steps.length - 1) {
-                    // Next call will be the final step — but this call completing means we're on track
+            const currentBundleForStrategy = worldModel.getState().focusedApp?.bundleId;
+            // Try auto-execution first (10+ successes, 0 failures)
+            // Guard: skip if another auto-execution is already in progress
+            const autoExec = autoExecutionInProgress ? null : memory.getAutoExecutableStrategy(recentTools, currentBundleForStrategy);
+            if (autoExec) {
+                autoExecutionInProgress = true;
+                activeStrategyFingerprint = autoExec.fingerprint;
+                const autoResults = [];
+                let allOk = true;
+                hints.push(`🚀 Auto-executing proven strategy "${autoExec.strategy.task}" (${autoExec.strategy.successCount} wins) — ${autoExec.remainingSteps.length} steps remaining`);
+                for (const step of autoExec.remainingSteps) {
+                    try {
+                        const stepResult = await toolRegistry.toExecutor()(step.tool, step.params);
+                        autoResults.push({ tool: step.tool, ...stepResult });
+                        // Record outcome for learning
+                        const target = typeof step.params.target === "string" ? step.params.target
+                            : typeof step.params.title === "string" ? step.params.title
+                                : typeof step.params.text === "string" ? step.params.text
+                                    : null;
+                        contextTracker.recordOutcome(step.tool, { target, text: typeof step.params.text === "string" ? step.params.text : null }, stepResult.ok, stepResult.ok ? null : (stepResult.error ?? null));
+                        if (!stepResult.ok) {
+                            allOk = false;
+                            hints.push(`  ✗ ${step.tool} failed: ${stepResult.error ?? "unknown"}`);
+                            break; // Stop auto-execution on first failure
+                        }
+                        hints.push(`  ✓ ${step.tool} — ok`);
+                    }
+                    catch (err) {
+                        allOk = false;
+                        hints.push(`  ✗ ${step.tool} threw: ${err instanceof Error ? err.message : String(err)}`);
+                        break;
+                    }
                 }
-            }
-            else if (activeStrategyFingerprint && recentTools.length > 0) {
-                // We were following a strategy but the sequence diverged — record success
-                // (the agent completed the strategy or went its own way after it)
-                memory.recordStrategyOutcome(activeStrategyFingerprint, true);
+                // Record strategy outcome
+                memory.recordStrategyOutcome(autoExec.fingerprint, allOk);
                 activeStrategyFingerprint = null;
+                autoExecutionInProgress = false;
+                // Append auto-execution results to the response
+                const autoSummary = autoResults.map((r) => `${r.tool}: ${r.ok ? "ok" : r.error}`).join("\n");
+                const resultContent = Array.isArray(result?.content) ? result.content : [];
+                resultContent.push({ type: "text", text: `\n── AUTO-EXECUTED (${autoResults.length} steps) ──\n${autoSummary}` });
+                result = { ...result, content: resultContent };
+            }
+            else {
+                // Fall back to strategy hint (suggest but don't execute)
+                const strategyHint = memory.quickStrategyHint(recentTools, currentBundleForStrategy);
+                if (strategyHint) {
+                    activeStrategyFingerprint = strategyHint.fingerprint;
+                    const nextParams = Object.keys(strategyHint.nextStep.params).length > 0
+                        ? `(${JSON.stringify(strategyHint.nextStep.params)})`
+                        : "";
+                    hints.push(`💡 Memory: This matches strategy "${strategyHint.strategy.task}" (${strategyHint.strategy.successCount} wins, ${strategyHint.strategy.failCount ?? 0} fails). Next step: ${strategyHint.nextStep.tool}${nextParams}`);
+                }
+                else if (activeStrategyFingerprint && recentTools.length > 0) {
+                    memory.recordStrategyOutcome(activeStrategyFingerprint, true);
+                    activeStrategyFingerprint = null;
+                }
             }
             // Attach hints in BOTH content (visible) and _meta (for programmatic access)
             if (hints.length > 0) {
@@ -1745,7 +1821,9 @@ server.tool("focus", "Focus/activate an application (or a specific window by win
                     targetApp = { bundleId, name: appWin.appName, pid: appWin.pid || appWin.ownerPid };
                 }
             }
-            catch { /* ignore */ }
+            catch (e) {
+                process.stderr.write(`[screenhand] focus window check for ${bundleId} failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
             if (!targetApp) {
                 return { content: [{ type: "text", text: `Error: ${bundleId} is not running. Use launch("${bundleId}") first.` }], isError: true };
             }
@@ -1816,10 +1894,14 @@ server.tool("focus", "Focus/activate an application (or a specific window by win
                     await perceptionManager.ensureStarted(ctx);
                     installSafariEnricher(bundleId);
                 }
-                catch { /* best-effort */ }
+                catch (e) {
+                    process.stderr.write(`[screenhand] perception ensureStarted in focus failed: ${e instanceof Error ? e.message : String(e)}\n`);
+                }
             }
         }
-        catch { /* app.list failed — world model update is best-effort */ }
+        catch (e) {
+            process.stderr.write(`[screenhand] focus world-model update failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
         return { content: [{ type: "text", text: focusMsg }] };
     }
     finally {
@@ -1883,7 +1965,9 @@ server.tool("launch", "Launch an application. Chrome/Chromium browsers are launc
         await perceptionManager.ensureStarted({ bundleId, appName: r.appName ?? bundleId, pid: r.pid, windowTitle: "", ...(windowId != null ? { windowId } : {}) });
         installSafariEnricher(bundleId);
     }
-    catch { /* perception start is best-effort */ }
+    catch (e) {
+        process.stderr.write(`[screenhand] perception start after launch failed: ${e instanceof Error ? e.message : String(e)}\n`);
+    }
     let msg = `Launched ${r.appName} pid=${r.pid}`;
     if (chromeAppName) {
         const port = cdpPort ?? 9222;
@@ -2112,7 +2196,9 @@ server.tool("ui_press", "PREFERRED: Find and press/click a UI element by its tit
                     return { content: [{ type: "text", text: `Element "${title}" not found in PID ${pid}. A system dialog from "${front.name}" (${front.bundleId}, PID ${front.pid}) may be blocking. Dismiss it first, or use click(x, y) to interact with the dialog directly.` }], isError: true };
                 }
             }
-            catch { /* ignore frontmost check failure */ }
+            catch (e) {
+                process.stderr.write(`[screenhand] frontmost check in ui_press failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
             throw new Error(`Element "${title}" not found (searched title, value, and description)`);
         }
     }
@@ -2333,7 +2419,9 @@ server.tool("type_text", "Type text using the keyboard. Auto-detects Electron ap
                 catch { /* not available on this port */ }
             }
         }
-        catch { /* auto-detect is best-effort */ }
+        catch (e) {
+            process.stderr.write(`[screenhand] CDP auto-detect failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     if (electronCdpPort) {
         // CDP path: click editor to ensure focus, then type via key events
@@ -2396,7 +2484,9 @@ server.tool("key", "Press a key combination", {
             const front = await bridge.call("app.frontmost", {});
             targetPid = front.pid;
         }
-        catch { /* fallback to global posting */ }
+        catch (e) {
+            process.stderr.write(`[screenhand] key frontmost PID resolve failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     const keys = combo.split("+");
     const hasModifier = keys.some(k => ["cmd", "ctrl", "alt", "shift"].includes(k.toLowerCase()));
@@ -2466,7 +2556,9 @@ async function getCDPClient(tabId, overridePort) {
     try {
         perceptionManager.activateCDP(client);
     }
-    catch { /* best-effort */ }
+    catch (e) {
+        process.stderr.write(`[screenhand] perception CDP activate failed: ${e instanceof Error ? e.message : String(e)}\n`);
+    }
     return { client, targetId: targetId, CDP: cdp, port };
 }
 // ── Random delay helper ──
@@ -3403,6 +3495,12 @@ server.tool("platform_explore", "Autonomously explore an app or website. Maps al
         // Compile and save
         const result = compileReference(platform, "web", tested, url);
         const filePath = saveExploreResult(referencesDir, result);
+        // Auto-merge explore selectors into main reference so data isn't fragmented
+        if (result.selectors && Object.keys(result.selectors).length > 0) {
+            referenceMerger.mergeExploreSelectors(result.selectors, result.errors, "", platform);
+        }
+        // Hot-reload: make new data immediately available to context tracker
+        _playbookStoreForContext.reload();
         return { content: [{ type: "text", text: `Exploration complete: ${filePath}\n\nElements found: ${elements.length}\nTested: ${result.testedElements}\nWorking selectors: ${result.workingSelectors}\nErrors: ${result.errors.length}\n\nKey discoveries:\n${result.keyDiscoveries.map(d => `  - ${d}`).join("\n")}` }] };
     }
     else if (bundleId) {
@@ -3424,6 +3522,12 @@ server.tool("platform_explore", "Autonomously explore an app or website. Maps al
             ...el, clickWorked: true, result: "discovered_not_tested",
         })), undefined, bundleId);
         const filePath = saveExploreResult(referencesDir, result);
+        // Auto-merge explore selectors into main reference so data isn't fragmented
+        if (result.selectors && Object.keys(result.selectors).length > 0) {
+            referenceMerger.mergeExploreSelectors(result.selectors, result.errors, bundleId, platform);
+        }
+        // Hot-reload: make new data immediately available to context tracker
+        _playbookStoreForContext.reload();
         return { content: [{ type: "text", text: `Native app exploration complete: ${filePath}\n\nElements discovered: ${elements.length}\n(Native elements discovered but not auto-clicked for safety. Use playbook_record to test interactively.)` }] };
     }
     else {
@@ -5037,7 +5141,9 @@ server.tool("scroll_with_fallback", "Scroll within an element or the active wind
                     return { content: [{ type: "text", text: `"${target}" is visible after ${i} scroll(s).` }] };
                 }
             }
-            catch { /* OCR failed, keep scrolling */ }
+            catch (e) {
+                process.stderr.write(`[screenhand] OCR during scroll search failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
             // Scroll once
             const deltaX = direction === "left" ? -scrollAmount : direction === "right" ? scrollAmount : 0;
             const deltaY = direction === "up" ? -scrollAmount : direction === "down" ? scrollAmount : 0;
@@ -5164,7 +5270,9 @@ server.tool("wait_for_state", "Wait until a condition is met on screen: text app
                     await client.close();
                 }
             }
-            catch { /* CDP unavailable */ }
+            catch (e) {
+                process.stderr.write(`[screenhand] wait_for_state CDP check failed: ${e instanceof Error ? e.message : String(e)}\n`);
+            }
         }
         const elapsed = Date.now() - (deadline - timeout);
         lastCheck = `${elapsed}ms`;
@@ -5450,6 +5558,18 @@ function getJobRunner() {
                 timeout: 15000,
             }).trim();
         });
+        // Wire learning feedback: PlaybookEngine reports step outcomes to context tracker + AppMap
+        playbookEngine.setOutcomeCallback((step, success, error) => {
+            const target = typeof step.target === "string" ? step.target : null;
+            contextTracker.recordOutcome(step.action, { target, text: step.text }, success, error);
+            const bid = worldModel.getState().focusedApp?.bundleId ?? lastKnownBundleId;
+            if (bid && target) {
+                try {
+                    appMap.recordElementOutcome(bid, "auto", target, success);
+                }
+                catch { /* non-critical */ }
+            }
+        });
         activeJobRunner = new JobRunner(bridge, jobManager, leaseManager, supervisor, (() => {
             const cfg = {
                 hasCDP: cdpPort !== null,
@@ -5626,10 +5746,22 @@ originalTool("plan_execute", "Run a plan automatically. Known steps (from playbo
     if (!goal) {
         return { content: [{ type: "text", text: `Goal not found: ${goalId}` }] };
     }
-    const adaptiveBudget = learningEngine.getAdaptiveBudget(worldModel.getState().focusedApp?.bundleId ?? "unknown");
+    const focusedBundleId = worldModel.getState().focusedApp?.bundleId ?? "unknown";
+    const adaptiveBudget = learningEngine.getAdaptiveBudget(focusedBundleId);
     const executor = new PlanExecutor(worldModel, planner, toolRegistry.toExecutor(), { postconditionWaitMs: adaptiveBudget.verifyMs, defaultStepTimeout: Math.max(30_000, adaptiveBudget.actMs * 2) }, recoveryEngine, learningEngine);
     executor.setAppMap(appMap);
-    const result = await executor.executeGoal(goal);
+    // Enable perception-triggered recovery during plan execution
+    perceptionManager.setExpectedApp(focusedBundleId);
+    perceptionManager.startStallDetection(30_000);
+    let result;
+    try {
+        result = await executor.executeGoal(goal);
+    }
+    finally {
+        // Disable reactive recovery after plan completes
+        perceptionManager.setExpectedApp(null);
+        perceptionManager.stopStallDetection();
+    }
     goalStore.update(goalId, goal);
     // Check if paused at an LLM step
     if ("paused" in result) {
@@ -5668,7 +5800,25 @@ originalTool("plan_execute", "Run a plan automatically. Known steps (from playbo
                 }
             }
         }
-        catch { /* strategy recording is best-effort */ }
+        catch (e) {
+            process.stderr.write(`[screenhand] strategy recording failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
+        // Self-improving plans: refine and check for graduation
+        try {
+            const refinement = planRefiner.refine(goal, result);
+            if (refinement.refinementCount > 0) {
+                process.stderr.write(`[plan-refiner] Refined plan for "${goal.description}" (${refinement.refinementCount}x)\n`);
+            }
+            // Check graduation to playbook (3+ refinements)
+            const playbook = planRefiner.checkGraduation(goal.description, focusedBundleId, worldModel.getState().focusedApp?.appName ?? focusedBundleId);
+            if (playbook) {
+                _playbookStoreForContext.save(playbook);
+                process.stderr.write(`[plan-refiner] Plan GRADUATED to playbook: ${playbook.id}\n`);
+            }
+        }
+        catch (e) {
+            process.stderr.write(`[plan-refiner] Refinement failed: ${e instanceof Error ? e.message : String(e)}\n`);
+        }
     }
     const lines = [
         result.success ? "Goal completed successfully." : `Goal failed: ${result.error}`,
@@ -6429,6 +6579,88 @@ server.tool("observer_ocr_roi", "Submit a targeted ROI OCR command to the runnin
     return { content: [{ type: "text", text: `ROI OCR command submitted: ${id}\nRegion: (${x}, ${y}, ${width}×${height})\nThe daemon will process this on its next cycle. Call observer_ocr_roi with commandId="${id}" to poll the result.` }] };
 });
 // ═══════════════════════════════════════════════
+// STATE WATCHER — Continuous observation event bus
+// ═══════════════════════════════════════════════
+server.tool("watch_start", "Start the state watcher polling loop. Evaluates registered watch rules every 2s against the world model.", {}, async () => {
+    stateWatcher.start();
+    const rules = stateWatcher.getRules();
+    return { content: [{ type: "text", text: `State watcher started. ${rules.length} rules registered.` }] };
+});
+server.tool("watch_stop", "Stop the state watcher polling loop.", {}, async () => {
+    stateWatcher.stop();
+    return { content: [{ type: "text", text: "State watcher stopped." }] };
+});
+server.tool("watch_register", "Register a watch rule: when element with matching title appears, execute an action. Use for automated responses to known UI states.", {
+    id: z.string().describe("Unique rule ID"),
+    elementTitle: z.string().describe("UI element title/label to watch for (case-insensitive substring match)"),
+    actionTool: z.string().describe("Tool to execute when element appears (e.g. click_text, key)"),
+    actionParams: z.record(z.string(), z.unknown()).describe("Params for the action tool"),
+    bundleId: z.string().optional().describe("Only match when this app is focused"),
+    maxFires: z.number().optional().describe("Max times to fire (0=unlimited, default=1)"),
+}, async ({ id, elementTitle, actionTool, actionParams, bundleId, maxFires }) => {
+    // Validate tool exists and is safe for automated execution
+    const BLOCKED_WATCH_TOOLS = new Set(["applescript", "browser_js", "browser_stealth"]);
+    if (BLOCKED_WATCH_TOOLS.has(actionTool)) {
+        return { content: [{ type: "text", text: `Tool "${actionTool}" is not allowed in watch rules (security: prevents arbitrary code execution)` }], isError: true };
+    }
+    if (!toolRegistry.has(actionTool)) {
+        return { content: [{ type: "text", text: `Unknown tool: "${actionTool}"` }], isError: true };
+    }
+    stateWatcher.watchForElement(id, elementTitle, { tool: actionTool, params: actionParams }, bundleId);
+    if (maxFires !== undefined) {
+        const rules = stateWatcher.getRules();
+        const rule = rules.find((r) => r.id === id);
+        if (rule) {
+            // Update maxFires on the registered rule
+            const ruleState = stateWatcher.rules.get(id);
+            if (ruleState)
+                ruleState.rule.maxFires = maxFires;
+        }
+    }
+    return { content: [{ type: "text", text: `Watch rule "${id}" registered: when "${elementTitle}" appears → ${actionTool}(${JSON.stringify(actionParams)})` }] };
+});
+server.tool("watch_dialog", "Register a dialog watch rule: when a dialog matching the pattern appears, auto-execute an action.", {
+    id: z.string().describe("Unique rule ID"),
+    titlePattern: z.string().describe("Regex pattern to match dialog titles"),
+    actionTool: z.string().describe("Tool to execute (e.g. click_text, key)"),
+    actionParams: z.record(z.string(), z.unknown()).describe("Params for the action tool"),
+}, async ({ id, titlePattern, actionTool, actionParams }) => {
+    // Validate regex — reject patterns that could cause ReDoS
+    let regex;
+    try {
+        regex = new RegExp(titlePattern, "i");
+        // Quick sanity check — if it takes >50ms on a test string, reject
+        const testStr = "a".repeat(100);
+        const t0 = Date.now();
+        regex.test(testStr);
+        if (Date.now() - t0 > 50) {
+            return { content: [{ type: "text", text: `Rejected: regex pattern "${titlePattern}" is too expensive (potential ReDoS)` }], isError: true };
+        }
+    }
+    catch (e) {
+        return { content: [{ type: "text", text: `Invalid regex: ${e instanceof Error ? e.message : String(e)}` }], isError: true };
+    }
+    stateWatcher.watchForDialog(id, regex, { tool: actionTool, params: actionParams });
+    return { content: [{ type: "text", text: `Dialog watch "${id}" registered: /${titlePattern}/i → ${actionTool}(${JSON.stringify(actionParams)})` }] };
+});
+server.tool("watch_unregister", "Remove a watch rule by ID.", {
+    id: z.string().describe("Rule ID to remove"),
+}, async ({ id }) => {
+    const removed = stateWatcher.unregister(id);
+    return { content: [{ type: "text", text: removed ? `Rule "${id}" removed.` : `Rule "${id}" not found.` }] };
+});
+server.tool("watch_status", "Get all registered watch rules and their fire counts.", {}, async () => {
+    const rules = stateWatcher.getRules();
+    const running = stateWatcher.isRunning;
+    const lines = [
+        `State watcher: ${running ? "running" : "stopped"}`,
+        `Rules: ${rules.length}`,
+        "",
+        ...rules.map((r) => `  [${r.id}] ${r.description} (fired ${r.fireCount}x)`),
+    ];
+    return { content: [{ type: "text", text: lines.join("\n") }] };
+});
+// ═══════════════════════════════════════════════
 // PHASE 6: TOOL MASTERY — Ingestion + Community
 // ═══════════════════════════════════════════════
 server.tool("scan_menu_bar", "Scan an app's menu bar via AX tree. Extracts all menu paths, keyboard shortcuts, and enabled/disabled states. Automatically merges discovered shortcuts into the reference file.", {
@@ -6490,6 +6722,8 @@ server.tool("scan_menu_bar", "Scan an app's menu bar via AX tree. Extracts all m
             });
         }
     }
+    // Hot-reload: make new reference data immediately available to context tracker
+    _playbookStoreForContext.reload();
     let output = lines.join("\n") + bootstrapInfo;
     output = redactUsername(output);
     output = output.replace(/Log Out [^\n:]+/g, "Log Out [USER]");
@@ -6553,6 +6787,8 @@ server.tool("ingest_documentation", "Parse a documentation page (HTML, markdown,
             }
         }
     }
+    // Hot-reload: make new reference data immediately available to context tracker
+    _playbookStoreForContext.reload();
     return { content: [{ type: "text", text: lines.join("\n") }] };
 });
 server.tool("ingest_tutorial", "Extract structured playbook steps from a video transcript (e.g. YouTube captions). Converts tutorial narration into actionable automation steps with tool mappings.", {
@@ -6643,6 +6879,190 @@ server.tool("discover_features", "Extract features from an app's official websit
             lines.push(`  [${f.category}] ${f.name}: ${f.description}`);
         }
     }
+    // Hot-reload: make new reference data immediately available to context tracker
+    _playbookStoreForContext.reload();
+    return { content: [{ type: "text", text: lines.join("\n") }] };
+});
+// ── Visual App Mapping (Phase 3) ─────────────────────────────────
+server.tool("map_app", "Visually map an app's UI by taking a screenshot, running OCR to identify interactive elements and zones with coordinates. Makes subsequent tool calls faster and more accurate. Runs quick scan (~500ms) inline, optional LLM enrichment in background if ANTHROPIC_API_KEY is set.", {
+    bundleId: z.string().describe("macOS bundle ID (e.g. com.apple.Notes)"),
+    appName: z.string().describe("Human-readable app name"),
+    force: z.boolean().optional().describe("Re-map even if a recent map exists (default: false)"),
+    depth: z.enum(["quick", "full"]).optional().describe("'quick' = OCR only (~500ms). 'full' = OCR + LLM enrichment (~15s). Default: quick"),
+}, async ({ bundleId, appName, force, depth }) => {
+    if (isSensitiveApp(bundleId)) {
+        return { content: [{ type: "text", text: `Blocked: ${bundleId} is a sensitive app (password manager, banking, etc.). Visual mapping is not allowed for privacy reasons.` }] };
+    }
+    // Check if already mapped (unless force)
+    if (!force) {
+        const existingMeta = appMap.getVisualMeta(bundleId);
+        if (existingMeta && !appMap.isVisualMapStale(bundleId)) {
+            return { content: [{ type: "text", text: `Visual map for ${appName} already exists (${existingMeta.screensMapped.length} screens, confidence: ${existingMeta.confidence.toFixed(2)}). Use force: true to re-map.` }] };
+        }
+    }
+    await ensureBridge();
+    // Get focused app PID
+    const apps = await bridge.call("app.list", {});
+    const matchedApp = apps?.find((a) => a.bundleId === bundleId);
+    if (!matchedApp?.pid) {
+        return { content: [{ type: "text", text: `App ${bundleId} is not running. Launch it first with focus("${bundleId}") or launch("${bundleId}").` }] };
+    }
+    const pid = matchedApp.pid;
+    // Get window bounds
+    let windowTitle = "";
+    let windowBounds;
+    try {
+        const wins = await bridge.call("window.list", {});
+        const appWins = wins?.filter((w) => w.pid === pid);
+        const mainWin = appWins?.find((w) => w.focused || w.frontmost || w.isMain) ?? appWins?.[0];
+        if (mainWin) {
+            windowTitle = mainWin.title ?? "";
+            windowBounds = mainWin.bounds ?? mainWin;
+        }
+    }
+    catch { /* use defaults */ }
+    // Phase A: Quick scan (OCR)
+    const scanResult = await quickScan(bridge, pid, windowBounds);
+    if (!scanResult) {
+        return { content: [{ type: "text", text: `Failed to capture screenshot of ${appName}. Make sure the app window is visible.` }] };
+    }
+    // Get app version for staleness tracking
+    let appVersion = "unknown";
+    try {
+        const infoResult = await bridge.call("app.info", { bundleId });
+        appVersion = infoResult?.version ?? infoResult?.shortVersion ?? "unknown";
+    }
+    catch { /* use default */ }
+    // Get display scale factor
+    let scaleFactor = 2;
+    try {
+        const screenInfo = await bridge.call("screen.info", {});
+        scaleFactor = screenInfo?.scaleFactor ?? 2;
+    }
+    catch { /* default to Retina */ }
+    const meta = buildVisualMeta(scanResult.hash, scanResult.captureSize, windowTitle, appVersion, scanResult.scan.confidence, scaleFactor);
+    // Populate into AppMap
+    const { added, updated } = appMap.populateFromVisualScan(bundleId, appName, scanResult.scan, meta);
+    const lines = [
+        `Visual map for ${appName} (${bundleId}):`,
+        `  Zones identified: ${scanResult.scan.zones.length}`,
+        `  Elements mapped: ${scanResult.scan.elements.length} (${added} new, ${updated} updated)`,
+        `  Map confidence: ${scanResult.scan.confidence.toFixed(2)}`,
+        `  App version: ${appVersion}`,
+    ];
+    if (scanResult.scan.zones.length > 0) {
+        lines.push("  Zones:");
+        for (const z of scanResult.scan.zones) {
+            const elCount = scanResult.scan.elements.filter(e => e.zone === z.label).length;
+            lines.push(`    ${z.label} (${z.type}): ${elCount} elements`);
+        }
+    }
+    // Phase B: LLM enrichment (background, if depth=full and API key exists)
+    if ((depth === "full") && process.env.ANTHROPIC_API_KEY) {
+        lines.push("  LLM enrichment: starting in background...");
+        // Fire and forget — don't block the response
+        (async () => {
+            try {
+                // Get screenshot as file, then read as base64 for LLM
+                const screenshotShot = await bridge.call("cg.captureScreen", {});
+                if (!screenshotShot?.path)
+                    return;
+                const fs = await import("node:fs");
+                const screenshotBase64 = fs.readFileSync(screenshotShot.path).toString("base64");
+                const screenshotData = { base64: screenshotBase64 };
+                // Get AX tree for cross-reference
+                let axTree = "";
+                try {
+                    const tree = await bridge.call("ax.tree", { pid, depth: 3 });
+                    axTree = JSON.stringify(tree, null, 1).slice(0, 3000);
+                }
+                catch { /* proceed without AX */ }
+                const enrichment = await llmEnrich(screenshotData.base64, axTree, appName, bundleId, windowTitle, scanResult.captureSize);
+                if (enrichment) {
+                    // Merge LLM results into AppMap (LLM confidence capped at 0.5)
+                    const llmScan = {
+                        zones: enrichment.zones,
+                        elements: enrichment.elements.map(e => ({
+                            ...e,
+                            confidence: Math.min(e.confidence, 0.5), // Cap — LLM is hypothesis
+                        })),
+                        confidence: Math.min(enrichment.confidence, 0.6),
+                    };
+                    appMap.populateFromVisualScan(bundleId, appName, llmScan, {
+                        ...meta,
+                        confidence: Math.min(enrichment.confidence, 0.6),
+                    });
+                    process.stderr.write(`[visual-mapper] LLM enrichment complete for ${appName}: ${enrichment.elements.length} elements, ${enrichment.zones.length} zones\n`);
+                }
+            }
+            catch (err) {
+                process.stderr.write(`[visual-mapper] Background enrichment failed: ${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        })().catch(() => { });
+    }
+    else if (depth === "full" && !process.env.ANTHROPIC_API_KEY) {
+        lines.push("  LLM enrichment: skipped (no ANTHROPIC_API_KEY set)");
+    }
+    return { content: [{ type: "text", text: lines.join("\n") }] };
+});
+originalTool("map_status", "Check the health of an app's visual map. Shows zones, element counts, confidence, staleness, and failure rates. Useful for debugging click failures.", {
+    bundleId: z.string().describe("macOS bundle ID"),
+}, async ({ bundleId }) => {
+    const meta = appMap.getVisualMeta(bundleId);
+    const data = appMap.getLoaded(bundleId) ?? appMap.load(bundleId);
+    if (!meta) {
+        return { content: [{ type: "text", text: `No visual map exists for ${bundleId}. Run map_app to create one.` }] };
+    }
+    const isStale = appMap.isVisualMapStale(bundleId);
+    const ageMs = Date.now() - new Date(meta.lastScannedAt).getTime();
+    const ageHours = Math.round(ageMs / 3_600_000);
+    const lines = [
+        `Visual Map Status: ${bundleId}`,
+        `  Last scanned: ${meta.lastScannedAt} (${ageHours}h ago)`,
+        `  App version: ${meta.appVersion}`,
+        `  Confidence: ${meta.confidence.toFixed(2)}`,
+        `  Staleness: ${isStale ? "STALE — consider re-mapping" : "fresh"}`,
+        `  Screens mapped: ${meta.screensMapped.join(", ") || "(none)"}`,
+        `  Scale factor: ${meta.scaleFactor}x`,
+        `  Capture size: ${meta.captureSize.w}x${meta.captureSize.h}`,
+    ];
+    // Count visual-scan elements
+    if (data) {
+        let visualElements = 0;
+        let axElements = 0;
+        let totalValidations = 0;
+        let totalMismatches = 0;
+        for (const zone of Object.values(data.zones)) {
+            for (const el of zone.elements) {
+                if (el.labelSource === "ocr" || el.labelSource === "llm") {
+                    visualElements++;
+                    totalValidations += el.validationCount ?? 0;
+                    totalMismatches += el.mismatchCount ?? 0;
+                }
+                else if (el.labelSource === "ax" || el.labelSource === "manual") {
+                    axElements++;
+                }
+            }
+        }
+        lines.push(`  Visual-scan elements: ${visualElements}`);
+        lines.push(`  AX-confirmed elements: ${axElements}`);
+        if (totalValidations + totalMismatches > 0) {
+            const matchRate = totalValidations / (totalValidations + totalMismatches);
+            lines.push(`  Position match rate: ${(matchRate * 100).toFixed(1)}% (${totalValidations} matches, ${totalMismatches} mismatches)`);
+        }
+        // Zone breakdown
+        const zoneKeys = Object.keys(data.zones);
+        if (zoneKeys.length > 0) {
+            lines.push("  Zones:");
+            for (const key of zoneKeys.slice(0, 15)) {
+                const zone = data.zones[key];
+                lines.push(`    ${key} (${zone.type}): ${zone.elements.length} elements`);
+            }
+            if (zoneKeys.length > 15) {
+                lines.push(`    ... and ${zoneKeys.length - 15} more`);
+            }
+        }
+    }
     return { content: [{ type: "text", text: lines.join("\n") }] };
 });
 server.tool("coverage_report", "Check what ScreenHand knows about an app: shortcuts, selectors, flows, playbooks, error patterns, and stability %. Useful before complex workflows to decide strategy: learn first (if empty), go fast (if high coverage), or use fallback tools (if error patterns exist). Optional for quick actions.", {
@@ -6751,10 +7171,14 @@ originalTool("community_fetch", "Search community playbooks for a platform or wo
 // START
 // ═══════════════════════════════════════════════
 async function main() {
-    // Flush playbook learnings on graceful shutdown
-    process.on("SIGINT", () => { void perceptionManager.stop(); contextTracker.flush(); learningEngine.flush(); appMap.flush(); process.exit(0); });
-    process.on("SIGTERM", () => { void perceptionManager.stop(); contextTracker.flush(); learningEngine.flush(); appMap.flush(); process.exit(0); });
-    process.on("beforeExit", () => { void perceptionManager.stop(); contextTracker.flush(); learningEngine.flush(); appMap.flush(); });
+    // Flush all learned state on shutdown (signals, stdin EOF, or normal exit)
+    const flushAll = () => { void perceptionManager.stop(); perceptionManager.stopStallDetection(); stateWatcher.stop(); contextTracker.flush(); learningEngine.flush(); appMap.flush(); };
+    process.on("SIGINT", () => { flushAll(); process.exit(0); });
+    process.on("SIGTERM", () => { flushAll(); process.exit(0); });
+    process.on("beforeExit", flushAll);
+    // MCP clients often close stdin without sending a signal — flush on stdin end too
+    process.stdin.on("end", () => { flushAll(); process.exit(0); });
+    process.stdin.on("close", () => { flushAll(); process.exit(0); });
     const transport = new StdioServerTransport();
     await server.connect(transport);
 }