screenhand 0.3.5 → 0.3.7

package/dist/mcp-desktop.js

@@ -267,134 +267,92 @@ async function ensureCDP(overridePort) {
     throw new Error("Chrome not running with --remote-debugging-port. Launch with: /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-debug");
 }
 const server = new McpServer({ name: "screenhand", version: "3.0.0" }, {
-    instructions: `ScreenHand gives you native desktop control on macOS/Windows. 111 tools. Never click blind — always follow: KNOW → SEE → NAVIGATE → ACT → VERIFY → STOP.
+    instructions: `ScreenHand gives you native desktop control on macOS/Windows. 111 tools.
 
-## The Golden Sequence (follow this order)
+## Quick Actions (just do it)
+For simple tasks, go direct — no setup needed:
 
-### 1. KNOW (before touching anything)
-platform_guide("figma")          → get selectors, flows, known errors for this app/site
-memory_recall("figma export")    → check if you've done this before — reuse past strategies
-scan_menu_bar()                  → discover all menu items in the current app
+focus("com.apple.Notes") → ui_press("New Note") → type_text("hello") → key("cmd+s")
+browser_navigate("https://...") → browser_click("#btn") → browser_js("return ...")
+
+## Tool Speed (fastest first)
+1. **ui_press / key / type_text** — native AX, ~50ms
+2. **browser_* tools** — CDP, ~10ms (background, no focus needed)
+3. ***_with_fallback** — auto-tries AX → CDP → OCR (~100-500ms)
+4. **screenshot + ocr** — visual, ~600ms (canvas apps only)
+5. **applescript** — macOS scripting (Finder, Mail, Safari)
 
-If platform_guide() has no data: platform_explore("bundleId") to auto-discover the app, or platform_learn("domain") for websites.
+## The Golden Sequence (for multi-step workflows)
+For complex tasks with 3+ steps, follow this order:
+
+### 1. KNOW (before touching anything)
+platform_guide("figma")          → get selectors, flows, known errors
+memory_recall("figma export")    → reuse past strategies
+If unknown app: platform_explore("bundleId") or platform_learn("domain")
 
 ### 2. SEE (understand current state)
 apps()                           → what's running?
-perception_start()               → turn on continuous monitoring (3-rate: 100ms/300ms/1000ms)
-world_state()                    → current app, windows, controls, dialogs
-screenshot()                     → visual confirmation if needed
+perception_start()               → continuous monitoring (for multi-step only)
+world_state()                    → current app, windows, controls
 
-perception_start() keeps world_state() continuously updated. Use it for complex multi-step workflows.
-
-### 3. NAVIGATE (get to the right place)
+### 3. NAVIGATE
 focus("com.figma.Desktop")       → bring app to front
-ui_tree()                        → see all clickable elements with roles and labels
-ui_find("Export")                → check if a specific target exists before clicking
+ui_tree()                        → see all clickable elements
+ui_find("Export")                → check if target exists
 
-### 4. ACT (do the thing)
-click_with_fallback("Export")    → click element (auto-tries AX → CDP → OCR → coordinates)
-type_with_fallback("filename")   → type text with auto-fallback
+### 4. ACT
+click_with_fallback("Export")    → click (auto-tries multiple methods)
+type_with_fallback("filename")   → type with fallback
 key("cmd+shift+e")               → keyboard shortcuts
-drag(fromX, fromY, toX, toY)     → drag and drop
-scroll(direction)                → scroll up/down/left/right
-
-Always prefer *_with_fallback tools over bare click/type — they auto-recover when one method fails.
-
-### 5. VERIFY (confirm it worked)
-world_state()                    → did UI change as expected?
-world_state_diff()               → what exactly changed since last check?
-screenshot()                     → visual proof
-
-### 6. STOP (clean up)
-perception_stop()                → stop monitoring (save resources)
-memory_save("figma_export", ...) → save successful strategy for next time
 
-## For Web/Browser (Chrome, Electron apps)
-browser_navigate("https://...")  → go to URL
-browser_stealth()                → activate FIRST if site has bot detection
-browser_dom()                    → read page structure (CSS selectors)
-browser_click("#submit")         → click element by CSS selector
-browser_type("input", "text")    → type into form field
-browser_fill_form({...})         → fill multiple fields at once (human-like timing)
-browser_js("return ...")         → run JavaScript for complex extraction/actions
-browser_wait("selector")         → wait for element to appear
-browser_human_click(x, y)        → human-like click with randomized timing
+### 5. VERIFY
+world_state()                    → did UI change?
+world_state_diff()               → what changed?
 
-All browser tools work in the background (~10ms) — no need to focus Chrome.
+### 6. STOP
+perception_stop()                → stop monitoring
+memory_save("task", ...)         → save strategy for next time
 
-## For Complex Multi-Step Tasks (let ScreenHand plan it)
-plan_goal("Export video as H.264")  → describe WHAT you want — ScreenHand generates steps from playbooks, strategies, and references
-plan_execute(goalId)                → auto-run deterministic steps, pauses at LLM steps for your judgment
-plan_step_resolve(goalId, tool, params) → you provide the tool+params for LLM steps
-plan_status(goalId)                 → check progress
-plan_cancel(goalId)                 → abort if needed
+## Strategy Selection (optional — for when you want to be smart about it)
+Use these tools to pick the best approach. Skip for quick one-off actions.
 
-On success, the strategy is auto-saved to memory for future reuse.
+**coverage_report(bundleId)** — what does ScreenHand know about this app?
+- Empty (0 selectors/flows) → learn first: scan_menu_bar() + platform_explore()
+- Has data + high stability → go fast: direct tools (ui_press, key)
+- Has error patterns → be careful: use *_with_fallback tools
 
-## For Repeatable Workflows (automate once, run forever)
-playbook_record()                → start recording your actions
-... do the work ...
-export_playbook()                → save as reusable playbook
-job_create("daily post", steps)  → make it a persistent job
-worker_start()                   → background daemon runs jobs autonomously
+**learning_status(bundleId)** — how experienced is ScreenHand with this app?
+- 100+ samples → app is well-known, direct tools are safe
+- 0 samples → unknown app, use *_with_fallback
+- AX score high → use ui_tree + ui_press
+- CDP score high → it's a web app, use browser_* tools
+- Vision score high → canvas app, use screenshot + ocr
 
-Jobs survive MCP client restarts. worker_start() runs independently.
+## Browser Automation
+browser_navigate/browser_click/browser_type/browser_js — all work in background (~10ms)
+browser_stealth() — activate before sites with bot detection
+browser_fill_form({...}) — human-like multi-field form filling
+browser_human_click(x, y) — randomized timing to avoid detection
 
-## For Multi-Agent Coordination
-session_claim()                  → claim exclusive access to an app window (lease-based)
-session_heartbeat()              → keep your lease alive (call periodically)
-session_release()                → release when done
-supervisor_start()               → daemon that detects stalled agents and auto-recovers
+## Planning (let ScreenHand figure out the steps)
+plan_goal("Export video as H.264")  → generates step-by-step plan from playbooks/strategies/references
+plan_execute(goalId)                → auto-runs known steps, pauses at LLM steps for your judgment
+plan_step_resolve(goalId, tool, params) → you resolve paused steps
+plan_status(goalId) / plan_list() / plan_cancel(goalId)
 
-## Self-Healing (automatic — no action needed)
-When any tool fails, ScreenHand automatically tries alternative strategies (AX → CDP → OCR → coordinates). Learning is also automatic — every tool call teaches which selectors work, optimal timing, and recovery rankings per app. Check with:
-- learning_status()              → see learned preferences per app
-- recovery_status()              → see active cooldowns and cached strategies
-- recovery_configure()           → tune recovery budget (max time, max retries)
+## Repeatable Workflows
+playbook_record() → do work → export_playbook() → job_create("name", steps) → worker_start()
+Jobs survive restarts. Worker daemon runs independently.
 
-## Tool Speed Priority
-1. **ui_tree + ui_press** — native Accessibility API, ~50ms (fastest, most reliable)
-2. **browser_* tools** — Chrome DevTools Protocol, ~10ms (background, no focus needed)
-3. ***_with_fallback** — auto-tries multiple methods (~100-500ms)
-4. **screenshot + ocr** — visual capture, ~600ms (only for canvas apps)
-5. **applescript** — macOS scripting (Finder, Mail, Safari, etc.)
+## Multi-Agent
+session_claim() → work → session_heartbeat() → session_release()
+supervisor_start() — auto-detects stalled agents and recovers
 
-## Decision Flow (run BEFORE step 1 of Golden Sequence)
-
-Before starting any automation, ask two questions to pick your strategy:
-
-### "Should I learn first or just go?" → coverage_report(bundleId)
-- 0 shortcuts, 0 selectors, 0 flows → LEARN FIRST: scan_menu_bar() + platform_explore() before acting
-- Has selectors + flows but 0 playbooks → CAN ACT, but start playbook_record() to save for next time
-- Has everything + high stability → GO FAST: use direct tools (ui_press, key, type_text)
-- Has error patterns for your tool → BE CAREFUL: use *_with_fallback tools
-
-### "Should I use fast or safe tools?" → learning_status(bundleId)
-- 100+ timing samples → FAST: app is well-known, use direct tools (ui_press, key, type_text ~50ms)
-- 1-99 timing samples → SAFE: use *_with_fallback tools (~100-500ms)
-- 0 timing samples → LEARN: platform_explore() first, then *_with_fallback
-- AX score > 0.9 → use ui_tree + ui_press (native accessibility, fastest)
-- AX low, CDP high → it's a web app, use browser_* tools
-- Both low, Vision high → canvas app, use screenshot + ocr + click_text
-
-### "Do I need perception?"
-- Single action (click a button) → NO, just ui_find + ui_press
-- Multi-step workflow (5+ steps) → YES, perception_start()
-- Visual app (Figma, DaVinci) → YES, with vision (default)
-- Text-heavy app (Notes, Terminal) → AX-only is enough
-
-### Decision Tree Summary
-1. coverage_report(bundleId) → do we know this app?
-   - YES (has references) → use known selectors/flows directly
-   - NO (empty) → scan_menu_bar + platform_explore FIRST
-2. learning_status(bundleId) → how well do we know it?
-   - 100+ samples → direct tools (fast)
-   - <100 samples → *_with_fallback (safe)
-   - 0 samples → learn first, then fallback
-3. Multi-step? → perception_start() : skip perception
-
-## Key Rule
-Never click blind. Always: coverage_report → learning_status → KNOW → SEE → NAVIGATE → ACT → VERIFY → STOP.
+## Self-Healing (automatic)
+Tool failures auto-retry with alternative strategies. Learning is automatic — every call improves selectors, timing, and recovery per app.
+- learning_status() — inspect learned knowledge
+- recovery_status() — check recovery state
+- recovery_configure() — tune recovery budget
 `,
 });
 // ═══════════════════════════════════════════════
@@ -536,7 +494,14 @@ const originalTool = ((...args) => {
         const wrappedHandler = async (params, extra) => {
             const sessionId = memory.getSessionId();
             if (sessionId && worldModel.getState().sessionId !== sessionId) {
-                worldModel.init(sessionId);
+                // If perception is running, rebind sessionId without clearing live state —
+                // a full init() wipes windows/controls that perception actively feeds.
+                if (perceptionManager.isRunning) {
+                    worldModel.rebindSession(sessionId);
+                }
+                else {
+                    worldModel.init(sessionId);
+                }
             }
             return handler(params, extra);
         };
@@ -574,7 +539,14 @@ server.tool = (...args) => {
         const start = Date.now();
         // ── PRE-CALL: lazy-init world model on first session ──
         if (sessionId && worldModel.getState().sessionId !== sessionId) {
-            worldModel.init(sessionId);
+            // If perception is running, rebind sessionId without clearing live state —
+            // a full init() wipes windows/controls that perception actively feeds.
+            if (perceptionManager.isRunning) {
+                worldModel.rebindSession(sessionId);
+            }
+            else {
+                worldModel.init(sessionId);
+            }
         }
         // ── PRE-CALL: notify perception to stay active (idle gating) ──
         perceptionManager.notifyToolCall();
@@ -634,6 +606,12 @@ server.tool = (...args) => {
         const postCallBundleId = preBundleId ?? lastKnownBundleId;
         // Capture pre-call window title for navigation edge tracking
         const preWindowTitle = worldModel.getFocusedWindow()?.title.value ?? null;
+        // Capture pre-call state snapshot for auto-diff (when perception is running)
+        const preState = perceptionManager.isRunning ? worldModel.getState() : null;
+        const preWindowCount = preState?.windows.size ?? 0;
+        const preControlCount = preState ? [...preState.windows.values()].reduce((s, w) => s + w.controls.size, 0) : 0;
+        const preDialogCount = preState?.activeDialogs.length ?? 0;
+        const preFocusedTitle = preState ? (worldModel.getFocusedWindow()?.title.value ?? "") : "";
         // Action tools = actually doing something. Navigation = just clicking around.
         const ACTION_TOOLS = new Set([
             "type_text", "key", "drag", "scroll", "menu_click", "applescript",
@@ -954,6 +932,20 @@ server.tool = (...args) => {
                                         hitFeatures.push(feature.id);
                                 }
                             }
+                            // Auto-discover features for apps on the generic ladder.
+                            // When a non-nav tool call doesn't match any existing feature, create a new
+                            // feature entry from the interaction target so distinct actions register separately.
+                            if (hitFeatures.length === 0 && !isNavTool && appMap.isGenericLadder(learnBundleId)) {
+                                const target = typeof locatorTarget === "string" ? locatorTarget
+                                    : typeof safeParams.text === "string" ? safeParams.text.slice(0, 40)
+                                        : typeof safeParams.combo === "string" ? safeParams.combo
+                                            : null;
+                                if (target) {
+                                    const discovered = appMap.discoverFeature(learnBundleId, toolName, target, 2);
+                                    if (discovered)
+                                        hitFeatures.push(discovered);
+                                }
+                            }
                             // Cross-feature workflow detection: track distinct features hit by action tools.
                             // When 3+ distinct features are hit in a rolling window, record a cross-feature workflow.
                             if (!crossFeatureBuffer.has(learnBundleId)) {
@@ -1281,6 +1273,26 @@ server.tool = (...args) => {
             if (perceptionManager.isRunning) {
                 hints.push(perceptionManager.getFreshnessSummary());
             }
+            // Auto world_state_diff: when perception is running and an action tool was used,
+            // show what changed so the agent gets instant feedback without manual world_state_diff calls
+            if (preState && ACTION_TOOLS.has(toolName)) {
+                const postWindowCount = worldModel.getState().windows.size;
+                const postControlCount = [...worldModel.getState().windows.values()].reduce((s, w) => s + w.controls.size, 0);
+                const postDialogCount = worldModel.getState().activeDialogs.length;
+                const postFocusedTitle = worldModel.getFocusedWindow()?.title.value ?? "";
+                const diffs = [];
+                if (postWindowCount !== preWindowCount)
+                    diffs.push(`windows: ${preWindowCount}→${postWindowCount}`);
+                if (postControlCount !== preControlCount)
+                    diffs.push(`controls: ${preControlCount}→${postControlCount}`);
+                if (postDialogCount !== preDialogCount)
+                    diffs.push(`dialogs: ${preDialogCount}→${postDialogCount}`);
+                if (postFocusedTitle !== preFocusedTitle && postFocusedTitle)
+                    diffs.push(`title: "${preFocusedTitle}"→"${postFocusedTitle}"`);
+                if (diffs.length > 0) {
+                    hints.push(`Δ ${diffs.join(", ")}`);
+                }
+            }
             // Learning engine recommendations
             const patternRec = learningEngine.recommendPattern(learnBundleId, toolName);
             if (patternRec) {
@@ -3073,13 +3085,14 @@ server.tool("export_playbook", "Generate a playbook JSON from your session. Extr
 // ═══════════════════════════════════════════════
 // PLAYBOOK RECORD — macro recorder for MCP tool calls
 // ═══════════════════════════════════════════════
-server.tool("playbook_record", "Macro recorder: start recording, do the flow, stop to save as executable playbook. Captures every click/type/navigate tool call as a PlaybookStep.", {
-    action: z.enum(["start", "stop", "cancel", "status"]).describe("start/stop/cancel/status"),
+server.tool("playbook_record", "Macro recorder: start/stop/trim/clean recorded playbooks. Use 'trim' to remove specific steps, 'clean' to auto-remove failed steps and retries before export.", {
+    action: z.enum(["start", "stop", "cancel", "status", "trim", "clean"]).describe("start/stop/cancel/status/trim/clean"),
     platform: z.string().optional().describe("Platform name (required for start)"),
     name: z.string().optional().describe("Playbook name (required for stop)"),
     description: z.string().optional().describe("Playbook description (for stop)"),
     cdpPort: z.number().min(9222).max(9999).optional().describe("CDP port if needed for browser_js steps (e.g. 9333 for Codex)"),
-}, async ({ action, platform, name, description, cdpPort }) => {
+    removeSteps: z.array(z.number()).optional().describe("Step indices to remove (0-based, for trim action)"),
+}, async ({ action, platform, name, description, cdpPort, removeSteps: removeIndices }) => {
     switch (action) {
         case "start": {
             if (!platform)
@@ -3087,7 +3100,7 @@ server.tool("playbook_record", "Macro recorder: start recording, do the flow, st
             if (mcpRecorder.isRecording)
                 return { content: [{ type: "text", text: "Already recording. Call stop or cancel first." }] };
             mcpRecorder.start(platform, cdpPort ?? undefined);
-            return { content: [{ type: "text", text: `Recording started for "${platform}". All subsequent tool calls will be captured.\nCall playbook_record(action="stop", name="...") when done.` }] };
+            return { content: [{ type: "text", text: `Recording started for "${platform}". All subsequent tool calls will be captured.\nCall playbook_record(action="stop", name="...") when done.\n\nTip: Before stopping, use action="clean" to auto-remove failed steps and retries, or action="trim" to remove specific steps by index.` }] };
         }
         case "stop": {
             if (!mcpRecorder.isRecording)
@@ -3114,8 +3127,29 @@ server.tool("playbook_record", "Macro recorder: start recording, do the flow, st
         case "status": {
             if (!mcpRecorder.isRecording)
                 return { content: [{ type: "text", text: "Not recording." }] };
-            const steps = mcpRecorder.getSteps().map((s, i) => `  ${i + 1}. [${s.action}] ${s.description ?? ""}`).join("\n");
-            return { content: [{ type: "text", text: `Recording active: ${mcpRecorder.stepCount} steps captured\n${steps}` }] };
+            const steps = mcpRecorder.getSteps().map((s, i) => {
+                const marker = s.optional ? " ⚠️FAILED" : "";
+                return `  ${i}. [${s.action}]${marker} ${s.description ?? ""}`;
+            }).join("\n");
+            return { content: [{ type: "text", text: `Recording active: ${mcpRecorder.stepCount} steps captured\n${steps}\n\nUse action="clean" to auto-remove failed steps and retries, or action="trim" with removeSteps=[0,3,5] to remove specific steps.` }] };
+        }
+        case "trim": {
+            if (!mcpRecorder.isRecording)
+                return { content: [{ type: "text", text: "No active recording to trim." }] };
+            if (!removeIndices || removeIndices.length === 0)
+                return { content: [{ type: "text", text: "Error: removeSteps array required (e.g. removeSteps=[0, 3, 5])" }] };
+            const removed = mcpRecorder.removeSteps(removeIndices);
+            const steps = mcpRecorder.getSteps().map((s, i) => `  ${i}. [${s.action}] ${s.description ?? ""}`).join("\n");
+            return { content: [{ type: "text", text: `Removed ${removed} step(s). ${mcpRecorder.stepCount} remaining:\n${steps}` }] };
+        }
+        case "clean": {
+            if (!mcpRecorder.isRecording)
+                return { content: [{ type: "text", text: "No active recording to clean." }] };
+            const failedRemoved = mcpRecorder.removeFailedSteps();
+            const retriesRemoved = mcpRecorder.removeRetries();
+            const total = failedRemoved + retriesRemoved;
+            const steps = mcpRecorder.getSteps().map((s, i) => `  ${i}. [${s.action}] ${s.description ?? ""}`).join("\n");
+            return { content: [{ type: "text", text: `Cleaned: removed ${failedRemoved} failed step(s) + ${retriesRemoved} retry(s) = ${total} total. ${mcpRecorder.stepCount} steps remaining:\n${steps}` }] };
         }
     }
 });
@@ -6282,7 +6316,7 @@ server.tool("ingest_tutorial", "Extract structured playbook steps from a video t
             }],
     };
 });
-server.tool("coverage_report", "CALL THIS FIRST before automating any app. Shows what ScreenHand knows: shortcuts, selectors, flows, playbooks, error patterns, and stability %. Use the result to decide your strategy: learn first (if empty), go fast (if high coverage), or use fallback tools (if error patterns exist). See Decision Flow in server instructions.", {
+server.tool("coverage_report", "Check what ScreenHand knows about an app: shortcuts, selectors, flows, playbooks, error patterns, and stability %. Useful before complex workflows to decide strategy: learn first (if empty), go fast (if high coverage), or use fallback tools (if error patterns exist). Optional for quick actions.", {
     bundleId: z.string().describe("macOS bundle ID (e.g. com.blackmagic-design.DaVinciResolveLite)"),
     appName: z.string().describe("Human-readable app name"),
     includeLiveMenuScan: z.boolean().optional().describe("Also scan the live menu bar for comparison (requires app to be running, needs pid)"),

package/dist/src/playbook/mcp-recorder.js

@@ -201,4 +201,54 @@ export class McpPlaybookRecorder {
         this.recording = false;
         this.steps = [];
     }
+    /**
+     * Remove steps by index (0-based). Accepts individual indices or ranges.
+     * Returns the number of steps removed.
+     */
+    removeSteps(indices) {
+        if (!this.recording || indices.length === 0)
+            return 0;
+        const toRemove = new Set(indices.filter((i) => i >= 0 && i < this.steps.length));
+        const before = this.steps.length;
+        this.steps = this.steps.filter((_, i) => !toRemove.has(i));
+        return before - this.steps.length;
+    }
+    /**
+     * Remove all failed/optional steps (recorded when tool calls failed).
+     * Returns the number of steps removed.
+     */
+    removeFailedSteps() {
+        if (!this.recording)
+            return 0;
+        const before = this.steps.length;
+        this.steps = this.steps.filter((s) => !s.optional);
+        return before - this.steps.length;
+    }
+    /**
+     * Remove consecutive duplicate steps that look like retries.
+     * A "retry" = same action + same target within 3 consecutive steps.
+     * Keeps the last occurrence (the one that presumably worked).
+     * Returns the number of steps removed.
+     */
+    removeRetries() {
+        if (!this.recording || this.steps.length < 2)
+            return 0;
+        const before = this.steps.length;
+        const cleaned = [];
+        for (let i = 0; i < this.steps.length; i++) {
+            const step = this.steps[i];
+            const next = this.steps[i + 1];
+            // If the next step is the same action+target, skip this one (keep the later one)
+            if (next &&
+                next.action === step.action &&
+                next.target === step.target &&
+                JSON.stringify(next.keys ?? []) === JSON.stringify(step.keys ?? []) &&
+                JSON.stringify(next.menuPath ?? []) === JSON.stringify(step.menuPath ?? [])) {
+                continue; // skip this duplicate, keep the next one
+            }
+            cleaned.push(step);
+        }
+        this.steps = cleaned;
+        return before - this.steps.length;
+    }
 }

package/dist/src/state/app-map.js

@@ -242,6 +242,70 @@ export class AppMap {
             return true;
         return this.loadGeneratedLadder(bundleId) !== null;
     }
+    /** Check if this app is using the 5-item generic fallback ladder (no builtin, no generated). */
+    isGenericLadder(bundleId) {
+        if (BUILTIN_LADDERS[bundleId])
+            return false;
+        if (this.generatedLadderCache.has(bundleId))
+            return false;
+        if (this.loadGeneratedLadder(bundleId) !== null)
+            return false;
+        return true;
+    }
+    /**
+     * Auto-discover a feature from a tool interaction and add it to the ladder.
+     * Called when a tool call doesn't match any existing ladder feature AND we're
+     * on the generic ladder. Derives a feature ID from the interaction context
+     * (menu path, target element, tool name) so distinct interactions register
+     * as distinct features instead of all collapsing into "core_action".
+     *
+     * Returns the new feature ID if created, null if skipped.
+     */
+    discoverFeature(bundleId, toolName, target, depth) {
+        if (!target || target.length < 2)
+            return null;
+        // Derive a feature ID from the target — normalize to snake_case
+        // "Format > Checklist" → "format_checklist"
+        // "New Folder" → "new_folder"
+        const featureId = target
+            .toLowerCase()
+            .replace(/[>→/\\|]/g, " ") // split menu paths
+            .replace(/[^a-z0-9\s]/g, "") // strip special chars
+            .trim()
+            .replace(/\s+/g, "_") // spaces to underscores
+            .slice(0, 40); // cap length
+        if (!featureId || featureId.length < 2)
+            return null;
+        // Don't create duplicates
+        const data = this.ensureLoaded(bundleId);
+        if (!data)
+            return null;
+        if (data.featureLadder.some((f) => f.id === featureId))
+            return null;
+        if (data.featureMastery[featureId])
+            return null;
+        // Cap auto-discovered features at 30 per app to prevent unbounded growth
+        const discoveredCount = data.featureLadder.filter((f) => f.description.startsWith("[auto]")).length;
+        if (discoveredCount >= 30)
+            return null;
+        // Assign level based on tool complexity
+        let level = "beginner";
+        if (toolName === "menu_click" || toolName === "key")
+            level = "pro";
+        if (toolName === "applescript" || toolName === "browser_js")
+            level = "expert";
+        const feature = {
+            id: featureId,
+            description: `[auto] ${target}`,
+            level,
+            weight: 1,
+            critical: false,
+        };
+        data.featureLadder.push(feature);
+        this.recordFeatureSignal(bundleId, featureId, depth, true);
+        this.save(data);
+        return featureId;
+    }
     /**
      * Set a custom feature ladder for an app. Useful for apps without built-in ladders.
      */

package/dist/src/state/world-model.js

@@ -261,6 +261,14 @@ export class WorldModel {
             this.entityTracker.rehydrate(this.state.trackedEntities);
         }
     }
+    /**
+     * Rebind session ID without clearing live state. Use when perception is actively
+     * feeding data — a full init() would wipe windows/controls that perception wrote.
+     * Only updates the sessionId tag so persistence targets the new session.
+     */
+    rebindSession(sessionId) {
+        this.state.sessionId = sessionId;
+    }
     /**
      * Merge an incoming control with an existing one using source confidence.
      * Higher-confidence sources win unless the existing data is very recent (<5s).

package/dist-app-maps/notion.id.json

@@ -3,8 +3,8 @@
   "appName": "Notion",
   "version": "unknown",
   "masteryLevel": "grandmaster",
-  "confidence": 0.999600599101348,
-  "lastValidated": "2026-03-21T09:29:49.708Z",
+  "confidence": 0.9996007984031936,
+  "lastValidated": "2026-03-22T08:48:48.328Z",
   "mapVersion": 1,
   "uiArchitecture": {
     "type": "other",
@@ -1279,6 +1279,17 @@
       "confidence": 0.9995235259082788,
       "zonesKnown": 1,
       "edgesVerified": 0
+    },
+    {
+      "date": "2026-03-22",
+      "level": "grandmaster",
+      "rating": {
+        "grade": "A",
+        "subTier": 1
+      },
+      "confidence": 0.9996007984031936,
+      "zonesKnown": 6,
+      "edgesVerified": 2
     }
   ],
   "totalTasksCompleted": 0,
@@ -1400,11 +1411,11 @@
     "new_page_options": {
       "depth": 4,
       "confidence": 1,
-      "repeatCount": 279,
-      "workflowCount": 261,
+      "repeatCount": 280,
+      "workflowCount": 262,
       "healingCount": 0,
       "failCount": 0,
-      "lastSeen": "2026-03-21T09:29:46.950Z",
+      "lastSeen": "2026-03-22T08:48:40.288Z",
       "lastVerified": "2026-03-21T09:29:08.915Z"
     },
     "database": {
@@ -1532,7 +1543,7 @@
     "breadth": 1,
     "workflowBreadth": 1,
     "outcomeBreadth": 0.7241379310344828,
-    "reliability": 0.9960059910134798,
+    "reliability": 0.9960079840319361,
     "healingRate": 1,
     "crossFeatureWorkflows": 60,
     "criticalFloor": 3,

package/dist-references/notes.json

@@ -1,27 +1,175 @@
 {
-  "id": "notes",
-  "name": "notes playbook",
-  "description": "",
   "platform": "notes",
-  "urlPatterns": [],
-  "steps": [],
-  "tags": [
-    "notes"
-  ],
-  "version": "1.0.0",
-  "successCount": 0,
-  "failCount": 0,
   "bundleId": "com.apple.Notes",
+  "version": "2.0.0",
+  "updated": "2026-03-22",
+  "description": "Apple Notes mastery: AppleScript bulk ops, HTML styling, inline images, website-quality notes",
+  "urls": {},
   "selectors": {
-    "auto_discovered": {}
+    "noteBody": "AXTextArea[identifier='Note Body Text View']",
+    "noteList": "AXTable — sidebar note list",
+    "folderList": "AXOutline — sidebar folder tree"
+  },
+  "flows": {
+    "create_html_note": {
+      "description": "Create a rich HTML-formatted note with tables, styled sections, colors, and emoji icons",
+      "steps": [
+        "applescript: make new note at folder 'Notes' of default account with properties {body:htmlString}",
+        "HTML supports: h1-h3, p, b, i, u, hr, ul/li, ol/li, table, a href, br",
+        "Inline CSS works: style='color:#00e5ff; background-color:#0a1a1c; border-radius:12px; font-size:28px;'",
+        "Tables create card layouts: cellpadding for spacing, cellspacing for gaps, vertical-align:top"
+      ],
+      "example": "set h to \"<h1>Title</h1><table border='0' cellpadding='15' cellspacing='8'><tr><td style='background-color:#0a1a1c; border:1px solid #00e5ff30; border-radius:12px;'><b>Card</b><br>Description</td></tr></table>\""
+    },
+    "inline_image_paste": {
+      "description": "Place images at exact positions inside a note using clipboard paste (not make new attachment which only appends to end)",
+      "steps": [
+        "Step 1: Create the full note content via AppleScript with HTML body",
+        "Step 2: Copy image to clipboard via Bash: osascript -e 'set the clipboard to (read (POSIX file \"/path/to/image.png\") as «class PNGf»)'",
+        "Step 3: Focus Notes app: focus({bundleId:'com.apple.Notes'})",
+        "Step 4: Use Find to position cursor: cmd+f → type target text → escape",
+        "Step 5: Navigate: cmd+left (start of line) → up (line above)",
+        "Step 6: Paste: cmd+v (image appears inline at cursor position)",
+        "Repeat steps 2-6 for each image at different positions"
+      ],
+      "gotchas": [
+        "make new attachment at end of attachments ONLY appends to bottom — never inline",
+        "cmd+right in Notes goes to end of ENTIRE note, not end of line — use arrow keys instead",
+        "Copy image via Bash osascript, not via ScreenHand applescript tool (read command is blocked)",
+        "Always focus Notes before pasting — switching apps moves cursor"
+      ]
+    },
+    "bulk_read_notes": {
+      "description": "Read all notes via AppleScript (100x faster than UI automation)",
+      "snippet": "tell application \"Notes\"\n    set output to \"\"\n    repeat with n in notes of default account\n        set output to output & \"---\" & return & name of n & return & plaintext of n & return\n    end repeat\n    return output\nend tell"
+    },
+    "bulk_delete_by_name": {
+      "description": "Delete specific notes by name (safer than pattern matching)",
+      "snippet": "tell application \"Notes\"\n    set noteNames to {\"Note One\", \"Note Two\", \"Note Three\"}\n    repeat with noteName in noteNames\n        try\n            delete note noteName of default account\n        end try\n    end repeat\nend tell",
+      "gotchas": [
+        "Deleting inside a loop shifts the list — wrap in 'repeat 5 times' outer loop",
+        "'items' is a reserved word — use taskList, noteList etc",
+        "container of note errors out — skip folder name, query folder-specific instead"
+      ]
+    },
+    "search_notes": {
+      "description": "Search note content via AppleScript",
+      "snippet": "tell application \"Notes\"\n    set results to {}\n    repeat with n in notes of default account\n        if plaintext of n contains \"keyword\" then\n            set end of results to name of n\n        end if\n    end repeat\n    return results\nend tell"
+    },
+    "move_note": {
+      "description": "Move a note between folders",
+      "snippet": "tell application \"Notes\"\n    set sourceNote to note \"My Note\" of default account\n    set targetFolder to folder \"Target Folder\" of default account\n    move sourceNote to targetFolder\nend tell"
+    },
+    "export_all_notes": {
+      "description": "Export all notes to files (two-step: AppleScript dumps, Bash writes files)",
+      "snippet_step1": "tell application \"Notes\"\n    set output to \"\"\n    repeat with n in notes of default account\n        set output to output & \"<<<NOTE_START>>>\" & name of n & \"<<<NOTE_SEP>>>\" & plaintext of n & \"<<<NOTE_END>>>\"\n    end repeat\n    return output\nend tell",
+      "snippet_step2": "# Bash/Python: parse output file, split by delimiters, write each note to ~/Desktop/notes-export/",
+      "gotchas": [
+        "do shell script is blocked in ScreenHand applescript tool — must use two-step approach"
+      ]
+    },
+    "count_per_folder": {
+      "description": "Count notes in each folder",
+      "snippet": "tell application \"Notes\"\n    set output to \"\"\n    repeat with f in folders of default account\n        set output to output & name of f & \": \" & (count of notes of f) & \" notes\" & return\n    end repeat\n    return output\nend tell"
+    },
+    "find_duplicates": {
+      "description": "Find duplicate note titles",
+      "snippet": "tell application \"Notes\"\n    set titles to {}\n    set dupes to {}\n    repeat with n in notes of default account\n        set t to name of n\n        if titles contains t then\n            if dupes does not contain t then set end of dupes to t\n        else\n            set end of titles to t\n        end if\n    end repeat\n    return dupes\nend tell"
+    },
+    "merge_notes": {
+      "description": "Merge multiple notes into one with HTML",
+      "snippet": "tell application \"Notes\"\n    set note1 to note \"First Note\" of default account\n    set note2 to note \"Second Note\" of default account\n    set combined to \"<h1>Merged Note</h1>\" & body of note1 & \"<hr>\" & body of note2\n    make new note at folder \"Notes\" of default account with properties {body:combined}\nend tell"
+    },
+    "bulk_tag": {
+      "description": "Add tag to note body",
+      "snippet": "tell application \"Notes\"\n    set n to note \"My Note\" of default account\n    set body of n to (body of n) & \"<br>#newtag\"\nend tell"
+    },
+    "list_titles_only": {
+      "description": "List just note titles (fast overview)",
+      "snippet": "tell application \"Notes\"\n    set output to {}\n    repeat with n in notes of default account\n        set end of output to name of n\n    end repeat\n    return output\nend tell"
+    },
+    "delete_by_pattern": {
+      "description": "Delete notes matching a pattern (wraps 5x for list shifting)",
+      "snippet": "tell application \"Notes\"\n    repeat 5 times\n        repeat with n in notes of default account\n            if name of n starts with \"Test\" then delete n\n        end repeat\n    end repeat\nend tell"
+    },
+    "move_from_folder": {
+      "description": "Move a note from a specific folder back to Notes",
+      "snippet": "tell application \"Notes\"\n    set sourceNote to note \"My Note\" of folder \"Source Folder\" of default account\n    set targetFolder to folder \"Notes\" of default account\n    move sourceNote to targetFolder\nend tell"
+    },
+    "list_with_dates": {
+      "description": "List notes with creation and modification dates",
+      "snippet": "tell application \"Notes\"\n    set output to \"\"\n    repeat with n in notes of default account\n        set output to output & name of n & \" | created: \" & (creation date of n as string) & \" | modified: \" & (modification date of n as string) & return\n    end repeat\n    return output\nend tell"
+    },
+    "bulk_rename": {
+      "description": "Rename a note by prepending a new h1 title to body",
+      "snippet": "tell application \"Notes\"\n    set n to note \"Old Title\" of default account\n    set body of n to \"<h1>New Title</h1>\" & body of n\nend tell"
+    },
+    "get_metadata": {
+      "description": "Get note metadata: name, ID, dates, password status",
+      "snippet": "tell application \"Notes\"\n    set n to note \"My Note\" of default account\n    return \"Name: \" & name of n & return & \"ID: \" & id of n & return & \"Created: \" & (creation date of n as string) & return & \"Modified: \" & (modification date of n as string) & return & \"Password protected: \" & (password protected of n as string)\nend tell"
+    },
+    "bulk_create_from_list": {
+      "description": "Create multiple notes from a list of titles",
+      "snippet": "tell application \"Notes\"\n    set taskList to {\"Task One\", \"Task Two\", \"Task Three\"}\n    repeat with taskItem in taskList\n        make new note at folder \"Notes\" of default account with properties {body:\"<h1>\" & taskItem & \"</h1><p>Bulk created</p>\"}\n    end repeat\nend tell"
+    },
+    "create_delete_folders": {
+      "description": "Create and delete folders",
+      "steps": [
+        "applescript: make new folder with properties {name:'Folder Name'}",
+        "applescript: delete folder 'Folder Name' of default account"
+      ]
+    },
+    "website_style_note": {
+      "description": "Create a website-quality note mirroring a Next.js site structure with Hero, Problem, Solution, Features, Comparison, Demo, CTA sections",
+      "steps": [
+        "Step 1: Build full HTML body with styled sections using tables for card layouts",
+        "Step 2: Use inline CSS for colors (color:#00e5ff), backgrounds (background-color:#0a1a1c), borders, border-radius",
+        "Step 3: Use <table border='0' cellpadding='15' cellspacing='8'> for card grids",
+        "Step 4: Use <hr> between sections, styled <p> tags for section labels",
+        "Step 5: Add comparison tables with emoji checkmarks (✅/❌)",
+        "Step 6: Add CTA with styled button-like table cells (background-color:#00e5ff; border-radius:25px)",
+        "Step 7: Paste images inline at exact positions using Find+cursor+clipboard technique"
+      ],
+      "html_tags_supported": ["h1", "h2", "h3", "p", "b", "i", "u", "hr", "br", "table", "tr", "td", "ul", "ol", "li", "a", "span", "img"],
+      "css_properties_supported": ["color", "background-color", "font-size", "border", "border-radius", "text-align", "vertical-align", "padding", "letter-spacing", "width"]
+    }
+  },
+  "detection": {
+    "is_logged_in": "Always logged in — native macOS app"
   },
-  "flows": {},
   "errors": [
     {
-      "error": "Not found: Element not found matching criteria",
-      "context": "tool: ui_find, domain: native:com.apple.Notes",
-      "solution": "No resolution yet — investigate and update this entry",
-      "severity": "medium"
+      "pattern": "string concatenation containing 'script' or 'shell'",
+      "solution": "Rephrase body text to avoid the word 'script' — use 'macOS native' instead of 'AppleScript'",
+      "tool": "applescript"
+    },
+    {
+      "pattern": "read command blocked",
+      "solution": "Use Bash osascript to copy images to clipboard, not ScreenHand applescript tool",
+      "tool": "applescript"
+    },
+    {
+      "pattern": "container of note errors -1728",
+      "solution": "Skip folder name in metadata queries — query folder-specific notes instead",
+      "tool": "applescript"
+    },
+    {
+      "pattern": "items reserved word",
+      "solution": "Use taskList, noteList etc instead of 'items' as variable names",
+      "tool": "applescript"
+    },
+    {
+      "pattern": "delete inside loop skips items",
+      "solution": "Wrap in 'repeat 5 times' outer loop, or delete by specific name with try blocks",
+      "tool": "applescript"
     }
-  ]
-}
+  ],
+  "_meta": {
+    "exported_from": "screenhand",
+    "actions_count": 446,
+    "strategies_count": 21,
+    "proven_techniques": 21,
+    "flows_count": 19,
+    "grade": "D3"
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screenhand",
-  "version": "0.3.5",
+  "version": "0.3.7",
   "mcpName": "io.github.manushi4/screenhand",
   "description": "Give AI eyes and hands on your desktop. ScreenHand is an open-source MCP server that lets Claude and other AI agents see your screen, click buttons, type text, and control any app on macOS and Windows.",
   "homepage": "https://screenhand.com",