screenhand 0.3.8 → 0.4.0

package/README.md

@@ -128,7 +128,7 @@ On Windows, use `npm run build:native:windows` instead.
 
 ## What It Does
 
-ScreenHand gives AI agents seven capabilities:
+ScreenHand gives AI agents eight capabilities:
 
 ### Desktop Control — 19 tools
 Click buttons, type text, read UI trees, navigate menus, drag, scroll — all via native Accessibility APIs in ~50ms. Works with any app: Finder, Notes, VS Code, Xcode, System Settings, etc.
@@ -145,6 +145,9 @@ Gets smarter every session. Logs tool calls, saves winning strategies, tracks er
 ### App Mastery Map — automatic per-app spatial understanding
 Builds a persistent reverse-engineered blueprint of every app from normal tool usage. 8 features record automatically: page zones, navigation graph (BFS pathfinding), hierarchy, I/O contracts, state machine, element visibility, timing profiles, and ready signals. Mastery levels (beginner → pro → expert → grandmaster) honestly reflect how well ScreenHand knows each app. Maps stored at `~/.screenhand/app-maps/`.
 
+### Website Feature Discovery — real features, not generic ladders
+`discover_features` fetches an app's official website and extracts real product features (headings, feature cards, definition lists). Assigns difficulty tiers automatically and generates value-add features only ScreenHand can provide: bulk operations, cross-app export, content summarization, auto-organize, and change monitoring. No LLM calls needed — pure rule-based extraction. Features merge into the reference file and enrich the mastery ladder.
+
 ### Jobs & Orchestration — 34 tools
 Queue multi-step jobs, run them via background worker daemon, coordinate multiple AI agents with session leases, detect stalls, auto-recover. Survives client restarts.
 
@@ -294,7 +297,7 @@ Accessibility: ~50ms. Chrome CDP: ~10ms (background, no focus needed). OCR: ~600
 ```bash
 git clone https://github.com/manushi4/screenhand.git
 cd screenhand && npm install && npm run build:native
-npm test   # 1331 tests, 54 files
+npm test   # 1405 tests, 56 files
 ```
 
 ## Contact

package/dist/mcp-desktop.js

@@ -69,6 +69,7 @@ import os from "node:os";
 import { MenuScanner } from "./src/ingestion/menu-scanner.js";
 import { DocParser } from "./src/ingestion/doc-parser.js";
 import { TutorialExtractor } from "./src/ingestion/tutorial-extractor.js";
+import { extractFeaturesFromHTML } from "./src/ingestion/feature-extractor.js";
 import { CoverageAuditor } from "./src/ingestion/coverage-auditor.js";
 import { ReferenceMerger } from "./src/ingestion/reference-merger.js";
 import { PlaybookPublisher } from "./src/community/publisher.js";
@@ -280,6 +281,7 @@ coverage_report(bundleId, appName) → tells you exactly what ScreenHand knows
 - "0 selectors, 0 flows" → LEARN FIRST (Step 0a)
 - "Has selectors + flows" → GO (skip to Step 1)
 - "Has error patterns for your tool" → use *_with_fallback tools
+- "Website features: 0" → run discover_features first (Step 0b)
 
 learning_status(bundleId) → tells you WHICH tools to use
 - AX score > 0.9 → use ui_press/ui_tree (fastest, ~50ms)
@@ -294,6 +296,16 @@ platform_guide("platform")       → load curated selectors/flows/errors
 memory_recall("task description") → reuse past strategies
 Then go to Step 1.
 
+### Step 0b: DISCOVER FEATURES (if website features = 0)
+discover_features(url, bundleId, appName) → fetch official app website, extract real features
+  → parses headings, feature cards, definition lists from HTML
+  → assigns levels: beginner/pro/expert/grandmaster
+  → generates value-add features: bulk ops, cross-app, summarize, organize, monitor
+  → merges into reference file, enriches the feature ladder
+  → coverage_report will now show real feature count
+  Priority: discover_features BEFORE scan_menu_bar (features give meaningful ladder)
+Then continue to Step 0a or Step 1.
+
 ### Step 1: SEE
 perception_start()               → turns on continuous monitoring (3 rates: AX 100ms, CDP 300ms, Vision 1s)
 world_state()                    → verify windows + controls are tracked
@@ -4219,6 +4231,27 @@ server.tool("click_with_fallback", "Click a target by text using the canonical f
                     }
                     throw new Error("Target not found via OCR");
                 }
+                case "window_buffer": {
+                    // Last resort: capture GPU window buffer (works even when window is hidden),
+                    // OCR it, find target text, translate window-relative to screen-absolute coords
+                    const wbWindowId = await resolveWindowId(targetPid);
+                    if (!wbWindowId)
+                        throw new Error("No window found for window_buffer capture");
+                    const wbShot = await bridge.call("cg.captureWindow", { windowId: wbWindowId });
+                    const wbMatches = await bridge.call("vision.findText", { imagePath: wbShot.path, searchText: target });
+                    const wbMatch = Array.isArray(wbMatches) ? wbMatches[0] : null;
+                    if (!wbMatch?.bounds)
+                        throw new Error("Target not found via window buffer OCR");
+                    // Translate window-relative coords to screen-absolute
+                    const allWins = await bridge.call("app.windows");
+                    const winInfo = allWins.find((w) => w.windowId === wbWindowId);
+                    const winX = winInfo?.bounds?.x ?? 0;
+                    const winY = winInfo?.bounds?.y ?? 0;
+                    const absX = winX + wbMatch.bounds.x + wbMatch.bounds.width / 2;
+                    const absY = winY + wbMatch.bounds.y + wbMatch.bounds.height / 2;
+                    await bridge.call("cg.mouseClick", { x: absX, y: absY });
+                    return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: `${target} at (${Math.round(absX)},${Math.round(absY)}) [window_buffer]` };
+                }
             }
             throw new Error(`Unknown method: ${method}`);
         }
@@ -4553,6 +4586,22 @@ server.tool("read_with_fallback", "Read text content from the screen or a specif
                     const ocr = await bridge.call("vision.ocr", { imagePath: shot.path });
                     return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: ocr.text?.slice(0, 4000) ?? "" };
                 }
+                case "window_buffer": {
+                    // GPU window buffer capture — reads content even when window is behind other apps
+                    const rbWindowId = await resolveWindowId(targetPid);
+                    if (!rbWindowId)
+                        throw new Error("No window found for window_buffer read");
+                    const rbShot = await bridge.call("cg.captureWindow", { windowId: rbWindowId });
+                    if (target) {
+                        const rbMatches = await bridge.call("vision.findText", { imagePath: rbShot.path, searchText: target });
+                        const rbMatch = Array.isArray(rbMatches) ? rbMatches[0] : null;
+                        if (!rbMatch)
+                            throw new Error("Text not found via window buffer OCR");
+                        return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: rbMatch.text };
+                    }
+                    const rbOcr = await bridge.call("vision.ocr", { imagePath: rbShot.path });
+                    return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: rbOcr.text?.slice(0, 4000) ?? "" };
+                }
             }
             throw new Error(`Method ${method} does not support read`);
         }
@@ -4668,6 +4717,29 @@ server.tool("locate_with_fallback", "Find an element's position on screen using
                     const b = match.bounds;
                     return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: `${target} at (${b.x},${b.y} ${b.width}x${b.height})` };
                 }
+                case "window_buffer": {
+                    // GPU window buffer capture + OCR — works even when window is hidden
+                    const lbWindowId = await resolveWindowId(targetPid);
+                    if (!lbWindowId)
+                        throw new Error("No window found for window_buffer locate");
+                    const lbShot = await bridge.call("cg.captureWindow", { windowId: lbWindowId });
+                    const lbMatches = await bridge.call("vision.findText", { imagePath: lbShot.path, searchText: target });
+                    const lbMatch = Array.isArray(lbMatches) ? lbMatches[0] : null;
+                    if (!lbMatch?.bounds)
+                        throw new Error("Target not found via window buffer OCR");
+                    // Translate window-relative to screen-absolute bounds
+                    const lbWins = await bridge.call("app.windows");
+                    const lbWinInfo = lbWins.find((w) => w.windowId === lbWindowId);
+                    const lbOffX = lbWinInfo?.bounds?.x ?? 0;
+                    const lbOffY = lbWinInfo?.bounds?.y ?? 0;
+                    const lbBounds = {
+                        x: lbOffX + lbMatch.bounds.x,
+                        y: lbOffY + lbMatch.bounds.y,
+                        width: lbMatch.bounds.width,
+                        height: lbMatch.bounds.height,
+                    };
+                    return { ok: true, method, durationMs: Date.now() - start, fallbackFrom: null, retries: attempt, error: null, target: `${target} at (${lbBounds.x},${lbBounds.y} ${lbBounds.width}x${lbBounds.height}) [window_buffer]` };
+                }
             }
             throw new Error(`Method ${method} does not support locate`);
         }
@@ -6353,6 +6425,65 @@ server.tool("ingest_tutorial", "Extract structured playbook steps from a video t
             }],
     };
 });
+server.tool("discover_features", "Extract features from an app's official website and generate ScreenHand value-add features. Fetches the page, parses feature headings/cards/lists, assigns difficulty levels, and generates bulk/cross-app/intelligence/organization/monitoring value-adds. Merges into the reference file and enriches the feature ladder.", {
+    url: z.string().url().describe("Official app website URL (e.g. https://www.apple.com/notes)"),
+    bundleId: z.string().describe("macOS bundle ID (e.g. com.apple.Notes)"),
+    appName: z.string().describe("Human-readable app name (e.g. Notes)"),
+}, async ({ url, bundleId, appName }) => {
+    // SSRF protection: only allow http/https to public hosts
+    const parsed = new URL(url);
+    if (!["http:", "https:"].includes(parsed.protocol)) {
+        throw new Error("Only http/https URLs are allowed");
+    }
+    const hostname = parsed.hostname.toLowerCase();
+    if (hostname === "localhost" ||
+        hostname === "metadata.google.internal" ||
+        /^(127\.|10\.|172\.(1[6-9]|2\d|3[01])\.|192\.168\.|169\.254\.|0\.|0x|::1|\[::1\])/.test(hostname) ||
+        /^\d+$/.test(hostname)) {
+        throw new Error("URL points to internal/private network — blocked for security");
+    }
+    const MAX_HTML_BYTES = 5 * 1024 * 1024; // 5MB
+    const resp = await fetch(url, {
+        headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" },
+        signal: AbortSignal.timeout(15000),
+        redirect: "follow",
+    });
+    if (!resp.ok)
+        throw new Error(`Failed to fetch ${url}: ${resp.status}`);
+    // Check Content-Length before buffering
+    const contentLength = resp.headers.get("content-length");
+    if (contentLength && parseInt(contentLength) > MAX_HTML_BYTES) {
+        throw new Error(`Response too large: ${contentLength} bytes (max ${MAX_HTML_BYTES})`);
+    }
+    const html = await resp.text();
+    if (html.length > MAX_HTML_BYTES) {
+        throw new Error(`Response body too large: ${html.length} chars (max ${MAX_HTML_BYTES})`);
+    }
+    const result = extractFeaturesFromHTML(html, appName, url);
+    const mergeResult = referenceMerger.mergeWebsiteFeatures(result, bundleId, appName);
+    const lines = [
+        `Feature discovery: ${appName} (${bundleId})`,
+        `Source: ${url}`,
+        `Website features: ${result.websiteFeatures.length}`,
+        `Value-add features: ${result.valueAddFeatures.length}`,
+        `Reference updated: ${mergeResult.filePath} (${mergeResult.added} new features added)`,
+        "",
+    ];
+    if (result.websiteFeatures.length > 0) {
+        lines.push("Website Features:");
+        for (const f of result.websiteFeatures) {
+            lines.push(`  [${f.level}] ${f.name}: ${f.description.slice(0, 80)}`);
+        }
+        lines.push("");
+    }
+    if (result.valueAddFeatures.length > 0) {
+        lines.push("ScreenHand Value-Adds:");
+        for (const f of result.valueAddFeatures) {
+            lines.push(`  [${f.category}] ${f.name}: ${f.description}`);
+        }
+    }
+    return { content: [{ type: "text", text: lines.join("\n") }] };
+});
 server.tool("coverage_report", "Check what ScreenHand knows about an app: shortcuts, selectors, flows, playbooks, error patterns, and stability %. Useful before complex workflows to decide strategy: learn first (if empty), go fast (if high coverage), or use fallback tools (if error patterns exist). Optional for quick actions.", {
     bundleId: z.string().describe("macOS bundle ID (e.g. com.blackmagic-design.DaVinciResolveLite)"),
     appName: z.string().describe("Human-readable app name"),
@@ -6375,6 +6506,7 @@ server.tool("coverage_report", "Check what ScreenHand knows about an app: shortc
         `  Flows: ${report.flowsKnown}`,
         `  Playbooks: ${report.playbooksAvailable}`,
         `  Error patterns: ${report.errorsDocumented}`,
+        `  Website features: ${report.websiteFeaturesKnown}`,
     ];
     if (report.selectorStabilityScore > 0) {
         lines.push(`  Selector stability: ${(report.selectorStabilityScore * 100).toFixed(0)}%`);

package/dist/src/ingestion/coverage-auditor.js

@@ -63,6 +63,13 @@ export class CoverageAuditor {
                 errorsDocumented += ref.errors.length;
             }
         }
+        // Count website features
+        let websiteFeaturesKnown = 0;
+        for (const ref of refs) {
+            const wf = ref.websiteFeatures;
+            if (Array.isArray(wf))
+                websiteFeaturesKnown += wf.length;
+        }
         // Compare menu scan against reference shortcuts
         const menuPathsNotCovered = [];
         const shortcutsNotInReference = [];
@@ -153,6 +160,9 @@ export class CoverageAuditor {
         if (errorsDocumented === 0) {
             highValueGaps.push("No error patterns documented — errors will be learned automatically over time");
         }
+        if (websiteFeaturesKnown === 0) {
+            highValueGaps.push("No website features extracted — run discover_features to learn app capabilities from official website");
+        }
         if (workflowsWithNoPlaybook.length > 0) {
             highValueGaps.push(`Common workflows without playbooks: ${workflowsWithNoPlaybook.join(", ")}`);
         }
@@ -167,6 +177,7 @@ export class CoverageAuditor {
             flowsKnown,
             playbooksAvailable: playbooks.length,
             errorsDocumented,
+            websiteFeaturesKnown,
             menuPathsNotCovered: menuPathsNotCovered.slice(0, 50),
             shortcutsNotInReference: shortcutsNotInReference.slice(0, 50),
             workflowsWithNoPlaybook,

package/dist/src/ingestion/feature-extractor.js

@@ -0,0 +1,366 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+// ── Level assignment keywords ─────────────────────────────────────
+/** Single-word keywords checked individually */
+const BEGINNER_WORDS = new Set([
+    "basic", "create", "view", "share", "read",
+    "browse", "search", "home", "start", "launch", "write",
+]);
+const PRO_WORDS = new Set([
+    "organize", "format", "customize", "template", "tag", "folder",
+    "sort", "filter", "pin", "archive", "move", "rename", "duplicate",
+    "favorites", "bookmark", "list", "table", "style", "font",
+]);
+const EXPERT_WORDS = new Set([
+    "automate", "shortcut", "export", "import", "collaborate", "sync",
+    "scan", "link", "mention", "embed", "attachment", "password",
+    "encrypt", "lock", "version", "history", "recover", "backup",
+]);
+const GRANDMASTER_WORDS = new Set([
+    "api", "integrate", "plugin", "advanced", "workflow", "script",
+    "extension", "developer", "sdk", "automation", "pipeline", "webhook",
+]);
+/** Multi-word phrases checked via substring match */
+const GRANDMASTER_PHRASES = [
+    "custom action", "get started",
+];
+// ── HTML entity decoding ──────────────────────────────────────────
+const HTML_ENTITIES = {
+    "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"',
+    "&#39;": "'", "&apos;": "'", "&nbsp;": " ",
+    "&#x27;": "'", "&#x2F;": "/",
+};
+function decodeHTMLEntities(text) {
+    let result = text;
+    for (const [entity, char] of Object.entries(HTML_ENTITIES)) {
+        result = result.replaceAll(entity, char);
+    }
+    // Numeric entities: &#123; or &#x1A;
+    // Only decode printable chars (>= 0x20), skip control chars
+    result = result.replace(/&#(\d+);/g, (orig, code) => {
+        const n = Number(code);
+        return n >= 0x20 && n !== 0x7F ? String.fromCharCode(n) : "";
+    });
+    result = result.replace(/&#x([0-9a-fA-F]+);/g, (orig, hex) => {
+        const n = parseInt(hex, 16);
+        return n >= 0x20 && n !== 0x7F ? String.fromCharCode(n) : "";
+    });
+    return result;
+}
+// ── Strip control characters ──────────────────────────────────────
+function stripControlChars(text) {
+    // Remove ASCII control chars (0x00-0x08, 0x0B, 0x0C, 0x0E-0x1F, 0x7F)
+    // Preserve \t (0x09), \n (0x0A), \r (0x0D)
+    return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "");
+}
+// ── Strip HTML tags — O(n) single-pass state machine ──────────────
+function stripTags(html) {
+    let result = "";
+    let inTag = false;
+    for (let i = 0; i < html.length; i++) {
+        const ch = html[i];
+        if (ch === "<") {
+            inTag = true;
+            continue;
+        }
+        if (ch === ">") {
+            inTag = false;
+            continue;
+        }
+        if (!inTag)
+            result += ch;
+    }
+    return result.trim();
+}
+// ── Strip script and style blocks before processing ───────────────
+function stripScriptsAndStyles(html) {
+    // Remove <script>...</script> and <style>...</style> blocks
+    // Use bounded match to prevent backtracking on malformed HTML
+    return html
+        .replace(/<script\b[^>]*>[\s\S]{0,100000}?<\/script>/gi, "")
+        .replace(/<style\b[^>]*>[\s\S]{0,100000}?<\/style>/gi, "")
+        .replace(/<!--[\s\S]{0,50000}?-->/g, "");
+}
+// ── Clean extracted text ──────────────────────────────────────────
+function cleanText(rawHtml) {
+    return stripControlChars(decodeHTMLEntities(stripTags(rawHtml)));
+}
+// ── Normalize feature name to ID ──────────────────────────────────
+function nameToId(name) {
+    return name
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "_")
+        .replace(/^_|_$/g, "")
+        .slice(0, 80);
+}
+// ── Assign level based on keywords ────────────────────────────────
+function assignLevel(name, description) {
+    const text = `${name} ${description}`.toLowerCase();
+    const words = text.split(/\s+/);
+    // Check multi-word grandmaster phrases first
+    for (const phrase of GRANDMASTER_PHRASES) {
+        if (text.includes(phrase))
+            return "grandmaster";
+    }
+    // Check single-word keywords
+    for (const w of words) {
+        if (GRANDMASTER_WORDS.has(w))
+            return "grandmaster";
+    }
+    for (const w of words) {
+        if (EXPERT_WORDS.has(w))
+            return "expert";
+    }
+    for (const w of words) {
+        if (PRO_WORDS.has(w))
+            return "pro";
+    }
+    for (const w of words) {
+        if (BEGINNER_WORDS.has(w))
+            return "beginner";
+    }
+    // Fallback: longer descriptions suggest complexity
+    if (description.length > 80)
+        return "pro";
+    return "beginner";
+}
+// ── Main: Extract features from HTML ──────────────────────────────
+export function extractFeaturesFromHTML(html, appName, url) {
+    const seen = new Map();
+    // Pre-process: strip scripts, styles, and comments to avoid content leakage
+    const cleanHtml = stripScriptsAndStyles(html);
+    // ── 1. Extract from headings (h2, h3, h4) ─────────────────────
+    const headingRegex = /<h([2-4])[^>]*>([\s\S]*?)<\/h\1>/gi;
+    let match;
+    while ((match = headingRegex.exec(cleanHtml)) !== null) {
+        const text = cleanText(match[2]).trim();
+        if (!text || text.length < 3 || text.length > 120)
+            continue;
+        // Skip navigation/generic headings
+        if (/^(menu|nav|footer|header|copyright|legal|privacy)$/i.test(text))
+            continue;
+        const id = nameToId(text);
+        if (!id || seen.has(id))
+            continue;
+        // Try to find a nearby paragraph for description
+        const afterHeading = cleanHtml.slice(match.index + match[0].length, match.index + match[0].length + 500);
+        const pMatch = afterHeading.match(/<p[^>]*>([\s\S]*?)<\/p>/i);
+        const description = pMatch
+            ? cleanText(pMatch[1]).trim().slice(0, 200)
+            : text;
+        seen.set(id, {
+            id,
+            name: text,
+            description,
+            sourceHeading: text,
+            level: assignLevel(text, description),
+        });
+    }
+    // ── 2. Extract from feature cards ──────────────────────────────
+    // Pattern: <div class="...feature..."> with a heading inside
+    // Bounded to 3000 chars to prevent backtracking on deeply nested divs
+    const cardRegex = /<div[^>]*class="[^"]*feature[^"]*"[^>]*>([\s\S]{0,3000}?)<\/div>/gi;
+    while ((match = cardRegex.exec(cleanHtml)) !== null) {
+        const cardHtml = match[1];
+        // Find heading inside card
+        const innerHeading = cardHtml.match(/<h[2-5][^>]*>([\s\S]*?)<\/h[2-5]>/i);
+        if (!innerHeading)
+            continue;
+        const name = cleanText(innerHeading[1]).trim();
+        if (!name || name.length < 3)
+            continue;
+        const id = nameToId(name);
+        if (seen.has(id))
+            continue;
+        // Description from paragraph
+        const pMatch = cardHtml.match(/<p[^>]*>([\s\S]*?)<\/p>/i);
+        const description = pMatch
+            ? cleanText(pMatch[1]).trim().slice(0, 200)
+            : name;
+        seen.set(id, {
+            id,
+            name,
+            description,
+            sourceHeading: name,
+            level: assignLevel(name, description),
+        });
+    }
+    // ── 3. Extract from list items (feature lists) ─────────────────
+    // Look for <ul> or <ol> near "feature" context
+    const listItemRegex = /<li[^>]*>([\s\S]*?)<\/li>/gi;
+    while ((match = listItemRegex.exec(cleanHtml)) !== null) {
+        const rawText = cleanText(match[1]).trim();
+        // Only accept list items that look like feature names (not too long, not too short)
+        if (!rawText || rawText.length < 5 || rawText.length > 100)
+            continue;
+        // Skip items that look like navigation
+        if (/^(home|about|contact|blog|pricing|sign up|log in|download)$/i.test(rawText))
+            continue;
+        // Skip items with too many sentences (likely paragraphs, not feature names)
+        if ((rawText.match(/\./g) ?? []).length > 2)
+            continue;
+        const id = nameToId(rawText);
+        if (seen.has(id))
+            continue;
+        // Only add if the surrounding context mentions "feature" (within 500 chars before)
+        const contextBefore = cleanHtml.slice(Math.max(0, match.index - 500), match.index).toLowerCase();
+        if (!contextBefore.includes("feature") && !contextBefore.includes("capability") && !contextBefore.includes("what you can"))
+            continue;
+        seen.set(id, {
+            id,
+            name: rawText,
+            description: rawText,
+            sourceHeading: rawText,
+            level: assignLevel(rawText, ""),
+        });
+    }
+    // ── 4. Extract from definition lists ───────────────────────────
+    const dtRegex = /<dt[^>]*>([\s\S]*?)<\/dt>\s*<dd[^>]*>([\s\S]*?)<\/dd>/gi;
+    while ((match = dtRegex.exec(cleanHtml)) !== null) {
+        const name = cleanText(match[1]).trim();
+        const desc = cleanText(match[2]).trim().slice(0, 200);
+        if (!name || name.length < 3)
+            continue;
+        const id = nameToId(name);
+        if (seen.has(id))
+            continue;
+        seen.set(id, {
+            id,
+            name,
+            description: desc || name,
+            sourceHeading: name,
+            level: assignLevel(name, desc),
+        });
+    }
+    const websiteFeatures = [...seen.values()];
+    // ── Generate value-add features ────────────────────────────────
+    const valueAddFeatures = generateValueAddFeatures(appName, websiteFeatures);
+    return {
+        appName,
+        url,
+        websiteFeatures,
+        valueAddFeatures,
+        extractedAt: new Date().toISOString(),
+    };
+}
+const VALUE_ADD_RULES = [
+    {
+        category: "bulk",
+        trigger: (fs) => fs.some((f) => /\b(creates?|adds?)\b/i.test(f.name)),
+        generate: (app) => ({
+            id: "bulk_create",
+            name: "Bulk Create",
+            description: `Create multiple ${app} items from a list or template`,
+            category: "bulk",
+            level: "pro",
+        }),
+    },
+    {
+        category: "bulk",
+        trigger: (fs) => fs.some((f) => /\b(deletes?|removes?|trash)\b/i.test(f.name)),
+        generate: (app) => ({
+            id: "bulk_delete",
+            name: "Bulk Delete",
+            description: `Delete multiple ${app} items matching criteria`,
+            category: "bulk",
+            level: "pro",
+        }),
+    },
+    {
+        category: "bulk",
+        trigger: (fs) => fs.some((f) => /\b(exports?|downloads?)\b/i.test(f.name)),
+        generate: (app) => ({
+            id: "bulk_export",
+            name: "Bulk Export",
+            description: `Export all ${app} items at once`,
+            category: "bulk",
+            level: "pro",
+        }),
+    },
+    {
+        category: "organization",
+        trigger: (fs) => fs.some((f) => /\b(folders?|tags?|labels?|categor(?:y|ies))\b/i.test(f.name)),
+        generate: (app) => ({
+            id: "auto_organize",
+            name: "Auto-Organize",
+            description: `Sort and organize ${app} items by content, date, or type`,
+            category: "organization",
+            level: "expert",
+        }),
+    },
+    {
+        category: "organization",
+        trigger: (fs) => fs.some((f) => /\b(search|finds?)\b/i.test(f.name)),
+        generate: (app) => ({
+            id: "smart_search",
+            name: "Smart Search",
+            description: `Search across all ${app} content with pattern matching`,
+            category: "organization",
+            level: "pro",
+        }),
+    },
+    {
+        category: "intelligence",
+        trigger: () => true, // Always available
+        generate: (app) => ({
+            id: "summarize_all",
+            name: "Summarize",
+            description: `Read and summarize all ${app} content`,
+            category: "intelligence",
+            level: "expert",
+        }),
+    },
+    {
+        category: "intelligence",
+        trigger: (fs) => fs.some((f) => /\b(duplicates?|similar)\b/i.test(f.name)),
+        generate: (app) => ({
+            id: "find_duplicates",
+            name: "Find Duplicates",
+            description: `Identify duplicate or near-duplicate ${app} items`,
+            category: "intelligence",
+            level: "expert",
+        }),
+    },
+    {
+        category: "cross_app",
+        trigger: (fs) => fs.some((f) => /\b(shares?|exports?|sends?)\b/i.test(f.name)),
+        generate: (app) => ({
+            id: "cross_app_export",
+            name: "Cross-App Export",
+            description: `Export ${app} content to other apps automatically`,
+            category: "cross_app",
+            level: "expert",
+        }),
+    },
+    {
+        category: "cross_app",
+        trigger: (fs) => fs.some((f) => /\bimports?\b/i.test(f.name)),
+        generate: (app) => ({
+            id: "cross_app_import",
+            name: "Cross-App Import",
+            description: `Import content from other apps into ${app}`,
+            category: "cross_app",
+            level: "expert",
+        }),
+    },
+    {
+        category: "monitoring",
+        trigger: () => true, // Always available
+        generate: (app) => ({
+            id: "change_monitor",
+            name: "Change Monitor",
+            description: `Monitor ${app} for changes and notify`,
+            category: "monitoring",
+            level: "grandmaster",
+        }),
+    },
+];
+export function generateValueAddFeatures(appName, websiteFeatures) {
+    const results = [];
+    for (const rule of VALUE_ADD_RULES) {
+        if (rule.trigger(websiteFeatures)) {
+            results.push(rule.generate(appName));
+        }
+    }
+    return results;
+}

package/dist/src/ingestion/reference-merger.js

@@ -95,6 +95,23 @@ export class ReferenceMerger {
         const filePath = this.save(ref);
         return { filePath, added };
     }
+    /**
+     * Merge website-extracted features into the reference file.
+     */
+    mergeWebsiteFeatures(result, bundleId, appName) {
+        const ref = this.loadOrCreate(bundleId, appName);
+        const existing = ref.websiteFeatures;
+        const existingIds = new Set((existing ?? []).map((f) => f.id));
+        const newFeatures = result.websiteFeatures.filter((f) => !existingIds.has(f.id));
+        ref.websiteFeatures = [...(existing ?? []), ...newFeatures];
+        // Merge value-add features by id (don't overwrite existing)
+        const existingVA = ref.valueAddFeatures;
+        const existingVAIds = new Set((existingVA ?? []).map((f) => f.id));
+        const newVA = result.valueAddFeatures.filter((f) => !existingVAIds.has(f.id));
+        ref.valueAddFeatures = [...(existingVA ?? []), ...newVA];
+        const filePath = this.save(ref);
+        return { filePath, added: newFeatures.length };
+    }
     /**
      * Merge errors/solutions into reference.
      */

package/dist/src/learning/engine.js

@@ -126,6 +126,26 @@ export class LearningEngine {
     rankSensors(bundleId) {
         return this.sensors.rank(bundleId);
     }
+    /**
+     * Detect whether an app is "vision-only" — AX can't see its content,
+     * so window buffer capture + OCR is the only viable perception source.
+     * Returns true when AX has failed enough times with a low score and
+     * at least one other source (vision/ocr) has succeeded.
+     */
+    isVisionOnlyApp(bundleId) {
+        const ranked = this.sensors.rank(bundleId);
+        if (ranked.length < 2)
+            return false;
+        const ax = ranked.find(r => r.sourceType === "ax");
+        const vision = ranked.find(r => r.sourceType === "vision" || r.sourceType === "ocr");
+        // AX score near zero + vision/ocr has some success
+        if (ax && ax.score < 0.15 && vision && vision.score > 0.3)
+            return true;
+        // No AX entry at all but vision works
+        if (!ax && vision && vision.score > 0.3)
+            return true;
+        return false;
+    }
     /**
      * Query verified UI patterns for a given app, optionally filtered by tool.
      */

package/dist/src/perception/coordinator.js

@@ -768,13 +768,18 @@ export class PerceptionCoordinator extends EventEmitter {
         // Safe CLI mode is already enabled via setSafeCLI() in start().
         // This allows vision/OCR for canvas-heavy apps like Canva in Chrome.
         // Skip vision if learning engine shows it consistently fails for this app,
-        // but retry every 20th cycle to re-evaluate (apps may gain windows later)
+        // but retry every 20th cycle to re-evaluate (apps may gain windows later).
+        // Exception: vision-only apps (AX blind) — vision/OCR is their ONLY perception
+        // source, so never skip it. Window buffer capture works even when window is hidden.
         if (this.learningEngine && this.activeAppContext) {
-            const ranked = this.learningEngine.rankSensors(this.activeAppContext.bundleId);
-            const visionRank = ranked.find(r => r.sourceType === "vision");
-            if (visionRank && visionRank.score < 0.1 && ranked.length >= 2 && this.stats.slowCycles % 20 !== 0) {
-                this.stats.slowCycles++;
-                return; // Vision consistently fails for this app — skip (retry every 20th cycle)
+            const isVisionOnly = this.learningEngine.isVisionOnlyApp(this.activeAppContext.bundleId);
+            if (!isVisionOnly) {
+                const ranked = this.learningEngine.rankSensors(this.activeAppContext.bundleId);
+                const visionRank = ranked.find(r => r.sourceType === "vision");
+                if (visionRank && visionRank.score < 0.1 && ranked.length >= 2 && this.stats.slowCycles % 20 !== 0) {
+                    this.stats.slowCycles++;
+                    return; // Vision consistently fails for this app — skip (retry every 20th cycle)
+                }
             }
         }
         const timestamp = new Date().toISOString();
@@ -860,14 +865,24 @@ export class PerceptionCoordinator extends EventEmitter {
                     },
                 });
             }
-            // Record vision sensor outcome
+            // Record vision sensor outcome — also record as window_buffer for vision-only apps
+            // so the fallback chain knows this source works for element location
             if (this.learningEngine && this.activeAppContext) {
+                const latencyMs = Date.now() - new Date(timestamp).getTime();
                 this.learningEngine.recordSensorOutcome({
                     bundleId: this.activeAppContext.bundleId,
                     sourceType: "vision",
                     success: !!diffEvent,
-                    latencyMs: Date.now() - new Date(timestamp).getTime(),
+                    latencyMs,
                 });
+                if (this.learningEngine.isVisionOnlyApp(this.activeAppContext.bundleId) && ocrEvent) {
+                    this.learningEngine.recordSensorOutcome({
+                        bundleId: this.activeAppContext.bundleId,
+                        sourceType: "window_buffer",
+                        success: true,
+                        latencyMs,
+                    });
+                }
             }
         }
         catch {

package/dist/src/perception/types.js

@@ -21,6 +21,7 @@ export const DEFAULT_PERCEPTION_CONFIG = {
     enableAX: true,
     enableCDP: true,
     enableVision: true,
+    enableWindowBuffer: true,
     maxROIsPerCycle: 3,
     skipCaptureLock: false,
 };

package/dist/src/runtime/execution-contract.js

@@ -23,7 +23,7 @@
  */
 // ── 1. Fallback Chain ──────────────────────────────────────────────────
 /** Ordered list of execution methods, from fastest/most reliable to slowest/least reliable */
-const EXECUTION_METHODS = ["ax", "cdp", "ocr", "coordinates"];
+const EXECUTION_METHODS = ["ax", "cdp", "ocr", "window_buffer", "coordinates"];
 const METHOD_CAPABILITIES = {
     ax: {
         method: "ax",
@@ -61,6 +61,18 @@ const METHOD_CAPABILITIES = {
         requiresBridge: true,
         requiresCDP: false,
     },
+    window_buffer: {
+        method: "window_buffer",
+        canClick: true,
+        canType: false,
+        canRead: true,
+        canLocate: true,
+        canSelect: false,
+        canScroll: false,
+        avgLatencyMs: 350,
+        requiresBridge: true,
+        requiresCDP: false,
+    },
     coordinates: {
         method: "coordinates",
         canClick: true,
@@ -90,6 +102,7 @@ const SENSOR_TO_METHOD = {
     chrome: "cdp",
     ocr: "ocr",
     vision: "ocr",
+    window_buffer: "window_buffer",
     coordinates: "coordinates",
 };
 /**

package/dist/src/state/ladder-generator.js

@@ -32,8 +32,10 @@ export function generateLadderFromReference(ref) {
     const selectorGroups = ref.selectors ?? {};
     const flows = ref.flows ?? {};
     // Minimum threshold: need at least 2 meaningful selector groups
+    // (but website features can stand alone — they come from official sources)
     const meaningfulGroups = Object.keys(selectorGroups).filter(k => !SKIP_GROUPS.has(k));
-    if (meaningfulGroups.length < 2 && Object.keys(flows).length < 2) {
+    const hasWebsiteFeatures = (ref.websiteFeatures?.length ?? 0) > 0 || (ref.valueAddFeatures?.length ?? 0) > 0;
+    if (meaningfulGroups.length < 2 && Object.keys(flows).length < 2 && !hasWebsiteFeatures) {
         return { ladder: [], signals: {}, hash: computeHash(ref) };
     }
     // Track which flow names are already covered by selector groups
@@ -76,6 +78,50 @@ export function generateLadderFromReference(ref) {
         const keywords = extractKeywordsFromFlow(flowName, flow);
         signals[featureId] = keywords;
     }
+    // ── Step 2.5: Features from website extraction ─────────────────
+    if (ref.websiteFeatures) {
+        for (const wf of ref.websiteFeatures) {
+            const featureId = `web_${wf.id}`;
+            if (features.some((f) => f.id === featureId))
+                continue;
+            // Skip if an exact selector group or flow already covers this feature
+            // (use exact key match, not fuzzy flowNamesRelated — avoids false positives
+            // where a short website feature id like "export" matches "export_pdf_flow")
+            if (selectorGroups[wf.id] !== undefined || flows[wf.id] !== undefined)
+                continue;
+            const level = (["beginner", "pro", "expert", "grandmaster"].includes(wf.level)
+                ? wf.level
+                : "beginner");
+            const weight = assignWeight(wf.id, 0, level);
+            features.push({
+                id: featureId,
+                description: wf.description || wf.name,
+                level,
+                weight,
+                critical: false,
+            });
+            signals[featureId] = extractKeywordsFromName(wf.name);
+        }
+    }
+    // ── Step 2.6: ScreenHand value-add features ───────────────────
+    if (ref.valueAddFeatures) {
+        for (const va of ref.valueAddFeatures) {
+            const featureId = `va_${va.id}`;
+            if (features.some((f) => f.id === featureId))
+                continue;
+            const level = (["pro", "expert", "grandmaster"].includes(va.level)
+                ? va.level
+                : "expert");
+            features.push({
+                id: featureId,
+                description: va.description,
+                level,
+                weight: 2,
+                critical: false,
+            });
+            signals[featureId] = extractKeywordsFromName(va.name);
+        }
+    }
     // ── Step 3: Sort by level progression ──────────────────────────
     const levelOrder = {
         beginner: 0, pro: 1, expert: 2, grandmaster: 3,
@@ -204,6 +250,17 @@ function extractKeywordsFromFlow(flowName, flow) {
     }
     return deduplicateArray(keywords);
 }
+// ── Keyword extraction from feature name ──────────────────────
+function extractKeywordsFromName(name) {
+    const keywords = [];
+    for (const part of name.split(/[\s_-]+/)) {
+        const lower = part.toLowerCase().replace(/[^a-z0-9]/g, "");
+        if (lower.length > 2 && !STOP_WORDS.has(lower)) {
+            keywords.push(lower);
+        }
+    }
+    return deduplicateArray(keywords);
+}
 // ── Flow-to-selector group name matching ─────────────────────────
 function flowNamesRelated(groupName, flowName) {
     const gParts = new Set(groupName.split("_"));
@@ -222,6 +279,8 @@ function computeHash(ref) {
     const keys = [
         ...Object.keys(ref.selectors ?? {}).sort(),
         ...Object.keys(ref.flows ?? {}).sort(),
+        ...(ref.websiteFeatures ?? []).map((f) => f.id).sort(),
+        ...(ref.valueAddFeatures ?? []).map((f) => f.id).sort(),
     ].join("|");
     // Simple string hash (djb2)
     let hash = 5381;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "screenhand",
-  "version": "0.3.8",
+  "version": "0.4.0",
   "mcpName": "io.github.manushi4/screenhand",
   "description": "Give AI eyes and hands on your desktop. ScreenHand is an open-source MCP server that lets Claude and other AI agents see your screen, click buttons, type text, and control any app on macOS and Windows.",
   "homepage": "https://screenhand.com",