screenhand 0.1.1 → 0.3.0

package/dist/src/ingestion/coverage-auditor.js

@@ -0,0 +1,233 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+import * as fs from "node:fs";
+import * as path from "node:path";
+/**
+ * CoverageAuditor — answers "How well do we know this app?"
+ *
+ * Compares reference files, playbooks, menu scans, and learning data
+ * to identify gaps and generate recommendations.
+ */
+/**
+ * Normalize a menu path shortcut name by extracting the last segment.
+ * "Edit > Undo" → "undo", "File.Save" → "save"
+ */
+function normalizeShortcutName(name) {
+    const parts = name.split(/[.>]/);
+    return (parts[parts.length - 1] ?? name).trim().toLowerCase();
+}
+/**
+ * Strip parenthetical descriptions from key combos.
+ * "Cmd+N (desktop app)" → "cmd+n"
+ */
+function normalizeKeyCombo(keys) {
+    return keys.replace(/\s*\([^)]*\)\s*/g, "").trim().toLowerCase();
+}
+export class CoverageAuditor {
+    referencesDir;
+    playbooksDir;
+    learningEngine;
+    goalStore;
+    constructor(referencesDir, playbooksDir, learningEngine, goalStore) {
+        this.referencesDir = referencesDir;
+        this.playbooksDir = playbooksDir;
+        this.learningEngine = learningEngine;
+        this.goalStore = goalStore;
+    }
+    /**
+     * Generate a full coverage report for an app.
+     */
+    audit(bundleId, appName, menuScan) {
+        const refs = this.loadReferences(bundleId);
+        const playbooks = this.loadPlaybooks(bundleId);
+        // Count what we know
+        let shortcutsKnown = 0;
+        let selectorsKnown = 0;
+        let flowsKnown = 0;
+        let errorsDocumented = 0;
+        for (const ref of refs) {
+            if (ref.shortcuts) {
+                for (const category of Object.values(ref.shortcuts)) {
+                    shortcutsKnown += Object.keys(category).length;
+                }
+            }
+            if (ref.selectors) {
+                for (const group of Object.values(ref.selectors)) {
+                    selectorsKnown += Object.keys(group).length;
+                }
+            }
+            if (ref.flows) {
+                flowsKnown += Object.keys(ref.flows).length;
+            }
+            if (ref.errors) {
+                errorsDocumented += ref.errors.length;
+            }
+        }
+        // Compare menu scan against reference shortcuts
+        const menuPathsNotCovered = [];
+        const shortcutsNotInReference = [];
+        if (menuScan) {
+            // Build a map keyed by normalized name → normalized keys for comparison
+            const refShortcuts = new Map();
+            for (const ref of refs) {
+                if (ref.shortcuts) {
+                    for (const category of Object.values(ref.shortcuts)) {
+                        for (const [name, keys] of Object.entries(category)) {
+                            refShortcuts.set(name.toLowerCase(), normalizeKeyCombo(keys));
+                        }
+                    }
+                }
+            }
+            for (const [menuPath, keys] of Object.entries(menuScan.shortcuts)) {
+                const normalizedName = normalizeShortcutName(menuPath);
+                const normalizedKeys = normalizeKeyCombo(keys);
+                const refKeys = refShortcuts.get(normalizedName);
+                if (refKeys !== normalizedKeys) {
+                    shortcutsNotInReference.push(`${menuPath}: ${keys}`);
+                }
+            }
+            // Find menu paths not covered at all
+            const flatPaths = this.flattenMenuPaths(menuScan.menuTree);
+            for (const p of flatPaths) {
+                const covered = refs.some((ref) => {
+                    if (!ref.flows)
+                        return false;
+                    return Object.values(ref.flows).some((f) => f.steps.some((s) => s.toLowerCase().includes(p.toLowerCase())));
+                });
+                if (!covered) {
+                    menuPathsNotCovered.push(p);
+                }
+            }
+        }
+        // Identify common workflows without playbooks
+        const COMMON_WORKFLOWS = [
+            "export", "import", "save as", "new project", "undo",
+            "preferences", "settings", "print", "share",
+        ];
+        const workflowsWithNoPlaybook = COMMON_WORKFLOWS.filter((w) => {
+            const hasPlaybook = playbooks.some((p) => p.name.toLowerCase().includes(w));
+            const hasFlow = refs.some((r) => r.flows && Object.keys(r.flows).some((k) => k.includes(w)));
+            return !hasPlaybook && !hasFlow;
+        });
+        // Quality scores from learning engine
+        let selectorStabilityScore = 0;
+        let playbookSuccessRate = 0;
+        let averageRecoveryTime = 0;
+        // Compute playbookSuccessRate from GoalStore
+        if (this.goalStore) {
+            const allGoals = this.goalStore.list();
+            const playbookGoals = allGoals.filter((g) => g.subgoals.some((sg) => sg.plan?.source === "playbook"));
+            if (playbookGoals.length > 0) {
+                const completed = playbookGoals.filter((g) => g.status === "completed").length;
+                playbookSuccessRate = completed / playbookGoals.length;
+            }
+        }
+        if (this.learningEngine) {
+            const summary = this.learningEngine.getAppSummary(bundleId);
+            if (summary.locatorEntries > 0) {
+                const entries = this.learningEngine.locators.getAllEntries()
+                    .filter((e) => e.key.startsWith(`${bundleId}::`));
+                if (entries.length > 0) {
+                    selectorStabilityScore =
+                        entries.reduce((sum, e) => sum + e.score, 0) / entries.length;
+                }
+            }
+            const recEntries = this.learningEngine.recovery.getAllEntries()
+                .filter((e) => e.key.endsWith(`::${bundleId}`));
+            if (recEntries.length > 0) {
+                averageRecoveryTime =
+                    recEntries.reduce((sum, e) => sum + e.avgDurationMs, 0) / recEntries.length;
+            }
+        }
+        // Generate recommendations
+        const highValueGaps = [];
+        if (shortcutsKnown === 0) {
+            highValueGaps.push("No shortcuts documented — run scan_menu_bar to extract keyboard shortcuts");
+        }
+        if (selectorsKnown === 0) {
+            highValueGaps.push("No selectors documented — run platform_explore to discover stable selectors");
+        }
+        if (playbooks.length === 0) {
+            highValueGaps.push("No playbooks available — record common workflows with playbook_record");
+        }
+        if (errorsDocumented === 0) {
+            highValueGaps.push("No error patterns documented — errors will be learned automatically over time");
+        }
+        if (workflowsWithNoPlaybook.length > 0) {
+            highValueGaps.push(`Common workflows without playbooks: ${workflowsWithNoPlaybook.join(", ")}`);
+        }
+        if (menuScan && shortcutsNotInReference.length > 10) {
+            highValueGaps.push(`${shortcutsNotInReference.length} shortcuts found in menu bar but missing from reference`);
+        }
+        return {
+            app: appName,
+            bundleId,
+            shortcutsKnown,
+            selectorsKnown,
+            flowsKnown,
+            playbooksAvailable: playbooks.length,
+            errorsDocumented,
+            menuPathsNotCovered: menuPathsNotCovered.slice(0, 50),
+            shortcutsNotInReference: shortcutsNotInReference.slice(0, 50),
+            workflowsWithNoPlaybook,
+            selectorStabilityScore,
+            playbookSuccessRate,
+            averageRecoveryTime,
+            highValueGaps,
+            generatedAt: new Date().toISOString(),
+        };
+    }
+    loadReferences(bundleId) {
+        const refs = [];
+        try {
+            const files = fs.readdirSync(this.referencesDir);
+            for (const file of files) {
+                if (!file.endsWith(".json"))
+                    continue;
+                try {
+                    const raw = fs.readFileSync(path.join(this.referencesDir, file), "utf-8");
+                    const ref = JSON.parse(raw);
+                    if (ref.bundleId === bundleId || ref.platform === bundleId) {
+                        refs.push(ref);
+                    }
+                }
+                catch { /* skip */ }
+            }
+        }
+        catch { /* dir not found */ }
+        return refs;
+    }
+    loadPlaybooks(bundleId) {
+        const playbooks = [];
+        try {
+            const files = fs.readdirSync(this.playbooksDir);
+            for (const file of files) {
+                if (!file.endsWith(".json"))
+                    continue;
+                try {
+                    const raw = fs.readFileSync(path.join(this.playbooksDir, file), "utf-8");
+                    const pb = JSON.parse(raw);
+                    if (pb.bundleId === bundleId || pb.platform === bundleId) {
+                        playbooks.push(pb);
+                    }
+                }
+                catch { /* skip */ }
+            }
+        }
+        catch { /* dir not found */ }
+        return playbooks;
+    }
+    flattenMenuPaths(nodes, prefix = []) {
+        const paths = [];
+        for (const node of nodes) {
+            const p = [...prefix, node.title];
+            if (!node.children || node.children.length === 0) {
+                paths.push(p.join(" > "));
+            }
+            else {
+                paths.push(...this.flattenMenuPaths(node.children, p));
+            }
+        }
+        return paths;
+    }
+}

package/dist/src/ingestion/doc-parser.js

@@ -0,0 +1,164 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+import { parseShortcutsFromHTML, parseShortcutsFromText, parseShortcutsFromMarkdown, } from "./shortcut-extractor.js";
+/**
+ * DocParser — extracts structured knowledge from documentation pages.
+ * Handles HTML, markdown, and plain text.
+ */
+export class DocParser {
+    /**
+     * Parse a documentation page and extract shortcuts, flows, and tips.
+     */
+    parse(content, url, format = "html") {
+        const title = this.extractTitle(content, format);
+        const shortcuts = this.extractShortcuts(content, format);
+        const flows = this.extractFlows(content, format);
+        const tips = this.extractTips(content, format);
+        return {
+            url,
+            title,
+            shortcuts,
+            flows,
+            tips,
+            parsedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Extract page title.
+     */
+    extractTitle(content, format) {
+        if (format === "html") {
+            const titleMatch = content.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+            if (titleMatch)
+                return titleMatch[1].replace(/<[^>]+>/g, "").trim();
+            const h1Match = content.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
+            if (h1Match)
+                return h1Match[1].replace(/<[^>]+>/g, "").trim();
+        }
+        else if (format === "markdown") {
+            const mdTitle = content.match(/^#\s+(.+)$/m);
+            if (mdTitle)
+                return mdTitle[1].trim();
+        }
+        return "Untitled";
+    }
+    /**
+     * Extract shortcuts from the content.
+     */
+    extractShortcuts(content, format) {
+        switch (format) {
+            case "html":
+                return parseShortcutsFromHTML(content);
+            case "markdown":
+                return parseShortcutsFromMarkdown(content);
+            case "text":
+                return parseShortcutsFromText(content);
+            default:
+                return [];
+        }
+    }
+    /**
+     * Extract workflow/how-to steps from documentation.
+     * Looks for numbered lists, step-by-step sections.
+     */
+    extractFlows(content, format) {
+        const flows = [];
+        const stripped = format === "html" ? this.stripHTML(content) : content;
+        // Find sections with numbered steps
+        // Pattern: heading followed by numbered list
+        const sectionRegex = /(?:^|\n)(?:#{1,3}\s+|<h[1-3][^>]*>)(.+?)(?:<\/h[1-3]>)?(?:\n|$)([\s\S]*?)(?=(?:\n(?:#{1,3}\s+|<h[1-3]))|$)/gi;
+        let match;
+        while ((match = sectionRegex.exec(stripped)) !== null) {
+            const heading = match[1].replace(/<[^>]+>/g, "").trim();
+            const body = match[2];
+            // Check if the section has numbered steps
+            const stepRegex = /(?:^|\n)\s*(?:(\d+)[.)]\s*|Step\s+\d+[:.]\s*)(.+)/gi;
+            const steps = [];
+            let stepMatch;
+            while ((stepMatch = stepRegex.exec(body)) !== null) {
+                const desc = stepMatch[2].trim();
+                if (desc.length > 5) {
+                    steps.push({
+                        description: desc,
+                        tool: this.inferTool(desc),
+                        params: this.inferParams(desc),
+                    });
+                }
+            }
+            if (steps.length >= 2) {
+                flows.push({ name: heading, steps });
+            }
+        }
+        return flows;
+    }
+    /**
+     * Extract tips/best practices from the content.
+     */
+    extractTips(content, format) {
+        const tips = [];
+        const stripped = format === "html" ? this.stripHTML(content) : content;
+        // Look for tip/note/important callouts
+        const tipRegex = /(?:tip|note|important|best practice|pro tip|hint)[:\s]*(.+?)(?:\n|$)/gi;
+        let match;
+        while ((match = tipRegex.exec(stripped)) !== null) {
+            const tip = match[1].trim();
+            if (tip.length > 10 && tip.length < 500) {
+                tips.push(tip);
+            }
+        }
+        return tips.slice(0, 20); // Cap tips
+    }
+    /**
+     * Infer the ScreenHand tool from a step description.
+     */
+    inferTool(description) {
+        const lower = description.toLowerCase();
+        if (lower.includes("click") && lower.includes("menu"))
+            return "menu_click";
+        if (lower.includes("click"))
+            return "click_text";
+        if (lower.includes("type") || lower.includes("enter"))
+            return "type_text";
+        if (lower.includes("press") || lower.includes("keyboard") || lower.includes("shortcut"))
+            return "key";
+        if (lower.includes("select"))
+            return "click_text";
+        if (lower.includes("drag"))
+            return "drag";
+        if (lower.includes("scroll"))
+            return "scroll";
+        if (lower.includes("navigate") || lower.includes("go to") || lower.includes("open"))
+            return "menu_click";
+        return undefined;
+    }
+    /**
+     * Infer tool params from a step description.
+     */
+    inferParams(description) {
+        // Extract quoted text as target
+        const quoteMatch = description.match(/["'"](.+?)["'"]/);
+        if (quoteMatch) {
+            return { text: quoteMatch[1] };
+        }
+        // Extract menu path: File > Export > Media
+        const menuMatch = description.match(/(?:click|go to|select|choose|open)\s+(.+?>.*)/i);
+        if (menuMatch) {
+            const path = menuMatch[1]
+                .split(/\s*>\s*/)
+                .map((s) => s.trim())
+                .filter(Boolean);
+            if (path.length >= 2) {
+                return { menuPath: path.join("/") };
+            }
+        }
+        return undefined;
+    }
+    stripHTML(html) {
+        return html
+            .replace(/<script[\s\S]*?<\/script>/gi, "")
+            .replace(/<style[\s\S]*?<\/style>/gi, "")
+            .replace(/<[^>]+>/g, " ")
+            .replace(/\s+/g, " ")
+            .trim();
+    }
+}

package/dist/src/ingestion/index.js

@@ -0,0 +1,8 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+export { MenuScanner } from "./menu-scanner.js";
+export { DocParser } from "./doc-parser.js";
+export { TutorialExtractor } from "./tutorial-extractor.js";
+export { ReferenceMerger } from "./reference-merger.js";
+export { CoverageAuditor } from "./coverage-auditor.js";
+export { parseShortcutsFromHTML, parseShortcutsFromText, parseShortcutsFromMarkdown, shortcutsToReferenceFormat, } from "./shortcut-extractor.js";

package/dist/src/ingestion/menu-scanner.js

@@ -0,0 +1,152 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * MenuScanner — scans an app's menu bar via AX tree, extracting
+ * all menu paths, keyboard shortcuts, and enabled/disabled states.
+ *
+ * Uses the native bridge to walk the AXMenuBar subtree.
+ */
+export class MenuScanner {
+    bridge;
+    constructor(bridge) {
+        this.bridge = bridge;
+    }
+    /**
+     * Scan the menu bar of a running app.
+     */
+    async scan(pid, bundleId, appName) {
+        const tree = await this.bridge.call("ax.getMenuBar", {
+            pid,
+            maxDepth: 10,
+        });
+        const menuTree = this.parseAXTree(tree);
+        const items = this.flattenTree(menuTree, []);
+        const shortcuts = {};
+        for (const item of items) {
+            if (item.shortcut) {
+                shortcuts[item.path.join(".")] = item.shortcut;
+            }
+        }
+        return {
+            bundleId,
+            appName,
+            totalMenus: menuTree.length,
+            totalItems: items.length,
+            shortcuts,
+            menuTree,
+            scannedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Convert scan result to reference JSON format (shortcuts + flows sections).
+     */
+    toReferenceFormat(result) {
+        const shortcuts = {};
+        const menuPaths = [];
+        for (const topMenu of result.menuTree) {
+            const category = topMenu.title;
+            const items = this.flattenTree([topMenu], []);
+            for (const item of items) {
+                menuPaths.push(item.path.join(" > "));
+                if (item.shortcut) {
+                    if (!shortcuts[category])
+                        shortcuts[category] = {};
+                    shortcuts[category][item.title] = item.shortcut;
+                }
+            }
+        }
+        return { shortcuts, menuPaths };
+    }
+    /**
+     * Parse raw AX tree response into MenuNode tree.
+     */
+    parseAXTree(tree) {
+        if (!tree || !Array.isArray(tree.children))
+            return [];
+        const nodes = [];
+        for (const child of tree.children) {
+            const node = this.parseNode(child);
+            if (node)
+                nodes.push(node);
+        }
+        return nodes;
+    }
+    parseNode(axNode) {
+        if (!axNode)
+            return null;
+        const role = axNode.role ?? axNode.AXRole ?? "";
+        const title = axNode.title ??
+            axNode.AXTitle ??
+            axNode.description ??
+            axNode.AXDescription ??
+            "";
+        // Skip separators
+        if (role === "AXSeparator" || title === "separator")
+            return null;
+        const shortcut = this.extractShortcut(axNode);
+        const enabled = axNode.enabled !== false && axNode.AXEnabled !== false;
+        const children = [];
+        if (Array.isArray(axNode.children)) {
+            for (const child of axNode.children) {
+                const childNode = this.parseNode(child);
+                if (childNode)
+                    children.push(childNode);
+            }
+        }
+        // AXMenu containers have no title but hold the actual menu items as children.
+        // Pass through their children instead of dropping the entire subtree.
+        if (!title && children.length > 0) {
+            return { title: role, shortcut: null, enabled: true, children };
+        }
+        // Skip leaf nodes with no title (not containers, not useful)
+        if (!title)
+            return null;
+        return {
+            title: String(title),
+            shortcut,
+            enabled,
+            children,
+        };
+    }
+    /**
+     * Extract keyboard shortcut from AX node attributes.
+     */
+    extractShortcut(axNode) {
+        // macOS AX provides shortcuts via AXMenuItemCmdChar, AXMenuItemCmdModifiers
+        const cmdChar = axNode.AXMenuItemCmdChar ?? axNode.cmdChar ?? null;
+        const cmdModifiers = axNode.AXMenuItemCmdModifiers ?? axNode.cmdModifiers ?? 0;
+        if (!cmdChar)
+            return null;
+        const parts = [];
+        // Modifier masks: Control=4, Option=2, Shift=1, Cmd=0 (always present)
+        if (cmdModifiers & 4)
+            parts.push("Ctrl");
+        if (cmdModifiers & 2)
+            parts.push("Option");
+        if (cmdModifiers & 1)
+            parts.push("Shift");
+        parts.push("Cmd");
+        parts.push(String(cmdChar));
+        return parts.join("+");
+    }
+    /**
+     * Flatten tree to list of items with full paths.
+     */
+    flattenTree(nodes, parentPath) {
+        const items = [];
+        for (const node of nodes) {
+            const path = [...parentPath, node.title];
+            items.push({
+                path,
+                title: node.title,
+                shortcut: node.shortcut,
+                enabled: node.enabled,
+                hasSubmenu: node.children.length > 0,
+            });
+            if (node.children.length > 0) {
+                items.push(...this.flattenTree(node.children, path));
+            }
+        }
+        return items;
+    }
+}