screenhand 0.1.1 → 0.3.0

package/dist/src/ingestion/reference-merger.js

@@ -0,0 +1,186 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { writeFileAtomicSync } from "../util/atomic-write.js";
+import { shortcutsToReferenceFormat } from "./shortcut-extractor.js";
+/**
+ * ReferenceMerger — merges ingested knowledge into existing reference files.
+ * Creates new reference files when no matching file exists.
+ */
+export class ReferenceMerger {
+    referencesDir;
+    constructor(referencesDir) {
+        this.referencesDir = referencesDir;
+    }
+    /**
+     * Merge shortcuts from a menu scan into the reference file.
+     */
+    mergeMenuScan(scan) {
+        const ref = this.loadOrCreate(scan.bundleId, scan.appName);
+        const { shortcuts: scannedShortcuts } = this.menuScanToShortcuts(scan);
+        let added = 0;
+        let updated = 0;
+        if (!ref.shortcuts)
+            ref.shortcuts = {};
+        for (const [category, entries] of Object.entries(scannedShortcuts)) {
+            if (!ref.shortcuts[category]) {
+                ref.shortcuts[category] = {};
+            }
+            for (const [name, keys] of Object.entries(entries)) {
+                if (!ref.shortcuts[category][name]) {
+                    added++;
+                }
+                else if (ref.shortcuts[category][name] !== keys) {
+                    updated++;
+                }
+                ref.shortcuts[category][name] = keys;
+            }
+        }
+        const filePath = this.save(ref);
+        return { filePath, added, updated };
+    }
+    /**
+     * Merge shortcuts from parsed documentation.
+     */
+    mergeDocShortcuts(shortcuts, bundleId, appName) {
+        const ref = this.loadOrCreate(bundleId, appName);
+        const formatted = shortcutsToReferenceFormat(shortcuts);
+        let added = 0;
+        let updated = 0;
+        if (!ref.shortcuts)
+            ref.shortcuts = {};
+        for (const [category, entries] of Object.entries(formatted)) {
+            if (!ref.shortcuts[category]) {
+                ref.shortcuts[category] = {};
+            }
+            for (const [name, keys] of Object.entries(entries)) {
+                if (!ref.shortcuts[category][name]) {
+                    added++;
+                }
+                else if (ref.shortcuts[category][name] !== keys) {
+                    updated++;
+                }
+                ref.shortcuts[category][name] = keys;
+            }
+        }
+        const filePath = this.save(ref);
+        return { filePath, added, updated };
+    }
+    /**
+     * Merge flows from parsed documentation.
+     */
+    mergeDocFlows(docResult, bundleId, appName) {
+        const ref = this.loadOrCreate(bundleId, appName);
+        if (!ref.flows)
+            ref.flows = {};
+        let added = 0;
+        for (const flow of docResult.flows) {
+            const key = flow.name
+                .toLowerCase()
+                .replace(/[^a-z0-9]+/g, "_")
+                .replace(/^_|_$/g, "");
+            if (!ref.flows[key]) {
+                const parsed = flow.steps
+                    .filter((s) => s.tool)
+                    .map((s) => ({ tool: s.tool, params: s.params ?? {} }));
+                ref.flows[key] = {
+                    steps: flow.steps.map((s) => s.description),
+                    description: flow.name,
+                    ...(parsed.length > 0 ? { _parsed: parsed } : {}),
+                };
+                added++;
+            }
+        }
+        const filePath = this.save(ref);
+        return { filePath, added };
+    }
+    /**
+     * Merge errors/solutions into reference.
+     */
+    mergeErrors(errors, bundleId, appName) {
+        const ref = this.loadOrCreate(bundleId, appName);
+        if (!ref.errors)
+            ref.errors = [];
+        let added = 0;
+        const existingErrors = new Set(ref.errors.map((e) => e.error.toLowerCase()));
+        for (const err of errors) {
+            if (!existingErrors.has(err.error.toLowerCase())) {
+                ref.errors.push(err);
+                existingErrors.add(err.error.toLowerCase());
+                added++;
+            }
+        }
+        const filePath = this.save(ref);
+        return { filePath, added };
+    }
+    /**
+     * Load existing reference file for a bundleId, or create a new one.
+     */
+    loadOrCreate(bundleId, appName) {
+        // Search for existing file by bundleId
+        try {
+            const files = fs.readdirSync(this.referencesDir);
+            for (const file of files) {
+                if (!file.endsWith(".json"))
+                    continue;
+                try {
+                    const raw = fs.readFileSync(path.join(this.referencesDir, file), "utf-8");
+                    const ref = JSON.parse(raw);
+                    if (ref.bundleId === bundleId)
+                        return ref;
+                }
+                catch {
+                    /* skip malformed */
+                }
+            }
+        }
+        catch {
+            /* dir doesn't exist */
+        }
+        // Create new reference
+        const platform = appName.toLowerCase().replace(/\s+/g, "-");
+        return {
+            id: platform,
+            name: `${appName} — Auto-Generated Reference`,
+            platform,
+            bundleId,
+            shortcuts: {},
+            selectors: {},
+            flows: {},
+            errors: [],
+        };
+    }
+    save(ref) {
+        fs.mkdirSync(this.referencesDir, { recursive: true });
+        const filePath = path.join(this.referencesDir, `${ref.id}.json`);
+        writeFileAtomicSync(filePath, JSON.stringify(ref, null, 2) + "\n");
+        return filePath;
+    }
+    menuScanToShortcuts(scan) {
+        const shortcuts = {};
+        for (const topMenu of scan.menuTree) {
+            const category = topMenu.title;
+            if (!shortcuts[category])
+                shortcuts[category] = {};
+            const items = this.flattenMenuNode(topMenu, []);
+            for (const item of items) {
+                if (item.shortcut) {
+                    shortcuts[category][item.label] = item.shortcut;
+                }
+            }
+        }
+        return { shortcuts };
+    }
+    flattenMenuNode(node, parentPath) {
+        const items = [];
+        const path = [...parentPath, node.title];
+        if (node.shortcut) {
+            items.push({ label: path.slice(1).join(" > "), shortcut: node.shortcut });
+        }
+        for (const child of node.children ?? []) {
+            items.push(...this.flattenMenuNode(child, path));
+        }
+        return items;
+    }
+}

package/dist/src/ingestion/shortcut-extractor.js

@@ -0,0 +1,180 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * ShortcutExtractor — parses keyboard shortcut lists from various formats
+ * (HTML tables, plain text lists, markdown) into structured data.
+ */
+/**
+ * Parse shortcuts from an HTML table.
+ * Expects columns like: Action/Name | Shortcut/Keys
+ */
+export function parseShortcutsFromHTML(html) {
+    const shortcuts = [];
+    // Extract table rows
+    const tableRegex = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
+    const cellRegex = /<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi;
+    let tableMatch;
+    while ((tableMatch = tableRegex.exec(html)) !== null) {
+        const row = tableMatch[1];
+        const cells = [];
+        let cellMatch;
+        while ((cellMatch = cellRegex.exec(row)) !== null) {
+            // Strip HTML tags and normalize whitespace
+            const text = cellMatch[1]
+                .replace(/<[^>]+>/g, "")
+                .replace(/&amp;/g, "&")
+                .replace(/&lt;/g, "<")
+                .replace(/&gt;/g, ">")
+                .replace(/&nbsp;/g, " ")
+                .replace(/&#\d+;/g, "")
+                .trim();
+            cells.push(text);
+        }
+        if (cells.length >= 2 && cells[0] && cells[1]) {
+            // Skip header rows
+            const lower0 = cells[0].toLowerCase();
+            if (lower0 === "action" ||
+                lower0 === "command" ||
+                lower0 === "shortcut" ||
+                lower0 === "name" ||
+                lower0 === "function") {
+                continue;
+            }
+            shortcuts.push({
+                name: cells[0],
+                keys: normalizeKeys(cells[1]),
+                context: cells.length > 2 ? cells[2] : undefined,
+            });
+        }
+    }
+    return shortcuts;
+}
+/**
+ * Parse shortcuts from plain text lines.
+ * Supports formats:
+ * - "Action: Cmd+K"
+ * - "Action — Cmd+K"
+ * - "Action\tCmd+K"
+ * - "Cmd+K  Action"
+ */
+export function parseShortcutsFromText(text) {
+    const shortcuts = [];
+    const lines = text.split("\n");
+    let currentCategory;
+    for (const rawLine of lines) {
+        const line = rawLine.trim();
+        if (!line)
+            continue;
+        // Detect category headers (all caps or ending with colon, no shortcut key)
+        if ((line.endsWith(":") && !SHORTCUT_PATTERN.test(line)) ||
+            (line === line.toUpperCase() && line.length > 3 && !SHORTCUT_PATTERN.test(line))) {
+            currentCategory = line.replace(/:$/, "").trim();
+            continue;
+        }
+        // Try "Name: Keys" or "Name — Keys" or "Name\tKeys"
+        const separatorMatch = line.match(/^(.+?)(?:\s*[:—–\-|]\s*|\t)(.+)$/);
+        if (separatorMatch && SHORTCUT_PATTERN.test(separatorMatch[2])) {
+            shortcuts.push({
+                name: separatorMatch[1].trim(),
+                keys: normalizeKeys(separatorMatch[2].trim()),
+                category: currentCategory,
+            });
+            continue;
+        }
+        // Try "Keys  Name" (shortcut first)
+        const keysFirstMatch = line.match(/^((?:Cmd|Ctrl|Alt|Option|Shift|Meta|⌘|⌃|⌥|⇧)[+\s].+?)\s{2,}(.+)$/i);
+        if (keysFirstMatch) {
+            shortcuts.push({
+                name: keysFirstMatch[2].trim(),
+                keys: normalizeKeys(keysFirstMatch[1].trim()),
+                category: currentCategory,
+            });
+        }
+    }
+    return shortcuts;
+}
+/**
+ * Parse shortcuts from markdown format.
+ */
+export function parseShortcutsFromMarkdown(md) {
+    const shortcuts = [];
+    const lines = md.split("\n");
+    let currentCategory;
+    for (const rawLine of lines) {
+        const line = rawLine.trim();
+        // Category from heading
+        const headingMatch = line.match(/^#{1,3}\s+(.+)$/);
+        if (headingMatch) {
+            currentCategory = headingMatch[1].trim();
+            continue;
+        }
+        // Table row: | Name | Keys |
+        const tableMatch = line.match(/^\|(.+)\|(.+)\|/);
+        if (tableMatch) {
+            const name = tableMatch[1].trim();
+            const keys = tableMatch[2].trim();
+            if (name &&
+                keys &&
+                name !== "---" &&
+                !name.startsWith("-") &&
+                SHORTCUT_PATTERN.test(keys)) {
+                shortcuts.push({
+                    name,
+                    keys: normalizeKeys(keys),
+                    category: currentCategory,
+                });
+            }
+            continue;
+        }
+        // List item: - Name: Keys or * Name — Keys
+        const listMatch = line.match(/^[*\-+]\s+(.+?)(?:\s*[:—–]\s*)(.+)$/);
+        if (listMatch && SHORTCUT_PATTERN.test(listMatch[2])) {
+            shortcuts.push({
+                name: listMatch[1].trim(),
+                keys: normalizeKeys(listMatch[2].trim()),
+                category: currentCategory,
+            });
+            continue;
+        }
+        // Inline code: `Cmd+K` — Description
+        const codeMatch = line.match(/`([^`]+)`\s*[-—:]\s*(.+)/);
+        if (codeMatch && SHORTCUT_PATTERN.test(codeMatch[1])) {
+            shortcuts.push({
+                name: codeMatch[2].trim(),
+                keys: normalizeKeys(codeMatch[1].trim()),
+                category: currentCategory,
+            });
+        }
+    }
+    return shortcuts;
+}
+/**
+ * Convert parsed shortcuts to reference JSON shortcuts format.
+ */
+export function shortcutsToReferenceFormat(shortcuts) {
+    const result = {};
+    for (const sc of shortcuts) {
+        const category = sc.category ?? "general";
+        if (!result[category])
+            result[category] = {};
+        result[category][sc.name] = sc.keys;
+    }
+    return result;
+}
+// Pattern to detect shortcut-like strings
+const SHORTCUT_PATTERN = /(?:Cmd|Ctrl|Alt|Option|Shift|Meta|⌘|⌃|⌥|⇧|Command|Control)[+\s]/i;
+/**
+ * Normalize keyboard shortcut notation.
+ * Converts symbols to names: ⌘→Cmd, ⌃→Ctrl, ⌥→Option, ⇧→Shift
+ */
+function normalizeKeys(keys) {
+    return keys
+        .replace(/⌘/g, "Cmd")
+        .replace(/⌃/g, "Ctrl")
+        .replace(/⌥/g, "Option")
+        .replace(/⇧/g, "Shift")
+        .replace(/Command/gi, "Cmd")
+        .replace(/Control/gi, "Ctrl")
+        .replace(/\s*\+\s*/g, "+")
+        .trim();
+}

package/dist/src/ingestion/tutorial-extractor.js

@@ -0,0 +1,170 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+// Patterns that indicate an action step in a tutorial
+const ACTION_PATTERNS = [
+    /\b(?:click|tap|press|hit)\s+(?:on\s+)?(?:the\s+)?(.+?)(?:\s+button|\s+tab|\s+menu|\s+icon)?(?:\.|,|$)/i,
+    /\b(?:go to|navigate to|open|switch to)\s+(.+?)(?:\.|,|$)/i,
+    /\b(?:select|choose|pick)\s+(.+?)(?:\.|,|$)/i,
+    /\b(?:type|enter|input|write)\s+(.+?)(?:\.|,|$)/i,
+    /\b(?:drag|move)\s+(.+?)\s+to\s+(.+?)(?:\.|,|$)/i,
+    /\b(?:right[- ]?click|double[- ]?click)\s+(?:on\s+)?(.+?)(?:\.|,|$)/i,
+    /\bpress\s+((?:cmd|ctrl|alt|shift|command|control|option)[+\s].+?)(?:\.|,|$)/i,
+    /\b(?:scroll|zoom)\s+(.+?)(?:\.|,|$)/i,
+    /\b(?:set|change|adjust)\s+(.+?)\s+to\s+(.+?)(?:\.|,|$)/i,
+];
+// Words that indicate non-action segments (skip these)
+const SKIP_PATTERNS = [
+    /\bhey guys\b/i,
+    /\bsubscribe\b/i,
+    /\blike (?:this|the) video\b/i,
+    /\bcomment below\b/i,
+    /\bsponsor/i,
+    /\bwhat's up\b/i,
+    /\bhello everyone\b/i,
+    /\bwelcome (?:back|to)\b/i,
+    /\blet me know\b/i,
+    /\bcheck out\b/i,
+    /\blink in (?:the )?description\b/i,
+];
+/**
+ * TutorialExtractor — extracts structured playbook steps from video
+ * transcripts (typically YouTube captions/subtitles).
+ */
+export class TutorialExtractor {
+    /**
+     * Extract action steps from a transcript.
+     */
+    extract(segments, title, platform) {
+        const steps = [];
+        for (const segment of segments) {
+            // Skip filler/promo segments
+            if (SKIP_PATTERNS.some((p) => p.test(segment.text)))
+                continue;
+            const parsedSteps = this.parseSegment(segment.text);
+            for (const step of parsedSteps) {
+                // Deduplicate consecutive identical steps
+                const last = steps[steps.length - 1];
+                if (last && last.description === step.description)
+                    continue;
+                steps.push(step);
+            }
+        }
+        return {
+            title,
+            platform,
+            steps,
+            rawSegments: segments.length,
+            actionSegments: steps.length,
+            extractedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Convert extracted steps to a playbook-ready format.
+     */
+    toPlaybookSteps(result) {
+        return result.steps
+            .filter((s) => s.tool)
+            .map((step) => ({
+            action: step.tool === "key" ? "press" : step.tool === "type_text" ? "type" : "click",
+            tool: step.tool,
+            params: step.params ?? {},
+            description: step.description,
+            postcondition: step.postcondition,
+        }));
+    }
+    /**
+     * Parse a single transcript segment for action steps.
+     */
+    parseSegment(text) {
+        const steps = [];
+        // Split on sentence boundaries
+        const sentences = text.split(/(?<=[.!?])\s+|(?:,\s+(?:then|and then|next|after that)\s+)/i);
+        for (const sentence of sentences) {
+            const trimmed = sentence.trim();
+            if (trimmed.length < 5)
+                continue;
+            for (const pattern of ACTION_PATTERNS) {
+                const match = trimmed.match(pattern);
+                if (match) {
+                    const step = this.mapToStep(trimmed, match);
+                    if (step) {
+                        steps.push(step);
+                        break; // Only take first matching pattern per sentence
+                    }
+                }
+            }
+        }
+        return steps;
+    }
+    /**
+     * Map a matched action to a ParsedFlowStep with tool and params.
+     */
+    mapToStep(fullText, match) {
+        const lower = fullText.toLowerCase();
+        const target = match[1]?.trim();
+        if (!target || target.length < 2)
+            return null;
+        // Determine tool and params
+        if (lower.includes("type") || lower.includes("enter") || lower.includes("input")) {
+            return {
+                description: fullText,
+                tool: "type_text",
+                params: { text: target },
+            };
+        }
+        if (lower.includes("press") && /(?:cmd|ctrl|alt|shift|command|control|option)/i.test(target)) {
+            return {
+                description: fullText,
+                tool: "key",
+                params: { combo: this.normalizeShortcut(target) },
+            };
+        }
+        if (lower.includes("drag") || lower.includes("move")) {
+            return {
+                description: fullText,
+                tool: "click_text",
+                params: { text: target },
+            };
+        }
+        if (lower.includes("scroll") || lower.includes("zoom")) {
+            return {
+                description: fullText,
+                tool: "scroll",
+                params: { direction: lower.includes("down") ? "down" : lower.includes("up") ? "up" : "down" },
+            };
+        }
+        if (lower.includes("menu") || target.includes(">")) {
+            const pathParts = target
+                .split(/\s*>\s*/)
+                .map((s) => s.trim())
+                .filter(Boolean);
+            if (pathParts.length >= 2) {
+                return {
+                    description: fullText,
+                    tool: "menu_click",
+                    params: { menuPath: pathParts.join("/") },
+                };
+            }
+        }
+        if (lower.includes("set") || lower.includes("change") || lower.includes("adjust")) {
+            return {
+                description: fullText,
+                tool: "click_text",
+                params: { text: target },
+            };
+        }
+        // Default: click on the target
+        return {
+            description: fullText,
+            tool: "click_text",
+            params: { text: target },
+        };
+    }
+    normalizeShortcut(keys) {
+        return keys
+            .replace(/Command/gi, "Cmd")
+            .replace(/Control/gi, "Ctrl")
+            .replace(/\s+/g, "+")
+            .trim();
+    }
+}

package/dist/src/ingestion/types.js

@@ -0,0 +1,3 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+export {};