screenhand 0.1.1 → 0.3.0

package/dist/src/memory/recall.js

@@ -0,0 +1,222 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { MemoryStore } from "./store.js";
+export class RecallEngine {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    /**
+     * Find strategies matching a task description (~0ms, in-memory).
+     * Strategies with high fail rates are penalized.
+     */
+    recallStrategies(query, limit = 5, currentBundleId) {
+        const strategies = this.store.readStrategies();
+        if (strategies.length === 0)
+            return [];
+        const queryTokens = tokenize(query);
+        if (queryTokens.length === 0)
+            return [];
+        const scored = strategies.map((s) => {
+            // Only match against task description, tags, and tool names.
+            // Step params (JS code, URLs, selectors) contain too many generic words
+            // and cause false positives against unrelated strategies.
+            const targetTokens = new Set([
+                ...tokenize(s.task),
+                ...s.tags,
+                ...s.steps.map((step) => step.tool),
+            ]);
+            let matches = 0;
+            for (const qt of queryTokens) {
+                for (const tt of targetTokens) {
+                    if (tt.includes(qt) || qt.includes(tt)) {
+                        matches++;
+                        break;
+                    }
+                }
+            }
+            const relevance = matches / queryTokens.length;
+            const ageMs = Date.now() - new Date(s.lastUsed).getTime();
+            const ageDays = Number.isFinite(ageMs) ? ageMs / (1000 * 60 * 60 * 24) : 0;
+            const recency = Math.max(0.5, 1.0 - ageDays / 365);
+            const successBoost = 1 + Math.log2(Math.max(1, s.successCount)) * 0.1;
+            // Penalty for strategies that have failed — reduces score proportionally
+            const failCount = s.failCount ?? 0;
+            const totalAttempts = s.successCount + failCount;
+            const reliabilityPenalty = totalAttempts > 0
+                ? s.successCount / totalAttempts
+                : 1;
+            // App-context filtering: penalize strategies whose steps target a different app
+            let appContextFactor = 1.0;
+            if (currentBundleId) {
+                const stepsStr = s.steps.map((step) => JSON.stringify(step.params)).join(" ").toLowerCase();
+                const taskStr = s.task.toLowerCase();
+                const bundleLower = currentBundleId.toLowerCase();
+                // Extract app name from bundleId (e.g. "com.apple.Safari" → "safari")
+                const appName = bundleLower.split(".").pop() ?? "";
+                const mentionsCurrentApp = taskStr.includes(bundleLower) || taskStr.includes(appName)
+                    || stepsStr.includes(bundleLower);
+                // Check if strategy targets a DIFFERENT app via focus/launch steps
+                const hasFocusStep = s.steps.some((step) => (step.tool === "focus" || step.tool === "launch") &&
+                    step.params && "bundleId" in step.params &&
+                    typeof step.params.bundleId === "string" &&
+                    step.params.bundleId.toLowerCase() !== bundleLower);
+                if (hasFocusStep && !mentionsCurrentApp) {
+                    appContextFactor = 0.1; // Heavy penalty for wrong-app strategies
+                }
+                else if (mentionsCurrentApp) {
+                    appContextFactor = 1.5; // Boost for matching strategies
+                }
+            }
+            // Require at least 50% token overlap to be considered relevant
+            if (relevance < 0.5)
+                return { ...s, score: 0 };
+            const score = relevance * recency * successBoost * reliabilityPenalty * appContextFactor;
+            return { ...s, score };
+        });
+        return scored
+            .filter((s) => s.score > 0)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    /**
+     * O(1) exact match by tool sequence fingerprint.
+     * Returns the strategy if found and it has a positive reliability score.
+     */
+    recallByFingerprint(tools) {
+        const fp = MemoryStore.makeFingerprint(tools);
+        const strategy = this.store.lookupByFingerprint(fp);
+        if (!strategy)
+            return null;
+        // Skip strategies that fail more than they succeed
+        const failCount = strategy.failCount ?? 0;
+        if (failCount > strategy.successCount)
+            return null;
+        return strategy;
+    }
+    /**
+     * Quick error lookup for a tool — used by interceptor on every call (~0ms).
+     * Returns the most relevant error pattern or null.
+     */
+    quickErrorCheck(tool) {
+        const errors = this.store.readErrors();
+        let best = null;
+        for (const e of errors) {
+            if (e.tool === tool && e.resolution) {
+                if (!best || e.occurrences > best.occurrences)
+                    best = e;
+            }
+        }
+        return best;
+    }
+    /**
+     * Quick strategy hint for a tool sequence — used by interceptor.
+     * Tries fingerprint prefix match first (O(1)), then falls back to scan.
+     * Skips unreliable strategies (failCount > successCount).
+     */
+    quickStrategyHint(recentTools, currentBundleId) {
+        if (recentTools.length === 0)
+            return null;
+        // Require at least 2 tools in the sequence to reduce false positives
+        // from single-tool matches (e.g. just "focus" matching every strategy)
+        if (recentTools.length < 2)
+            return null;
+        const strategies = this.store.readStrategies();
+        for (const s of strategies) {
+            if (s.steps.length <= recentTools.length)
+                continue;
+            // Skip unreliable strategies
+            const failCount = s.failCount ?? 0;
+            if (failCount > s.successCount)
+                continue;
+            // If we know the current app, prefer strategies that mention it
+            // and skip strategies clearly for a different app
+            if (currentBundleId) {
+                const taskLower = s.task.toLowerCase();
+                const bundleLower = currentBundleId.toLowerCase();
+                // Extract app name from bundleId (e.g. "com.apple.Safari" → "safari")
+                const appName = bundleLower.split(".").pop() ?? "";
+                const mentionsCurrentApp = taskLower.includes(appName) || taskLower.includes(bundleLower);
+                const mentionsOtherApp = !mentionsCurrentApp && /com\.\w+\.\w+/.test(s.task);
+                if (mentionsOtherApp)
+                    continue; // strategy is for a different app
+            }
+            const strategyToolPrefix = s.steps.slice(0, recentTools.length).map((st) => st.tool);
+            const matches = recentTools.every((t, i) => t === strategyToolPrefix[i]);
+            if (matches) {
+                return {
+                    strategy: s,
+                    nextStep: s.steps[recentTools.length],
+                    fingerprint: s.fingerprint ?? MemoryStore.makeFingerprint(s.steps.map((st) => st.tool)),
+                };
+            }
+        }
+        return null;
+    }
+    /** Find error patterns for a specific tool or all tools */
+    recallErrors(tool, params) {
+        const errors = this.store.readErrors();
+        if (!tool)
+            return errors;
+        let filtered = errors.filter((e) => e.tool === tool);
+        if (params && filtered.length > 1) {
+            const paramStr = JSON.stringify(params).toLowerCase();
+            filtered.sort((a, b) => {
+                const aScore = stringSimilarity(paramStr, JSON.stringify(a.params).toLowerCase());
+                const bScore = stringSimilarity(paramStr, JSON.stringify(b.params).toLowerCase());
+                return bScore - aScore;
+            });
+        }
+        return filtered;
+    }
+}
+/** Tokenize a string into lowercase keywords (3+ chars) */
+/** Common automation verbs/nouns that match almost any strategy — filter them out */
+const RECALL_STOPWORDS = new Set([
+    "open", "close", "click", "set", "get", "the", "and", "for", "from",
+    "into", "with", "then", "this", "that", "use", "run", "start", "stop",
+    "new", "add", "app", "settings", "window", "button", "text", "page",
+    "file", "menu", "tab", "navigate", "type", "select", "find", "wait",
+    "send", "save", "copy", "paste", "delete", "create", "edit", "view",
+    "show", "hide", "move", "drag", "drop", "enter", "press", "about",
+    "input", "form", "link", "image", "video", "upload", "download",
+    "first", "last", "next", "take", "result", "search",
+]);
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .split(/[\W_]+/)
+        .filter((w) => w.length >= 2 && !RECALL_STOPWORDS.has(w));
+}
+/** Simple string similarity: shared character bigrams / total bigrams */
+function stringSimilarity(a, b) {
+    if (a === b)
+        return 1;
+    if (a.length < 2 || b.length < 2)
+        return 0;
+    const bigramsA = new Set();
+    for (let i = 0; i < a.length - 1; i++)
+        bigramsA.add(a.slice(i, i + 2));
+    let matches = 0;
+    let total = 0;
+    for (let i = 0; i < b.length - 1; i++) {
+        total++;
+        if (bigramsA.has(b.slice(i, i + 2)))
+            matches++;
+    }
+    return total > 0 ? matches / total : 0;
+}

package/dist/src/memory/research.js

@@ -0,0 +1,104 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export function backgroundResearch(store, tool, params, errorMessage) {
+    // Fire-and-forget — never blocks, never throws
+    doResearch(store, tool, params, errorMessage).catch(() => { });
+}
+async function doResearch(store, tool, params, errorMessage) {
+    const query = `macOS automation: "${tool}" failed with "${errorMessage.slice(0, 200)}"`;
+    let resolution = null;
+    const apiKey = process.env["ANTHROPIC_API_KEY"];
+    if (apiKey) {
+        resolution = await tryClaudeAPI(apiKey, query);
+    }
+    if (!resolution) {
+        resolution = await tryDuckDuckGo(query);
+    }
+    if (!resolution)
+        return;
+    // Save resolution to error cache
+    store.appendError({
+        id: "err_research_" + Date.now().toString(36),
+        tool,
+        params,
+        error: errorMessage,
+        resolution,
+        occurrences: 1,
+        lastSeen: new Date().toISOString(),
+    });
+    // Save as a reusable strategy
+    store.appendStrategy({
+        id: "str_research_" + Date.now().toString(36),
+        task: `Fix: ${tool} — ${errorMessage.slice(0, 100)}`,
+        steps: [{ tool, params }],
+        totalDurationMs: 0,
+        successCount: 1,
+        failCount: 0,
+        lastUsed: new Date().toISOString(),
+        tags: [tool, "research", "fix"],
+        fingerprint: "",
+    });
+}
+async function tryClaudeAPI(apiKey, query) {
+    try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 10000);
+        const response = await fetch("https://api.anthropic.com/v1/messages", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "x-api-key": apiKey,
+                "anthropic-version": "2023-06-01",
+            },
+            body: JSON.stringify({
+                model: "claude-haiku-4-5-20251001",
+                max_tokens: 200,
+                messages: [{
+                        role: "user",
+                        content: `You are a macOS automation expert. Give a brief fix (1-2 sentences) for this error:\n\n${query}`,
+                    }],
+            }),
+            signal: controller.signal,
+        });
+        clearTimeout(timeout);
+        if (!response.ok)
+            return null;
+        const data = await response.json();
+        const text = data.content?.[0]?.text;
+        return text && text.length > 10 ? text.trim() : null;
+    }
+    catch {
+        return null;
+    }
+}
+async function tryDuckDuckGo(query) {
+    try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 5000);
+        const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1`;
+        const response = await fetch(url, { signal: controller.signal });
+        clearTimeout(timeout);
+        if (!response.ok)
+            return null;
+        const data = await response.json();
+        const text = data.AbstractText || data.Abstract || data.Answer;
+        return text && text.length > 10 ? text.trim() : null;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/src/memory/seeds.js

@@ -0,0 +1,101 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+let seedCounter = 0;
+function makeFingerprint(tools) {
+    return tools.join("→");
+}
+function seed(task, steps, tags) {
+    seedCounter++;
+    return {
+        id: `seed_${String(seedCounter).padStart(3, "0")}`,
+        task,
+        steps,
+        totalDurationMs: 0,
+        successCount: 10,
+        failCount: 0,
+        lastUsed: new Date().toISOString(),
+        tags,
+        fingerprint: makeFingerprint(steps.map((s) => s.tool)),
+    };
+}
+export const SEED_STRATEGIES = [
+    // 1. Take a photo with Photo Booth
+    seed("Take a photo with Photo Booth", [
+        { tool: "launch", params: { bundleId: "com.apple.PhotoBooth" } },
+        { tool: "ui_press", params: { title: "Take Photo" } },
+    ], ["photo", "booth", "camera"]),
+    // 2. Open a URL in Chrome
+    seed("Open a URL in Chrome", [
+        { tool: "launch", params: { bundleId: "com.google.Chrome" } },
+        { tool: "browser_navigate", params: { url: "" } },
+    ], ["chrome", "browse", "url"]),
+    // 3. Save current document
+    seed("Save current document", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+s" } },
+    ], ["save", "document"]),
+    // 4. Copy from one app and paste into another
+    seed("Copy from one app and paste into another", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+c" } },
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+v" } },
+    ], ["copy", "paste"]),
+    // 5. Navigate to a folder in Finder
+    seed("Navigate to a folder in Finder", [
+        { tool: "focus", params: { bundleId: "com.apple.finder" } },
+        { tool: "key", params: { combo: "cmd+shift+g" } },
+        { tool: "type_text", params: { text: "" } },
+    ], ["finder", "folder", "navigate"]),
+    // 6. Create a new folder in Finder
+    seed("Create a new folder in Finder", [
+        { tool: "focus", params: { bundleId: "com.apple.finder" } },
+        { tool: "key", params: { combo: "cmd+shift+n" } },
+        { tool: "type_text", params: { text: "" } },
+    ], ["finder", "folder", "create"]),
+    // 7. Close the current window
+    seed("Close the current window", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+w" } },
+    ], ["close", "window"]),
+    // 8. Select all and copy
+    seed("Select all content and copy", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "key", params: { combo: "cmd+a" } },
+        { tool: "key", params: { combo: "cmd+c" } },
+    ], ["select", "all", "copy"]),
+    // 9. List running apps
+    seed("List all running applications", [
+        { tool: "apps", params: {} },
+    ], ["apps", "list", "running"]),
+    // 10. Inspect app UI tree
+    seed("Inspect an app's UI element tree", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "ui_tree", params: { pid: 0 } },
+    ], ["inspect", "tree", "accessibility"]),
+    // 11. Open a new tab in Chrome and navigate
+    seed("Open a new Chrome tab and navigate to URL", [
+        { tool: "focus", params: { bundleId: "com.google.Chrome" } },
+        { tool: "key", params: { combo: "cmd+t" } },
+        { tool: "browser_navigate", params: { url: "" } },
+    ], ["chrome", "tab", "new"]),
+    // 12. Export as PDF via menu
+    seed("Export document as PDF", [
+        { tool: "focus", params: { bundleId: "" } },
+        { tool: "menu_click", params: { menuPath: "File/Export as PDF" } },
+    ], ["export", "pdf"]),
+];