screenhand 0.1.0 → 0.1.1

package/src/memory/research.ts

@@ -0,0 +1,146 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+
+/**
+ * Background web research — fire-and-forget resolution lookup.
+ *
+ * When a tool fails and no resolution exists, this searches for a fix
+ * in the background (non-blocking) and saves it for next time.
+ *
+ * Resolution paths:
+ * 1. Claude API (haiku) — if ANTHROPIC_API_KEY is set
+ * 2. DuckDuckGo instant answer — free fallback, no auth
+ */
+
+import type { MemoryStore } from "./store.js";
+
+export function backgroundResearch(
+  store: MemoryStore,
+  tool: string,
+  params: Record<string, unknown>,
+  errorMessage: string,
+): void {
+  // Fire-and-forget — never blocks, never throws
+  doResearch(store, tool, params, errorMessage).catch(() => {});
+}
+
+async function doResearch(
+  store: MemoryStore,
+  tool: string,
+  params: Record<string, unknown>,
+  errorMessage: string,
+): Promise<void> {
+  const query = `macOS automation: "${tool}" failed with "${errorMessage.slice(0, 200)}"`;
+
+  let resolution: string | null = null;
+
+  const apiKey = process.env["ANTHROPIC_API_KEY"];
+  if (apiKey) {
+    resolution = await tryClaudeAPI(apiKey, query);
+  }
+
+  if (!resolution) {
+    resolution = await tryDuckDuckGo(query);
+  }
+
+  if (!resolution) return;
+
+  // Save resolution to error cache
+  store.appendError({
+    id: "err_research_" + Date.now().toString(36),
+    tool,
+    params,
+    error: errorMessage,
+    resolution,
+    occurrences: 1,
+    lastSeen: new Date().toISOString(),
+  });
+
+  // Save as a reusable strategy
+  store.appendStrategy({
+    id: "str_research_" + Date.now().toString(36),
+    task: `Fix: ${tool} — ${errorMessage.slice(0, 100)}`,
+    steps: [{ tool, params }],
+    totalDurationMs: 0,
+    successCount: 1,
+    failCount: 0,
+    lastUsed: new Date().toISOString(),
+    tags: [tool, "research", "fix"],
+    fingerprint: "",
+  });
+}
+
+async function tryClaudeAPI(apiKey: string, query: string): Promise<string | null> {
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 10000);
+
+    const response = await fetch("https://api.anthropic.com/v1/messages", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "x-api-key": apiKey,
+        "anthropic-version": "2023-06-01",
+      },
+      body: JSON.stringify({
+        model: "claude-haiku-4-5-20251001",
+        max_tokens: 200,
+        messages: [{
+          role: "user",
+          content: `You are a macOS automation expert. Give a brief fix (1-2 sentences) for this error:\n\n${query}`,
+        }],
+      }),
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeout);
+
+    if (!response.ok) return null;
+
+    const data = await response.json() as {
+      content?: Array<{ type: string; text?: string }>;
+    };
+    const text = data.content?.[0]?.text;
+    return text && text.length > 10 ? text.trim() : null;
+  } catch {
+    return null;
+  }
+}
+
+async function tryDuckDuckGo(query: string): Promise<string | null> {
+  try {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 5000);
+
+    const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1`;
+    const response = await fetch(url, { signal: controller.signal });
+
+    clearTimeout(timeout);
+
+    if (!response.ok) return null;
+
+    const data = await response.json() as {
+      AbstractText?: string;
+      Abstract?: string;
+      Answer?: string;
+    };
+    const text = data.AbstractText || data.Abstract || data.Answer;
+    return text && text.length > 10 ? text.trim() : null;
+  } catch {
+    return null;
+  }
+}

package/src/memory/seeds.ts

@@ -0,0 +1,123 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+
+/**
+ * Predefined seed strategies — common macOS desktop workflows.
+ * Loaded on first boot so the memory system has knowledge from day one.
+ */
+
+import type { Strategy, StrategyStep } from "./types.js";
+
+let seedCounter = 0;
+
+function makeFingerprint(tools: string[]): string {
+  return tools.join("→");
+}
+
+function seed(task: string, steps: StrategyStep[], tags: string[]): Strategy {
+  seedCounter++;
+  return {
+    id: `seed_${String(seedCounter).padStart(3, "0")}`,
+    task,
+    steps,
+    totalDurationMs: 0,
+    successCount: 10,
+    failCount: 0,
+    lastUsed: new Date().toISOString(),
+    tags,
+    fingerprint: makeFingerprint(steps.map((s) => s.tool)),
+  };
+}
+
+export const SEED_STRATEGIES: Strategy[] = [
+  // 1. Take a photo with Photo Booth
+  seed("Take a photo with Photo Booth", [
+    { tool: "launch", params: { bundleId: "com.apple.PhotoBooth" } },
+    { tool: "ui_press", params: { title: "Take Photo" } },
+  ], ["photo", "booth", "camera"]),
+
+  // 2. Open a URL in Chrome
+  seed("Open a URL in Chrome", [
+    { tool: "launch", params: { bundleId: "com.google.Chrome" } },
+    { tool: "browser_navigate", params: { url: "" } },
+  ], ["chrome", "browse", "url"]),
+
+  // 3. Save current document
+  seed("Save current document", [
+    { tool: "focus", params: { bundleId: "" } },
+    { tool: "key", params: { combo: "cmd+s" } },
+  ], ["save", "document"]),
+
+  // 4. Copy from one app and paste into another
+  seed("Copy from one app and paste into another", [
+    { tool: "focus", params: { bundleId: "" } },
+    { tool: "key", params: { combo: "cmd+c" } },
+    { tool: "focus", params: { bundleId: "" } },
+    { tool: "key", params: { combo: "cmd+v" } },
+  ], ["copy", "paste"]),
+
+  // 5. Navigate to a folder in Finder
+  seed("Navigate to a folder in Finder", [
+    { tool: "focus", params: { bundleId: "com.apple.finder" } },
+    { tool: "key", params: { combo: "cmd+shift+g" } },
+    { tool: "type_text", params: { text: "" } },
+  ], ["finder", "folder", "navigate"]),
+
+  // 6. Create a new folder in Finder
+  seed("Create a new folder in Finder", [
+    { tool: "focus", params: { bundleId: "com.apple.finder" } },
+    { tool: "key", params: { combo: "cmd+shift+n" } },
+    { tool: "type_text", params: { text: "" } },
+  ], ["finder", "folder", "create"]),
+
+  // 7. Close the current window
+  seed("Close the current window", [
+    { tool: "focus", params: { bundleId: "" } },
+    { tool: "key", params: { combo: "cmd+w" } },
+  ], ["close", "window"]),
+
+  // 8. Select all and copy
+  seed("Select all content and copy", [
+    { tool: "focus", params: { bundleId: "" } },
+    { tool: "key", params: { combo: "cmd+a" } },
+    { tool: "key", params: { combo: "cmd+c" } },
+  ], ["select", "all", "copy"]),
+
+  // 9. List running apps
+  seed("List all running applications", [
+    { tool: "apps", params: {} },
+  ], ["apps", "list", "running"]),
+
+  // 10. Inspect app UI tree
+  seed("Inspect an app's UI element tree", [
+    { tool: "focus", params: { bundleId: "" } },
+    { tool: "ui_tree", params: { pid: 0 } },
+  ], ["inspect", "tree", "accessibility"]),
+
+  // 11. Open a new tab in Chrome and navigate
+  seed("Open a new Chrome tab and navigate to URL", [
+    { tool: "focus", params: { bundleId: "com.google.Chrome" } },
+    { tool: "key", params: { combo: "cmd+t" } },
+    { tool: "browser_navigate", params: { url: "" } },
+  ], ["chrome", "tab", "new"]),
+
+  // 12. Export as PDF via menu
+  seed("Export document as PDF", [
+    { tool: "focus", params: { bundleId: "" } },
+    { tool: "menu_click", params: { menuPath: "File/Export as PDF" } },
+  ], ["export", "pdf"]),
+];

package/src/memory/session.ts

@@ -0,0 +1,201 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+
+/**
+ * Learning Memory — Session tracking with auto-save
+ *
+ * Tracks a rolling buffer of actions within a "task session".
+ * Auto-saves strategies when a successful sequence is detected:
+ * - 3+ consecutive successes followed by a gap (>60s) or session end
+ * - Or explicit endSession() call
+ */
+
+import type { ActionEntry, Strategy, StrategyStep } from "./types.js";
+import { MemoryStore } from "./store.js";
+import type { RecallEngine } from "./recall.js";
+
+const SESSION_GAP_MS = 60_000; // 60s gap = new session
+const MAX_BUFFER_SIZE = 100;
+const MIN_AUTO_SAVE_STEPS = 3; // Need at least 3 successful steps to auto-save
+
+export class SessionTracker {
+  private store: MemoryStore;
+  private sessionId: string;
+  private taskDescription: string | null = null;
+  private buffer: ActionEntry[] = [];
+  private lastActionTime = 0;
+
+  constructor(store: MemoryStore) {
+    this.store = store;
+    this.sessionId = SessionTracker.generateId();
+  }
+
+  private static generateId(): string {
+    return "s_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
+  }
+
+  /** Start (or restart) a named task session */
+  startSession(taskDescription?: string): string {
+    // Auto-save previous session if it had successful actions
+    this.tryAutoSave();
+
+    this.sessionId = SessionTracker.generateId();
+    this.taskDescription = taskDescription ?? null;
+    this.buffer = [];
+    this.lastActionTime = Date.now();
+    return this.sessionId;
+  }
+
+  /** Get the current session ID, auto-rotating if stale */
+  getSessionId(): string {
+    const now = Date.now();
+    if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
+      // Session gap detected — auto-save previous sequence then start fresh
+      this.tryAutoSave();
+      this.sessionId = SessionTracker.generateId();
+      this.buffer = [];
+      this.taskDescription = null;
+    }
+    return this.sessionId;
+  }
+
+  /** Record an action into the current session buffer */
+  recordAction(entry: ActionEntry): void {
+    const now = Date.now();
+    if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
+      // Gap detected — auto-save then start new session
+      this.tryAutoSave();
+      this.sessionId = SessionTracker.generateId();
+      this.buffer = [];
+      this.taskDescription = null;
+    }
+    this.lastActionTime = now;
+
+    this.buffer.push(entry);
+    if (this.buffer.length > MAX_BUFFER_SIZE) {
+      this.buffer.shift();
+    }
+  }
+
+  /** End the session and save a strategy if successful */
+  endSession(success: boolean, taskDescription?: string): Strategy | null {
+    const task = taskDescription ?? this.taskDescription;
+    if (!success || !task || this.buffer.length === 0) {
+      this.buffer = [];
+      return null;
+    }
+
+    const strategy = this.buildStrategy(task, this.buffer);
+    this.store.appendStrategy(strategy);
+    this.buffer = [];
+    return strategy;
+  }
+
+  /** Get the current session's action buffer */
+  getBuffer(): ActionEntry[] {
+    return [...this.buffer];
+  }
+
+  /** Get recent tool names (for strategy hint matching) */
+  getRecentToolNames(limit = 10): string[] {
+    return this.buffer.slice(-limit).map((a) => a.tool);
+  }
+
+  /** Get current task description */
+  getTaskDescription(): string | null {
+    return this.taskDescription;
+  }
+
+  // ── auto-save logic ────────────────────────────
+
+  /**
+   * Try to auto-save the current buffer as a strategy.
+   * Only saves if there are MIN_AUTO_SAVE_STEPS+ consecutive successes.
+   * Uses tool sequence as task description if no explicit one was given.
+   */
+  private tryAutoSave(): void {
+    if (this.buffer.length < MIN_AUTO_SAVE_STEPS) return;
+
+    // Find the longest trailing streak of successes
+    let successStreak: ActionEntry[] = [];
+    for (let i = this.buffer.length - 1; i >= 0; i--) {
+      if (this.buffer[i]!.success) {
+        successStreak.unshift(this.buffer[i]!);
+      } else {
+        break;
+      }
+    }
+
+    if (successStreak.length < MIN_AUTO_SAVE_STEPS) return;
+
+    // Build a task description from the tool sequence if none provided
+    const task = this.taskDescription ?? this.inferTaskDescription(successStreak);
+
+    const strategy = this.buildStrategy(task, successStreak);
+    this.store.appendStrategy(strategy);
+  }
+
+  /** Infer a task description from a sequence of actions */
+  private inferTaskDescription(actions: ActionEntry[]): string {
+    const tools = [...new Set(actions.map((a) => a.tool))];
+    // Extract key param values (bundle IDs, titles, URLs, etc.)
+    const keyParams: string[] = [];
+    for (const a of actions) {
+      for (const [key, val] of Object.entries(a.params)) {
+        if (typeof val === "string" && val.length > 2 && val.length < 60) {
+          if (["bundleId", "title", "url", "text", "script", "selector", "menuPath"].includes(key)) {
+            keyParams.push(val);
+          }
+        }
+      }
+    }
+    const paramHint = keyParams.length > 0 ? ` (${keyParams.slice(0, 3).join(", ")})` : "";
+    return `${tools.join(" → ")}${paramHint}`;
+  }
+
+  private buildStrategy(task: string, actions: ActionEntry[]): Strategy {
+    const steps: StrategyStep[] = actions.map((a) => ({
+      tool: a.tool,
+      params: a.params,
+    }));
+
+    const totalDurationMs = actions.reduce((sum, a) => sum + a.durationMs, 0);
+    const tags = extractTags(task, steps);
+    const fingerprint = MemoryStore.makeFingerprint(steps.map((s) => s.tool));
+
+    return {
+      id: "str_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
+      task,
+      steps,
+      totalDurationMs,
+      successCount: 1,
+      failCount: 0,
+      lastUsed: new Date().toISOString(),
+      tags,
+      fingerprint,
+    };
+  }
+}
+
+/** Extract tags from task description and tool names */
+function extractTags(task: string, steps: StrategyStep[]): string[] {
+  const tags = new Set<string>();
+  const words = task.toLowerCase().split(/\W+/).filter((w) => w.length >= 3);
+  for (const w of words) tags.add(w);
+  for (const s of steps) tags.add(s.tool);
+  return [...tags];
+}