screenhand 0.1.1 → 0.3.0

package/src/memory/research.ts

@@ -1,146 +0,0 @@
-// Copyright (C) 2025 Clazro Technology Private Limited
-// SPDX-License-Identifier: AGPL-3.0-only
-//
-// This file is part of ScreenHand.
-//
-// ScreenHand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, version 3.
-//
-// ScreenHand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
-
-/**
- * Background web research — fire-and-forget resolution lookup.
- *
- * When a tool fails and no resolution exists, this searches for a fix
- * in the background (non-blocking) and saves it for next time.
- *
- * Resolution paths:
- * 1. Claude API (haiku) — if ANTHROPIC_API_KEY is set
- * 2. DuckDuckGo instant answer — free fallback, no auth
- */
-
-import type { MemoryStore } from "./store.js";
-
-export function backgroundResearch(
-  store: MemoryStore,
-  tool: string,
-  params: Record<string, unknown>,
-  errorMessage: string,
-): void {
-  // Fire-and-forget — never blocks, never throws
-  doResearch(store, tool, params, errorMessage).catch(() => {});
-}
-
-async function doResearch(
-  store: MemoryStore,
-  tool: string,
-  params: Record<string, unknown>,
-  errorMessage: string,
-): Promise<void> {
-  const query = `macOS automation: "${tool}" failed with "${errorMessage.slice(0, 200)}"`;
-
-  let resolution: string | null = null;
-
-  const apiKey = process.env["ANTHROPIC_API_KEY"];
-  if (apiKey) {
-    resolution = await tryClaudeAPI(apiKey, query);
-  }
-
-  if (!resolution) {
-    resolution = await tryDuckDuckGo(query);
-  }
-
-  if (!resolution) return;
-
-  // Save resolution to error cache
-  store.appendError({
-    id: "err_research_" + Date.now().toString(36),
-    tool,
-    params,
-    error: errorMessage,
-    resolution,
-    occurrences: 1,
-    lastSeen: new Date().toISOString(),
-  });
-
-  // Save as a reusable strategy
-  store.appendStrategy({
-    id: "str_research_" + Date.now().toString(36),
-    task: `Fix: ${tool} — ${errorMessage.slice(0, 100)}`,
-    steps: [{ tool, params }],
-    totalDurationMs: 0,
-    successCount: 1,
-    failCount: 0,
-    lastUsed: new Date().toISOString(),
-    tags: [tool, "research", "fix"],
-    fingerprint: "",
-  });
-}
-
-async function tryClaudeAPI(apiKey: string, query: string): Promise<string | null> {
-  try {
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 10000);
-
-    const response = await fetch("https://api.anthropic.com/v1/messages", {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "x-api-key": apiKey,
-        "anthropic-version": "2023-06-01",
-      },
-      body: JSON.stringify({
-        model: "claude-haiku-4-5-20251001",
-        max_tokens: 200,
-        messages: [{
-          role: "user",
-          content: `You are a macOS automation expert. Give a brief fix (1-2 sentences) for this error:\n\n${query}`,
-        }],
-      }),
-      signal: controller.signal,
-    });
-
-    clearTimeout(timeout);
-
-    if (!response.ok) return null;
-
-    const data = await response.json() as {
-      content?: Array<{ type: string; text?: string }>;
-    };
-    const text = data.content?.[0]?.text;
-    return text && text.length > 10 ? text.trim() : null;
-  } catch {
-    return null;
-  }
-}
-
-async function tryDuckDuckGo(query: string): Promise<string | null> {
-  try {
-    const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(), 5000);
-
-    const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1`;
-    const response = await fetch(url, { signal: controller.signal });
-
-    clearTimeout(timeout);
-
-    if (!response.ok) return null;
-
-    const data = await response.json() as {
-      AbstractText?: string;
-      Abstract?: string;
-      Answer?: string;
-    };
-    const text = data.AbstractText || data.Abstract || data.Answer;
-    return text && text.length > 10 ? text.trim() : null;
-  } catch {
-    return null;
-  }
-}

package/src/memory/seeds.ts

@@ -1,123 +0,0 @@
-// Copyright (C) 2025 Clazro Technology Private Limited
-// SPDX-License-Identifier: AGPL-3.0-only
-//
-// This file is part of ScreenHand.
-//
-// ScreenHand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, version 3.
-//
-// ScreenHand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
-
-/**
- * Predefined seed strategies — common macOS desktop workflows.
- * Loaded on first boot so the memory system has knowledge from day one.
- */
-
-import type { Strategy, StrategyStep } from "./types.js";
-
-let seedCounter = 0;
-
-function makeFingerprint(tools: string[]): string {
-  return tools.join("→");
-}
-
-function seed(task: string, steps: StrategyStep[], tags: string[]): Strategy {
-  seedCounter++;
-  return {
-    id: `seed_${String(seedCounter).padStart(3, "0")}`,
-    task,
-    steps,
-    totalDurationMs: 0,
-    successCount: 10,
-    failCount: 0,
-    lastUsed: new Date().toISOString(),
-    tags,
-    fingerprint: makeFingerprint(steps.map((s) => s.tool)),
-  };
-}
-
-export const SEED_STRATEGIES: Strategy[] = [
-  // 1. Take a photo with Photo Booth
-  seed("Take a photo with Photo Booth", [
-    { tool: "launch", params: { bundleId: "com.apple.PhotoBooth" } },
-    { tool: "ui_press", params: { title: "Take Photo" } },
-  ], ["photo", "booth", "camera"]),
-
-  // 2. Open a URL in Chrome
-  seed("Open a URL in Chrome", [
-    { tool: "launch", params: { bundleId: "com.google.Chrome" } },
-    { tool: "browser_navigate", params: { url: "" } },
-  ], ["chrome", "browse", "url"]),
-
-  // 3. Save current document
-  seed("Save current document", [
-    { tool: "focus", params: { bundleId: "" } },
-    { tool: "key", params: { combo: "cmd+s" } },
-  ], ["save", "document"]),
-
-  // 4. Copy from one app and paste into another
-  seed("Copy from one app and paste into another", [
-    { tool: "focus", params: { bundleId: "" } },
-    { tool: "key", params: { combo: "cmd+c" } },
-    { tool: "focus", params: { bundleId: "" } },
-    { tool: "key", params: { combo: "cmd+v" } },
-  ], ["copy", "paste"]),
-
-  // 5. Navigate to a folder in Finder
-  seed("Navigate to a folder in Finder", [
-    { tool: "focus", params: { bundleId: "com.apple.finder" } },
-    { tool: "key", params: { combo: "cmd+shift+g" } },
-    { tool: "type_text", params: { text: "" } },
-  ], ["finder", "folder", "navigate"]),
-
-  // 6. Create a new folder in Finder
-  seed("Create a new folder in Finder", [
-    { tool: "focus", params: { bundleId: "com.apple.finder" } },
-    { tool: "key", params: { combo: "cmd+shift+n" } },
-    { tool: "type_text", params: { text: "" } },
-  ], ["finder", "folder", "create"]),
-
-  // 7. Close the current window
-  seed("Close the current window", [
-    { tool: "focus", params: { bundleId: "" } },
-    { tool: "key", params: { combo: "cmd+w" } },
-  ], ["close", "window"]),
-
-  // 8. Select all and copy
-  seed("Select all content and copy", [
-    { tool: "focus", params: { bundleId: "" } },
-    { tool: "key", params: { combo: "cmd+a" } },
-    { tool: "key", params: { combo: "cmd+c" } },
-  ], ["select", "all", "copy"]),
-
-  // 9. List running apps
-  seed("List all running applications", [
-    { tool: "apps", params: {} },
-  ], ["apps", "list", "running"]),
-
-  // 10. Inspect app UI tree
-  seed("Inspect an app's UI element tree", [
-    { tool: "focus", params: { bundleId: "" } },
-    { tool: "ui_tree", params: { pid: 0 } },
-  ], ["inspect", "tree", "accessibility"]),
-
-  // 11. Open a new tab in Chrome and navigate
-  seed("Open a new Chrome tab and navigate to URL", [
-    { tool: "focus", params: { bundleId: "com.google.Chrome" } },
-    { tool: "key", params: { combo: "cmd+t" } },
-    { tool: "browser_navigate", params: { url: "" } },
-  ], ["chrome", "tab", "new"]),
-
-  // 12. Export as PDF via menu
-  seed("Export document as PDF", [
-    { tool: "focus", params: { bundleId: "" } },
-    { tool: "menu_click", params: { menuPath: "File/Export as PDF" } },
-  ], ["export", "pdf"]),
-];

package/src/memory/session.ts

@@ -1,201 +0,0 @@
-// Copyright (C) 2025 Clazro Technology Private Limited
-// SPDX-License-Identifier: AGPL-3.0-only
-//
-// This file is part of ScreenHand.
-//
-// ScreenHand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, version 3.
-//
-// ScreenHand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
-
-/**
- * Learning Memory — Session tracking with auto-save
- *
- * Tracks a rolling buffer of actions within a "task session".
- * Auto-saves strategies when a successful sequence is detected:
- * - 3+ consecutive successes followed by a gap (>60s) or session end
- * - Or explicit endSession() call
- */
-
-import type { ActionEntry, Strategy, StrategyStep } from "./types.js";
-import { MemoryStore } from "./store.js";
-import type { RecallEngine } from "./recall.js";
-
-const SESSION_GAP_MS = 60_000; // 60s gap = new session
-const MAX_BUFFER_SIZE = 100;
-const MIN_AUTO_SAVE_STEPS = 3; // Need at least 3 successful steps to auto-save
-
-export class SessionTracker {
-  private store: MemoryStore;
-  private sessionId: string;
-  private taskDescription: string | null = null;
-  private buffer: ActionEntry[] = [];
-  private lastActionTime = 0;
-
-  constructor(store: MemoryStore) {
-    this.store = store;
-    this.sessionId = SessionTracker.generateId();
-  }
-
-  private static generateId(): string {
-    return "s_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
-  }
-
-  /** Start (or restart) a named task session */
-  startSession(taskDescription?: string): string {
-    // Auto-save previous session if it had successful actions
-    this.tryAutoSave();
-
-    this.sessionId = SessionTracker.generateId();
-    this.taskDescription = taskDescription ?? null;
-    this.buffer = [];
-    this.lastActionTime = Date.now();
-    return this.sessionId;
-  }
-
-  /** Get the current session ID, auto-rotating if stale */
-  getSessionId(): string {
-    const now = Date.now();
-    if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
-      // Session gap detected — auto-save previous sequence then start fresh
-      this.tryAutoSave();
-      this.sessionId = SessionTracker.generateId();
-      this.buffer = [];
-      this.taskDescription = null;
-    }
-    return this.sessionId;
-  }
-
-  /** Record an action into the current session buffer */
-  recordAction(entry: ActionEntry): void {
-    const now = Date.now();
-    if (this.lastActionTime > 0 && now - this.lastActionTime > SESSION_GAP_MS) {
-      // Gap detected — auto-save then start new session
-      this.tryAutoSave();
-      this.sessionId = SessionTracker.generateId();
-      this.buffer = [];
-      this.taskDescription = null;
-    }
-    this.lastActionTime = now;
-
-    this.buffer.push(entry);
-    if (this.buffer.length > MAX_BUFFER_SIZE) {
-      this.buffer.shift();
-    }
-  }
-
-  /** End the session and save a strategy if successful */
-  endSession(success: boolean, taskDescription?: string): Strategy | null {
-    const task = taskDescription ?? this.taskDescription;
-    if (!success || !task || this.buffer.length === 0) {
-      this.buffer = [];
-      return null;
-    }
-
-    const strategy = this.buildStrategy(task, this.buffer);
-    this.store.appendStrategy(strategy);
-    this.buffer = [];
-    return strategy;
-  }
-
-  /** Get the current session's action buffer */
-  getBuffer(): ActionEntry[] {
-    return [...this.buffer];
-  }
-
-  /** Get recent tool names (for strategy hint matching) */
-  getRecentToolNames(limit = 10): string[] {
-    return this.buffer.slice(-limit).map((a) => a.tool);
-  }
-
-  /** Get current task description */
-  getTaskDescription(): string | null {
-    return this.taskDescription;
-  }
-
-  // ── auto-save logic ────────────────────────────
-
-  /**
-   * Try to auto-save the current buffer as a strategy.
-   * Only saves if there are MIN_AUTO_SAVE_STEPS+ consecutive successes.
-   * Uses tool sequence as task description if no explicit one was given.
-   */
-  private tryAutoSave(): void {
-    if (this.buffer.length < MIN_AUTO_SAVE_STEPS) return;
-
-    // Find the longest trailing streak of successes
-    let successStreak: ActionEntry[] = [];
-    for (let i = this.buffer.length - 1; i >= 0; i--) {
-      if (this.buffer[i]!.success) {
-        successStreak.unshift(this.buffer[i]!);
-      } else {
-        break;
-      }
-    }
-
-    if (successStreak.length < MIN_AUTO_SAVE_STEPS) return;
-
-    // Build a task description from the tool sequence if none provided
-    const task = this.taskDescription ?? this.inferTaskDescription(successStreak);
-
-    const strategy = this.buildStrategy(task, successStreak);
-    this.store.appendStrategy(strategy);
-  }
-
-  /** Infer a task description from a sequence of actions */
-  private inferTaskDescription(actions: ActionEntry[]): string {
-    const tools = [...new Set(actions.map((a) => a.tool))];
-    // Extract key param values (bundle IDs, titles, URLs, etc.)
-    const keyParams: string[] = [];
-    for (const a of actions) {
-      for (const [key, val] of Object.entries(a.params)) {
-        if (typeof val === "string" && val.length > 2 && val.length < 60) {
-          if (["bundleId", "title", "url", "text", "script", "selector", "menuPath"].includes(key)) {
-            keyParams.push(val);
-          }
-        }
-      }
-    }
-    const paramHint = keyParams.length > 0 ? ` (${keyParams.slice(0, 3).join(", ")})` : "";
-    return `${tools.join(" → ")}${paramHint}`;
-  }
-
-  private buildStrategy(task: string, actions: ActionEntry[]): Strategy {
-    const steps: StrategyStep[] = actions.map((a) => ({
-      tool: a.tool,
-      params: a.params,
-    }));
-
-    const totalDurationMs = actions.reduce((sum, a) => sum + a.durationMs, 0);
-    const tags = extractTags(task, steps);
-    const fingerprint = MemoryStore.makeFingerprint(steps.map((s) => s.tool));
-
-    return {
-      id: "str_" + Date.now().toString(36) + Math.random().toString(36).slice(2, 6),
-      task,
-      steps,
-      totalDurationMs,
-      successCount: 1,
-      failCount: 0,
-      lastUsed: new Date().toISOString(),
-      tags,
-      fingerprint,
-    };
-  }
-}
-
-/** Extract tags from task description and tool names */
-function extractTags(task: string, steps: StrategyStep[]): string[] {
-  const tags = new Set<string>();
-  const words = task.toLowerCase().split(/\W+/).filter((w) => w.length >= 3);
-  for (const w of words) tags.add(w);
-  for (const s of steps) tags.add(s.tool);
-  return [...tags];
-}