screenhand 0.1.1 → 0.2.0

package/dist/src/playbook/runner.js

@@ -0,0 +1,408 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * Playbook Runner — the brain
+ *
+ * 1. Match task → playbook
+ * 2. Execute playbook steps (fast, no AI)
+ * 3. If step fails → ask AI to recover
+ * 4. Save AI's recovery steps back into playbook
+ * 5. Loop forever in monitor mode
+ */
+import Anthropic from "@anthropic-ai/sdk";
+import { PlaybookEngine } from "./engine.js";
+import { PlaybookStore } from "./store.js";
+export class PlaybookRunner {
+    runtime;
+    engine;
+    store;
+    ai;
+    model;
+    maxRecovery;
+    log;
+    constructor(runtime, playbookDir, options = {}) {
+        this.runtime = runtime;
+        this.engine = new PlaybookEngine(runtime);
+        this.store = new PlaybookStore(playbookDir);
+        this.store.load();
+        this.ai = new Anthropic();
+        this.model = options.model ?? "claude-sonnet-4-20250514";
+        this.maxRecovery = options.maxRecoveryAttempts ?? 3;
+        this.log = options.onLog ?? ((msg) => console.error(`[PlaybookRunner] ${msg}`));
+    }
+    /**
+     * Execute a task. Tries playbook first, falls back to AI.
+     */
+    async execute(sessionId, task) {
+        // 1. Find matching playbook
+        const playbook = this.store.matchByTask(task);
+        if (playbook && playbook.steps.length > 0) {
+            this.log(`Found playbook: ${playbook.name} (${playbook.successCount} successes)`);
+            // 2. Run playbook
+            const result = await this.engine.run(sessionId, playbook, {
+                onStep: (i, step, res) => {
+                    this.log(`  Step ${i + 1}/${playbook.steps.length}: ${step.description ?? step.action} → ${res}`);
+                },
+            });
+            if (result.success) {
+                this.store.recordOutcome(playbook.id, true);
+                this.log(`Playbook completed successfully in ${result.durationMs}ms`);
+                return result;
+            }
+            // 3. Playbook failed at a step — try AI recovery
+            this.log(`Playbook failed at step ${result.failedAtStep}: ${result.error}`);
+            const recovery = await this.aiRecover(sessionId, playbook, result);
+            if (recovery) {
+                this.store.recordOutcome(playbook.id, true);
+                return { ...result, success: true, aiRecovery: recovery };
+            }
+            this.store.recordOutcome(playbook.id, false);
+            return result;
+        }
+        // Playbook found but has no executable steps (legacy format with flows/selectors)
+        // → Use AI mode but feed it the playbook's rich metadata as context
+        if (playbook) {
+            this.log(`Found reference playbook: ${playbook.name} (flows/selectors, no executable steps)`);
+            return this.aiExecute(sessionId, task, playbook);
+        }
+        // No playbook found — pure AI mode
+        this.log(`No playbook found for: "${task}". Using AI.`);
+        return this.aiExecute(sessionId, task);
+    }
+    /**
+     * AI recovery — when a playbook step fails, ask AI to fix it.
+     */
+    async aiRecover(sessionId, playbook, failResult) {
+        const failedStep = playbook.steps[failResult.failedAtStep];
+        if (!failedStep)
+            return null;
+        // Take screenshot for context
+        let screenshotInfo = "";
+        try {
+            const shot = await this.runtime.screenshot({ sessionId });
+            if (shot.ok)
+                screenshotInfo = `Screenshot saved to: ${shot.data.path}`;
+        }
+        catch { /* ignore */ }
+        // Get current page state
+        let pageState = "";
+        try {
+            const tree = await this.runtime.elementTree({ sessionId, maxDepth: 4 });
+            if (tree.ok) {
+                pageState = JSON.stringify(tree.data).slice(0, 3000);
+            }
+        }
+        catch { /* ignore */ }
+        // Build rich context from playbook metadata
+        const playbookContext = buildPlaybookContext(playbook);
+        const prompt = `A playbook automation failed. Help me recover.
+
+Playbook: ${playbook.name}
+Platform: ${playbook.platform}
+Failed at step ${failResult.failedAtStep + 1}/${playbook.steps.length}:
+  Action: ${failedStep.action}
+  Target: ${JSON.stringify(failedStep.target)}
+  Error: ${failResult.error}
+
+Steps completed before failure:
+${playbook.steps.slice(0, failResult.failedAtStep).map((s, i) => `  ${i + 1}. ${s.description ?? s.action}`).join("\n")}
+
+Remaining steps after failure:
+${playbook.steps.slice(failResult.failedAtStep).map((s, i) => `  ${failResult.failedAtStep + i + 1}. ${s.description ?? s.action}`).join("\n")}
+
+${playbookContext}
+
+Current UI state (accessibility tree):
+${pageState}
+
+${screenshotInfo}
+
+What should I do to recover? Respond with a JSON array of recovery steps:
+[
+  { "action": "press", "target": "...", "description": "..." },
+  { "action": "wait", "ms": 1000 }
+]
+
+Or if unrecoverable, respond with: { "unrecoverable": true, "reason": "..." }`;
+        for (let attempt = 0; attempt < this.maxRecovery; attempt++) {
+            try {
+                const resp = await this.ai.messages.create({
+                    model: this.model,
+                    max_tokens: 1024,
+                    messages: [{ role: "user", content: prompt }],
+                });
+                const text = resp.content[0]?.type === "text" ? resp.content[0].text : "";
+                const jsonMatch = text.match(/\[[\s\S]*\]|\{[\s\S]*\}/);
+                if (!jsonMatch)
+                    continue;
+                const parsed = JSON.parse(jsonMatch[0]);
+                // Check if unrecoverable
+                if (parsed.unrecoverable) {
+                    this.log(`AI says unrecoverable: ${parsed.reason}`);
+                    return null;
+                }
+                // Execute recovery steps
+                const recoverySteps = Array.isArray(parsed) ? parsed : [parsed];
+                this.log(`AI suggests ${recoverySteps.length} recovery steps`);
+                for (const step of recoverySteps) {
+                    const stepResult = await this.engine.run(sessionId, {
+                        id: "recovery",
+                        name: "AI Recovery",
+                        description: "",
+                        platform: playbook.platform,
+                        steps: [step],
+                        version: "0",
+                        tags: [],
+                        successCount: 0,
+                        failCount: 0,
+                    });
+                    if (!stepResult.success) {
+                        this.log(`Recovery step failed: ${stepResult.error}`);
+                        continue;
+                    }
+                }
+                // Now try remaining playbook steps
+                const remaining = {
+                    ...playbook,
+                    id: `${playbook.id}_remaining`,
+                    steps: playbook.steps.slice(failResult.failedAtStep + 1),
+                };
+                if (remaining.steps.length > 0) {
+                    const remainingResult = await this.engine.run(sessionId, remaining, {
+                        onStep: (i, step, res) => {
+                            const globalIdx = failResult.failedAtStep + 1 + i;
+                            this.log(`  Step ${globalIdx + 1}/${playbook.steps.length}: ${step.description ?? step.action} → ${res}`);
+                        },
+                    });
+                    if (!remainingResult.success) {
+                        this.log(`Remaining steps failed at ${remainingResult.failedAtStep}`);
+                        return null;
+                    }
+                }
+                // Save recovery steps back into playbook for next time
+                this.patchPlaybook(playbook, failResult.failedAtStep, recoverySteps);
+                return `AI recovered with ${recoverySteps.length} steps, then completed remaining ${remaining.steps.length} steps`;
+            }
+            catch (err) {
+                this.log(`AI recovery attempt ${attempt + 1} failed: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        return null;
+    }
+    /**
+     * AI execution — optionally guided by a reference playbook's metadata.
+     * When a playbook has selectors/flows/errors but no executable steps,
+     * AI uses that knowledge to make smarter decisions.
+     * After success, saves the steps as a new playbook.
+     */
+    async aiExecute(sessionId, task, refPlaybook) {
+        const start = Date.now();
+        const executedSteps = [];
+        const playbookContext = refPlaybook ? buildPlaybookContext(refPlaybook) : "";
+        // Simple AI loop — observe, decide, act
+        for (let i = 0; i < 20; i++) {
+            let pageState = "";
+            try {
+                const tree = await this.runtime.elementTree({ sessionId, maxDepth: 4 });
+                if (tree.ok)
+                    pageState = JSON.stringify(tree.data).slice(0, 4000);
+            }
+            catch { /* ignore */ }
+            const prompt = `Task: ${task}
+
+Steps taken so far:
+${executedSteps.map((s, idx) => `${idx + 1}. ${s.description ?? s.action}`).join("\n") || "(none)"}
+
+Current UI state:
+${pageState}
+${playbookContext ? `\n--- PLAYBOOK REFERENCE ---\n${playbookContext}\nUse the selectors, flows, and error solutions above to guide your actions. Prefer data-testid selectors over text matching.\n---\n` : ""}
+What's the next step? Respond with ONE step as JSON:
+{ "action": "press|type_into|navigate|key_combo|scroll|wait", "target": "...", "text": "...", "url": "...", "keys": [...], "ms": 1000, "description": "..." }
+
+Or if done: { "action": "done", "description": "Task complete" }`;
+            try {
+                const resp = await this.ai.messages.create({
+                    model: this.model,
+                    max_tokens: 512,
+                    messages: [{ role: "user", content: prompt }],
+                });
+                const text = resp.content[0]?.type === "text" ? resp.content[0].text : "";
+                const jsonMatch = text.match(/\{[\s\S]*\}/);
+                if (!jsonMatch)
+                    continue;
+                const step = JSON.parse(jsonMatch[0]);
+                if (step.action === "done") {
+                    // Save as new playbook for next time
+                    if (executedSteps.length > 0) {
+                        this.saveNewPlaybook(task, executedSteps);
+                    }
+                    return {
+                        playbook: "ai_generated",
+                        success: true,
+                        stepsCompleted: executedSteps.length,
+                        totalSteps: executedSteps.length,
+                        failedAtStep: -1,
+                        durationMs: Date.now() - start,
+                    };
+                }
+                // Execute the step
+                const stepResult = await this.engine.run(sessionId, {
+                    id: "ai_step",
+                    name: "AI Step",
+                    description: "",
+                    platform: "unknown",
+                    steps: [step],
+                    version: "0",
+                    tags: [],
+                    successCount: 0,
+                    failCount: 0,
+                });
+                if (stepResult.success) {
+                    executedSteps.push(step);
+                    this.log(`AI step ${executedSteps.length}: ${step.description ?? step.action}`);
+                }
+                else {
+                    this.log(`AI step failed: ${stepResult.error}`);
+                }
+                await sleep(300);
+            }
+            catch (err) {
+                this.log(`AI step error: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        return {
+            playbook: "ai_generated",
+            success: false,
+            stepsCompleted: executedSteps.length,
+            totalSteps: -1,
+            failedAtStep: executedSteps.length,
+            error: "Max AI steps reached",
+            durationMs: Date.now() - start,
+        };
+    }
+    /**
+     * Patch a playbook — insert recovery steps at the failure point.
+     */
+    patchPlaybook(playbook, failedAt, recoverySteps) {
+        const patched = {
+            ...playbook,
+            steps: [
+                ...playbook.steps.slice(0, failedAt),
+                ...recoverySteps,
+                ...playbook.steps.slice(failedAt),
+            ],
+            version: bumpVersion(playbook.version),
+        };
+        this.store.save(patched);
+        this.log(`Patched playbook ${playbook.id}: inserted ${recoverySteps.length} recovery steps at position ${failedAt}`);
+    }
+    /**
+     * Save AI-generated steps as a new playbook.
+     */
+    saveNewPlaybook(task, steps) {
+        const id = `auto_${Date.now()}`;
+        const playbook = {
+            id,
+            name: task.slice(0, 80),
+            description: `Auto-generated from AI execution: ${task}`,
+            platform: "unknown",
+            steps,
+            version: "1.0.0",
+            tags: task.toLowerCase().split(/\W+/).filter((w) => w.length >= 3),
+            successCount: 1,
+            failCount: 0,
+            lastRun: new Date().toISOString(),
+        };
+        this.store.save(playbook);
+        this.log(`Saved new playbook: ${id} (${steps.length} steps)`);
+    }
+    /** Get all loaded playbooks. */
+    listPlaybooks() {
+        return this.store.getAll();
+    }
+    /** Reload playbooks from disk. */
+    reload() {
+        this.store.load();
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function bumpVersion(version) {
+    const parts = version.split(".").map(Number);
+    if (parts.length === 3) {
+        parts[2]++;
+        return parts.join(".");
+    }
+    return version + ".1";
+}
+/**
+ * Build a context string from playbook metadata for AI consumption.
+ * Includes selectors, flows, known errors, detection expressions.
+ */
+function buildPlaybookContext(playbook) {
+    const parts = [];
+    if (playbook.urls && Object.keys(playbook.urls).length > 0) {
+        parts.push("URLS:\n" + Object.entries(playbook.urls).map(([k, v]) => `  ${k}: ${v}`).join("\n"));
+    }
+    if (playbook.selectors && Object.keys(playbook.selectors).length > 0) {
+        parts.push("SELECTORS:");
+        for (const [group, sels] of Object.entries(playbook.selectors)) {
+            parts.push(`  [${group}]`);
+            for (const [name, sel] of Object.entries(sels)) {
+                parts.push(`    ${name}: ${sel}`);
+            }
+        }
+    }
+    if (playbook.flows && Object.keys(playbook.flows).length > 0) {
+        parts.push("FLOWS:");
+        for (const [name, flow] of Object.entries(playbook.flows)) {
+            parts.push(`  [${name}]`);
+            for (const step of flow.steps) {
+                parts.push(`    - ${step}`);
+            }
+            if (flow.guards) {
+                parts.push(`    Guards:`);
+                for (const g of flow.guards)
+                    parts.push(`      ! ${g}`);
+            }
+        }
+    }
+    if (playbook.detection && Object.keys(playbook.detection).length > 0) {
+        parts.push("DETECTION (JS expressions):");
+        for (const [name, expr] of Object.entries(playbook.detection)) {
+            parts.push(`  ${name}: ${expr}`);
+        }
+    }
+    if (playbook.errors && playbook.errors.length > 0) {
+        parts.push("KNOWN ERRORS & SOLUTIONS:");
+        for (const e of playbook.errors) {
+            parts.push(`  [${e.severity}] ${e.error}`);
+            parts.push(`    Context: ${e.context}`);
+            parts.push(`    Solution: ${e.solution}`);
+        }
+    }
+    if (playbook.policyNotes && Object.keys(playbook.policyNotes).length > 0) {
+        parts.push("POLICY NOTES:");
+        for (const [cat, notes] of Object.entries(playbook.policyNotes)) {
+            parts.push(`  [${cat}]`);
+            for (const n of notes)
+                parts.push(`    - ${n}`);
+        }
+    }
+    return parts.join("\n");
+}

package/dist/src/playbook/store.js

@@ -0,0 +1,183 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+/**
+ * Playbook Store — load, save, match playbooks from disk
+ *
+ * Playbooks are stored as JSON files in the playbooks/ directory.
+ * Each file can contain a Playbook directly, or a legacy format
+ * that gets converted.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { writeFileAtomicSync } from "../util/atomic-write.js";
+export class PlaybookStore {
+    dir;
+    playbooks = new Map();
+    constructor(dir) {
+        this.dir = dir;
+    }
+    /** Load all playbooks from disk into memory. */
+    load() {
+        this.playbooks.clear();
+        if (!fs.existsSync(this.dir))
+            return;
+        const files = fs.readdirSync(this.dir).filter((f) => f.endsWith(".json"));
+        for (const file of files) {
+            try {
+                const raw = JSON.parse(fs.readFileSync(path.join(this.dir, file), "utf-8"));
+                const playbook = this.normalize(raw, file);
+                if (playbook) {
+                    this.playbooks.set(playbook.id, playbook);
+                }
+            }
+            catch {
+                // Skip unparseable files
+            }
+        }
+    }
+    /** Get all loaded playbooks. */
+    getAll() {
+        return [...this.playbooks.values()];
+    }
+    /** Get a playbook by ID. */
+    get(id) {
+        return this.playbooks.get(id);
+    }
+    /** Find playbooks matching a URL. */
+    matchByUrl(url) {
+        return this.getAll().filter((p) => {
+            if (!p.urlPatterns || p.urlPatterns.length === 0)
+                return false;
+            return p.urlPatterns.some((pattern) => {
+                try {
+                    return new RegExp(pattern).test(url);
+                }
+                catch {
+                    return url.includes(pattern);
+                }
+            });
+        });
+    }
+    /** Find playbooks matching tags. */
+    matchByTags(tags) {
+        const tagSet = new Set(tags.map((t) => t.toLowerCase()));
+        return this.getAll()
+            .filter((p) => p.tags.some((t) => tagSet.has(t.toLowerCase())))
+            .sort((a, b) => b.successCount - a.successCount);
+    }
+    /** Find playbooks by platform. */
+    matchByPlatform(platform) {
+        return this.getAll()
+            .filter((p) => p.platform.toLowerCase() === platform.toLowerCase())
+            .sort((a, b) => b.successCount - a.successCount);
+    }
+    /** Find best playbook for a task description (simple keyword matching). */
+    matchByTask(task) {
+        const tokens = task.toLowerCase().split(/\W+/).filter((w) => w.length >= 3);
+        if (tokens.length === 0)
+            return null;
+        let best = null;
+        let bestScore = 0;
+        for (const p of this.playbooks.values()) {
+            const haystack = `${p.name} ${p.description} ${p.tags.join(" ")} ${p.platform}`.toLowerCase();
+            let score = 0;
+            for (const token of tokens) {
+                if (haystack.includes(token))
+                    score++;
+            }
+            // Weight by reliability
+            const reliability = p.successCount + p.failCount > 0
+                ? p.successCount / (p.successCount + p.failCount)
+                : 0.5;
+            score *= reliability;
+            if (score > bestScore) {
+                bestScore = score;
+                best = p;
+            }
+        }
+        return bestScore > 0 ? best : null;
+    }
+    /** Save a playbook to disk. */
+    save(playbook) {
+        if (!fs.existsSync(this.dir)) {
+            fs.mkdirSync(this.dir, { recursive: true });
+        }
+        const filename = `${playbook.id}.json`;
+        writeFileAtomicSync(path.join(this.dir, filename), JSON.stringify(playbook, null, 2) + "\n");
+        this.playbooks.set(playbook.id, playbook);
+    }
+    /** Record a run outcome. */
+    recordOutcome(id, success) {
+        const playbook = this.playbooks.get(id);
+        if (!playbook)
+            return;
+        if (success) {
+            playbook.successCount++;
+        }
+        else {
+            playbook.failCount++;
+        }
+        playbook.lastRun = new Date().toISOString();
+        this.save(playbook);
+    }
+    /**
+     * Normalize raw JSON into a Playbook.
+     * Handles both new format (has steps array) and legacy format (has flows object).
+     */
+    normalize(raw, filename) {
+        // Already in new format
+        if (Array.isArray(raw.steps)) {
+            return raw;
+        }
+        // Legacy format: has flows with steps arrays (like instagram_v2.json)
+        if (raw.flows && typeof raw.flows === "object") {
+            return this.convertLegacy(raw, filename);
+        }
+        return null;
+    }
+    /**
+     * Convert legacy playbook format to new format.
+     * Preserves all rich metadata: selectors, flows, errors, detection, policy notes.
+     */
+    convertLegacy(raw, filename) {
+        const platform = raw.platform ?? filename.replace(".json", "");
+        const id = raw.playbook ?? filename.replace(".json", "");
+        const description = raw.description ?? "";
+        return {
+            id,
+            name: `${platform} playbook`,
+            description,
+            platform,
+            urlPatterns: raw.urls ? Object.values(raw.urls).map(escapeRegex) : [],
+            steps: [], // Legacy playbooks use flows instead of direct steps
+            tags: [platform, ...(raw.playbook ? [raw.playbook] : [])],
+            version: raw.version ?? "1.0.0",
+            successCount: 0,
+            failCount: 0,
+            // Preserve rich metadata
+            ...(raw.urls ? { urls: raw.urls } : {}),
+            ...(raw.selectors ? { selectors: raw.selectors } : {}),
+            ...(raw.flows ? { flows: raw.flows } : {}),
+            ...(raw.detection ? { detection: raw.detection } : {}),
+            ...(raw.errors ? { errors: raw.errors } : {}),
+            ...(raw.policy_notes ? { policyNotes: raw.policy_notes } : {}),
+        };
+    }
+}
+function escapeRegex(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

package/dist/src/playbook/types.js

@@ -0,0 +1,17 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export {};