screenhand 0.1.1 → 0.3.0

package/dist/src/playbook/mcp-recorder.js

@@ -0,0 +1,204 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * McpPlaybookRecorder — captures MCP tool calls as PlaybookSteps.
+ *
+ * Start recording → agent does the flow → stop → saves as executable playbook.
+ * Like a macro recorder, but for AI tool calls.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { writeFileAtomicSync } from "../util/atomic-write.js";
+import { redactPII } from "../util/sanitize.js";
+/** Tools that are observation-only — not recorded as steps */
+const SKIP_TOOLS = new Set([
+    "screenshot", "screenshot_file",
+    "ui_tree", "ui_find", "browser_dom", "browser_page_info", "browser_tabs",
+    "ocr", "apps", "windows", "memory_recall", "memory_save", "memory_snapshot",
+    "memory_stats", "memory_errors", "memory_query_patterns", "memory_record_error",
+    "memory_record_learning", "memory_clear", "platform_guide", "playbook_preflight",
+    "export_playbook", "playbook_record", "job_create", "job_status", "job_list",
+    "job_run", "job_run_all", "job_create_chain", "job_remove", "job_transition",
+    "job_step_done", "job_step_fail", "job_resume", "job_dequeue",
+    "worker_start", "worker_stop", "worker_status",
+    "supervisor_status", "supervisor_start", "supervisor_stop",
+    "supervisor_pause", "supervisor_resume", "supervisor_install", "supervisor_uninstall",
+    "session_claim", "session_heartbeat", "session_release",
+    "recovery_queue_add", "recovery_queue_list",
+    "codex_monitor_start", "codex_monitor_status", "codex_monitor_add_task",
+    "codex_monitor_tasks", "codex_monitor_assign_now", "codex_monitor_stop",
+    "platform_learn", "platform_explore",
+]);
+/** Map MCP tool names to PlaybookStep actions */
+function mapToolToAction(toolName) {
+    switch (toolName) {
+        case "browser_navigate": return "navigate";
+        case "click":
+        case "click_text":
+        case "browser_click":
+        case "click_with_fallback":
+        case "ui_press": return "press";
+        case "type_text":
+        case "browser_type":
+        case "type_with_fallback": return "type_into";
+        case "key": return "key";
+        case "menu_click": return "menu_click";
+        case "scroll":
+        case "scroll_with_fallback": return "scroll";
+        case "browser_js": return "browser_js";
+        case "screenshot":
+        case "screenshot_file": return "screenshot";
+        case "browser_wait":
+        case "wait_for_state": return "wait";
+        case "focus":
+        case "launch": return null; // useful context but not a step
+        case "drag": return null; // drag is complex, skip for now
+        default: return null;
+    }
+}
+/** Build a PlaybookStep from an MCP tool call */
+function buildStep(toolName, params, success) {
+    const action = mapToolToAction(toolName);
+    if (!action)
+        return null;
+    const step = { action };
+    switch (action) {
+        case "navigate":
+            step.url = String(params.url ?? "");
+            step.description = `Navigate to ${step.url}`;
+            break;
+        case "press":
+            step.target = String(params.selector ?? params.text ?? params.title ?? params.target ?? "");
+            step.description = `Click ${step.target}`;
+            break;
+        case "type_into":
+            step.target = String(params.selector ?? params.target ?? params.field ?? "");
+            step.text = String(params.text ?? params.value ?? "");
+            step.description = `Type "${step.text.substring(0, 50)}" into ${step.target}`;
+            break;
+        case "key":
+        case "key_combo": {
+            const combo = String(params.combo ?? params.key ?? "");
+            step.keys = combo.split("+").map(k => k.trim());
+            step.description = `${action === "key" ? "Key" : "Key combo"}: ${combo}`;
+            break;
+        }
+        case "menu_click": {
+            const menuPath = Array.isArray(params.menuPath)
+                ? params.menuPath.map((part) => String(part).trim()).filter(Boolean)
+                : String(params.menuPath ?? "").split("/").map((part) => part.trim()).filter(Boolean);
+            step.menuPath = menuPath;
+            step.description = `Menu click: ${menuPath.join(" > ")}`;
+            break;
+        }
+        case "scroll":
+            step.direction = params.direction ?? "down";
+            if (params.amount != null)
+                step.amount = Number(params.amount);
+            step.description = `Scroll ${step.direction}`;
+            break;
+        case "browser_js":
+            step.code = String(params.code ?? "");
+            step.description = `Execute JS: ${step.code.substring(0, 60)}...`;
+            break;
+        case "screenshot":
+            step.description = "Take screenshot";
+            break;
+        case "wait":
+            step.ms = Number(params.timeout ?? params.ms ?? params.timeoutMs ?? 1000);
+            step.description = `Wait ${step.ms}ms`;
+            break;
+    }
+    if (!success) {
+        step.optional = true;
+    }
+    return step;
+}
+export class McpPlaybookRecorder {
+    playbooksDir;
+    recording = false;
+    platform = "";
+    steps = [];
+    startTime = "";
+    cdpPort;
+    constructor(playbooksDir) {
+        this.playbooksDir = playbooksDir;
+    }
+    get isRecording() { return this.recording; }
+    get stepCount() { return this.steps.length; }
+    getSteps() { return [...this.steps]; }
+    start(platform, cdpPort) {
+        this.recording = true;
+        this.platform = platform;
+        this.steps = [];
+        this.startTime = new Date().toISOString();
+        if (cdpPort !== undefined)
+            this.cdpPort = cdpPort;
+    }
+    captureToolCall(toolName, params, success, _result, _durationMs) {
+        if (!this.recording)
+            return;
+        if (SKIP_TOOLS.has(toolName))
+            return;
+        const step = buildStep(toolName, params, success);
+        if (!step)
+            return;
+        // Deduplicate consecutive identical steps
+        const last = this.steps[this.steps.length - 1];
+        if (last &&
+            last.action === step.action &&
+            last.target === step.target &&
+            last.text === step.text &&
+            last.code === step.code &&
+            JSON.stringify(last.keys ?? []) === JSON.stringify(step.keys ?? []) &&
+            JSON.stringify(last.menuPath ?? []) === JSON.stringify(step.menuPath ?? [])) {
+            return; // skip duplicate
+        }
+        this.steps.push(step);
+    }
+    stop(name, description) {
+        this.recording = false;
+        // Sanitize platform to prevent path traversal in filename
+        const safePlatform = this.platform.replace(/[^a-zA-Z0-9_\-]/g, "_").slice(0, 100);
+        const id = safePlatform + "-" + Date.now().toString(36);
+        // S75 Option C: Redact PII in persisted playbook steps and metadata
+        const redactedSteps = this.steps.map(s => ({
+            ...s,
+            ...(s.text ? { text: redactPII(s.text) } : {}),
+            ...(typeof s.target === "string" ? { target: redactPII(s.target) } : {}),
+            ...(s.url ? { url: redactPII(s.url) } : {}),
+            ...(s.code ? { code: redactPII(s.code) } : {}),
+            ...(s.description ? { description: redactPII(s.description) } : {}),
+        }));
+        const playbook = {
+            id,
+            name: redactPII(name),
+            description: redactPII(description),
+            platform: this.platform,
+            version: "1.0.0",
+            tags: [this.platform, "recorded"],
+            successCount: 0,
+            failCount: 0,
+            steps: redactedSteps,
+        };
+        if (this.cdpPort) {
+            playbook.cdpPort = this.cdpPort;
+        }
+        // Save to playbooks dir
+        if (!fs.existsSync(this.playbooksDir)) {
+            fs.mkdirSync(this.playbooksDir, { recursive: true });
+        }
+        const safeFilename = `${id}.json`;
+        const resolved = path.resolve(path.join(this.playbooksDir, safeFilename));
+        if (!resolved.startsWith(path.resolve(this.playbooksDir))) {
+            throw new Error("Invalid playbook path — refusing to write outside playbooks directory");
+        }
+        writeFileAtomicSync(resolved, JSON.stringify(playbook, null, 2));
+        this.steps = [];
+        return playbook;
+    }
+    cancel() {
+        this.recording = false;
+        this.steps = [];
+    }
+}