screenhand 0.2.0 → 0.3.1

package/dist/src/perception/vision-source.js

@@ -0,0 +1,399 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import fs from "node:fs";
+import { FrameDiffer } from "./frame-differ.js";
+/**
+ * Vision perception source — screenshot diff + ROI OCR.
+ *
+ * Uses the native bridge for capture and OCR. Keeps last frame in memory
+ * to avoid file I/O on the diff path. Falls back to file-based capture
+ * if in-memory buffer capture is not available from the bridge.
+ */
+export class VisionSource {
+    bridge;
+    differ;
+    /** When true, always use CLI fallback for captures (avoids CG API SIGSEGV on browser windows) */
+    safeCLI = false;
+    constructor(bridge, cellSize = 128) {
+        this.bridge = bridge;
+        this.differ = new FrameDiffer(cellSize);
+    }
+    /** Enable safe CLI mode for browser windows that crash CGWindowListCreateImage */
+    setSafeCLI(enabled) {
+        this.safeCLI = enabled;
+    }
+    /**
+     * SLOW rate: capture window and diff against last frame.
+     * Returns changed status and regions needing OCR.
+     */
+    async captureAndDiff(windowId) {
+        const start = Date.now();
+        try {
+            // Try in-memory buffer capture first (cg.captureWindowBuffer — not yet in native bridges),
+            // fall back to file-based capture (cg.captureWindow — always available)
+            let buffer;
+            let width;
+            let height;
+            try {
+                const result = await this.bridge.call("cg.captureWindowBuffer", { windowId, safeCLI: this.safeCLI });
+                buffer = Buffer.from(result.base64, "base64");
+                width = result.width;
+                height = result.height;
+            }
+            catch {
+                // Fallback: file-based capture
+                const fileResult = await this.bridge.call("cg.captureWindow", { windowId, safeCLI: this.safeCLI });
+                buffer = fs.readFileSync(fileResult.path);
+                width = fileResult.width;
+                height = fileResult.height;
+                // Clean up temp file
+                try {
+                    fs.unlinkSync(fileResult.path);
+                }
+                catch {
+                    /* ignore */
+                }
+            }
+            const diffResult = this.differ.diff(buffer, width, height);
+            const captureMs = Date.now() - start;
+            return {
+                source: "vision_diff",
+                rate: "slow",
+                timestamp: new Date().toISOString(),
+                data: {
+                    type: "vision_diff",
+                    changed: diffResult.changed,
+                    hash: diffResult.hash,
+                    changedRegions: diffResult.changedRegions,
+                    captureMs,
+                },
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * OCR a specific region of interest.
+     * Uses bridge's vision.ocrRegion if available (not yet in native bridges),
+     * falls back to full capture + full OCR via vision.ocr.
+     */
+    async ocrRegion(windowId, roi, mode = "fast") {
+        const start = Date.now();
+        try {
+            let text;
+            let regions;
+            try {
+                // Try ROI-specific OCR
+                const result = await this.bridge.call("vision.ocrRegion", {
+                    windowId,
+                    region: { x: roi.x, y: roi.y, width: roi.width, height: roi.height },
+                    mode,
+                });
+                text = result.text;
+                regions = result.regions;
+            }
+            catch {
+                // Fallback: full capture + OCR (less efficient)
+                const shot = await this.bridge.call("cg.captureWindow", { windowId, safeCLI: this.safeCLI });
+                const ocrResult = await this.bridge.call("vision.ocr", { imagePath: shot.path, mode: "fast" });
+                text = ocrResult.text;
+                regions = ocrResult.regions ?? [];
+                try {
+                    fs.unlinkSync(shot.path);
+                }
+                catch {
+                    /* ignore */
+                }
+            }
+            const latencyMs = Date.now() - start;
+            return {
+                source: "vision_ocr",
+                rate: "slow",
+                timestamp: new Date().toISOString(),
+                data: {
+                    type: "vision_ocr",
+                    roi,
+                    text,
+                    regions,
+                    latencyMs,
+                },
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Capture window to a temp file (no base64 round-trip).
+     * Returns the file path and dimensions, or null on failure.
+     */
+    async captureToFile(windowId) {
+        try {
+            return await this.bridge.call("cg.captureWindow", { windowId, safeCLI: this.safeCLI });
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * OCR an existing image file (no new capture needed).
+     */
+    async ocrFile(imagePath, roi, mode = "accurate") {
+        const start = Date.now();
+        try {
+            const result = await this.bridge.call("vision.ocr", { imagePath, mode });
+            return {
+                source: "vision_ocr", rate: "slow",
+                timestamp: new Date().toISOString(),
+                data: {
+                    type: "vision_ocr",
+                    roi: roi ?? { x: 0, y: 0, width: 0, height: 0, reason: "changed_pixels" },
+                    text: result.text,
+                    regions: result.regions ?? [],
+                    latencyMs: Date.now() - start,
+                },
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Optimized single-capture pipeline: capture ONCE to file, diff for change
+     * detection with region extraction, OCR only changed regions.
+     *
+     * Performance: unchanged ~113ms, changed ~175ms (vs ~370ms before Phase 2).
+     * Phase 1 (FAST OCR) + Phase 2 (region OCR) combined.
+     */
+    async captureAndDiffOptimized(windowId, maxROIs = 3) {
+        const start = Date.now();
+        // 1. Capture: stream frame (~0ms) or one-shot (~112ms)
+        const capture = await this.captureToFileOrStream(windowId);
+        if (!capture)
+            return { diffEvent: null, ocrEvent: null };
+        try {
+            // 2. Full diff with grid hashing — detects changed regions (~5ms)
+            const { changed, hash, changedRegions } = this.differ.diffFile(capture.path, capture.width, capture.height);
+            const captureMs = Date.now() - start;
+            const diffEvent = {
+                source: "vision_diff", rate: "slow",
+                timestamp: new Date().toISOString(),
+                data: { type: "vision_diff", changed, hash, changedRegions, captureMs },
+            };
+            // 3. If unchanged → done (~113ms total)
+            if (!changed) {
+                if (!capture.fromStream) {
+                    try {
+                        fs.unlinkSync(capture.path);
+                    }
+                    catch { }
+                }
+                return { diffEvent, ocrEvent: null };
+            }
+            // 4. Run OCR and YOLO in parallel on the same captured frame
+            const mergedRegions = FrameDiffer.mergeRegions(changedRegions, maxROIs, 64, capture.width, capture.height);
+            // OCR (region-based or full)
+            const ocrPromise = (mergedRegions.length > 0 && mergedRegions.length <= maxROIs)
+                ? (async () => {
+                    const regionResults = [];
+                    for (const roi of mergedRegions) {
+                        const regionEvent = await this.ocrRegion(windowId, roi);
+                        if (regionEvent?.data.type === "vision_ocr" && regionEvent.data.regions) {
+                            regionResults.push(...regionEvent.data.regions);
+                        }
+                    }
+                    const fullText = regionResults.map((r) => r.text).join("\n");
+                    return {
+                        source: "vision_ocr", rate: "slow",
+                        timestamp: new Date().toISOString(),
+                        data: {
+                            type: "vision_ocr",
+                            roi: mergedRegions[0] ?? { x: 0, y: 0, width: 0, height: 0, reason: "changed_pixels" },
+                            text: fullText,
+                            regions: regionResults,
+                            latencyMs: Date.now() - start - captureMs,
+                        },
+                    };
+                })()
+                : this.ocrFile(capture.path, undefined, "fast");
+            // YOLO element detection (~2-5ms on ANE, runs parallel with OCR)
+            const yoloPromise = this.detectElements(capture.path, 0.25);
+            const [ocrEvent, yoloElements] = await Promise.all([ocrPromise, yoloPromise]);
+            // 5. Cleanup (don't delete stream frames — they're owned by the native bridge)
+            if (!capture.fromStream) {
+                try {
+                    fs.unlinkSync(capture.path);
+                }
+                catch { }
+            }
+            return { diffEvent, ocrEvent, yoloElements };
+        }
+        catch {
+            if (!capture.fromStream) {
+                try {
+                    fs.unlinkSync(capture.path);
+                }
+                catch { }
+            }
+            return { diffEvent: null, ocrEvent: null };
+        }
+    }
+    // ── YOLO element detection ──
+    /**
+     * Detect UI elements in an image using the YOLO CoreML model.
+     * Returns classified elements (button, field, heading, image, label, link, text)
+     * with bounding boxes and confidence scores.
+     */
+    async detectElements(imagePath, confidence = 0.25) {
+        try {
+            const result = await this.bridge.call("vision.detectElements", { imagePath, confidence });
+            return result.elements;
+        }
+        catch {
+            return []; // Model not available — degrade gracefully
+        }
+    }
+    /**
+     * Fuse OCR text regions with YOLO element detections.
+     * Matches YOLO bounding boxes to nearest OCR text to produce labeled elements.
+     * E.g., YOLO "button at (450,200)" + OCR "Submit at (460,205)" → "Submit button"
+     */
+    static fuseOcrAndYolo(ocrRegions, yoloElements, maxDistance = 50) {
+        const results = [];
+        const usedOcr = new Set();
+        // For each YOLO detection, find nearest OCR text
+        for (const elem of yoloElements) {
+            const elemCenterX = elem.bounds.x + elem.bounds.width / 2;
+            const elemCenterY = elem.bounds.y + elem.bounds.height / 2;
+            let bestIdx = -1;
+            let bestDist = maxDistance + 1;
+            for (let i = 0; i < ocrRegions.length; i++) {
+                if (usedOcr.has(i))
+                    continue;
+                const ocr = ocrRegions[i];
+                const ocrCenterX = ocr.bounds.x + ocr.bounds.width / 2;
+                const ocrCenterY = ocr.bounds.y + ocr.bounds.height / 2;
+                const dist = Math.sqrt((elemCenterX - ocrCenterX) ** 2 + (elemCenterY - ocrCenterY) ** 2);
+                if (dist < bestDist) {
+                    bestDist = dist;
+                    bestIdx = i;
+                }
+            }
+            if (bestIdx >= 0) {
+                // Fused: YOLO class + OCR text
+                usedOcr.add(bestIdx);
+                results.push({
+                    text: ocrRegions[bestIdx].text,
+                    class: elem.class,
+                    confidence: elem.confidence,
+                    bounds: elem.bounds,
+                    source: "fused",
+                });
+            }
+            else {
+                // YOLO only (no nearby text — unlabeled icon/button)
+                results.push({
+                    text: "",
+                    class: elem.class,
+                    confidence: elem.confidence,
+                    bounds: elem.bounds,
+                    source: "yolo",
+                });
+            }
+        }
+        // Add remaining OCR-only elements (text not matched to any YOLO detection)
+        for (let i = 0; i < ocrRegions.length; i++) {
+            if (usedOcr.has(i))
+                continue;
+            const ocr = ocrRegions[i];
+            results.push({
+                text: ocr.text,
+                class: "text",
+                confidence: 0.7, // Default OCR confidence
+                bounds: ocr.bounds,
+                source: "ocr",
+            });
+        }
+        return results;
+    }
+    // ── Stream capture mode ──
+    streamRunning = false;
+    /**
+     * Start continuous SCStream capture for a window.
+     * Frames are buffered in the native bridge — getLatestFrame returns instantly.
+     */
+    async startStream(windowId, fps = 30) {
+        try {
+            await this.bridge.call("vision.startStream", { windowId, fps });
+            this.streamRunning = true;
+            return true;
+        }
+        catch {
+            this.streamRunning = false;
+            return false;
+        }
+    }
+    /**
+     * Stop the continuous capture stream.
+     */
+    async stopStream() {
+        try {
+            await this.bridge.call("vision.stopStream", {});
+        }
+        catch { /* ignore */ }
+        this.streamRunning = false;
+    }
+    get isStreaming() {
+        return this.streamRunning;
+    }
+    /**
+     * Get the latest frame from the running stream (~0ms, already captured).
+     * Returns null if stream is not running or no frame available yet.
+     */
+    async getStreamFrame() {
+        if (!this.streamRunning)
+            return null;
+        try {
+            return await this.bridge.call("vision.latestFrame", {});
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Optimized capture: use stream frame if available (~0ms), else fall back to one-shot (~200ms).
+     */
+    async captureToFileOrStream(windowId) {
+        // Try stream frame first (instant)
+        if (this.streamRunning) {
+            const frame = await this.getStreamFrame();
+            if (frame && frame.ageMs < 200) { // Only use if fresh (<200ms old)
+                return { path: frame.path, width: frame.width, height: frame.height, fromStream: true };
+            }
+        }
+        // Fall back to one-shot capture
+        const result = await this.captureToFile(windowId);
+        return result ? { ...result, fromStream: false } : null;
+    }
+    /**
+     * Reset differ state (e.g., on context switch to new window).
+     */
+    reset() {
+        this.differ.reset();
+    }
+}

package/dist/src/planner/deterministic.js

@@ -0,0 +1,298 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+import { DEFAULT_PLANNER_CONFIG } from "./types.js";
+/** Minimum score threshold for a learned locator to override a playbook/strategy locator */
+const LEARNED_LOCATOR_MIN_SCORE = 0.7;
+/**
+ * Safe MCP tool names for flow step parsing — excludes dangerous tools
+ * that allow arbitrary code execution (browser_js, applescript).
+ * These tools should only come from trusted playbooks, not parsed flow descriptions.
+ */
+const KNOWN_TOOLS = new Set([
+    "browser_navigate", "browser_click", "browser_type", "browser_wait",
+    "browser_dom", "browser_open", "browser_tabs",
+    "browser_page_info", "browser_fill_form", "browser_human_click", "browser_stealth",
+    "screenshot", "screenshot_file", "ocr", "ui_tree", "ui_find", "ui_press",
+    "ui_set_value", "click_text", "click", "click_with_fallback", "type_text",
+    "type_with_fallback", "key", "drag", "scroll", "scroll_with_fallback",
+    "launch", "focus", "menu_click", "wait_for_state",
+    "select_with_fallback", "read_with_fallback", "locate_with_fallback",
+    // Excluded: "browser_js", "applescript" — arbitrary code execution risk
+]);
+/**
+ * Maps PlaybookStep action types to MCP tool names.
+ */
+const ACTION_TO_TOOL = {
+    navigate: "browser_navigate",
+    press: "click_with_fallback",
+    type_into: "type_with_fallback",
+    key: "key",
+    scroll: "scroll_with_fallback",
+    wait: "wait_for_state",
+    screenshot: "screenshot",
+    extract: "browser_js",
+    menu_click: "menu_click",
+    browser_js: "browser_js",
+    browser_click: "browser_click",
+    browser_type: "browser_type",
+    cdp_key_event: "browser_js",
+    key_combo: "key",
+};
+/**
+ * Converts a Playbook into an ActionPlan for deterministic execution.
+ * No LLM calls needed — all steps come from the playbook.
+ */
+export function playbookToPlan(playbook, config = DEFAULT_PLANNER_CONFIG, learningEngine, bundleId) {
+    const steps = playbook.steps.map((step, i) => playbookStepToPlanStep(step, i, config, learningEngine, bundleId));
+    const reliability = playbook.successCount + playbook.failCount > 0
+        ? playbook.successCount / (playbook.successCount + playbook.failCount)
+        : 0.5;
+    return {
+        steps,
+        currentStepIndex: 0,
+        confidence: reliability,
+        source: "playbook",
+        sourceId: playbook.id,
+    };
+}
+/**
+ * Converts a Strategy (from memory recall) into an ActionPlan.
+ */
+export function strategyToPlan(strategy, config = DEFAULT_PLANNER_CONFIG, learningEngine, bundleId) {
+    const steps = strategy.steps.map((step, i) => strategyStepToPlanStep(step, i, config, learningEngine, bundleId));
+    const reliability = strategy.successCount + strategy.failCount > 0
+        ? strategy.successCount / (strategy.successCount + strategy.failCount)
+        : 0.5;
+    return {
+        steps,
+        currentStepIndex: 0,
+        confidence: reliability,
+        source: "strategy",
+        sourceId: strategy.id,
+    };
+}
+/**
+ * Try to parse a flow step description into a concrete tool + params.
+ * Many flow steps embed tool names (e.g. "browser_navigate to canva.com").
+ * Returns null if the step is too vague to parse.
+ */
+function parseFlowStep(stepDesc, ctx) {
+    const desc = stepDesc.trim();
+    // Pattern 1: function call syntax — tool(key: 'value')
+    // Only accept known MCP tool names to prevent arbitrary tool injection
+    const funcMatch = desc.match(/^(\w+)\((.+)\)$/);
+    if (funcMatch) {
+        const tool = funcMatch[1];
+        if (!KNOWN_TOOLS.has(tool.toLowerCase()))
+            return null;
+        const argsStr = funcMatch[2];
+        const params = {};
+        const argPattern = /(\w+)\s*:\s*'([^']+)'/g;
+        let m;
+        while ((m = argPattern.exec(argsStr)) !== null) {
+            params[m[1]] = m[2];
+        }
+        return { tool, params };
+    }
+    // Pattern 2: tool_name at start of description
+    const toolPrefixMatch = desc.match(/^(browser_navigate|browser_click|browser_type|browser_wait|browser_dom|browser_open|browser_tabs|browser_page_info|browser_fill_form|screenshot|screenshot_file|ocr|ui_tree|ui_find|ui_press|ui_set_value|click_text|click|type_text|key|drag|scroll|launch|focus|menu_click|wait_for_state)\b/i);
+    if (toolPrefixMatch) {
+        const tool = toolPrefixMatch[1].toLowerCase();
+        const rest = desc.slice(toolPrefixMatch[0].length).trim();
+        const params = {};
+        if (tool === "browser_navigate") {
+            const urlMatch = rest.match(/(?:to\s+)?(\S+\.(?:com|org|net|io|dev|app|co)\S*)/i);
+            if (urlMatch)
+                params.url = urlMatch[1].startsWith("http") ? urlMatch[1] : `https://${urlMatch[1]}`;
+        }
+        else if (tool === "browser_click") {
+            const quoteMatch = rest.match(/'([^']+)'|"([^"]+)"/);
+            if (quoteMatch)
+                params.selector = quoteMatch[1] ?? quoteMatch[2];
+        }
+        else if (tool === "browser_type") {
+            const quoteMatch = rest.match(/'([^']+)'|"([^"]+)"/);
+            if (quoteMatch)
+                params.text = quoteMatch[1] ?? quoteMatch[2];
+        }
+        else if (tool === "click_text") {
+            const quoteMatch = rest.match(/'([^']+)'|"([^"]+)"/);
+            if (quoteMatch) {
+                params.text = quoteMatch[1] ?? quoteMatch[2];
+                if (ctx.windowId)
+                    params.windowId = ctx.windowId;
+            }
+        }
+        else if (tool === "ui_press") {
+            const quoteMatch = rest.match(/'([^']+)'|"([^"]+)"/);
+            if (quoteMatch) {
+                params.title = quoteMatch[1] ?? quoteMatch[2];
+                if (ctx.pid)
+                    params.pid = ctx.pid;
+            }
+        }
+        else if (tool === "launch") {
+            const bundleMatch = rest.match(/'([^']+)'|"([^"]+)"/);
+            if (bundleMatch)
+                params.bundleId = bundleMatch[1] ?? bundleMatch[2];
+        }
+        else if (tool === "focus") {
+            const appMatch = rest.match(/'([^']+)'|"([^"]+)"|(\S+)/);
+            if (appMatch)
+                params.bundleId = (appMatch[1] ?? appMatch[2] ?? appMatch[3]).replace(/['"]/g, "");
+        }
+        else if (tool === "key") {
+            const keyMatch = rest.match(/'([^']+)'|"([^"]+)"|(\S+)/);
+            if (keyMatch)
+                params.combo = keyMatch[1] ?? keyMatch[2] ?? keyMatch[3];
+        }
+        // screenshot, ocr, ui_tree need no extra params
+        return { tool, params };
+    }
+    return null;
+}
+/**
+ * Converts a reference flow (from references/*.json) into an ActionPlan.
+ * Parses tool names and params from step descriptions where possible.
+ * Steps that can't be parsed are marked requiresLLM=true for client resolution.
+ */
+export function flowToPlan(flowName, flow, config = DEFAULT_PLANNER_CONFIG, runtimeContext) {
+    const ctx = runtimeContext ?? {};
+    const steps = flow.steps.map((stepDesc) => {
+        const parsed = parseFlowStep(stepDesc, ctx);
+        if (parsed) {
+            return {
+                tool: parsed.tool,
+                params: parsed.params,
+                expectedPostcondition: null,
+                timeout: config.defaultStepTimeout,
+                fallbackTool: null,
+                requiresLLM: false,
+                status: "pending",
+                description: stepDesc,
+            };
+        }
+        return {
+            tool: "",
+            params: {},
+            expectedPostcondition: null,
+            timeout: config.defaultStepTimeout,
+            fallbackTool: null,
+            requiresLLM: true,
+            status: "pending",
+            description: stepDesc,
+        };
+    });
+    const executableCount = steps.filter((s) => !s.requiresLLM).length;
+    const confidence = steps.length > 0 ? 0.3 + 0.4 * (executableCount / steps.length) : 0;
+    return {
+        steps,
+        currentStepIndex: 0,
+        confidence,
+        source: "reference_flow",
+        sourceId: flowName,
+    };
+}
+function playbookStepToPlanStep(step, _index, config, learningEngine, bundleId) {
+    const tool = ACTION_TO_TOOL[step.action] ?? step.action;
+    const params = {};
+    if (step.target)
+        params.target = step.target;
+    if (step.text)
+        params.text = step.text;
+    if (step.url)
+        params.url = step.url;
+    if (step.keys)
+        params.keys = step.keys;
+    if (step.code)
+        params.code = step.code;
+    if (step.format)
+        params.format = step.format;
+    if (step.amount !== undefined)
+        params.amount = step.amount;
+    if (step.locateByOcr) {
+        params.locateByOcr = step.locateByOcr;
+        if (step.offsetX !== undefined)
+            params.offsetX = step.offsetX;
+        if (step.offsetY !== undefined)
+            params.offsetY = step.offsetY;
+    }
+    if (step.keyEvent)
+        params.keyEvent = step.keyEvent;
+    if (step.menuPath)
+        params.menuPath = step.menuPath;
+    if (step.ms !== undefined)
+        params.ms = step.ms;
+    // Normalize keys array → combo string for the key tool
+    if (tool === "key" && Array.isArray(params.keys)) {
+        params.combo = params.keys.join("+");
+        delete params.keys;
+    }
+    // Normalize menuPath array → "/" string for menu_click
+    if (tool === "menu_click" && Array.isArray(params.menuPath)) {
+        params.menuPath = params.menuPath.join("/");
+    }
+    // Overlay learned locator if confidence is high enough
+    applyLearnedLocator(params, tool, learningEngine, bundleId);
+    return {
+        tool,
+        params,
+        expectedPostcondition: step.verify
+            ? { type: "control_exists", target: step.verify }
+            : null,
+        timeout: step.verifyTimeoutMs ?? config.defaultStepTimeout,
+        fallbackTool: null,
+        requiresLLM: false,
+        status: "pending",
+        description: step.description ?? `${step.action} ${step.target ?? ""}`.trim(),
+    };
+}
+function strategyStepToPlanStep(step, _index, config, learningEngine, bundleId) {
+    const params = { ...step.params };
+    // Overlay learned locator if confidence is high enough
+    applyLearnedLocator(params, step.tool, learningEngine, bundleId);
+    return {
+        tool: step.tool,
+        params,
+        expectedPostcondition: null,
+        timeout: config.defaultStepTimeout,
+        fallbackTool: null,
+        requiresLLM: false,
+        status: "pending",
+        description: `${step.tool} (from strategy)`,
+    };
+}
+/**
+ * If the learning engine has a proven locator for this tool×app pair,
+ * override the step's target/selector with the learned one.
+ */
+function applyLearnedLocator(params, tool, learningEngine, bundleId) {
+    if (!learningEngine || !bundleId)
+        return;
+    const rec = learningEngine.recommendLocator(bundleId, tool);
+    if (!rec || rec.score < LEARNED_LOCATOR_MIN_SCORE)
+        return;
+    // Only override target-based params — don't replace url, keys, code, etc.
+    if (params.target !== undefined || params.selector !== undefined) {
+        params._originalTarget = params.target ?? params.selector;
+        params.target = rec.locator;
+        params._learnedLocator = true;
+        if (rec.method === "cdp") {
+            params.selector = rec.locator;
+        }
+    }
+}