screenhand 0.3.2 → 0.3.4

package/dist/src/recovery/engine.js

@@ -17,7 +17,7 @@
 import fs from "node:fs";
 import path from "node:path";
 import { detectBlockers } from "./detectors.js";
-import { getBuiltinStrategies, parseReferenceStrategies, buildStrategyWithContext, } from "./strategies.js";
+import { getBuiltinStrategies, parseReferenceStrategies, buildStrategyWithContext, parseSolutionToSteps, } from "./strategies.js";
 const DEFAULT_CONFIG = {
     referencesDir: path.join(process.cwd(), "references"),
 };
@@ -31,6 +31,7 @@ export class RecoveryEngine {
     /** Map of "blockerType:strategyId" → cooldown entry */
     strategyCooldowns = new Map();
     learningEngine = null;
+    appMap = null;
     constructor(worldModel, executeTool, memory, config) {
         this.worldModel = worldModel;
         this.executeTool = executeTool;
@@ -44,6 +45,12 @@ export class RecoveryEngine {
     setLearningEngine(engine) {
         this.learningEngine = engine;
     }
+    /**
+     * Wire #7: L7→L4 — Inject AppMap for contract-based recovery validation.
+     */
+    setAppMap(map) {
+        this.appMap = map;
+    }
     /**
      * Get the current status of the recovery engine.
      */
@@ -95,7 +102,16 @@ export class RecoveryEngine {
      */
     selectStrategies(blocker, budget) {
         const candidates = [];
-        // Reference strategies first (app-specific)
+        // Wire #7: L7→L4 — Try contract undo paths first (most specific)
+        if (this.appMap && blocker.bundleId && blocker.description) {
+            try {
+                const undoStrategy = this.buildUndoStrategy(blocker);
+                if (undoStrategy)
+                    candidates.push(undoStrategy);
+            }
+            catch { /* best-effort */ }
+        }
+        // Reference strategies second (app-specific)
         if (blocker.bundleId) {
             const refErrors = this.loadReferenceErrors(blocker.bundleId);
             candidates.push(...parseReferenceStrategies(refErrors, blocker.type));
@@ -181,7 +197,7 @@ export class RecoveryEngine {
                 return { recovered: false, reason: "all_strategies_failed" };
             }
         }
-        // Verify recovery
+        // Verify recovery (Wire #7: includes contract-based validation when available)
         await sleep(300);
         const verified = this.verifyRecovery(blocker);
         const durationMs = Date.now() - start;
@@ -282,6 +298,37 @@ export class RecoveryEngine {
             }
         }
     }
+    /**
+     * Wire #7: L7→L4 — Build an undo strategy from AppMap contract undo paths.
+     * If the blocker's description mentions an element that has a contract with an undoPath,
+     * create a recovery strategy that executes the undo action.
+     */
+    buildUndoStrategy(blocker) {
+        if (!this.appMap || !blocker.bundleId)
+            return null;
+        // Extract element name from blocker description
+        // Typical descriptions: 'Dialog appeared after click_text', 'Element not found: "Submit"'
+        const quotedMatch = blocker.description.match(/["']([^"']{1,60})["']/);
+        const elementLabel = quotedMatch?.[1];
+        if (!elementLabel)
+            return null;
+        const contractInfo = this.appMap.getContract(blocker.bundleId, elementLabel);
+        if (!contractInfo?.contract.undoPath)
+            return null;
+        const undoPath = contractInfo.contract.undoPath;
+        // Parse undoPath into recovery steps (e.g. "key cmd+z", "click Cancel")
+        const steps = parseSolutionToSteps(undoPath);
+        if (steps.length === 0)
+            return null;
+        return {
+            id: `undo_contract_${elementLabel}`,
+            blockerType: blocker.type,
+            label: `Undo via contract: ${undoPath}`,
+            steps,
+            postcondition: null,
+            source: "reference",
+        };
+    }
     /**
      * Load and cache reference errors for a bundleId.
      */

package/dist/src/runtime/execution-contract.js

@@ -82,6 +82,16 @@ const ACTION_TO_CAPABILITY = {
     select: "canSelect",
     scroll: "canScroll",
 };
+/** Maps sensor sourceType names to ExecutionMethod names */
+const SENSOR_TO_METHOD = {
+    ax: "ax",
+    accessibility: "ax",
+    cdp: "cdp",
+    chrome: "cdp",
+    ocr: "ocr",
+    vision: "ocr",
+    coordinates: "coordinates",
+};
 /**
  * Given an action type and available capabilities, returns the ordered
  * list of methods to try.
@@ -89,11 +99,15 @@ const ACTION_TO_CAPABILITY = {
  * Filters EXECUTION_METHODS to only those that:
  *   1. Support the requested action
  *   2. Have their infrastructure requirements met
- * Returns in canonical order (ax -> cdp -> ocr -> coordinates).
+ *
+ * When sensorRanking is provided (from LearningEngine.rankSensors()),
+ * reorders methods by learned success scores instead of using the
+ * hardcoded canonical order. Methods not present in the ranking
+ * are appended at the end in canonical order.
  */
-function planExecution(action, available) {
+function planExecution(action, available, sensorRanking) {
     const capKey = ACTION_TO_CAPABILITY[action];
-    return EXECUTION_METHODS.filter((method) => {
+    const eligible = EXECUTION_METHODS.filter((method) => {
         const cap = METHOD_CAPABILITIES[method];
         // Must support the requested action
         if (!cap[capKey])
@@ -105,6 +119,42 @@ function planExecution(action, available) {
             return false;
         return true;
     });
+    // Without sensor data, return canonical order
+    if (!sensorRanking || sensorRanking.length === 0)
+        return eligible;
+    // Build score map: ExecutionMethod → { score, latency }
+    // When multiple sourceTypes alias to the same method (e.g. "vision" + "ocr" → "ocr"),
+    // keep the higher score to avoid silent overwrite.
+    const scoreMap = new Map();
+    for (const entry of sensorRanking) {
+        const method = SENSOR_TO_METHOD[entry.sourceType];
+        if (method && eligible.includes(method)) {
+            const existing = scoreMap.get(method);
+            if (!existing || entry.score > existing.score) {
+                scoreMap.set(method, { score: entry.score, latencyMs: entry.avgLatencyMs });
+            }
+        }
+    }
+    // Sort: ranked methods first (by score desc, latency asc tiebreak),
+    // unranked methods last (canonical order preserved by stable sort)
+    return eligible.slice().sort((a, b) => {
+        const sa = scoreMap.get(a);
+        const sb = scoreMap.get(b);
+        if (sa != null && sb != null) {
+            const scoreDiff = sb.score - sa.score;
+            // Mirror SensorPolicy's 0.05 band: treat <5% score gaps as noise,
+            // let latency decide. Prevents planExecution from reversing the
+            // policy's latency-preferred orderings for near-equal scores.
+            if (Math.abs(scoreDiff) > 0.05)
+                return scoreDiff; // meaningful score gap → score wins
+            return sa.latencyMs - sb.latencyMs; // within noise band → lower latency first
+        }
+        if (sa != null)
+            return -1; // a ranked, b not → a first
+        if (sb != null)
+            return 1; // b ranked, a not → b first
+        return 0; // both unranked → keep canonical order (V8 stable sort)
+    });
 }
 const DEFAULT_RETRY_POLICY = {
     maxRetriesPerMethod: 2,
@@ -125,6 +175,17 @@ function delay(ms) {
  * Returns the result from whichever method succeeded (or the last failure).
  */
 async function executeWithFallback(action, plan, policy, executor) {
+    if (plan.length === 0) {
+        return {
+            ok: false,
+            method: "ax",
+            durationMs: 0,
+            fallbackFrom: null,
+            retries: 0,
+            error: "No execution methods available",
+            target: null,
+        };
+    }
     let totalRetries = 0;
     let lastResult = null;
     let previousMethod = null;
@@ -134,8 +195,8 @@ async function executeWithFallback(action, plan, policy, executor) {
                 // Exhausted total retry budget — return whatever we have
                 return lastResult;
             }
-            // Delay between retries (not before the very first attempt)
-            if (totalRetries > 0) {
+            // Delay between retries (not before the very first attempt of this method)
+            if (attempt > 0) {
                 await delay(policy.delayBetweenRetriesMs);
             }
             const result = await executor(method, attempt);
@@ -147,6 +208,7 @@ async function executeWithFallback(action, plan, policy, executor) {
             if (result.ok) {
                 return result;
             }
+            // Only count failed attempts toward total retry budget
             totalRetries++;
         }
         // This method is exhausted — record it so the next method knows

package/dist/src/runtime/executor.js

@@ -19,16 +19,56 @@ export class Executor {
     adapter;
     cache;
     logger;
+    appMap = null;
     constructor(adapter, cache, logger) {
         this.adapter = adapter;
         this.cache = cache;
         this.logger = logger;
     }
+    /**
+     * Wire #15: Set AppMap for skip-verify optimization.
+     * BundleId is resolved dynamically per-call from the adapter.
+     */
+    setAppMap(appMap) {
+        this.appMap = appMap;
+    }
+    /**
+     * Wire #15: Check if an element is well-known enough to skip verify.
+     * Requires 3+ prior successes and last interaction within 5 minutes.
+     * Never skips on retry (retry > 0) — retries need verification.
+     */
+    shouldSkipVerify(target, bundleId, retry) {
+        if (retry > 0)
+            return false; // Bug #3 fix: always verify on retry
+        if (!this.appMap || !bundleId)
+            return false;
+        let label = null;
+        if (target.type === "text")
+            label = target.value;
+        else if (target.type === "selector")
+            label = target.value;
+        else if (target.type === "role")
+            label = target.name;
+        else if (target.type === "ax_attribute")
+            label = `${target.attribute}=${target.value}`;
+        else if (target.type === "ax_path")
+            label = target.path.join("/");
+        if (!label)
+            return false;
+        return this.appMap.isElementVerified(bundleId, label);
+    }
     async press(input) {
         const telemetry = this.logger.start("press", input.sessionId);
         const budget = this.resolveBudget(input.budget);
         const attempts = [];
         let lastError;
+        // Wire #15: resolve bundleId dynamically for skip-verify
+        let pressBundleId = null;
+        try {
+            const ctx = await this.adapter.getAppContext(input.sessionId);
+            pressBundleId = ctx.bundleId ?? null;
+        }
+        catch { /* non-fatal — skip-verify just won't activate */ }
         for (let retry = 0; retry <= budget.maxRetries; retry += 1) {
             telemetry.retries = retry;
             try {
@@ -49,7 +89,7 @@ export class Executor {
                     await this.adapter.click(input.sessionId, locateResult.element);
                 }, "ACTION_FAILED");
                 telemetry.actMs += budget.actMs;
-                if (input.verify) {
+                if (input.verify && !this.shouldSkipVerify(input.target, pressBundleId, retry)) {
                     const verified = await this.timed(budget.verifyMs, () => this.adapter.waitFor(input.sessionId, input.verify, budget.verifyMs), "VERIFY_FAILED");
                     telemetry.verifyMs += budget.verifyMs;
                     if (!verified) {

package/dist/src/runtime/service.js

@@ -36,6 +36,13 @@ export class AutomationRuntimeService {
     setWorldModel(model) {
         this.worldModel = model;
     }
+    /**
+     * Wire #15: Pass AppMap to Executor for skip-verify optimization.
+     * BundleId is resolved dynamically per-call from the adapter.
+     */
+    setAppMap(appMap) {
+        this.executor.setAppMap(appMap);
+    }
     async sessionStart(profile = DEFAULT_PROFILE) {
         return this.sessions.sessionStart(profile);
     }