screenhand 0.2.0 → 0.3.1

package/dist/src/learning/locator-policy.js

@@ -0,0 +1,120 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * LocatorPolicy — tracks per-app×action locator reliability.
+ *
+ * After enough data points, recommends the highest-scoring locator
+ * for a given action in a given app. Uses Bayesian scoring so
+ * a locator with 3/3 successes doesn't beat one with 50/55.
+ */
+export class LocatorPolicy {
+    /** Map<compoundKey, LocatorEntry[]> — multiple locators per key */
+    entries = new Map();
+    priorStrength;
+    constructor(priorStrength = 2) {
+        this.priorStrength = priorStrength;
+    }
+    /**
+     * Record a locator outcome and update the score.
+     */
+    static makeKey(bundleId, actionKey) {
+        return `${bundleId.length}:${bundleId}\0${actionKey}`;
+    }
+    record(outcome) {
+        const compoundKey = LocatorPolicy.makeKey(outcome.bundleId, outcome.actionKey);
+        let list = this.entries.get(compoundKey);
+        if (!list) {
+            list = [];
+            this.entries.set(compoundKey, list);
+        }
+        let entry = list.find((e) => e.locator === outcome.locator && e.method === outcome.method);
+        if (!entry) {
+            entry = {
+                key: compoundKey,
+                locator: outcome.locator,
+                method: outcome.method,
+                successCount: 0,
+                failCount: 0,
+                score: 0.5,
+                lastUsed: new Date().toISOString(),
+            };
+            list.push(entry);
+        }
+        if (outcome.success) {
+            entry.successCount++;
+        }
+        else {
+            entry.failCount++;
+        }
+        entry.score = this.bayesianScore(entry.successCount, entry.failCount);
+        entry.lastUsed = new Date().toISOString();
+    }
+    /**
+     * Get the best locator for a given app×action.
+     * Returns null if no data or insufficient samples.
+     */
+    recommend(bundleId, actionKey, minSamples = 5) {
+        const compoundKey = LocatorPolicy.makeKey(bundleId, actionKey);
+        const list = this.entries.get(compoundKey);
+        if (!list || list.length === 0)
+            return null;
+        const qualified = list.filter((e) => e.successCount + e.failCount >= minSamples);
+        if (qualified.length === 0)
+            return null;
+        qualified.sort((a, b) => b.score - a.score);
+        return qualified[0];
+    }
+    /**
+     * Get all entries for a given app×action (for inspection/debugging).
+     */
+    getEntries(bundleId, actionKey) {
+        const compoundKey = LocatorPolicy.makeKey(bundleId, actionKey);
+        return this.entries.get(compoundKey) ?? [];
+    }
+    /**
+     * Clear all entries.
+     */
+    clear() {
+        this.entries.clear();
+    }
+    /**
+     * Get all entries across all keys (for persistence).
+     */
+    getAllEntries() {
+        const result = [];
+        for (const list of this.entries.values()) {
+            result.push(...list);
+        }
+        return result;
+    }
+    /**
+     * Load entries from persisted data.
+     */
+    loadEntries(entries) {
+        for (const entry of entries) {
+            let list = this.entries.get(entry.key);
+            if (!list) {
+                list = [];
+                this.entries.set(entry.key, list);
+            }
+            const existing = list.find((e) => e.locator === entry.locator && e.method === entry.method);
+            if (existing) {
+                existing.successCount = entry.successCount;
+                existing.failCount = entry.failCount;
+                existing.score = entry.score;
+                existing.lastUsed = entry.lastUsed;
+            }
+            else {
+                list.push({ ...entry });
+            }
+        }
+    }
+    /**
+     * Bayesian score: (successes + prior) / (total + 2*prior)
+     * With prior=2: starts at 0.5, converges to true rate with more data.
+     */
+    bayesianScore(successes, failures) {
+        return ((successes + this.priorStrength) /
+            (successes + failures + 2 * this.priorStrength));
+    }
+}

package/dist/src/learning/pattern-policy.js

@@ -0,0 +1,89 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * PatternPolicy — learns which tool+locator combos work for each app.
+ *
+ * Persisted to `patterns.jsonl`. Each entry tracks success/fail counts
+ * for a specific bundleId×tool×locator triple, scored with Bayesian averaging.
+ *
+ * Used by the intelligence wrapper to recommend known-good selectors
+ * and warn about known-bad ones.
+ */
+export class PatternPolicy {
+    entries = new Map();
+    priorStrength;
+    constructor(priorStrength = 2) {
+        this.priorStrength = priorStrength;
+    }
+    /**
+     * Record a pattern outcome (tool+locator success/failure for an app).
+     */
+    record(outcome) {
+        const key = `${outcome.bundleId}::${outcome.tool}::${outcome.locator}`;
+        let entry = this.entries.get(key);
+        if (!entry) {
+            entry = {
+                key,
+                bundleId: outcome.bundleId,
+                tool: outcome.tool,
+                locator: outcome.locator,
+                method: outcome.method,
+                successCount: 0,
+                failCount: 0,
+                score: 0.5,
+                lastSeen: new Date().toISOString(),
+            };
+            this.entries.set(key, entry);
+        }
+        if (outcome.success) {
+            entry.successCount++;
+        }
+        else {
+            entry.failCount++;
+        }
+        entry.score = this.bayesianScore(entry.successCount, entry.failCount);
+        entry.lastSeen = new Date().toISOString();
+    }
+    /**
+     * Query patterns for a given app, optionally filtered by tool.
+     * Returns entries sorted by score descending.
+     */
+    query(bundleId, tool) {
+        const results = [];
+        for (const entry of this.entries.values()) {
+            if (entry.bundleId !== bundleId)
+                continue;
+            if (tool && entry.tool !== tool)
+                continue;
+            results.push(entry);
+        }
+        return results.sort((a, b) => b.score - a.score);
+    }
+    /**
+     * Get the best pattern for a given app×tool, or null if insufficient data.
+     */
+    recommend(bundleId, tool, minSamples = 3) {
+        const candidates = this.query(bundleId, tool);
+        for (const entry of candidates) {
+            if (entry.successCount + entry.failCount >= minSamples && entry.score > 0.5) {
+                return entry;
+            }
+        }
+        return null;
+    }
+    clear() {
+        this.entries.clear();
+    }
+    getAllEntries() {
+        return [...this.entries.values()];
+    }
+    loadEntries(entries) {
+        for (const entry of entries) {
+            this.entries.set(entry.key, { ...entry });
+        }
+    }
+    bayesianScore(successes, failures) {
+        return ((successes + this.priorStrength) /
+            (successes + failures + 2 * this.priorStrength));
+    }
+}

package/dist/src/learning/recovery-policy.js

@@ -0,0 +1,116 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * RecoveryPolicy — ranks recovery strategies per blocker×app.
+ *
+ * When the RecoveryEngine needs to pick a strategy, this policy
+ * provides a ranked list based on past success rates. Over time,
+ * the best strategy surfaces to the top.
+ */
+export class RecoveryPolicy {
+    /** Map<compoundKey, RecoveryPolicyEntry[]> */
+    entries = new Map();
+    priorStrength;
+    constructor(priorStrength = 2) {
+        this.priorStrength = priorStrength;
+    }
+    /**
+     * Record a recovery outcome.
+     */
+    record(outcome) {
+        const compoundKey = `${outcome.blockerType}::${outcome.bundleId}`;
+        let list = this.entries.get(compoundKey);
+        if (!list) {
+            list = [];
+            this.entries.set(compoundKey, list);
+        }
+        let entry = list.find((e) => e.strategyId === outcome.strategyId);
+        if (!entry) {
+            entry = {
+                key: compoundKey,
+                strategyId: outcome.strategyId,
+                successCount: 0,
+                failCount: 0,
+                score: 0.5,
+                avgDurationMs: 0,
+                lastUsed: new Date().toISOString(),
+            };
+            list.push(entry);
+        }
+        if (outcome.success) {
+            entry.successCount++;
+        }
+        else {
+            entry.failCount++;
+        }
+        // Running average for duration — guard against NaN
+        const duration = Number.isFinite(outcome.durationMs) ? outcome.durationMs : 0;
+        const total = entry.successCount + entry.failCount;
+        entry.avgDurationMs =
+            entry.avgDurationMs * ((total - 1) / total) +
+                duration / total;
+        entry.score = this.bayesianScore(entry.successCount, entry.failCount);
+        entry.lastUsed = new Date().toISOString();
+    }
+    /**
+     * Rank strategies for a given blocker×app, best first.
+     * Returns strategy IDs sorted by score (descending).
+     */
+    rank(blockerType, bundleId) {
+        const compoundKey = `${blockerType}::${bundleId}`;
+        const list = this.entries.get(compoundKey);
+        if (!list || list.length === 0)
+            return [];
+        return [...list]
+            .sort((a, b) => b.score - a.score)
+            .map((e) => ({ strategyId: e.strategyId, score: e.score }));
+    }
+    /**
+     * Get the best strategy for a blocker×app pair, or null if no data.
+     */
+    recommend(blockerType, bundleId, minSamples = 3) {
+        const compoundKey = `${blockerType}::${bundleId}`;
+        const list = this.entries.get(compoundKey);
+        if (!list || list.length === 0)
+            return null;
+        const qualified = list.filter((e) => e.successCount + e.failCount >= minSamples);
+        if (qualified.length === 0)
+            return null;
+        qualified.sort((a, b) => b.score - a.score);
+        return qualified[0].strategyId;
+    }
+    clear() {
+        this.entries.clear();
+    }
+    getAllEntries() {
+        const result = [];
+        for (const list of this.entries.values()) {
+            result.push(...list);
+        }
+        return result;
+    }
+    loadEntries(entries) {
+        for (const entry of entries) {
+            let list = this.entries.get(entry.key);
+            if (!list) {
+                list = [];
+                this.entries.set(entry.key, list);
+            }
+            const existing = list.find((e) => e.strategyId === entry.strategyId);
+            if (existing) {
+                existing.successCount = entry.successCount;
+                existing.failCount = entry.failCount;
+                existing.score = entry.score;
+                existing.avgDurationMs = entry.avgDurationMs;
+                existing.lastUsed = entry.lastUsed;
+            }
+            else {
+                list.push({ ...entry });
+            }
+        }
+    }
+    bayesianScore(successes, failures) {
+        return ((successes + this.priorStrength) /
+            (successes + failures + 2 * this.priorStrength));
+    }
+}

package/dist/src/learning/sensor-policy.js

@@ -0,0 +1,115 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * SensorPolicy — learns which perception source works best per app.
+ *
+ * Some apps are AX-friendly (VS Code, Finder), some need CDP (web apps),
+ * some need OCR (canvas-heavy apps like Canva, Premiere Pro).
+ * This policy tracks success rates and latency per source per app,
+ * so the perception coordinator can prioritize the best source.
+ */
+export class SensorPolicy {
+    entries = new Map();
+    priorStrength;
+    constructor(priorStrength = 2) {
+        this.priorStrength = priorStrength;
+    }
+    /**
+     * Record a sensor outcome.
+     */
+    record(outcome) {
+        const key = `${outcome.bundleId}::${outcome.sourceType}`;
+        let entry = this.entries.get(key);
+        if (!entry) {
+            entry = {
+                key,
+                bundleId: outcome.bundleId,
+                sourceType: outcome.sourceType,
+                successCount: 0,
+                failCount: 0,
+                score: 0.5,
+                avgLatencyMs: 0,
+                lastUsed: new Date().toISOString(),
+            };
+            this.entries.set(key, entry);
+        }
+        if (outcome.success) {
+            entry.successCount++;
+        }
+        else {
+            entry.failCount++;
+        }
+        // Running average for latency — guard against NaN
+        const latency = Number.isFinite(outcome.latencyMs) ? outcome.latencyMs : 0;
+        const total = entry.successCount + entry.failCount;
+        entry.avgLatencyMs =
+            entry.avgLatencyMs * ((total - 1) / total) +
+                latency / total;
+        entry.score = this.bayesianScore(entry.successCount, entry.failCount);
+        entry.lastUsed = new Date().toISOString();
+    }
+    /** Known browser bundle IDs where CDP should be boosted */
+    static BROWSER_BUNDLES = new Set([
+        "com.apple.Safari", "com.google.Chrome", "com.brave.Browser",
+        "org.mozilla.firefox", "com.microsoft.edgemac",
+        "org.chromium.Chromium", "com.vivaldi.Vivaldi",
+        "com.operasoftware.Opera",
+    ]);
+    /**
+     * Rank perception sources for a given app, best first.
+     * Score combines reliability (Bayesian) and speed (lower latency = better).
+     * Browser apps get a CDP bootstrap boost when no CDP data exists yet.
+     */
+    rank(bundleId) {
+        const results = [];
+        for (const entry of this.entries.values()) {
+            if (entry.bundleId === bundleId) {
+                results.push({
+                    sourceType: entry.sourceType,
+                    score: entry.score,
+                    avgLatencyMs: entry.avgLatencyMs,
+                });
+            }
+        }
+        // Bootstrap CDP for browser apps: if no CDP data exists yet,
+        // inject a prior so CDP isn't ranked at 0 for browsers
+        const isBrowser = SensorPolicy.BROWSER_BUNDLES.has(bundleId);
+        if (isBrowser && !results.some((r) => r.sourceType === "cdp")) {
+            results.push({ sourceType: "cdp", score: 0.7, avgLatencyMs: 100 });
+        }
+        // Sort by score descending, then by latency ascending for ties
+        results.sort((a, b) => {
+            const scoreDiff = b.score - a.score;
+            if (Math.abs(scoreDiff) > 0.05)
+                return scoreDiff;
+            return a.avgLatencyMs - b.avgLatencyMs;
+        });
+        return results;
+    }
+    /**
+     * Get the best source for a given app, or null if no data.
+     */
+    recommend(bundleId, minSamples = 5) {
+        const ranked = this.rank(bundleId);
+        const qualified = ranked.filter((r) => {
+            const entry = this.entries.get(`${bundleId}::${r.sourceType}`);
+            return entry && entry.successCount + entry.failCount >= minSamples;
+        });
+        return qualified.length > 0 ? qualified[0].sourceType : null;
+    }
+    clear() {
+        this.entries.clear();
+    }
+    getAllEntries() {
+        return [...this.entries.values()];
+    }
+    loadEntries(entries) {
+        for (const entry of entries) {
+            this.entries.set(entry.key, { ...entry });
+        }
+    }
+    bayesianScore(successes, failures) {
+        return ((successes + this.priorStrength) /
+            (successes + failures + 2 * this.priorStrength));
+    }
+}

package/dist/src/learning/timing-model.js

@@ -0,0 +1,204 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/** Default budgets from src/config.ts — used when insufficient data. */
+const DEFAULT_LOCATE_MS = 800;
+const DEFAULT_ACT_MS = 200;
+const DEFAULT_VERIFY_MS = 2000;
+/** Tools categorized by their role in the locate→act→verify pipeline. */
+const LOCATE_TOOLS = new Set([
+    "ui_find", "locate_with_fallback", "browser_dom",
+]);
+const ACT_TOOLS = new Set([
+    "click", "click_text", "click_with_fallback", "type_text",
+    "type_with_fallback", "key", "drag", "scroll", "scroll_with_fallback",
+    "browser_click", "browser_type", "browser_human_click",
+    "select_with_fallback", "ui_press", "ui_set_value", "menu_click",
+]);
+const VERIFY_TOOLS = new Set([
+    "screenshot", "screenshot_file", "ocr", "ui_tree",
+    "browser_wait", "wait_for_state", "read_with_fallback",
+    "browser_page_info",
+]);
+/**
+ * TimingModel — learns per-tool×app timing distributions and produces
+ * adaptive budgets that replace fixed defaults.
+ *
+ * Keeps a sliding window of samples per key. Computes p50/p95 lazily
+ * when a budget is requested.
+ */
+export class TimingModel {
+    /** Map<compoundKey, TimingSample[]> — sliding window */
+    samples = new Map();
+    /** Cached distributions — invalidated on new sample */
+    distributions = new Map();
+    maxSamples;
+    constructor(maxSamples = 100) {
+        this.maxSamples = maxSamples;
+    }
+    /**
+     * Record a timing event.
+     */
+    record(event) {
+        const key = `${event.tool}::${event.bundleId}`;
+        let list = this.samples.get(key);
+        if (!list) {
+            list = [];
+            this.samples.set(key, list);
+        }
+        // Cap individual samples at 10s to prevent outliers (timeouts, stalls)
+        // from poisoning the adaptive budgets
+        const MAX_SAMPLE_MS = 10_000;
+        const rawDur = Number.isFinite(event.durationMs) && event.durationMs >= 0
+            ? event.durationMs
+            : 0;
+        const dur = Math.min(rawDur, MAX_SAMPLE_MS);
+        list.push({
+            tool: event.tool,
+            bundleId: event.bundleId,
+            durationMs: dur,
+            success: event.success,
+            timestamp: new Date().toISOString(),
+        });
+        // Sliding window: keep only recent samples
+        if (list.length > this.maxSamples) {
+            list.splice(0, list.length - this.maxSamples);
+        }
+        // Invalidate cached distribution
+        this.distributions.delete(key);
+    }
+    /**
+     * Get the timing distribution for a specific tool×app pair.
+     */
+    getDistribution(tool, bundleId) {
+        const key = `${tool}::${bundleId}`;
+        const cached = this.distributions.get(key);
+        if (cached)
+            return cached;
+        const list = this.samples.get(key);
+        if (!list || list.length === 0)
+            return null;
+        // Only use successful samples for timing (failures may have arbitrary durations)
+        const successDurations = list
+            .filter((s) => s.success)
+            .map((s) => s.durationMs);
+        if (successDurations.length === 0)
+            return null;
+        successDurations.sort((a, b) => a - b);
+        const dist = {
+            key,
+            sampleCount: successDurations.length,
+            p50: percentile(successDurations, 0.5),
+            p95: percentile(successDurations, 0.95),
+            mean: successDurations.reduce((a, b) => a + b, 0) / successDurations.length,
+            min: successDurations[0],
+            max: successDurations[successDurations.length - 1],
+            lastUpdated: new Date().toISOString(),
+        };
+        this.distributions.set(key, dist);
+        return dist;
+    }
+    /**
+     * Compute adaptive budgets for a given app by aggregating
+     * timing data across all tools of each category (locate/act/verify).
+     *
+     * Returns defaults for categories with insufficient data.
+     */
+    getAdaptiveBudget(bundleId, minSamples = 5) {
+        return {
+            locateMs: this.budgetForCategory(LOCATE_TOOLS, bundleId, DEFAULT_LOCATE_MS, minSamples),
+            actMs: this.budgetForCategory(ACT_TOOLS, bundleId, DEFAULT_ACT_MS, minSamples),
+            verifyMs: this.budgetForCategory(VERIFY_TOOLS, bundleId, DEFAULT_VERIFY_MS, minSamples),
+        };
+    }
+    /**
+     * Clear all samples and cached distributions.
+     */
+    clear() {
+        this.samples.clear();
+        this.distributions.clear();
+    }
+    /**
+     * Get all timing distributions (for persistence/inspection).
+     */
+    getAllDistributions() {
+        // Ensure all distributions are computed
+        for (const key of this.samples.keys()) {
+            if (!this.distributions.has(key)) {
+                const [tool, bundleId] = key.split("::");
+                if (tool && bundleId) {
+                    this.getDistribution(tool, bundleId);
+                }
+            }
+        }
+        return [...this.distributions.values()];
+    }
+    /**
+     * Get all raw samples (for persistence).
+     */
+    getAllSamples() {
+        const result = [];
+        for (const list of this.samples.values()) {
+            result.push(...list);
+        }
+        return result;
+    }
+    /**
+     * Load samples from persisted data.
+     */
+    loadSamples(samples) {
+        const MAX_SAMPLE_MS = 10_000;
+        for (const sample of samples) {
+            const key = `${sample.tool}::${sample.bundleId}`;
+            let list = this.samples.get(key);
+            if (!list) {
+                list = [];
+                this.samples.set(key, list);
+            }
+            // Cap loaded samples to prevent old poisoned data from inflating budgets
+            list.push({ ...sample, durationMs: Math.min(sample.durationMs, MAX_SAMPLE_MS) });
+            if (list.length > this.maxSamples) {
+                list.splice(0, list.length - this.maxSamples);
+            }
+        }
+        // Clear all cached distributions
+        this.distributions.clear();
+    }
+    /**
+     * Compute budget for a category of tools by taking the max p95
+     * across all tools in that category for the given app.
+     */
+    budgetForCategory(toolSet, bundleId, defaultMs, minSamples) {
+        let maxP95 = 0;
+        let hasData = false;
+        for (const tool of toolSet) {
+            const dist = this.getDistribution(tool, bundleId);
+            if (dist && dist.sampleCount >= minSamples) {
+                maxP95 = Math.max(maxP95, dist.p95);
+                hasData = true;
+            }
+        }
+        if (!hasData)
+            return defaultMs;
+        // Use p95 with a 20% margin, but never below the minimum sensible value
+        // and never above 5x the default to prevent budget explosion from outliers
+        const minFloor = defaultMs * 0.25;
+        const maxCeiling = defaultMs * 5;
+        return Math.min(Math.max(Math.ceil(maxP95 * 1.2), minFloor), maxCeiling);
+    }
+}
+/**
+ * Compute the p-th percentile of a sorted array.
+ */
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    if (sorted.length === 1)
+        return sorted[0];
+    const idx = p * (sorted.length - 1);
+    const lower = Math.floor(idx);
+    const upper = Math.ceil(idx);
+    if (lower === upper)
+        return sorted[lower];
+    const frac = idx - lower;
+    return sorted[lower] * (1 - frac) + sorted[upper] * frac;
+}