screenhand 0.1.1 → 0.3.0

package/dist/src/planner/tool-registry.js

@@ -0,0 +1,71 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export class ToolRegistry {
+    handlers = new Map();
+    register(name, handler) {
+        this.handlers.set(name, handler);
+    }
+    has(name) {
+        return this.handlers.has(name);
+    }
+    getToolNames() {
+        return [...this.handlers.keys()];
+    }
+    async execute(name, params) {
+        const handler = this.handlers.get(name);
+        if (!handler) {
+            return { ok: false, error: `Unknown tool: ${name}` };
+        }
+        try {
+            const result = await handler(params);
+            // Extract text from MCP result format { content: [{ type: "text", text: "..." }] }
+            let text = "";
+            if (result?.content && Array.isArray(result.content)) {
+                text = result.content
+                    .filter((c) => c.type === "text")
+                    .map((c) => c.text)
+                    .join("\n");
+            }
+            // Detect failures: explicit isError flag OR soft failure patterns in text
+            if (result?.isError) {
+                return { ok: false, error: text };
+            }
+            const lower = text.toLowerCase();
+            const isSoftFailure = (lower.includes("not found") &&
+                !lower.includes("clicked") &&
+                !lower.includes("using default") &&
+                !lower.includes("created") &&
+                !lower.includes("using fallback")) ||
+                lower.includes("window not found") ||
+                (lower.startsWith("error:") &&
+                    !/^error:\s*(none|no |nothing|0 )/.test(lower));
+            if (isSoftFailure) {
+                return { ok: false, error: text };
+            }
+            return { ok: true, result: text };
+        }
+        catch (err) {
+            return {
+                ok: false,
+                error: err instanceof Error ? err.message : String(err),
+            };
+        }
+    }
+    toExecutor() {
+        return (tool, params) => this.execute(tool, params);
+    }
+}

package/dist/src/planner/types.js

@@ -0,0 +1,22 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+//
+// This file is part of ScreenHand.
+//
+// ScreenHand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, version 3.
+//
+// ScreenHand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with ScreenHand. If not, see <https://www.gnu.org/licenses/>.
+export const DEFAULT_PLANNER_CONFIG = {
+    defaultMaxAttempts: 3,
+    defaultStepTimeout: 10_000,
+    postconditionWaitMs: 2_000,
+    minConfidenceForExecution: 0.3,
+};

package/dist/src/platform/explorer.js

@@ -0,0 +1,213 @@
+// Copyright (C) 2025 Clazro Technology Private Limited
+// SPDX-License-Identifier: AGPL-3.0-only
+/**
+ * PlatformExplorer — autonomously explore an app or website.
+ *
+ * Maps all interactive elements, tries each one, records working selectors,
+ * broken paths, and errors. Outputs a reference JSON.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { writeFileAtomicSync } from "../util/atomic-write.js";
+/** Patterns for buttons that should never be clicked */
+const DANGEROUS_PATTERNS = /delete|logout|log.?out|remove|cancel|unsubscribe|deactivate|sign.?out|close.?account|erase|destroy|terminate|disconnect|revoke/i;
+/** Discover all interactive elements on a web page via CDP */
+export async function discoverWebElements(cdpEvaluate, maxElements) {
+    const result = await cdpEvaluate(`(() => {
+    const elements = [];
+    const selectors = new Set();
+    const interactive = document.querySelectorAll(
+      'a[href], button, input, select, textarea, [role="button"], [role="tab"], [role="menuitem"], [role="link"], [onclick], [data-testid], summary'
+    );
+    for (const el of interactive) {
+      if (elements.length >= ${maxElements}) break;
+      const rect = el.getBoundingClientRect();
+      if (rect.width === 0 || rect.height === 0) continue;
+      if (rect.top < 0 || rect.left < 0) continue;
+
+      let selector = '';
+      if (el.getAttribute('data-testid')) selector = '[data-testid="' + el.getAttribute('data-testid') + '"]';
+      else if (el.id) selector = '#' + el.id;
+      else if (el.getAttribute('aria-label')) selector = '[aria-label="' + el.getAttribute('aria-label') + '"]';
+      else {
+        const tag = el.tagName.toLowerCase();
+        const text = (el.textContent || '').trim().substring(0, 30);
+        selector = tag + (text ? ':text("' + text + '")' : ':nth(' + elements.length + ')');
+      }
+
+      if (selectors.has(selector)) continue;
+      selectors.add(selector);
+
+      elements.push({
+        selector,
+        text: (el.textContent || '').trim().substring(0, 100),
+        tag: el.tagName.toLowerCase(),
+        role: el.getAttribute('role') || undefined,
+        ariaLabel: el.getAttribute('aria-label') || undefined,
+      });
+    }
+    return elements;
+  })()`);
+    return result.result?.value ?? [];
+}
+/** Test clicking an element and record what happens */
+export async function testWebElement(cdpEvaluate, element) {
+    // Check if dangerous
+    if (DANGEROUS_PATTERNS.test(element.text) || DANGEROUS_PATTERNS.test(element.ariaLabel ?? "")) {
+        return { ...element, clickWorked: false, result: "skipped_dangerous", error: "Skipped: potentially destructive action" };
+    }
+    try {
+        // Get pre-click state
+        const preState = await cdpEvaluate(`(() => ({ url: location.href, title: document.title }))()`);
+        const pre = preState.result?.value ?? { url: "", title: "" };
+        // Click the element
+        const clickResult = await cdpEvaluate(`(() => {
+      const el = document.querySelector('${element.selector.replace(/'/g, "\\'")}');
+      if (!el) return { found: false };
+      el.click();
+      return { found: true };
+    })()`);
+        if (!clickResult.result?.value?.found) {
+            return { ...element, clickWorked: false, result: "element_not_found" };
+        }
+        // Wait for UI to settle
+        await new Promise(r => setTimeout(r, 800));
+        // Get post-click state
+        const postState = await cdpEvaluate(`(() => ({ url: location.href, title: document.title }))()`);
+        const post = postState.result?.value ?? { url: "", title: "" };
+        const urlChanged = pre.url !== post.url;
+        const titleChanged = pre.title !== post.title;
+        const stateChange = urlChanged ? `url_changed: ${post.url}` : titleChanged ? `title_changed: ${post.title}` : "no_visible_change";
+        // Navigate back if URL changed
+        if (urlChanged) {
+            await cdpEvaluate(`history.back()`);
+            await new Promise(r => setTimeout(r, 500));
+        }
+        return {
+            ...element,
+            clickWorked: true,
+            result: stateChange,
+            newUrl: urlChanged ? post.url : undefined,
+            stateChange,
+        };
+    }
+    catch (err) {
+        return {
+            ...element,
+            clickWorked: false,
+            result: "error",
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+/** Discover all interactive elements in a native app via AX bridge */
+export async function discoverNativeElements(bridge, pid, maxElements) {
+    const tree = await bridge.call("ax.getElementTree", { pid, maxDepth: 5 });
+    const elements = [];
+    const CLICKABLE_ROLES = new Set([
+        "AXButton", "AXLink", "AXMenuItem", "AXTab", "AXCheckBox",
+        "AXPopUpButton", "AXRadioButton", "AXSwitch", "AXDisclosureTriangle",
+    ]);
+    function walk(node, path = []) {
+        if (elements.length >= maxElements)
+            return;
+        if (node.role && CLICKABLE_ROLES.has(node.role)) {
+            const title = node.title || node.description || node.value || "";
+            if (title && !DANGEROUS_PATTERNS.test(title)) {
+                elements.push({
+                    selector: path.join("."),
+                    text: String(title).substring(0, 100),
+                    tag: node.role,
+                    role: node.role,
+                    ariaLabel: node.description,
+                });
+            }
+        }
+        if (Array.isArray(node.children)) {
+            node.children.forEach((child, i) => walk(child, [...path, i]));
+        }
+    }
+    walk(tree);
+    return elements;
+}
+/** Compile exploration results into a reference JSON */
+export function compileReference(platform, source, tested, url, bundleId) {
+    const working = tested.filter(t => t.clickWorked);
+    const broken = tested.filter(t => !t.clickWorked && t.result !== "skipped_dangerous");
+    const skipped = tested.filter(t => t.result === "skipped_dangerous");
+    // Group working selectors by tag/role
+    const selectors = {};
+    for (const el of working) {
+        const group = el.role ?? el.tag;
+        if (!selectors[group])
+            selectors[group] = {};
+        const key = el.text.replace(/[^a-zA-Z0-9_]/g, "_").substring(0, 40) || `el_${Object.keys(selectors[group]).length}`;
+        selectors[group][key] = el.selector;
+    }
+    const navigation = working
+        .filter((t) => t.newUrl !== undefined)
+        .map(t => ({ text: t.text, selector: t.selector, worked: true, url: t.newUrl }));
+    const errors = broken.map(t => ({
+        error: t.error ?? t.result,
+        context: `Clicking "${t.text}" (${t.selector})`,
+        solution: "Element may be hidden, disabled, or require prior interaction",
+        severity: "low",
+    }));
+    const keyDiscoveries = [];
+    keyDiscoveries.push(`Found ${tested.length} interactive elements, ${working.length} clickable, ${broken.length} broken, ${skipped.length} skipped (dangerous)`);
+    if (navigation.length > 0)
+        keyDiscoveries.push(`${navigation.length} navigation links discovered`);
+    if (Object.keys(selectors).length > 0)
+        keyDiscoveries.push(`Selector groups: ${Object.keys(selectors).join(", ")}`);
+    return {
+        platform,
+        exploredAt: new Date().toISOString(),
+        source,
+        ...(url !== undefined ? { url } : {}),
+        ...(bundleId !== undefined ? { bundleId } : {}),
+        totalElements: tested.length,
+        testedElements: tested.filter(t => t.result !== "skipped_dangerous").length,
+        workingSelectors: working.length,
+        selectors,
+        navigation,
+        errors,
+        keyDiscoveries,
+    };
+}
+/** Save exploration result as a reference JSON */
+export function saveExploreResult(referencesDir, result) {
+    if (!fs.existsSync(referencesDir)) {
+        fs.mkdirSync(referencesDir, { recursive: true });
+    }
+    const safePlatform = result.platform.replace(/[^a-zA-Z0-9_\-]/g, "_").slice(0, 100);
+    const filePath = path.join(referencesDir, `${safePlatform}-explore.json`);
+    const resolved = path.resolve(filePath);
+    if (!resolved.startsWith(path.resolve(referencesDir))) {
+        throw new Error("Invalid platform name — path traversal detected");
+    }
+    // Build reference format matching existing references
+    const reference = {
+        id: `${result.platform}-explore`,
+        name: `${result.platform} — Auto-Explored`,
+        description: `Auto-explored ${result.source === "web" ? result.url : result.bundleId}. Found ${result.workingSelectors} working selectors out of ${result.totalElements} elements.`,
+        platform: result.platform,
+        version: "1.0.0",
+        tags: [result.platform, "auto-explored"],
+        successCount: result.workingSelectors,
+        failCount: result.errors.length,
+        selectors: result.selectors,
+        errors: result.errors,
+        _meta: {
+            exploredAt: result.exploredAt,
+            source: result.source,
+            url: result.url,
+            bundleId: result.bundleId,
+            totalElements: result.totalElements,
+            testedElements: result.testedElements,
+            workingSelectors: result.workingSelectors,
+            keyDiscoveries: result.keyDiscoveries,
+        },
+    };
+    writeFileAtomicSync(filePath, JSON.stringify(reference, null, 2));
+    return filePath;
+}