react-native-ai-debugger 1.0.6 → 1.0.8

package/build/core/android.js

@@ -5,6 +5,8 @@ import path from "path";
 import os from "os";
 import sharp from "sharp";
 const execAsync = promisify(exec);
+// XML parsing for uiautomator dump
+import { XMLParser } from "fast-xml-parser";
 // ADB command timeout in milliseconds
 const ADB_TIMEOUT = 30000;
 /**
@@ -639,6 +641,245 @@ export async function androidKeyEvent(keyCode, deviceId) {
         };
     }
 }
+/**
+ * Parse bounds string like "[0,0][1080,1920]" to AndroidUIElement bounds
+ */
+function parseBoundsForUIElement(boundsStr) {
+    const match = boundsStr.match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/);
+    if (!match)
+        return null;
+    const left = parseInt(match[1], 10);
+    const top = parseInt(match[2], 10);
+    const right = parseInt(match[3], 10);
+    const bottom = parseInt(match[4], 10);
+    return {
+        left,
+        top,
+        right,
+        bottom,
+        width: right - left,
+        height: bottom - top
+    };
+}
+/**
+ * Parse uiautomator XML dump into element array
+ */
+function parseUIAutomatorXML(xml) {
+    const elements = [];
+    // Match all node elements with their attributes
+    const nodeRegex = /<node\s+([^>]+)\/?>|<node\s+([^>]+)>/g;
+    let match;
+    while ((match = nodeRegex.exec(xml)) !== null) {
+        const attrStr = match[1] || match[2];
+        if (!attrStr)
+            continue;
+        // Extract attributes
+        const getAttr = (name) => {
+            const attrMatch = attrStr.match(new RegExp(`${name}="([^"]*)"`));
+            return attrMatch ? attrMatch[1] : "";
+        };
+        const boundsStr = getAttr("bounds");
+        const bounds = parseBoundsForUIElement(boundsStr);
+        if (!bounds)
+            continue;
+        const element = {
+            text: getAttr("text"),
+            contentDesc: getAttr("content-desc"),
+            resourceId: getAttr("resource-id"),
+            className: getAttr("class"),
+            bounds,
+            center: {
+                x: Math.round((bounds.left + bounds.right) / 2),
+                y: Math.round((bounds.top + bounds.bottom) / 2)
+            },
+            clickable: getAttr("clickable") === "true",
+            enabled: getAttr("enabled") === "true",
+            focused: getAttr("focused") === "true",
+            scrollable: getAttr("scrollable") === "true",
+            selected: getAttr("selected") === "true"
+        };
+        elements.push(element);
+    }
+    return elements;
+}
+/**
+ * Match element against find options
+ */
+function matchesElement(element, options) {
+    if (options.text !== undefined) {
+        if (element.text !== options.text)
+            return false;
+    }
+    if (options.textContains !== undefined) {
+        if (!element.text.toLowerCase().includes(options.textContains.toLowerCase()))
+            return false;
+    }
+    if (options.contentDesc !== undefined) {
+        if (element.contentDesc !== options.contentDesc)
+            return false;
+    }
+    if (options.contentDescContains !== undefined) {
+        if (!element.contentDesc.toLowerCase().includes(options.contentDescContains.toLowerCase()))
+            return false;
+    }
+    if (options.resourceId !== undefined) {
+        // Support both full "com.app:id/button" and short "button" forms
+        const shortId = element.resourceId.split("/").pop() || "";
+        if (element.resourceId !== options.resourceId && shortId !== options.resourceId)
+            return false;
+    }
+    return true;
+}
+/**
+ * Get UI accessibility tree from Android device using uiautomator
+ */
+export async function androidGetUITree(deviceId) {
+    try {
+        const adbAvailable = await isAdbAvailable();
+        if (!adbAvailable) {
+            return {
+                success: false,
+                error: "ADB is not installed or not in PATH. Install Android SDK Platform Tools."
+            };
+        }
+        const deviceArg = buildDeviceArg(deviceId);
+        const device = deviceId || (await getDefaultAndroidDevice());
+        if (!device) {
+            return {
+                success: false,
+                error: "No Android device connected. Connect a device or start an emulator."
+            };
+        }
+        // Dump UI hierarchy to device
+        const remotePath = "/sdcard/ui_dump.xml";
+        await execAsync(`adb ${deviceArg} shell uiautomator dump ${remotePath}`, {
+            timeout: ADB_TIMEOUT
+        });
+        // Read the XML content
+        const { stdout } = await execAsync(`adb ${deviceArg} shell cat ${remotePath}`, {
+            timeout: ADB_TIMEOUT
+        });
+        // Clean up remote file
+        await execAsync(`adb ${deviceArg} shell rm ${remotePath}`, {
+            timeout: ADB_TIMEOUT
+        }).catch(() => { });
+        const elements = parseUIAutomatorXML(stdout);
+        return {
+            success: true,
+            elements,
+            rawXml: stdout
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: `Failed to get UI tree: ${error instanceof Error ? error.message : String(error)}`
+        };
+    }
+}
+/**
+ * Find element(s) in the UI tree matching the given criteria
+ */
+export async function androidFindElement(options, deviceId) {
+    try {
+        // Validate that at least one search criteria is provided
+        if (!options.text && !options.textContains && !options.contentDesc &&
+            !options.contentDescContains && !options.resourceId) {
+            return {
+                success: false,
+                found: false,
+                error: "At least one search criteria (text, textContains, contentDesc, contentDescContains, or resourceId) must be provided"
+            };
+        }
+        const treeResult = await androidGetUITree(deviceId);
+        if (!treeResult.success || !treeResult.elements) {
+            return {
+                success: false,
+                found: false,
+                error: treeResult.error
+            };
+        }
+        // Find matching elements
+        const matches = treeResult.elements.filter(el => matchesElement(el, options));
+        if (matches.length === 0) {
+            return {
+                success: true,
+                found: false,
+                matchCount: 0
+            };
+        }
+        // Select the element at the specified index (default 0)
+        const index = options.index ?? 0;
+        const selectedElement = matches[index];
+        if (!selectedElement) {
+            return {
+                success: true,
+                found: false,
+                matchCount: matches.length,
+                error: `Index ${index} out of bounds. Found ${matches.length} matching element(s).`
+            };
+        }
+        return {
+            success: true,
+            found: true,
+            element: selectedElement,
+            allMatches: matches,
+            matchCount: matches.length
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            found: false,
+            error: `Failed to find element: ${error instanceof Error ? error.message : String(error)}`
+        };
+    }
+}
+/**
+ * Wait for element to appear on screen with polling
+ */
+export async function androidWaitForElement(options, deviceId) {
+    const timeoutMs = options.timeoutMs ?? 10000;
+    const pollIntervalMs = options.pollIntervalMs ?? 500;
+    const startTime = Date.now();
+    // Validate that at least one search criteria is provided
+    if (!options.text && !options.textContains && !options.contentDesc &&
+        !options.contentDescContains && !options.resourceId) {
+        return {
+            success: false,
+            found: false,
+            timedOut: false,
+            error: "At least one search criteria (text, textContains, contentDesc, contentDescContains, or resourceId) must be provided"
+        };
+    }
+    while (Date.now() - startTime < timeoutMs) {
+        const result = await androidFindElement(options, deviceId);
+        if (result.found && result.element) {
+            return {
+                ...result,
+                elapsedMs: Date.now() - startTime,
+                timedOut: false
+            };
+        }
+        // If there was an error (not just "not found"), return it
+        if (!result.success) {
+            return {
+                ...result,
+                elapsedMs: Date.now() - startTime,
+                timedOut: false
+            };
+        }
+        // Wait before next poll
+        await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
+    }
+    return {
+        success: true,
+        found: false,
+        elapsedMs: Date.now() - startTime,
+        timedOut: true,
+        error: `Timed out after ${timeoutMs}ms waiting for element`
+    };
+}
 /**
  * Get device screen size
  */
@@ -683,4 +924,393 @@ export async function androidGetScreenSize(deviceId) {
         };
     }
 }
+/**
+ * Simplify Android class name for display
+ * android.widget.Button -> Button
+ * android.widget.TextView -> TextView
+ */
+function simplifyClassName(className) {
+    if (!className)
+        return "Unknown";
+    const parts = className.split(".");
+    return parts[parts.length - 1];
+}
+/**
+ * Parse bounds string "[left,top][right,bottom]" to object
+ */
+function parseBounds(boundsStr) {
+    const match = boundsStr?.match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/);
+    if (!match)
+        return null;
+    return {
+        left: parseInt(match[1], 10),
+        top: parseInt(match[2], 10),
+        right: parseInt(match[3], 10),
+        bottom: parseInt(match[4], 10)
+    };
+}
+/**
+ * Parse a single node from uiautomator XML
+ */
+function parseUiNode(node) {
+    const attrs = node["@_bounds"]
+        ? node
+        : node.node
+            ? (Array.isArray(node.node) ? node.node[0] : node.node)
+            : null;
+    if (!attrs)
+        return null;
+    const boundsStr = attrs["@_bounds"];
+    const bounds = parseBounds(boundsStr);
+    if (!bounds)
+        return null;
+    const width = bounds.right - bounds.left;
+    const height = bounds.bottom - bounds.top;
+    const centerX = Math.round(bounds.left + width / 2);
+    const centerY = Math.round(bounds.top + height / 2);
+    const element = {
+        class: simplifyClassName(attrs["@_class"] || ""),
+        bounds,
+        frame: {
+            x: bounds.left,
+            y: bounds.top,
+            width,
+            height
+        },
+        tap: {
+            x: centerX,
+            y: centerY
+        },
+        children: []
+    };
+    // Add optional attributes
+    if (attrs["@_text"])
+        element.text = attrs["@_text"];
+    if (attrs["@_content-desc"])
+        element.contentDesc = attrs["@_content-desc"];
+    if (attrs["@_resource-id"])
+        element.resourceId = attrs["@_resource-id"];
+    if (attrs["@_checkable"] === "true")
+        element.checkable = true;
+    if (attrs["@_checked"] === "true")
+        element.checked = true;
+    if (attrs["@_clickable"] === "true")
+        element.clickable = true;
+    if (attrs["@_enabled"] === "true")
+        element.enabled = true;
+    if (attrs["@_focusable"] === "true")
+        element.focusable = true;
+    if (attrs["@_focused"] === "true")
+        element.focused = true;
+    if (attrs["@_scrollable"] === "true")
+        element.scrollable = true;
+    if (attrs["@_selected"] === "true")
+        element.selected = true;
+    return element;
+}
+/**
+ * Recursively parse UI hierarchy from XML node
+ */
+function parseHierarchy(node) {
+    const results = [];
+    // Handle the node itself
+    if (node["@_bounds"]) {
+        const element = parseUiNode(node);
+        if (element) {
+            // Parse children
+            if (node.node) {
+                const children = Array.isArray(node.node) ? node.node : [node.node];
+                for (const child of children) {
+                    element.children.push(...parseHierarchy(child));
+                }
+            }
+            results.push(element);
+        }
+    }
+    else if (node.node) {
+        // This is a container without bounds (like hierarchy root)
+        const children = Array.isArray(node.node) ? node.node : [node.node];
+        for (const child of children) {
+            results.push(...parseHierarchy(child));
+        }
+    }
+    return results;
+}
+/**
+ * Format accessibility tree for display (similar to iOS format)
+ */
+function formatAndroidAccessibilityTree(elements, indent = 0) {
+    const lines = [];
+    const prefix = "  ".repeat(indent);
+    for (const element of elements) {
+        const parts = [];
+        // [ClassName] "text" or "content-desc"
+        parts.push(`[${element.class}]`);
+        // Add label (text or content-desc)
+        const label = element.text || element.contentDesc;
+        if (label) {
+            parts.push(`"${label}"`);
+        }
+        // Add frame and tap coordinates
+        const f = element.frame;
+        parts.push(`frame=(${f.x}, ${f.y}, ${f.width}x${f.height}) tap=(${element.tap.x}, ${element.tap.y})`);
+        lines.push(`${prefix}${parts.join(" ")}`);
+        // Recurse into children
+        if (element.children.length > 0) {
+            lines.push(formatAndroidAccessibilityTree(element.children, indent + 1));
+        }
+    }
+    return lines.join("\n");
+}
+/**
+ * Flatten element tree to array for searching
+ */
+function flattenElements(elements) {
+    const result = [];
+    for (const element of elements) {
+        result.push(element);
+        if (element.children.length > 0) {
+            result.push(...flattenElements(element.children));
+        }
+    }
+    return result;
+}
+/**
+ * Get the UI hierarchy from the connected Android device using uiautomator dump
+ */
+export async function androidDescribeAll(deviceId) {
+    try {
+        const adbAvailable = await isAdbAvailable();
+        if (!adbAvailable) {
+            return {
+                success: false,
+                error: "ADB is not installed or not in PATH. Install Android SDK Platform Tools."
+            };
+        }
+        const deviceArg = buildDeviceArg(deviceId);
+        const device = deviceId || (await getDefaultAndroidDevice());
+        if (!device) {
+            return {
+                success: false,
+                error: "No Android device connected. Connect a device or start an emulator."
+            };
+        }
+        // Use file-based approach (most reliable across devices)
+        // /dev/tty doesn't work on most emulators/devices
+        const remotePath = "/sdcard/ui_dump.xml";
+        await execAsync(`adb ${deviceArg} shell uiautomator dump ${remotePath}`, {
+            timeout: ADB_TIMEOUT
+        });
+        const { stdout } = await execAsync(`adb ${deviceArg} shell cat ${remotePath}`, {
+            timeout: ADB_TIMEOUT,
+            maxBuffer: 10 * 1024 * 1024
+        });
+        const xmlContent = stdout.trim();
+        // Clean up
+        await execAsync(`adb ${deviceArg} shell rm ${remotePath}`, {
+            timeout: 5000
+        }).catch(() => { });
+        if (!xmlContent || !xmlContent.includes("<hierarchy")) {
+            return {
+                success: false,
+                error: "Failed to get UI hierarchy. Make sure the device screen is unlocked and the app is in foreground."
+            };
+        }
+        // Parse XML
+        const parser = new XMLParser({
+            ignoreAttributes: false,
+            attributeNamePrefix: "@_"
+        });
+        const parsed = parser.parse(xmlContent);
+        if (!parsed.hierarchy) {
+            return {
+                success: false,
+                error: "Invalid UI hierarchy XML structure"
+            };
+        }
+        const elements = parseHierarchy(parsed.hierarchy);
+        const formatted = formatAndroidAccessibilityTree(elements);
+        return {
+            success: true,
+            elements,
+            formatted
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: `Failed to get UI hierarchy: ${error instanceof Error ? error.message : String(error)}`
+        };
+    }
+}
+/**
+ * Get accessibility info for the UI element at specific coordinates
+ */
+export async function androidDescribePoint(x, y, deviceId) {
+    try {
+        // First get the full hierarchy
+        const result = await androidDescribeAll(deviceId);
+        if (!result.success || !result.elements) {
+            return result;
+        }
+        // Flatten and find elements containing the point
+        const allElements = flattenElements(result.elements);
+        // Find all elements whose bounds contain the point
+        const matchingElements = allElements.filter((el) => {
+            const b = el.bounds;
+            return x >= b.left && x <= b.right && y >= b.top && y <= b.bottom;
+        });
+        if (matchingElements.length === 0) {
+            return {
+                success: true,
+                formatted: `No element found at (${x}, ${y})`
+            };
+        }
+        // Return the deepest (smallest) element that contains the point
+        // Sort by area (smallest first) to get the most specific element
+        matchingElements.sort((a, b) => {
+            const areaA = a.frame.width * a.frame.height;
+            const areaB = b.frame.width * b.frame.height;
+            return areaA - areaB;
+        });
+        const element = matchingElements[0];
+        // Format detailed output
+        const lines = [];
+        const label = element.text || element.contentDesc;
+        lines.push(`[${element.class}]${label ? ` "${label}"` : ""} frame=(${element.frame.x}, ${element.frame.y}, ${element.frame.width}x${element.frame.height}) tap=(${element.tap.x}, ${element.tap.y})`);
+        if (element.resourceId) {
+            lines.push(`  resource-id: ${element.resourceId}`);
+        }
+        if (element.contentDesc && element.text) {
+            // Show content-desc separately if we showed text as label
+            lines.push(`  content-desc: ${element.contentDesc}`);
+        }
+        if (element.text && element.contentDesc) {
+            // Show text separately if we showed content-desc as label
+            lines.push(`  text: ${element.text}`);
+        }
+        // Show state flags
+        const flags = [];
+        if (element.clickable)
+            flags.push("clickable");
+        if (element.enabled)
+            flags.push("enabled");
+        if (element.focusable)
+            flags.push("focusable");
+        if (element.focused)
+            flags.push("focused");
+        if (element.scrollable)
+            flags.push("scrollable");
+        if (element.selected)
+            flags.push("selected");
+        if (element.checked)
+            flags.push("checked");
+        if (flags.length > 0) {
+            lines.push(`  state: ${flags.join(", ")}`);
+        }
+        return {
+            success: true,
+            elements: [element],
+            formatted: lines.join("\n")
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: `Failed to describe point: ${error instanceof Error ? error.message : String(error)}`
+        };
+    }
+}
+/**
+ * Tap an element by its text, content-description, or resource-id
+ */
+export async function androidTapElement(options) {
+    try {
+        const { text, textContains, contentDesc, contentDescContains, resourceId, index = 0, deviceId } = options;
+        // Validate that at least one search criterion is provided
+        if (!text && !textContains && !contentDesc && !contentDescContains && !resourceId) {
+            return {
+                success: false,
+                error: "At least one of text, textContains, contentDesc, contentDescContains, or resourceId must be provided"
+            };
+        }
+        // Get the UI hierarchy
+        const result = await androidDescribeAll(deviceId);
+        if (!result.success || !result.elements) {
+            return {
+                success: false,
+                error: result.error || "Failed to get UI hierarchy"
+            };
+        }
+        // Flatten and search
+        const allElements = flattenElements(result.elements);
+        // Filter elements based on search criteria
+        const matchingElements = allElements.filter((el) => {
+            if (text && el.text !== text)
+                return false;
+            if (textContains && (!el.text || !el.text.toLowerCase().includes(textContains.toLowerCase())))
+                return false;
+            if (contentDesc && el.contentDesc !== contentDesc)
+                return false;
+            if (contentDescContains && (!el.contentDesc || !el.contentDesc.toLowerCase().includes(contentDescContains.toLowerCase())))
+                return false;
+            if (resourceId) {
+                // Support both full resource-id and short form
+                if (!el.resourceId)
+                    return false;
+                if (el.resourceId !== resourceId && !el.resourceId.endsWith(`:id/${resourceId}`))
+                    return false;
+            }
+            return true;
+        });
+        if (matchingElements.length === 0) {
+            const criteria = [];
+            if (text)
+                criteria.push(`text="${text}"`);
+            if (textContains)
+                criteria.push(`textContains="${textContains}"`);
+            if (contentDesc)
+                criteria.push(`contentDesc="${contentDesc}"`);
+            if (contentDescContains)
+                criteria.push(`contentDescContains="${contentDescContains}"`);
+            if (resourceId)
+                criteria.push(`resourceId="${resourceId}"`);
+            return {
+                success: false,
+                error: `Element not found: ${criteria.join(", ")}`
+            };
+        }
+        if (index >= matchingElements.length) {
+            return {
+                success: false,
+                error: `Index ${index} out of range. Found ${matchingElements.length} matching element(s).`
+            };
+        }
+        const element = matchingElements[index];
+        const label = element.text || element.contentDesc || element.resourceId || element.class;
+        // Log if multiple matches
+        let resultMessage;
+        if (matchingElements.length > 1) {
+            resultMessage = `Found ${matchingElements.length} elements, tapping "${label}" (index ${index}) at (${element.tap.x}, ${element.tap.y})`;
+        }
+        else {
+            resultMessage = `Tapped "${label}" at (${element.tap.x}, ${element.tap.y})`;
+        }
+        // Perform the tap
+        const tapResult = await androidTap(element.tap.x, element.tap.y, deviceId);
+        if (!tapResult.success) {
+            return tapResult;
+        }
+        return {
+            success: true,
+            result: resultMessage
+        };
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: `Failed to tap element: ${error instanceof Error ? error.message : String(error)}`
+        };
+    }
+}
 //# sourceMappingURL=android.js.map