replicant-mcp 1.0.0 → 1.1.0

package/dist/tools/ui.js

@@ -1,23 +1,99 @@
 import { z } from "zod";
 import { CACHE_TTLS } from "../types/index.js";
+import { flattenTree } from "../parsers/ui-dump.js";
 export const uiInputSchema = z.object({
-    operation: z.enum(["dump", "find", "tap", "input", "screenshot", "accessibility-check"]),
+    operation: z.enum(["dump", "find", "tap", "input", "screenshot", "accessibility-check", "visual-snapshot"]),
     selector: z.object({
         resourceId: z.string().optional(),
         text: z.string().optional(),
         textContains: z.string().optional(),
         className: z.string().optional(),
+        nearestTo: z.string().optional(),
     }).optional(),
     x: z.number().optional(),
     y: z.number().optional(),
     elementIndex: z.number().optional(),
     text: z.string().optional(),
     localPath: z.string().optional(),
+    inline: z.boolean().optional(),
+    debug: z.boolean().optional(),
+    gridCell: z.number().min(1).max(24).optional(),
+    gridPosition: z.number().min(1).max(5).optional(),
 });
 // Store last find results for elementIndex reference
+// Updated to support accessibility, OCR, and grid elements
 let lastFindResults = [];
-export async function handleUiTool(input, context) {
-    const deviceId = context.deviceState.requireCurrentDevice().id;
+// Type guards for different element types
+function isAccessibilityNode(el) {
+    return "centerX" in el && "className" in el;
+}
+function isOcrElement(el) {
+    return "confidence" in el && "center" in el;
+}
+function isGridElement(el) {
+    return "center" in el && "bounds" in el && !("confidence" in el) && !("centerX" in el);
+}
+// Helper to get center coordinates from any element type
+function getElementCenter(element) {
+    if (isAccessibilityNode(element)) {
+        return { x: element.centerX, y: element.centerY };
+    }
+    else {
+        // OcrElement or GridElement - both have center property
+        return element.center;
+    }
+}
+// Calculate Euclidean distance between two points
+function calculateDistance(p1, p2) {
+    return Math.sqrt(Math.pow(p2.x - p1.x, 2) + Math.pow(p2.y - p1.y, 2));
+}
+// Check if a point is inside element bounds
+function isPointInBounds(point, bounds) {
+    return (point.x >= bounds.left &&
+        point.x <= bounds.right &&
+        point.y >= bounds.top &&
+        point.y <= bounds.bottom);
+}
+// Calculate area of bounds
+function boundsArea(bounds) {
+    return (bounds.right - bounds.left) * (bounds.bottom - bounds.top);
+}
+// Find targets whose smallest containing ViewGroup also contains the anchor point
+function findContainingSiblingTargets(tree, anchorPoint, targetElements) {
+    const flat = flattenTree(tree);
+    const containingTargets = [];
+    for (const target of targetElements) {
+        const targetCenter = { x: target.centerX, y: target.centerY };
+        // Find the smallest ViewGroup that contains the target
+        let smallestContainerForTarget = null;
+        let smallestArea = Infinity;
+        for (const node of flat) {
+            if (!node.className?.includes("ViewGroup"))
+                continue;
+            if (!isPointInBounds(targetCenter, node.bounds))
+                continue;
+            const area = boundsArea(node.bounds);
+            if (area < smallestArea) {
+                smallestArea = area;
+                smallestContainerForTarget = node;
+            }
+        }
+        // Check if that smallest container also contains the anchor point
+        if (smallestContainerForTarget && isPointInBounds(anchorPoint, smallestContainerForTarget.bounds)) {
+            containingTargets.push(target);
+        }
+    }
+    return containingTargets;
+}
+export async function handleUiTool(input, context, uiConfig) {
+    const device = await context.deviceState.ensureDevice(context.adb);
+    const deviceId = device.id;
+    // Get config - use provided or defaults
+    const config = uiConfig ?? {
+        visualModePackages: [],
+        autoFallbackScreenshot: true,
+        includeBase64: false,
+    };
     switch (input.operation) {
         case "dump": {
             const tree = await context.ui.dump(deviceId);
@@ -43,9 +119,149 @@ export async function handleUiTool(input, context) {
             if (!input.selector) {
                 throw new Error("selector is required for find operation");
             }
+            const debug = input.debug ?? false;
+            const nearestTo = input.selector.nearestTo;
+            // Use findWithFallbacks for text-based selectors
+            if (input.selector.text || input.selector.textContains) {
+                // If nearestTo is specified, first find the anchor element
+                let anchorCenter = null;
+                if (nearestTo) {
+                    const anchorResult = await context.ui.findWithFallbacks(deviceId, { text: nearestTo }, {
+                        debug: false,
+                        includeVisualFallback: false,
+                    });
+                    if (anchorResult.elements.length > 0) {
+                        anchorCenter = getElementCenter(anchorResult.elements[0]);
+                    }
+                }
+                const result = await context.ui.findWithFallbacks(deviceId, input.selector, {
+                    debug,
+                    includeVisualFallback: config.autoFallbackScreenshot,
+                    includeBase64: config.includeBase64,
+                    gridCell: input.gridCell,
+                    gridPosition: input.gridPosition,
+                });
+                // If we have an anchor, use containment-based matching
+                let usedContainment = false;
+                if (anchorCenter && result.elements.length > 0) {
+                    // Filter to AccessibilityNode elements for containment check
+                    const accessibilityElements = result.elements.filter(isAccessibilityNode);
+                    if (accessibilityElements.length > 0) {
+                        // Get the full tree for containment analysis
+                        const tree = await context.ui.dump(deviceId);
+                        // Find elements whose parent container contains the anchor point
+                        const containingMatches = findContainingSiblingTargets(tree, anchorCenter, accessibilityElements);
+                        if (containingMatches.length > 0) {
+                            // Prioritize containment matches, then sort remaining by distance
+                            usedContainment = true;
+                            const containingCenters = new Set(containingMatches.map((el) => `${el.centerX},${el.centerY}`));
+                            result.elements.sort((a, b) => {
+                                const aCenter = getElementCenter(a);
+                                const bCenter = getElementCenter(b);
+                                const aContains = containingCenters.has(`${aCenter.x},${aCenter.y}`);
+                                const bContains = containingCenters.has(`${bCenter.x},${bCenter.y}`);
+                                // Containment matches come first
+                                if (aContains && !bContains)
+                                    return -1;
+                                if (!aContains && bContains)
+                                    return 1;
+                                // Within same group, sort by distance
+                                const distA = calculateDistance(aCenter, anchorCenter);
+                                const distB = calculateDistance(bCenter, anchorCenter);
+                                return distA - distB;
+                            });
+                        }
+                        else {
+                            // Fallback to pure distance sorting if no containment matches
+                            result.elements.sort((a, b) => {
+                                const distA = calculateDistance(getElementCenter(a), anchorCenter);
+                                const distB = calculateDistance(getElementCenter(b), anchorCenter);
+                                return distA - distB;
+                            });
+                        }
+                    }
+                }
+                lastFindResults = result.elements;
+                const response = {
+                    elements: result.elements.map((el, index) => {
+                        if (isAccessibilityNode(el)) {
+                            return {
+                                index,
+                                text: el.text,
+                                resourceId: el.resourceId,
+                                className: el.className,
+                                centerX: el.centerX,
+                                centerY: el.centerY,
+                                bounds: el.bounds,
+                                clickable: el.clickable,
+                            };
+                        }
+                        else if (isOcrElement(el)) {
+                            return {
+                                index,
+                                text: el.text,
+                                center: el.center,
+                                bounds: el.bounds,
+                                confidence: debug ? el.confidence : undefined,
+                            };
+                        }
+                        else {
+                            // GridElement
+                            return {
+                                index,
+                                center: el.center,
+                                bounds: el.bounds,
+                            };
+                        }
+                    }),
+                    count: result.elements.length,
+                    deviceId,
+                };
+                // Always include tier and confidence when available
+                if (result.tier !== undefined)
+                    response.tier = result.tier;
+                if (result.confidence)
+                    response.confidence = result.confidence;
+                if (debug) {
+                    response.source = result.source;
+                    if (result.fallbackReason) {
+                        response.fallbackReason = result.fallbackReason;
+                    }
+                }
+                // Include nearestTo info when used
+                if (nearestTo && anchorCenter) {
+                    response.sortedByProximityTo = {
+                        query: nearestTo,
+                        anchor: anchorCenter,
+                        method: usedContainment ? "containment" : "distance",
+                    };
+                }
+                else if (nearestTo && !anchorCenter) {
+                    response.nearestToWarning = `Could not find anchor element: "${nearestTo}"`;
+                }
+                // Include Tier 4 visual candidates if present
+                if (result.candidates) {
+                    response.candidates = result.candidates;
+                    if (result.truncated)
+                        response.truncated = result.truncated;
+                    if (result.totalCandidates)
+                        response.totalCandidates = result.totalCandidates;
+                }
+                // Include Tier 5 grid fields if present
+                if (result.gridImage)
+                    response.gridImage = result.gridImage;
+                if (result.gridPositions)
+                    response.gridPositions = result.gridPositions;
+                // Include visual fallback if present (when count is 0 and autoFallbackScreenshot is enabled)
+                if (result.visualFallback) {
+                    response.visualFallback = result.visualFallback;
+                }
+                return response;
+            }
+            // Non-text selectors use regular find (no OCR fallback)
             const elements = await context.ui.find(deviceId, input.selector);
             lastFindResults = elements;
-            return {
+            const response = {
                 elements: elements.map((el, index) => ({
                     index,
                     text: el.text,
@@ -59,6 +275,17 @@ export async function handleUiTool(input, context) {
                 count: elements.length,
                 deviceId,
             };
+            // Include visual fallback for non-text selectors when no results and config allows
+            if (elements.length === 0 && config.autoFallbackScreenshot) {
+                const snapshot = await context.ui.visualSnapshot(deviceId, {
+                    includeBase64: config.includeBase64,
+                });
+                response.visualFallback = {
+                    ...snapshot,
+                    hint: "No elements matched selector. Use screenshot to identify tap coordinates.",
+                };
+            }
+            return response;
         }
         case "tap": {
             let x, y;
@@ -67,8 +294,9 @@ export async function handleUiTool(input, context) {
                     throw new Error(`Element at index ${input.elementIndex} not found. Run 'find' first.`);
                 }
                 const element = lastFindResults[input.elementIndex];
-                x = element.centerX;
-                y = element.centerY;
+                const center = getElementCenter(element);
+                x = center.x;
+                y = center.y;
             }
             else if (input.x !== undefined && input.y !== undefined) {
                 x = input.x;
@@ -88,27 +316,35 @@ export async function handleUiTool(input, context) {
             return { input: input.text, deviceId };
         }
         case "screenshot": {
-            const localPath = input.localPath || `/tmp/screenshot-${Date.now()}.png`;
-            await context.ui.screenshot(deviceId, localPath);
-            return { path: localPath, deviceId };
+            const result = await context.ui.screenshot(deviceId, {
+                localPath: input.localPath,
+                inline: input.inline,
+            });
+            return { ...result, deviceId };
         }
         case "accessibility-check": {
             const result = await context.ui.accessibilityCheck(deviceId);
             return { ...result, deviceId };
         }
+        case "visual-snapshot": {
+            const snapshot = await context.ui.visualSnapshot(deviceId, {
+                includeBase64: input.inline ?? config.includeBase64,
+            });
+            return { ...snapshot, deviceId };
+        }
         default:
             throw new Error(`Unknown operation: ${input.operation}`);
     }
 }
 export const uiToolDefinition = {
     name: "ui",
-    description: "Interact with app UI via accessibility tree. Operations: dump, find, tap, input, screenshot, accessibility-check.",
+    description: "Interact with app UI via accessibility tree. Auto-selects device if only one connected. Operations: dump, find, tap, input, screenshot, accessibility-check, visual-snapshot.",
     inputSchema: {
         type: "object",
         properties: {
             operation: {
                 type: "string",
-                enum: ["dump", "find", "tap", "input", "screenshot", "accessibility-check"],
+                enum: ["dump", "find", "tap", "input", "screenshot", "accessibility-check", "visual-snapshot"],
             },
             selector: {
                 type: "object",
@@ -117,6 +353,7 @@ export const uiToolDefinition = {
                     text: { type: "string" },
                     textContains: { type: "string" },
                     className: { type: "string" },
+                    nearestTo: { type: "string", description: "Find elements nearest to this text (spatial proximity)" },
                 },
                 description: "Element selector (for find)",
             },
@@ -124,7 +361,11 @@ export const uiToolDefinition = {
             y: { type: "number", description: "Y coordinate (for tap)" },
             elementIndex: { type: "number", description: "Element index from last find (for tap)" },
             text: { type: "string", description: "Text to input" },
-            localPath: { type: "string", description: "Local path for screenshot" },
+            localPath: { type: "string", description: "Local path for screenshot (default: .replicant/screenshots/screenshot-{timestamp}.png)" },
+            inline: { type: "boolean", description: "Return base64 instead of file path (token-heavy, use sparingly)" },
+            debug: { type: "boolean", description: "Include source (accessibility/ocr) and confidence in response" },
+            gridCell: { type: "number", minimum: 1, maximum: 24, description: "Grid cell number (1-24) for Tier 5 refinement" },
+            gridPosition: { type: "number", minimum: 1, maximum: 5, description: "Position within cell (1=TL, 2=TR, 3=Center, 4=BL, 5=BR)" },
         },
         required: ["operation"],
     },

package/dist/types/config.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Configuration types for replicant-mcp
+ * Loaded from REPLICANT_CONFIG environment variable path
+ */
+export interface UiConfig {
+    /** Always skip accessibility and use visual mode for these packages */
+    visualModePackages: string[];
+    /** Auto-include screenshot when find returns no results (default: true) */
+    autoFallbackScreenshot: boolean;
+    /** Include base64-encoded screenshot in response (default: false) */
+    includeBase64: boolean;
+}
+export interface ReplicantConfig {
+    ui: UiConfig;
+}
+export declare const DEFAULT_CONFIG: ReplicantConfig;
+/**
+ * Visual snapshot response returned when accessibility fails
+ * or when visual-snapshot operation is explicitly requested
+ */
+export interface VisualSnapshot {
+    screenshotPath: string;
+    screenshotBase64?: string;
+    screen: {
+        width: number;
+        height: number;
+        density: number;
+    };
+    app: {
+        packageName: string;
+        activityName: string;
+    };
+    hint?: string;
+}

package/dist/types/config.js

@@ -0,0 +1,11 @@
+/**
+ * Configuration types for replicant-mcp
+ * Loaded from REPLICANT_CONFIG environment variable path
+ */
+export const DEFAULT_CONFIG = {
+    ui: {
+        visualModePackages: [],
+        autoFallbackScreenshot: true,
+        includeBase64: false,
+    },
+};

package/dist/types/errors.d.ts

@@ -9,13 +9,30 @@ export declare const ErrorCode: {
     readonly PACKAGE_NOT_FOUND: "PACKAGE_NOT_FOUND";
     readonly INSTALL_FAILED: "INSTALL_FAILED";
     readonly AVD_NOT_FOUND: "AVD_NOT_FOUND";
+    readonly EMULATOR_NOT_FOUND: "EMULATOR_NOT_FOUND";
     readonly EMULATOR_START_FAILED: "EMULATOR_START_FAILED";
     readonly SNAPSHOT_NOT_FOUND: "SNAPSHOT_NOT_FOUND";
     readonly COMMAND_BLOCKED: "COMMAND_BLOCKED";
     readonly TIMEOUT: "TIMEOUT";
     readonly CACHE_MISS: "CACHE_MISS";
+    readonly SDK_NOT_FOUND: "SDK_NOT_FOUND";
+    readonly ADB_NOT_FOUND: "ADB_NOT_FOUND";
+    readonly ADB_NOT_EXECUTABLE: "ADB_NOT_EXECUTABLE";
+    readonly ADB_SERVER_ERROR: "ADB_SERVER_ERROR";
+    readonly NO_DEVICES: "NO_DEVICES";
+    readonly MULTIPLE_DEVICES: "MULTIPLE_DEVICES";
+    readonly SCREENSHOT_FAILED: "SCREENSHOT_FAILED";
+    readonly PULL_FAILED: "PULL_FAILED";
+    readonly HEALTH_CHECK_FAILED: "HEALTH_CHECK_FAILED";
 };
 export type ErrorCode = (typeof ErrorCode)[keyof typeof ErrorCode];
+export interface ErrorContext {
+    command?: string;
+    exitCode?: number;
+    stderr?: string;
+    checkedPaths?: string[];
+    [key: string]: unknown;
+}
 export interface ToolError {
     error: ErrorCode;
     message: string;
@@ -25,7 +42,13 @@ export interface ToolError {
 export declare class ReplicantError extends Error {
     readonly code: ErrorCode;
     readonly suggestion?: string | undefined;
-    readonly details?: Record<string, unknown> | undefined;
-    constructor(code: ErrorCode, message: string, suggestion?: string | undefined, details?: Record<string, unknown> | undefined);
+    readonly context?: ErrorContext | undefined;
+    constructor(code: ErrorCode, message: string, suggestion?: string | undefined, context?: ErrorContext | undefined);
+    toJSON(): {
+        error: ErrorCode;
+        message: string;
+        suggestion: string | undefined;
+        context: ErrorContext | undefined;
+    };
     toToolError(): ToolError;
 }

package/dist/types/errors.js

@@ -13,6 +13,7 @@ export const ErrorCode = {
     INSTALL_FAILED: "INSTALL_FAILED",
     // Emulator errors
     AVD_NOT_FOUND: "AVD_NOT_FOUND",
+    EMULATOR_NOT_FOUND: "EMULATOR_NOT_FOUND",
     EMULATOR_START_FAILED: "EMULATOR_START_FAILED",
     SNAPSHOT_NOT_FOUND: "SNAPSHOT_NOT_FOUND",
     // Safety errors
@@ -20,24 +21,42 @@ export const ErrorCode = {
     TIMEOUT: "TIMEOUT",
     // Cache errors
     CACHE_MISS: "CACHE_MISS",
+    // New "Just Works" UX error codes
+    SDK_NOT_FOUND: "SDK_NOT_FOUND",
+    ADB_NOT_FOUND: "ADB_NOT_FOUND",
+    ADB_NOT_EXECUTABLE: "ADB_NOT_EXECUTABLE",
+    ADB_SERVER_ERROR: "ADB_SERVER_ERROR",
+    NO_DEVICES: "NO_DEVICES",
+    MULTIPLE_DEVICES: "MULTIPLE_DEVICES",
+    SCREENSHOT_FAILED: "SCREENSHOT_FAILED",
+    PULL_FAILED: "PULL_FAILED",
+    HEALTH_CHECK_FAILED: "HEALTH_CHECK_FAILED",
 };
 export class ReplicantError extends Error {
     code;
     suggestion;
-    details;
-    constructor(code, message, suggestion, details) {
+    context;
+    constructor(code, message, suggestion, context) {
         super(message);
         this.code = code;
         this.suggestion = suggestion;
-        this.details = details;
+        this.context = context;
         this.name = "ReplicantError";
     }
+    toJSON() {
+        return {
+            error: this.code,
+            message: this.message,
+            suggestion: this.suggestion,
+            context: this.context,
+        };
+    }
     toToolError() {
         return {
             error: this.code,
             message: this.message,
             suggestion: this.suggestion,
-            details: this.details,
+            details: this.context,
         };
     }
 }

package/dist/types/icon-recognition.d.ts

@@ -0,0 +1,50 @@
+export type ConfidenceLevel = "high" | "medium" | "low";
+export type FindSource = "accessibility" | "ocr" | "visual" | "grid";
+export type FindTier = 1 | 2 | 3 | 4 | 5;
+export interface VisualCandidate {
+    index: number;
+    bounds: string;
+    center: {
+        x: number;
+        y: number;
+    };
+    image: string;
+}
+export interface GridPosition {
+    cell: number;
+    position: 1 | 2 | 3 | 4 | 5;
+    x: number;
+    y: number;
+}
+import { AccessibilityNode } from "../parsers/ui-dump.js";
+import { OcrElement } from "./ocr.js";
+export interface GridElement {
+    index: number;
+    bounds: string;
+    center: {
+        x: number;
+        y: number;
+    };
+}
+export type FindElement = AccessibilityNode | OcrElement | GridElement;
+export interface FindWithFallbacksResult {
+    elements: FindElement[];
+    source: FindSource;
+    tier?: FindTier;
+    confidence?: ConfidenceLevel;
+    fallbackReason?: string;
+    candidates?: VisualCandidate[];
+    truncated?: boolean;
+    totalCandidates?: number;
+    gridImage?: string;
+    gridCell?: number;
+    gridPositions?: string[];
+    visualFallback?: import("./config.js").VisualSnapshot;
+}
+export interface FindOptions {
+    debug?: boolean;
+    includeVisualFallback?: boolean;
+    includeBase64?: boolean;
+    gridCell?: number;
+    gridPosition?: 1 | 2 | 3 | 4 | 5;
+}

package/dist/types/icon-recognition.js

@@ -0,0 +1 @@
+export {};

package/dist/types/index.d.ts

@@ -1,3 +1,6 @@
 export * from "./errors.js";
 export * from "./cache.js";
 export * from "./device.js";
+export * from "./ocr.js";
+export * from "./config.js";
+export * from "./icon-recognition.js";

package/dist/types/index.js

@@ -1,3 +1,6 @@
 export * from "./errors.js";
 export * from "./cache.js";
 export * from "./device.js";
+export * from "./ocr.js";
+export * from "./config.js";
+export * from "./icon-recognition.js";

package/dist/types/ocr.d.ts

@@ -0,0 +1,21 @@
+export interface OcrBounds {
+    x0: number;
+    y0: number;
+    x1: number;
+    y1: number;
+}
+export interface OcrResult {
+    text: string;
+    confidence: number;
+    bounds: OcrBounds;
+}
+export interface OcrElement {
+    index: number;
+    text: string;
+    bounds: string;
+    center: {
+        x: number;
+        y: number;
+    };
+    confidence: number;
+}

package/dist/types/ocr.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicant-mcp",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Android MCP server for AI-assisted Android development",
   "type": "module",
   "main": "dist/index.js",
@@ -20,7 +20,6 @@
     "test:device": "tsx scripts/real-device-test.ts",
     "start": "node dist/index.js",
     "validate": "npm run build && npm run test -- --run",
-    "install-skill": "bash scripts/install-skill.sh",
     "prepublishOnly": "npm run build && npm test -- --run"
   },
   "keywords": [
@@ -52,10 +51,14 @@
     "@modelcontextprotocol/sdk": "^1.25.3",
     "commander": "^14.0.2",
     "execa": "^9.6.1",
+    "sharp": "^0.34.5",
+    "tesseract.js": "^7.0.0",
+    "yaml": "^2.8.2",
     "zod": "^4.3.5"
   },
   "devDependencies": {
     "@types/node": "^25.0.9",
+    "@types/sharp": "^0.31.1",
     "@vitest/coverage-v8": "^4.0.17",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",