replicant-mcp 1.0.4 → 1.1.0

package/dist/tools/ui.js

@@ -1,12 +1,14 @@
 import { z } from "zod";
 import { CACHE_TTLS } from "../types/index.js";
+import { flattenTree } from "../parsers/ui-dump.js";
 export const uiInputSchema = z.object({
-    operation: z.enum(["dump", "find", "tap", "input", "screenshot", "accessibility-check"]),
+    operation: z.enum(["dump", "find", "tap", "input", "screenshot", "accessibility-check", "visual-snapshot"]),
     selector: z.object({
         resourceId: z.string().optional(),
         text: z.string().optional(),
         textContains: z.string().optional(),
         className: z.string().optional(),
+        nearestTo: z.string().optional(),
     }).optional(),
     x: z.number().optional(),
     y: z.number().optional(),
@@ -15,24 +17,83 @@ export const uiInputSchema = z.object({
     localPath: z.string().optional(),
     inline: z.boolean().optional(),
     debug: z.boolean().optional(),
+    gridCell: z.number().min(1).max(24).optional(),
+    gridPosition: z.number().min(1).max(5).optional(),
 });
 // Store last find results for elementIndex reference
-// Updated to support both accessibility and OCR elements
+// Updated to support accessibility, OCR, and grid elements
 let lastFindResults = [];
-// Helper to get center coordinates from either element type
+// Type guards for different element types
+function isAccessibilityNode(el) {
+    return "centerX" in el && "className" in el;
+}
+function isOcrElement(el) {
+    return "confidence" in el && "center" in el;
+}
+function isGridElement(el) {
+    return "center" in el && "bounds" in el && !("confidence" in el) && !("centerX" in el);
+}
+// Helper to get center coordinates from any element type
 function getElementCenter(element) {
-    if ("centerX" in element) {
-        // AccessibilityNode
+    if (isAccessibilityNode(element)) {
         return { x: element.centerX, y: element.centerY };
     }
     else {
-        // OcrElement
+        // OcrElement or GridElement - both have center property
         return element.center;
     }
 }
-export async function handleUiTool(input, context) {
+// Calculate Euclidean distance between two points
+function calculateDistance(p1, p2) {
+    return Math.sqrt(Math.pow(p2.x - p1.x, 2) + Math.pow(p2.y - p1.y, 2));
+}
+// Check if a point is inside element bounds
+function isPointInBounds(point, bounds) {
+    return (point.x >= bounds.left &&
+        point.x <= bounds.right &&
+        point.y >= bounds.top &&
+        point.y <= bounds.bottom);
+}
+// Calculate area of bounds
+function boundsArea(bounds) {
+    return (bounds.right - bounds.left) * (bounds.bottom - bounds.top);
+}
+// Find targets whose smallest containing ViewGroup also contains the anchor point
+function findContainingSiblingTargets(tree, anchorPoint, targetElements) {
+    const flat = flattenTree(tree);
+    const containingTargets = [];
+    for (const target of targetElements) {
+        const targetCenter = { x: target.centerX, y: target.centerY };
+        // Find the smallest ViewGroup that contains the target
+        let smallestContainerForTarget = null;
+        let smallestArea = Infinity;
+        for (const node of flat) {
+            if (!node.className?.includes("ViewGroup"))
+                continue;
+            if (!isPointInBounds(targetCenter, node.bounds))
+                continue;
+            const area = boundsArea(node.bounds);
+            if (area < smallestArea) {
+                smallestArea = area;
+                smallestContainerForTarget = node;
+            }
+        }
+        // Check if that smallest container also contains the anchor point
+        if (smallestContainerForTarget && isPointInBounds(anchorPoint, smallestContainerForTarget.bounds)) {
+            containingTargets.push(target);
+        }
+    }
+    return containingTargets;
+}
+export async function handleUiTool(input, context, uiConfig) {
     const device = await context.deviceState.ensureDevice(context.adb);
     const deviceId = device.id;
+    // Get config - use provided or defaults
+    const config = uiConfig ?? {
+        visualModePackages: [],
+        autoFallbackScreenshot: true,
+        includeBase64: false,
+    };
     switch (input.operation) {
         case "dump": {
             const tree = await context.ui.dump(deviceId);
@@ -59,14 +120,71 @@ export async function handleUiTool(input, context) {
                 throw new Error("selector is required for find operation");
             }
             const debug = input.debug ?? false;
-            // Use findWithOcrFallback for text-based selectors
+            const nearestTo = input.selector.nearestTo;
+            // Use findWithFallbacks for text-based selectors
             if (input.selector.text || input.selector.textContains) {
-                const result = await context.ui.findWithOcrFallback(deviceId, input.selector, { debug });
+                // If nearestTo is specified, first find the anchor element
+                let anchorCenter = null;
+                if (nearestTo) {
+                    const anchorResult = await context.ui.findWithFallbacks(deviceId, { text: nearestTo }, {
+                        debug: false,
+                        includeVisualFallback: false,
+                    });
+                    if (anchorResult.elements.length > 0) {
+                        anchorCenter = getElementCenter(anchorResult.elements[0]);
+                    }
+                }
+                const result = await context.ui.findWithFallbacks(deviceId, input.selector, {
+                    debug,
+                    includeVisualFallback: config.autoFallbackScreenshot,
+                    includeBase64: config.includeBase64,
+                    gridCell: input.gridCell,
+                    gridPosition: input.gridPosition,
+                });
+                // If we have an anchor, use containment-based matching
+                let usedContainment = false;
+                if (anchorCenter && result.elements.length > 0) {
+                    // Filter to AccessibilityNode elements for containment check
+                    const accessibilityElements = result.elements.filter(isAccessibilityNode);
+                    if (accessibilityElements.length > 0) {
+                        // Get the full tree for containment analysis
+                        const tree = await context.ui.dump(deviceId);
+                        // Find elements whose parent container contains the anchor point
+                        const containingMatches = findContainingSiblingTargets(tree, anchorCenter, accessibilityElements);
+                        if (containingMatches.length > 0) {
+                            // Prioritize containment matches, then sort remaining by distance
+                            usedContainment = true;
+                            const containingCenters = new Set(containingMatches.map((el) => `${el.centerX},${el.centerY}`));
+                            result.elements.sort((a, b) => {
+                                const aCenter = getElementCenter(a);
+                                const bCenter = getElementCenter(b);
+                                const aContains = containingCenters.has(`${aCenter.x},${aCenter.y}`);
+                                const bContains = containingCenters.has(`${bCenter.x},${bCenter.y}`);
+                                // Containment matches come first
+                                if (aContains && !bContains)
+                                    return -1;
+                                if (!aContains && bContains)
+                                    return 1;
+                                // Within same group, sort by distance
+                                const distA = calculateDistance(aCenter, anchorCenter);
+                                const distB = calculateDistance(bCenter, anchorCenter);
+                                return distA - distB;
+                            });
+                        }
+                        else {
+                            // Fallback to pure distance sorting if no containment matches
+                            result.elements.sort((a, b) => {
+                                const distA = calculateDistance(getElementCenter(a), anchorCenter);
+                                const distB = calculateDistance(getElementCenter(b), anchorCenter);
+                                return distA - distB;
+                            });
+                        }
+                    }
+                }
                 lastFindResults = result.elements;
                 const response = {
                     elements: result.elements.map((el, index) => {
-                        if ("centerX" in el) {
-                            // AccessibilityNode
+                        if (isAccessibilityNode(el)) {
                             return {
                                 index,
                                 text: el.text,
@@ -78,8 +196,7 @@ export async function handleUiTool(input, context) {
                                 clickable: el.clickable,
                             };
                         }
-                        else {
-                            // OcrElement
+                        else if (isOcrElement(el)) {
                             return {
                                 index,
                                 text: el.text,
@@ -88,22 +205,63 @@ export async function handleUiTool(input, context) {
                                 confidence: debug ? el.confidence : undefined,
                             };
                         }
+                        else {
+                            // GridElement
+                            return {
+                                index,
+                                center: el.center,
+                                bounds: el.bounds,
+                            };
+                        }
                     }),
                     count: result.elements.length,
                     deviceId,
                 };
+                // Always include tier and confidence when available
+                if (result.tier !== undefined)
+                    response.tier = result.tier;
+                if (result.confidence)
+                    response.confidence = result.confidence;
                 if (debug) {
                     response.source = result.source;
                     if (result.fallbackReason) {
                         response.fallbackReason = result.fallbackReason;
                     }
                 }
+                // Include nearestTo info when used
+                if (nearestTo && anchorCenter) {
+                    response.sortedByProximityTo = {
+                        query: nearestTo,
+                        anchor: anchorCenter,
+                        method: usedContainment ? "containment" : "distance",
+                    };
+                }
+                else if (nearestTo && !anchorCenter) {
+                    response.nearestToWarning = `Could not find anchor element: "${nearestTo}"`;
+                }
+                // Include Tier 4 visual candidates if present
+                if (result.candidates) {
+                    response.candidates = result.candidates;
+                    if (result.truncated)
+                        response.truncated = result.truncated;
+                    if (result.totalCandidates)
+                        response.totalCandidates = result.totalCandidates;
+                }
+                // Include Tier 5 grid fields if present
+                if (result.gridImage)
+                    response.gridImage = result.gridImage;
+                if (result.gridPositions)
+                    response.gridPositions = result.gridPositions;
+                // Include visual fallback if present (when count is 0 and autoFallbackScreenshot is enabled)
+                if (result.visualFallback) {
+                    response.visualFallback = result.visualFallback;
+                }
                 return response;
             }
             // Non-text selectors use regular find (no OCR fallback)
             const elements = await context.ui.find(deviceId, input.selector);
             lastFindResults = elements;
-            return {
+            const response = {
                 elements: elements.map((el, index) => ({
                     index,
                     text: el.text,
@@ -117,6 +275,17 @@ export async function handleUiTool(input, context) {
                 count: elements.length,
                 deviceId,
             };
+            // Include visual fallback for non-text selectors when no results and config allows
+            if (elements.length === 0 && config.autoFallbackScreenshot) {
+                const snapshot = await context.ui.visualSnapshot(deviceId, {
+                    includeBase64: config.includeBase64,
+                });
+                response.visualFallback = {
+                    ...snapshot,
+                    hint: "No elements matched selector. Use screenshot to identify tap coordinates.",
+                };
+            }
+            return response;
         }
         case "tap": {
             let x, y;
@@ -157,19 +326,25 @@ export async function handleUiTool(input, context) {
             const result = await context.ui.accessibilityCheck(deviceId);
             return { ...result, deviceId };
         }
+        case "visual-snapshot": {
+            const snapshot = await context.ui.visualSnapshot(deviceId, {
+                includeBase64: input.inline ?? config.includeBase64,
+            });
+            return { ...snapshot, deviceId };
+        }
         default:
             throw new Error(`Unknown operation: ${input.operation}`);
     }
 }
 export const uiToolDefinition = {
     name: "ui",
-    description: "Interact with app UI via accessibility tree. Auto-selects device if only one connected. Operations: dump, find, tap, input, screenshot, accessibility-check.",
+    description: "Interact with app UI via accessibility tree. Auto-selects device if only one connected. Operations: dump, find, tap, input, screenshot, accessibility-check, visual-snapshot.",
     inputSchema: {
         type: "object",
         properties: {
             operation: {
                 type: "string",
-                enum: ["dump", "find", "tap", "input", "screenshot", "accessibility-check"],
+                enum: ["dump", "find", "tap", "input", "screenshot", "accessibility-check", "visual-snapshot"],
             },
             selector: {
                 type: "object",
@@ -178,6 +353,7 @@ export const uiToolDefinition = {
                     text: { type: "string" },
                     textContains: { type: "string" },
                     className: { type: "string" },
+                    nearestTo: { type: "string", description: "Find elements nearest to this text (spatial proximity)" },
                 },
                 description: "Element selector (for find)",
             },
@@ -185,9 +361,11 @@ export const uiToolDefinition = {
             y: { type: "number", description: "Y coordinate (for tap)" },
             elementIndex: { type: "number", description: "Element index from last find (for tap)" },
             text: { type: "string", description: "Text to input" },
-            localPath: { type: "string", description: "Local path for screenshot (default: /tmp/replicant-screenshot-{timestamp}.png)" },
+            localPath: { type: "string", description: "Local path for screenshot (default: .replicant/screenshots/screenshot-{timestamp}.png)" },
             inline: { type: "boolean", description: "Return base64 instead of file path (token-heavy, use sparingly)" },
             debug: { type: "boolean", description: "Include source (accessibility/ocr) and confidence in response" },
+            gridCell: { type: "number", minimum: 1, maximum: 24, description: "Grid cell number (1-24) for Tier 5 refinement" },
+            gridPosition: { type: "number", minimum: 1, maximum: 5, description: "Position within cell (1=TL, 2=TR, 3=Center, 4=BL, 5=BR)" },
         },
         required: ["operation"],
     },

package/dist/types/config.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Configuration types for replicant-mcp
+ * Loaded from REPLICANT_CONFIG environment variable path
+ */
+export interface UiConfig {
+    /** Always skip accessibility and use visual mode for these packages */
+    visualModePackages: string[];
+    /** Auto-include screenshot when find returns no results (default: true) */
+    autoFallbackScreenshot: boolean;
+    /** Include base64-encoded screenshot in response (default: false) */
+    includeBase64: boolean;
+}
+export interface ReplicantConfig {
+    ui: UiConfig;
+}
+export declare const DEFAULT_CONFIG: ReplicantConfig;
+/**
+ * Visual snapshot response returned when accessibility fails
+ * or when visual-snapshot operation is explicitly requested
+ */
+export interface VisualSnapshot {
+    screenshotPath: string;
+    screenshotBase64?: string;
+    screen: {
+        width: number;
+        height: number;
+        density: number;
+    };
+    app: {
+        packageName: string;
+        activityName: string;
+    };
+    hint?: string;
+}

package/dist/types/config.js

@@ -0,0 +1,11 @@
+/**
+ * Configuration types for replicant-mcp
+ * Loaded from REPLICANT_CONFIG environment variable path
+ */
+export const DEFAULT_CONFIG = {
+    ui: {
+        visualModePackages: [],
+        autoFallbackScreenshot: true,
+        includeBase64: false,
+    },
+};

package/dist/types/icon-recognition.d.ts

@@ -0,0 +1,50 @@
+export type ConfidenceLevel = "high" | "medium" | "low";
+export type FindSource = "accessibility" | "ocr" | "visual" | "grid";
+export type FindTier = 1 | 2 | 3 | 4 | 5;
+export interface VisualCandidate {
+    index: number;
+    bounds: string;
+    center: {
+        x: number;
+        y: number;
+    };
+    image: string;
+}
+export interface GridPosition {
+    cell: number;
+    position: 1 | 2 | 3 | 4 | 5;
+    x: number;
+    y: number;
+}
+import { AccessibilityNode } from "../parsers/ui-dump.js";
+import { OcrElement } from "./ocr.js";
+export interface GridElement {
+    index: number;
+    bounds: string;
+    center: {
+        x: number;
+        y: number;
+    };
+}
+export type FindElement = AccessibilityNode | OcrElement | GridElement;
+export interface FindWithFallbacksResult {
+    elements: FindElement[];
+    source: FindSource;
+    tier?: FindTier;
+    confidence?: ConfidenceLevel;
+    fallbackReason?: string;
+    candidates?: VisualCandidate[];
+    truncated?: boolean;
+    totalCandidates?: number;
+    gridImage?: string;
+    gridCell?: number;
+    gridPositions?: string[];
+    visualFallback?: import("./config.js").VisualSnapshot;
+}
+export interface FindOptions {
+    debug?: boolean;
+    includeVisualFallback?: boolean;
+    includeBase64?: boolean;
+    gridCell?: number;
+    gridPosition?: 1 | 2 | 3 | 4 | 5;
+}

package/dist/types/icon-recognition.js

@@ -0,0 +1 @@
+export {};

package/dist/types/index.d.ts

@@ -2,3 +2,5 @@ export * from "./errors.js";
 export * from "./cache.js";
 export * from "./device.js";
 export * from "./ocr.js";
+export * from "./config.js";
+export * from "./icon-recognition.js";

package/dist/types/index.js

@@ -2,3 +2,5 @@ export * from "./errors.js";
 export * from "./cache.js";
 export * from "./device.js";
 export * from "./ocr.js";
+export * from "./config.js";
+export * from "./icon-recognition.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicant-mcp",
-  "version": "1.0.4",
+  "version": "1.1.0",
   "description": "Android MCP server for AI-assisted Android development",
   "type": "module",
   "main": "dist/index.js",
@@ -51,11 +51,14 @@
     "@modelcontextprotocol/sdk": "^1.25.3",
     "commander": "^14.0.2",
     "execa": "^9.6.1",
+    "sharp": "^0.34.5",
     "tesseract.js": "^7.0.0",
+    "yaml": "^2.8.2",
     "zod": "^4.3.5"
   },
   "devDependencies": {
     "@types/node": "^25.0.9",
+    "@types/sharp": "^0.31.1",
     "@vitest/coverage-v8": "^4.0.17",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",