replicant-mcp 1.0.4 → 1.1.0

package/dist/services/grid.js

@@ -0,0 +1,98 @@
+// src/services/grid.ts
+import sharp from "sharp";
+export const GRID_COLS = 4;
+export const GRID_ROWS = 6;
+export const TOTAL_CELLS = GRID_COLS * GRID_ROWS; // 24
+/**
+ * Calculate the pixel bounds for a grid cell (1-24).
+ * Grid is 4 columns x 6 rows, numbered left-to-right, top-to-bottom.
+ */
+export function calculateGridCellBounds(cell, screenWidth, screenHeight) {
+    if (cell < 1 || cell > TOTAL_CELLS) {
+        throw new Error(`Invalid cell number: ${cell}. Must be 1-${TOTAL_CELLS}`);
+    }
+    const cellWidth = screenWidth / GRID_COLS;
+    const cellHeight = screenHeight / GRID_ROWS;
+    // Convert 1-based cell to 0-based row/col
+    const index = cell - 1;
+    const col = index % GRID_COLS;
+    const row = Math.floor(index / GRID_COLS);
+    return {
+        x0: Math.round(col * cellWidth),
+        y0: Math.round(row * cellHeight),
+        x1: Math.round((col + 1) * cellWidth),
+        y1: Math.round((row + 1) * cellHeight),
+    };
+}
+/**
+ * Calculate tap coordinates for a position within a cell.
+ * Position: 1=TL, 2=TR, 3=Center, 4=BL, 5=BR
+ */
+export function calculatePositionCoordinates(position, cellBounds) {
+    const width = cellBounds.x1 - cellBounds.x0;
+    const height = cellBounds.y1 - cellBounds.y0;
+    // Position multipliers (fraction of width/height from x0/y0)
+    const positionMap = {
+        1: { xMult: 0.25, yMult: 0.25 }, // Top-left
+        2: { xMult: 0.75, yMult: 0.25 }, // Top-right
+        3: { xMult: 0.5, yMult: 0.5 }, // Center
+        4: { xMult: 0.25, yMult: 0.75 }, // Bottom-left
+        5: { xMult: 0.75, yMult: 0.75 }, // Bottom-right
+    };
+    const { xMult, yMult } = positionMap[position];
+    return {
+        x: Math.round(cellBounds.x0 + width * xMult),
+        y: Math.round(cellBounds.y0 + height * yMult),
+    };
+}
+export const POSITION_LABELS = [
+    "Top-left",
+    "Top-right",
+    "Center",
+    "Bottom-left",
+    "Bottom-right",
+];
+const GRID_LINE_WIDTH = 3;
+const LABEL_FONT_SIZE = 36;
+/**
+ * Create a 4x6 numbered grid overlay on a screenshot.
+ * Returns base64 PNG.
+ */
+export async function createGridOverlay(imagePath) {
+    const image = sharp(imagePath);
+    const metadata = await image.metadata();
+    const width = metadata.width;
+    const height = metadata.height;
+    const cellWidth = width / GRID_COLS;
+    const cellHeight = height / GRID_ROWS;
+    // Create SVG overlay with grid lines and numbers
+    const svgParts = [];
+    // Grid lines (vertical)
+    for (let col = 1; col < GRID_COLS; col++) {
+        const x = Math.round(col * cellWidth);
+        svgParts.push(`<line x1="${x}" y1="0" x2="${x}" y2="${height}" stroke="rgba(255,165,0,0.8)" stroke-width="${GRID_LINE_WIDTH}"/>`);
+    }
+    // Grid lines (horizontal)
+    for (let row = 1; row < GRID_ROWS; row++) {
+        const y = Math.round(row * cellHeight);
+        svgParts.push(`<line x1="0" y1="${y}" x2="${width}" y2="${y}" stroke="rgba(255,165,0,0.8)" stroke-width="${GRID_LINE_WIDTH}"/>`);
+    }
+    // Cell numbers
+    for (let cell = 1; cell <= TOTAL_CELLS; cell++) {
+        const index = cell - 1;
+        const col = index % GRID_COLS;
+        const row = Math.floor(index / GRID_COLS);
+        const centerX = Math.round((col + 0.5) * cellWidth);
+        const centerY = Math.round((row + 0.5) * cellHeight);
+        // Background circle for visibility
+        svgParts.push(`<circle cx="${centerX}" cy="${centerY}" r="25" fill="rgba(0,0,0,0.6)"/>`);
+        // Number text
+        svgParts.push(`<text x="${centerX}" y="${centerY}" font-family="Arial, sans-serif" font-size="${LABEL_FONT_SIZE}" font-weight="bold" fill="white" text-anchor="middle" dominant-baseline="central">${cell}</text>`);
+    }
+    const svgOverlay = `<svg width="${width}" height="${height}">${svgParts.join("")}</svg>`;
+    const buffer = await image
+        .composite([{ input: Buffer.from(svgOverlay), top: 0, left: 0 }])
+        .png()
+        .toBuffer();
+    return buffer.toString("base64");
+}

package/dist/services/icon-patterns.d.ts

@@ -0,0 +1,10 @@
+export declare const ICON_PATTERNS: Record<string, string[]>;
+/**
+ * Match a user query to icon resourceId patterns.
+ * Returns array of patterns to search for, or null if no match.
+ */
+export declare function matchIconPattern(query: string): string[] | null;
+/**
+ * Check if a resourceId matches any of the given patterns.
+ */
+export declare function matchesResourceId(resourceId: string, patterns: string[]): boolean;

package/dist/services/icon-patterns.js

@@ -0,0 +1,51 @@
+export const ICON_PATTERNS = {
+    // Navigation
+    overflow: ["overflow", "more", "options", "menu", "dots", "kabob", "meatball"],
+    back: ["back", "navigate_up", "arrow_back", "return", "nav_back"],
+    close: ["close", "dismiss", "cancel", "ic_close", "btn_close"],
+    home: ["home", "nav_home", "ic_home"],
+    // Actions
+    search: ["search", "find", "magnify", "ic_search"],
+    settings: ["settings", "gear", "config", "preferences", "ic_settings"],
+    share: ["share", "ic_share", "btn_share"],
+    edit: ["edit", "pencil", "ic_edit", "btn_edit"],
+    delete: ["delete", "trash", "remove", "ic_delete"],
+    add: ["add", "plus", "create", "ic_add", "fab"],
+    // Media
+    play: ["play", "ic_play", "btn_play"],
+    pause: ["pause", "ic_pause"],
+    refresh: ["refresh", "reload", "sync", "ic_refresh"],
+    // Social
+    favorite: ["favorite", "heart", "like", "star", "ic_favorite"],
+    bookmark: ["bookmark", "save", "ic_bookmark"],
+    notification: ["notification", "bell", "ic_notification", "ic_notify"],
+    // Misc
+    filter: ["filter", "ic_filter", "btn_filter"],
+    sort: ["sort", "ic_sort", "btn_sort"],
+    download: ["download", "ic_download"],
+    upload: ["upload", "ic_upload"],
+    profile: ["profile", "account", "avatar", "user", "ic_profile"],
+    hamburger: ["hamburger", "drawer", "nav_drawer", "ic_menu"],
+};
+/**
+ * Match a user query to icon resourceId patterns.
+ * Returns array of patterns to search for, or null if no match.
+ */
+export function matchIconPattern(query) {
+    const lowerQuery = query.toLowerCase();
+    // Check each icon category
+    for (const [category, patterns] of Object.entries(ICON_PATTERNS)) {
+        // Match if query contains the category name or any pattern
+        if (lowerQuery.includes(category) || patterns.some((p) => lowerQuery.includes(p))) {
+            return patterns;
+        }
+    }
+    return null;
+}
+/**
+ * Check if a resourceId matches any of the given patterns.
+ */
+export function matchesResourceId(resourceId, patterns) {
+    const lowerId = resourceId.toLowerCase();
+    return patterns.some((pattern) => lowerId.includes(pattern));
+}

package/dist/services/index.d.ts

@@ -2,4 +2,8 @@ export { ProcessRunner, type RunOptions, type RunResult } from "./process-runner
 export { CacheManager, type CacheStats } from "./cache-manager.js";
 export { DeviceStateManager } from "./device-state.js";
 export { EnvironmentService, type Environment } from "./environment.js";
+export { ConfigManager, loadConfig } from "./config.js";
 export * from "./ocr.js";
+export * from "./icon-patterns.js";
+export * from "./grid.js";
+export * from "./visual-candidates.js";

package/dist/services/index.js

@@ -2,4 +2,8 @@ export { ProcessRunner } from "./process-runner.js";
 export { CacheManager } from "./cache-manager.js";
 export { DeviceStateManager } from "./device-state.js";
 export { EnvironmentService } from "./environment.js";
+export { ConfigManager, loadConfig } from "./config.js";
 export * from "./ocr.js";
+export * from "./icon-patterns.js";
+export * from "./grid.js";
+export * from "./visual-candidates.js";

package/dist/services/visual-candidates.d.ts

@@ -0,0 +1,24 @@
+import { AccessibilityNode, Bounds } from "../parsers/ui-dump.js";
+export declare const MIN_ICON_SIZE = 16;
+export declare const MAX_ICON_SIZE = 200;
+export declare const MIN_ASPECT_RATIO = 0.5;
+export declare const MAX_ASPECT_RATIO = 2;
+export declare const MAX_CANDIDATES = 6;
+/**
+ * Check if dimensions are within icon size constraints.
+ */
+export declare function isIconSized(width: number, height: number): boolean;
+/**
+ * Filter accessibility nodes to find unlabeled clickable elements that are icon-sized.
+ * Returns max 6 candidates sorted by position (top-to-bottom, left-to-right).
+ */
+export declare function filterIconCandidates(nodes: AccessibilityNode[]): AccessibilityNode[];
+/**
+ * Format candidate bounds as string "[x0,y0][x1,y1]"
+ */
+export declare function formatBounds(node: AccessibilityNode): string;
+/**
+ * Crop a region from an image and return as base64 JPEG.
+ * Scales down to max 128x128 if larger.
+ */
+export declare function cropCandidateImage(imagePath: string, bounds: Bounds): Promise<string>;

package/dist/services/visual-candidates.js

@@ -0,0 +1,78 @@
+import sharp from "sharp";
+export const MIN_ICON_SIZE = 16;
+export const MAX_ICON_SIZE = 200;
+export const MIN_ASPECT_RATIO = 0.5;
+export const MAX_ASPECT_RATIO = 2.0;
+export const MAX_CANDIDATES = 6;
+/**
+ * Check if dimensions are within icon size constraints.
+ */
+export function isIconSized(width, height) {
+    if (width < MIN_ICON_SIZE || width > MAX_ICON_SIZE)
+        return false;
+    if (height < MIN_ICON_SIZE || height > MAX_ICON_SIZE)
+        return false;
+    const aspectRatio = width / height;
+    if (aspectRatio < MIN_ASPECT_RATIO || aspectRatio > MAX_ASPECT_RATIO)
+        return false;
+    return true;
+}
+/**
+ * Filter accessibility nodes to find unlabeled clickable elements that are icon-sized.
+ * Returns max 6 candidates sorted by position (top-to-bottom, left-to-right).
+ */
+export function filterIconCandidates(nodes) {
+    const candidates = nodes.filter((node) => {
+        // Must be clickable
+        if (!node.clickable)
+            return false;
+        // Must not have text or contentDesc (we're looking for unlabeled icons)
+        if (node.text || node.contentDesc)
+            return false;
+        // Must be icon-sized
+        const width = node.bounds.right - node.bounds.left;
+        const height = node.bounds.bottom - node.bounds.top;
+        if (!isIconSized(width, height))
+            return false;
+        return true;
+    });
+    // Sort by Y first (top-to-bottom), then X (left-to-right)
+    candidates.sort((a, b) => {
+        if (a.centerY !== b.centerY)
+            return a.centerY - b.centerY;
+        return a.centerX - b.centerX;
+    });
+    // Limit to max candidates
+    return candidates.slice(0, MAX_CANDIDATES);
+}
+/**
+ * Format candidate bounds as string "[x0,y0][x1,y1]"
+ */
+export function formatBounds(node) {
+    return `[${node.bounds.left},${node.bounds.top}][${node.bounds.right},${node.bounds.bottom}]`;
+}
+const MAX_CROP_SIZE = 128;
+const JPEG_QUALITY = 70;
+/**
+ * Crop a region from an image and return as base64 JPEG.
+ * Scales down to max 128x128 if larger.
+ */
+export async function cropCandidateImage(imagePath, bounds) {
+    const width = bounds.right - bounds.left;
+    const height = bounds.bottom - bounds.top;
+    let pipeline = sharp(imagePath).extract({
+        left: bounds.left,
+        top: bounds.top,
+        width,
+        height,
+    });
+    // Scale down if larger than max size
+    if (width > MAX_CROP_SIZE || height > MAX_CROP_SIZE) {
+        pipeline = pipeline.resize(MAX_CROP_SIZE, MAX_CROP_SIZE, {
+            fit: "inside",
+            withoutEnlargement: true,
+        });
+    }
+    const buffer = await pipeline.jpeg({ quality: JPEG_QUALITY }).toBuffer();
+    return buffer.toString("base64");
+}

package/dist/tools/emulator-device.d.ts

@@ -3,8 +3,8 @@ import { ServerContext } from "../server.js";
 export declare const emulatorDeviceInputSchema: z.ZodObject<{
     operation: z.ZodEnum<{
         kill: "kill";
-        list: "list";
         create: "create";
+        list: "list";
         start: "start";
         wipe: "wipe";
         "snapshot-save": "snapshot-save";

package/dist/tools/ui.d.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import { ServerContext } from "../server.js";
+import { UiConfig } from "../types/index.js";
 export declare const uiInputSchema: z.ZodObject<{
     operation: z.ZodEnum<{
         find: "find";
@@ -8,12 +9,14 @@ export declare const uiInputSchema: z.ZodObject<{
         input: "input";
         screenshot: "screenshot";
         "accessibility-check": "accessibility-check";
+        "visual-snapshot": "visual-snapshot";
     }>;
     selector: z.ZodOptional<z.ZodObject<{
         resourceId: z.ZodOptional<z.ZodString>;
         text: z.ZodOptional<z.ZodString>;
         textContains: z.ZodOptional<z.ZodString>;
         className: z.ZodOptional<z.ZodString>;
+        nearestTo: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>;
     x: z.ZodOptional<z.ZodNumber>;
     y: z.ZodOptional<z.ZodNumber>;
@@ -22,9 +25,11 @@ export declare const uiInputSchema: z.ZodObject<{
     localPath: z.ZodOptional<z.ZodString>;
     inline: z.ZodOptional<z.ZodBoolean>;
     debug: z.ZodOptional<z.ZodBoolean>;
+    gridCell: z.ZodOptional<z.ZodNumber>;
+    gridPosition: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 export type UiInput = z.infer<typeof uiInputSchema>;
-export declare function handleUiTool(input: UiInput, context: ServerContext): Promise<Record<string, unknown>>;
+export declare function handleUiTool(input: UiInput, context: ServerContext, uiConfig?: UiConfig): Promise<Record<string, unknown>>;
 export declare const uiToolDefinition: {
     name: string;
     description: string;
@@ -50,6 +55,10 @@ export declare const uiToolDefinition: {
                     className: {
                         type: string;
                     };
+                    nearestTo: {
+                        type: string;
+                        description: string;
+                    };
                 };
                 description: string;
             };
@@ -81,6 +90,18 @@ export declare const uiToolDefinition: {
                 type: string;
                 description: string;
             };
+            gridCell: {
+                type: string;
+                minimum: number;
+                maximum: number;
+                description: string;
+            };
+            gridPosition: {
+                type: string;
+                minimum: number;
+                maximum: number;
+                description: string;
+            };
         };
         required: string[];
     };