replicant-mcp 1.1.0 → 1.2.0

package/README.md

@@ -63,6 +63,9 @@ replicant-mcp wraps all of this into a clean interface that AI can understand an
 | **Developer Experience** | Simplified tool authoring with `defineTool()` helper | Future |
 | | Auto-generate JSON schema from Zod via `zod-to-json-schema` | Future |
 | | Convention-based tool auto-discovery (no manual wiring) | Future |
+| **Screenshot Scaling** | Auto-resize screenshots to prevent API context limits | Planned |
+| | Transparent coordinate conversion (image ↔ device space) | Planned |
+| | Raw mode for external context management | Planned |
 
 ---
 

package/dist/adapters/ui-automator.d.ts

@@ -14,18 +14,48 @@ export interface CurrentApp {
 export interface ScreenshotOptions {
     localPath?: string;
     inline?: boolean;
+    maxDimension?: number;
+    raw?: boolean;
 }
 export interface ScreenshotResult {
     mode: "file" | "inline";
     path?: string;
     base64?: string;
     sizeBytes?: number;
+    device?: {
+        width: number;
+        height: number;
+    };
+    image?: {
+        width: number;
+        height: number;
+    };
+    scaleFactor?: number;
+    warning?: string;
 }
 export type FindWithOcrResult = FindWithFallbacksResult;
 export type FindOptions = IconFindOptions;
+/**
+ * Tracks the current scaling state between device and image coordinates.
+ * Updated on every screenshot operation.
+ */
+export interface ScalingState {
+    scaleFactor: number;
+    deviceWidth: number;
+    deviceHeight: number;
+    imageWidth: number;
+    imageHeight: number;
+}
 export declare class UiAutomatorAdapter {
     private adb;
+    private scalingState;
     constructor(adb?: AdbAdapter);
+    getScalingState(): ScalingState | null;
+    /**
+     * Transforms accessibility tree nodes from device space to image space.
+     * This ensures bounds/coordinates match the scaled screenshot when scaling is active.
+     */
+    private transformTreeToImageSpace;
     dump(deviceId: string): Promise<AccessibilityNode[]>;
     find(deviceId: string, selector: {
         resourceId?: string;

package/dist/adapters/ui-automator.js

@@ -1,5 +1,6 @@
 import * as path from "path";
 import * as fs from "fs";
+import sharp from "sharp";
 import { AdbAdapter } from "./adb.js";
 import { parseUiDump, findElements, flattenTree } from "../parsers/ui-dump.js";
 import { ReplicantError, ErrorCode } from "../types/index.js";
@@ -7,6 +8,7 @@ import { extractText, searchText } from "../services/ocr.js";
 import { matchIconPattern, matchesResourceId } from "../services/icon-patterns.js";
 import { filterIconCandidates, formatBounds, cropCandidateImage } from "../services/visual-candidates.js";
 import { calculateGridCellBounds, calculatePositionCoordinates, createGridOverlay, POSITION_LABELS, } from "../services/grid.js";
+import { calculateScaleFactor, toImageSpace, toDeviceSpace, boundsToImageSpace } from "../services/scaling.js";
 /**
  * Get default screenshot path in project-relative .replicant/screenshots directory.
  * Creates the directory if it doesn't exist.
@@ -18,9 +20,35 @@ function getDefaultScreenshotPath() {
 }
 export class UiAutomatorAdapter {
     adb;
+    scalingState = null;
     constructor(adb = new AdbAdapter()) {
         this.adb = adb;
     }
+    // Getter for tests
+    getScalingState() {
+        return this.scalingState;
+    }
+    /**
+     * Transforms accessibility tree nodes from device space to image space.
+     * This ensures bounds/coordinates match the scaled screenshot when scaling is active.
+     */
+    transformTreeToImageSpace(nodes) {
+        if (!this.scalingState || this.scalingState.scaleFactor === 1.0) {
+            return nodes;
+        }
+        const sf = this.scalingState.scaleFactor;
+        return nodes.map((node) => {
+            const newBounds = boundsToImageSpace(node.bounds, sf);
+            const center = toImageSpace(node.centerX, node.centerY, sf);
+            return {
+                ...node,
+                bounds: newBounds,
+                centerX: center.x,
+                centerY: center.y,
+                children: node.children ? this.transformTreeToImageSpace(node.children) : [],
+            };
+        });
+    }
     async dump(deviceId) {
         // Dump UI hierarchy to device
         await this.adb.shell(deviceId, "uiautomator dump /sdcard/ui-dump.xml");
@@ -28,14 +56,23 @@ export class UiAutomatorAdapter {
         const result = await this.adb.shell(deviceId, "cat /sdcard/ui-dump.xml");
         // Clean up
         await this.adb.shell(deviceId, "rm /sdcard/ui-dump.xml");
-        return parseUiDump(result.stdout);
+        const tree = parseUiDump(result.stdout);
+        return this.transformTreeToImageSpace(tree);
     }
     async find(deviceId, selector) {
         const tree = await this.dump(deviceId);
         return findElements(tree, selector);
     }
     async tap(deviceId, x, y) {
-        await this.adb.shell(deviceId, `input tap ${x} ${y}`);
+        // Convert from image space to device space if scaling is active
+        let tapX = x;
+        let tapY = y;
+        if (this.scalingState && this.scalingState.scaleFactor !== 1.0) {
+            const converted = toDeviceSpace(x, y, this.scalingState.scaleFactor);
+            tapX = converted.x;
+            tapY = converted.y;
+        }
+        await this.adb.shell(deviceId, `input tap ${tapX} ${tapY}`);
     }
     async tapElement(deviceId, element) {
         await this.tap(deviceId, element.centerX, element.centerY);
@@ -47,6 +84,7 @@ export class UiAutomatorAdapter {
     }
     async screenshot(deviceId, options = {}) {
         const remotePath = "/sdcard/replicant-screenshot.png";
+        const maxDimension = options.maxDimension ?? 1000;
         // Capture screenshot on device
         const captureResult = await this.adb.shell(deviceId, `screencap -p ${remotePath}`);
         if (captureResult.exitCode !== 0) {
@@ -54,7 +92,9 @@ export class UiAutomatorAdapter {
         }
         try {
             if (options.inline) {
-                // Inline mode: return base64
+                // Inline mode: return base64 (no scaling support for inline mode)
+                // Clear scaling state since inline mode doesn't support coordinate conversion
+                this.scalingState = null;
                 const base64Result = await this.adb.shell(deviceId, `base64 ${remotePath}`);
                 const sizeResult = await this.adb.shell(deviceId, `stat -c%s ${remotePath}`);
                 return {
@@ -64,10 +104,74 @@ export class UiAutomatorAdapter {
                 };
             }
             else {
-                // File mode (default): pull to local
+                // File mode: pull to local, then optionally scale
                 const localPath = options.localPath || getDefaultScreenshotPath();
                 await this.adb.pull(deviceId, remotePath, localPath);
-                return { mode: "file", path: localPath };
+                // Get image dimensions
+                const metadata = await sharp(localPath).metadata();
+                const deviceWidth = metadata.width;
+                const deviceHeight = metadata.height;
+                // Handle raw mode
+                if (options.raw) {
+                    this.scalingState = {
+                        scaleFactor: 1.0,
+                        deviceWidth,
+                        deviceHeight,
+                        imageWidth: deviceWidth,
+                        imageHeight: deviceHeight,
+                    };
+                    return {
+                        mode: "file",
+                        path: localPath,
+                        device: { width: deviceWidth, height: deviceHeight },
+                        image: { width: deviceWidth, height: deviceHeight },
+                        scaleFactor: 1.0,
+                        warning: "Raw mode: no scaling applied. May exceed API limits with multiple images.",
+                    };
+                }
+                // Calculate scale factor
+                const scaleFactor = calculateScaleFactor(deviceWidth, deviceHeight, maxDimension);
+                if (scaleFactor === 1.0) {
+                    // No scaling needed
+                    this.scalingState = {
+                        scaleFactor: 1.0,
+                        deviceWidth,
+                        deviceHeight,
+                        imageWidth: deviceWidth,
+                        imageHeight: deviceHeight,
+                    };
+                    return {
+                        mode: "file",
+                        path: localPath,
+                        device: { width: deviceWidth, height: deviceHeight },
+                        image: { width: deviceWidth, height: deviceHeight },
+                        scaleFactor: 1.0,
+                    };
+                }
+                // Scale the image
+                const imageWidth = Math.round(deviceWidth / scaleFactor);
+                const imageHeight = Math.round(deviceHeight / scaleFactor);
+                await sharp(localPath)
+                    .resize(imageWidth, imageHeight)
+                    .toFile(localPath + ".tmp");
+                // Replace original with scaled version
+                const fsPromises = await import("fs/promises");
+                await fsPromises.rename(localPath + ".tmp", localPath);
+                // Update scaling state
+                this.scalingState = {
+                    scaleFactor,
+                    deviceWidth,
+                    deviceHeight,
+                    imageWidth,
+                    imageHeight,
+                };
+                return {
+                    mode: "file",
+                    path: localPath,
+                    device: { width: deviceWidth, height: deviceHeight },
+                    image: { width: imageWidth, height: imageHeight },
+                    scaleFactor,
+                };
             }
         }
         finally {

package/dist/server.js

@@ -40,7 +40,7 @@ Tool mapping:
 - Emulator control → emulator-device (not \`emulator\` CLI)
 - Builds → gradle-build (not \`./gradlew\`)
 - Tests → gradle-test (not \`./gradlew test\`)
-- UI automation → ui (not \`adb shell uiautomator\` or \`screencap\`)
+- UI automation → ui (accessibility-first, screenshots auto-scaled to 1000px)
 
 Start with \`adb-device list\` to see connected devices.
 Use \`rtfm\` for detailed documentation on any tool.`,

package/dist/services/config.js

@@ -40,6 +40,7 @@ function mergeUiConfig(defaults, overrides) {
         visualModePackages: overrides.visualModePackages ?? defaults.visualModePackages,
         autoFallbackScreenshot: overrides.autoFallbackScreenshot ?? defaults.autoFallbackScreenshot,
         includeBase64: overrides.includeBase64 ?? defaults.includeBase64,
+        maxImageDimension: overrides.maxImageDimension ?? defaults.maxImageDimension,
     };
 }
 /**

package/dist/services/index.d.ts

@@ -7,3 +7,4 @@ export * from "./ocr.js";
 export * from "./icon-patterns.js";
 export * from "./grid.js";
 export * from "./visual-candidates.js";
+export * from "./scaling.js";

package/dist/services/index.js

@@ -7,3 +7,4 @@ export * from "./ocr.js";
 export * from "./icon-patterns.js";
 export * from "./grid.js";
 export * from "./visual-candidates.js";
+export * from "./scaling.js";

package/dist/services/scaling.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Calculate the scale factor needed to fit device dimensions within max dimension.
+ * Returns 1.0 if no scaling needed.
+ */
+export declare function calculateScaleFactor(deviceWidth: number, deviceHeight: number, maxDimension: number): number;
+export interface Bounds {
+    left: number;
+    top: number;
+    right: number;
+    bottom: number;
+}
+/**
+ * Convert device coordinates to image coordinates.
+ */
+export declare function toImageSpace(deviceX: number, deviceY: number, scaleFactor: number): {
+    x: number;
+    y: number;
+};
+/**
+ * Convert image coordinates to device coordinates.
+ */
+export declare function toDeviceSpace(imageX: number, imageY: number, scaleFactor: number): {
+    x: number;
+    y: number;
+};
+/**
+ * Convert bounds from device space to image space.
+ */
+export declare function boundsToImageSpace(bounds: Bounds, scaleFactor: number): Bounds;

package/dist/services/scaling.js

@@ -0,0 +1,49 @@
+/**
+ * Calculate the scale factor needed to fit device dimensions within max dimension.
+ * Returns 1.0 if no scaling needed.
+ */
+export function calculateScaleFactor(deviceWidth, deviceHeight, maxDimension) {
+    const longestSide = Math.max(deviceWidth, deviceHeight);
+    if (longestSide <= maxDimension) {
+        return 1.0;
+    }
+    return longestSide / maxDimension;
+}
+/**
+ * Convert device coordinates to image coordinates.
+ */
+export function toImageSpace(deviceX, deviceY, scaleFactor) {
+    if (scaleFactor === 1.0) {
+        return { x: deviceX, y: deviceY };
+    }
+    return {
+        x: Math.round(deviceX / scaleFactor),
+        y: Math.round(deviceY / scaleFactor),
+    };
+}
+/**
+ * Convert image coordinates to device coordinates.
+ */
+export function toDeviceSpace(imageX, imageY, scaleFactor) {
+    if (scaleFactor === 1.0) {
+        return { x: imageX, y: imageY };
+    }
+    return {
+        x: Math.round(imageX * scaleFactor),
+        y: Math.round(imageY * scaleFactor),
+    };
+}
+/**
+ * Convert bounds from device space to image space.
+ */
+export function boundsToImageSpace(bounds, scaleFactor) {
+    if (scaleFactor === 1.0) {
+        return bounds;
+    }
+    return {
+        left: Math.round(bounds.left / scaleFactor),
+        top: Math.round(bounds.top / scaleFactor),
+        right: Math.round(bounds.right / scaleFactor),
+        bottom: Math.round(bounds.bottom / scaleFactor),
+    };
+}

package/dist/tools/ui.d.ts

@@ -27,6 +27,8 @@ export declare const uiInputSchema: z.ZodObject<{
     debug: z.ZodOptional<z.ZodBoolean>;
     gridCell: z.ZodOptional<z.ZodNumber>;
     gridPosition: z.ZodOptional<z.ZodNumber>;
+    maxDimension: z.ZodOptional<z.ZodNumber>;
+    raw: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
 export type UiInput = z.infer<typeof uiInputSchema>;
 export declare function handleUiTool(input: UiInput, context: ServerContext, uiConfig?: UiConfig): Promise<Record<string, unknown>>;
@@ -102,6 +104,14 @@ export declare const uiToolDefinition: {
                 maximum: number;
                 description: string;
             };
+            maxDimension: {
+                type: string;
+                description: string;
+            };
+            raw: {
+                type: string;
+                description: string;
+            };
         };
         required: string[];
     };

package/dist/tools/ui.js

@@ -19,6 +19,8 @@ export const uiInputSchema = z.object({
     debug: z.boolean().optional(),
     gridCell: z.number().min(1).max(24).optional(),
     gridPosition: z.number().min(1).max(5).optional(),
+    maxDimension: z.number().optional(),
+    raw: z.boolean().optional(),
 });
 // Store last find results for elementIndex reference
 // Updated to support accessibility, OCR, and grid elements
@@ -93,6 +95,7 @@ export async function handleUiTool(input, context, uiConfig) {
         visualModePackages: [],
         autoFallbackScreenshot: true,
         includeBase64: false,
+        maxImageDimension: 1000,
     };
     switch (input.operation) {
         case "dump": {
@@ -319,6 +322,8 @@ export async function handleUiTool(input, context, uiConfig) {
             const result = await context.ui.screenshot(deviceId, {
                 localPath: input.localPath,
                 inline: input.inline,
+                maxDimension: input.maxDimension ?? config.maxImageDimension,
+                raw: input.raw,
             });
             return { ...result, deviceId };
         }
@@ -366,6 +371,14 @@ export const uiToolDefinition = {
             debug: { type: "boolean", description: "Include source (accessibility/ocr) and confidence in response" },
             gridCell: { type: "number", minimum: 1, maximum: 24, description: "Grid cell number (1-24) for Tier 5 refinement" },
             gridPosition: { type: "number", minimum: 1, maximum: 5, description: "Position within cell (1=TL, 2=TR, 3=Center, 4=BL, 5=BR)" },
+            maxDimension: {
+                type: "number",
+                description: "Max image dimension in pixels (default: 1000). Higher = better quality, more tokens.",
+            },
+            raw: {
+                type: "boolean",
+                description: "Skip scaling, return full device resolution. Warning: may exceed API limits.",
+            },
         },
         required: ["operation"],
     },

package/dist/types/config.d.ts

@@ -9,6 +9,8 @@ export interface UiConfig {
     autoFallbackScreenshot: boolean;
     /** Include base64-encoded screenshot in response (default: false) */
     includeBase64: boolean;
+    /** Maximum dimension (width or height) for screenshots in pixels (default: 1000) */
+    maxImageDimension: number;
 }
 export interface ReplicantConfig {
     ui: UiConfig;

package/dist/types/config.js

@@ -7,5 +7,6 @@ export const DEFAULT_CONFIG = {
         visualModePackages: [],
         autoFallbackScreenshot: true,
         includeBase64: false,
+        maxImageDimension: 1000,
     },
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicant-mcp",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Android MCP server for AI-assisted Android development",
   "type": "module",
   "main": "dist/index.js",
@@ -18,7 +18,7 @@
     "check-prereqs": "bash scripts/check-prerequisites.sh",
     "smoke-test": "bash scripts/smoke-test.sh",
     "test:device": "tsx scripts/real-device-test.ts",
-    "start": "node dist/index.js",
+    "start": "npm run build && node dist/index.js",
     "validate": "npm run build && npm run test -- --run",
     "prepublishOnly": "npm run build && npm test -- --run"
   },