replicant-mcp 1.3.2 → 1.4.0

package/README.md

@@ -40,7 +40,8 @@ replicant-mcp wraps all of this into a clean interface that AI can understand an
 | **Device Control** | List connected devices, select active device, query device properties |
 | **App Management** | Install, uninstall, launch, stop apps; clear app data; list installed packages |
 | **Log Analysis** | Filter logcat by package, tag, level, time; configurable line limits |
-| **UI Automation** | Accessibility-first element finding with multi-tier fallback (accessibility → OCR → visual), spatial proximity search (`nearestTo`), grid-based precision tapping, tap, text input, screenshots |
+| **UI Automation** | Accessibility-first element finding with multi-tier fallback (accessibility → OCR → visual), spatial proximity search (`nearestTo`), grid-based precision tapping, tap (with deviceSpace option), text input, screenshots |
+| **Screenshot Scaling** | Auto-resize to 1000px max, JPEG compression (~94% size reduction), transparent coordinate conversion (image ↔ device space) |
 | **Configuration** | YAML config via `REPLICANT_CONFIG` for UI behavior customization |
 | **Utilities** | Response caching with progressive disclosure, on-demand documentation |
 
@@ -63,8 +64,8 @@ replicant-mcp wraps all of this into a clean interface that AI can understand an
 | **Developer Experience** | Simplified tool authoring with `defineTool()` helper | Future |
 | | Auto-generate JSON schema from Zod via `zod-to-json-schema` | Future |
 | | Convention-based tool auto-discovery (no manual wiring) | Future |
-| **Screenshot Scaling** | Auto-resize screenshots to prevent API context limits | Planned |
-| | Transparent coordinate conversion (image ↔ device space) | Planned |
+| **Screenshot Scaling** | Auto-resize screenshots to prevent API context limits | ✅ |
+| | Transparent coordinate conversion (image ↔ device space) | ✅ |
 | | Raw mode for external context management | Planned |
 
 ---

package/dist/adapters/ui-automator.d.ts

@@ -21,6 +21,7 @@ export interface ScreenshotResult {
     mode: "file" | "inline";
     path?: string;
     base64?: string;
+    mimeType?: string;
     sizeBytes?: number;
     device?: {
         width: number;
@@ -63,7 +64,7 @@ export declare class UiAutomatorAdapter {
         textContains?: string;
         className?: string;
     }): Promise<AccessibilityNode[]>;
-    tap(deviceId: string, x: number, y: number): Promise<void>;
+    tap(deviceId: string, x: number, y: number, deviceSpace?: boolean): Promise<void>;
     tapElement(deviceId: string, element: AccessibilityNode): Promise<void>;
     input(deviceId: string, text: string): Promise<void>;
     scroll(deviceId: string, direction: "up" | "down" | "left" | "right", amount?: number): Promise<void>;

package/dist/adapters/ui-automator.js

@@ -1,5 +1,6 @@
 import * as path from "path";
 import * as fs from "fs";
+import * as os from "os";
 import sharp from "sharp";
 import { AdbAdapter } from "./adb.js";
 import { parseUiDump, findElements, flattenTree } from "../parsers/ui-dump.js";
@@ -63,11 +64,12 @@ export class UiAutomatorAdapter {
         const tree = await this.dump(deviceId);
         return findElements(tree, selector);
     }
-    async tap(deviceId, x, y) {
+    async tap(deviceId, x, y, deviceSpace) {
         // Convert from image space to device space if scaling is active
+        // Skip conversion if deviceSpace=true (coordinates are already in device space)
         let tapX = x;
         let tapY = y;
-        if (this.scalingState && this.scalingState.scaleFactor !== 1.0) {
+        if (!deviceSpace && this.scalingState && this.scalingState.scaleFactor !== 1.0) {
             const converted = toDeviceSpace(x, y, this.scalingState.scaleFactor);
             tapX = converted.x;
             tapY = converted.y;
@@ -133,16 +135,47 @@ export class UiAutomatorAdapter {
         }
         try {
             if (options.inline) {
-                // Inline mode: return base64 (no scaling support for inline mode)
-                // Clear scaling state since inline mode doesn't support coordinate conversion
-                this.scalingState = null;
-                const base64Result = await this.adb.shell(deviceId, `base64 ${remotePath}`);
-                const sizeResult = await this.adb.shell(deviceId, `stat -c%s ${remotePath}`);
-                return {
-                    mode: "inline",
-                    base64: base64Result.stdout.trim(),
-                    sizeBytes: parseInt(sizeResult.stdout.trim(), 10),
-                };
+                // Inline mode: pull to temp, scale, convert to JPEG, return base64
+                const tempPath = path.join(os.tmpdir(), `replicant-inline-${Date.now()}.png`);
+                try {
+                    // Pull to temp file
+                    await this.adb.pull(deviceId, remotePath, tempPath);
+                    // Get dimensions
+                    const metadata = await sharp(tempPath).metadata();
+                    const deviceWidth = metadata.width;
+                    const deviceHeight = metadata.height;
+                    // Calculate scale factor
+                    const scaleFactor = calculateScaleFactor(deviceWidth, deviceHeight, maxDimension);
+                    const imageWidth = Math.round(deviceWidth / scaleFactor);
+                    const imageHeight = Math.round(deviceHeight / scaleFactor);
+                    // Scale and convert to JPEG
+                    const buffer = await sharp(tempPath)
+                        .resize(imageWidth, imageHeight)
+                        .jpeg({ quality: 70 })
+                        .toBuffer();
+                    // Update scaling state (now supported for inline!)
+                    this.scalingState = {
+                        scaleFactor,
+                        deviceWidth,
+                        deviceHeight,
+                        imageWidth,
+                        imageHeight,
+                    };
+                    return {
+                        mode: "inline",
+                        base64: buffer.toString("base64"),
+                        mimeType: "image/jpeg",
+                        sizeBytes: buffer.length,
+                        device: { width: deviceWidth, height: deviceHeight },
+                        image: { width: imageWidth, height: imageHeight },
+                        scaleFactor,
+                    };
+                }
+                finally {
+                    // Clean up temp file
+                    const fsPromises = await import("fs/promises");
+                    await fsPromises.unlink(tempPath).catch(() => { });
+                }
             }
             else {
                 // File mode: pull to local, then optionally scale

package/dist/tools/ui.d.ts

@@ -28,6 +28,7 @@ export declare const uiInputSchema: z.ZodObject<{
     debug: z.ZodOptional<z.ZodBoolean>;
     gridCell: z.ZodOptional<z.ZodNumber>;
     gridPosition: z.ZodOptional<z.ZodNumber>;
+    deviceSpace: z.ZodOptional<z.ZodBoolean>;
     maxDimension: z.ZodOptional<z.ZodNumber>;
     raw: z.ZodOptional<z.ZodBoolean>;
     compact: z.ZodOptional<z.ZodBoolean>;
@@ -113,6 +114,10 @@ export declare const uiToolDefinition: {
                 maximum: number;
                 description: string;
             };
+            deviceSpace: {
+                type: string;
+                description: string;
+            };
             maxDimension: {
                 type: string;
                 description: string;

package/dist/tools/ui.js

@@ -19,6 +19,7 @@ export const uiInputSchema = z.object({
     debug: z.boolean().optional(),
     gridCell: z.number().min(1).max(24).optional(),
     gridPosition: z.number().min(1).max(5).optional(),
+    deviceSpace: z.boolean().optional(),
     maxDimension: z.number().optional(),
     raw: z.boolean().optional(),
     compact: z.boolean().optional(),
@@ -324,8 +325,8 @@ export async function handleUiTool(input, context, uiConfig) {
             else {
                 throw new Error("Either x/y coordinates or elementIndex is required for tap");
             }
-            await context.ui.tap(deviceId, x, y);
-            return { tapped: { x, y }, deviceId };
+            await context.ui.tap(deviceId, x, y, input.deviceSpace);
+            return { tapped: { x, y, deviceSpace: input.deviceSpace ?? false }, deviceId };
         }
         case "input": {
             if (!input.text) {
@@ -395,6 +396,10 @@ export const uiToolDefinition = {
             debug: { type: "boolean", description: "Include source (accessibility/ocr) and confidence in response" },
             gridCell: { type: "number", minimum: 1, maximum: 24, description: "Grid cell number (1-24) for Tier 5 refinement" },
             gridPosition: { type: "number", minimum: 1, maximum: 5, description: "Position within cell (1=TL, 2=TR, 3=Center, 4=BL, 5=BR)" },
+            deviceSpace: {
+                type: "boolean",
+                description: "For tap: treat x/y as device coordinates (skip image→device scaling). Use when coordinates come from adb shell input tap testing.",
+            },
             maxDimension: {
                 type: "number",
                 description: "Max image dimension in pixels (default: 1000). Higher = better quality, more tokens.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "replicant-mcp",
-  "version": "1.3.2",
+  "version": "1.4.0",
   "description": "Android MCP server for AI-assisted Android development",
   "type": "module",
   "main": "dist/index.js",