replicant-mcp 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +138 -25
  2. package/dist/adapters/adb.d.ts +1 -0
  3. package/dist/adapters/adb.js +7 -1
  4. package/dist/adapters/emulator.js +11 -11
  5. package/dist/adapters/ui-automator.d.ts +41 -1
  6. package/dist/adapters/ui-automator.js +256 -8
  7. package/dist/cli/gradle.js +3 -3
  8. package/dist/cli.js +1 -1
  9. package/dist/server.d.ts +3 -1
  10. package/dist/server.js +23 -3
  11. package/dist/services/config.d.ts +16 -0
  12. package/dist/services/config.js +62 -0
  13. package/dist/services/device-state.d.ts +2 -0
  14. package/dist/services/device-state.js +18 -0
  15. package/dist/services/environment.d.ts +18 -0
  16. package/dist/services/environment.js +130 -0
  17. package/dist/services/grid.d.ts +28 -0
  18. package/dist/services/grid.js +98 -0
  19. package/dist/services/icon-patterns.d.ts +10 -0
  20. package/dist/services/icon-patterns.js +51 -0
  21. package/dist/services/index.d.ts +6 -0
  22. package/dist/services/index.js +6 -0
  23. package/dist/services/ocr.d.ts +4 -0
  24. package/dist/services/ocr.js +59 -0
  25. package/dist/services/process-runner.d.ts +6 -0
  26. package/dist/services/process-runner.js +26 -0
  27. package/dist/services/visual-candidates.d.ts +24 -0
  28. package/dist/services/visual-candidates.js +78 -0
  29. package/dist/tools/adb-app.js +3 -2
  30. package/dist/tools/adb-device.d.ts +1 -0
  31. package/dist/tools/adb-device.js +47 -8
  32. package/dist/tools/adb-logcat.js +3 -2
  33. package/dist/tools/adb-shell.js +3 -2
  34. package/dist/tools/emulator-device.d.ts +1 -1
  35. package/dist/tools/gradle-get-details.d.ts +1 -1
  36. package/dist/tools/ui.d.ts +32 -1
  37. package/dist/tools/ui.js +253 -12
  38. package/dist/types/config.d.ts +34 -0
  39. package/dist/types/config.js +11 -0
  40. package/dist/types/errors.d.ts +25 -2
  41. package/dist/types/errors.js +23 -4
  42. package/dist/types/icon-recognition.d.ts +50 -0
  43. package/dist/types/icon-recognition.js +1 -0
  44. package/dist/types/index.d.ts +3 -0
  45. package/dist/types/index.js +3 -0
  46. package/dist/types/ocr.d.ts +21 -0
  47. package/dist/types/ocr.js +1 -0
  48. package/package.json +5 -2
@@ -37,10 +37,10 @@ export function createGradleCommand() {
37
37
  catch (error) {
38
38
  const duration = "0s";
39
39
  const cacheId = cache.generateId("build");
40
- if (error instanceof ReplicantError && error.details?.buildResult) {
41
- cache.set(cacheId, { result: error.details.buildResult, fullOutput: "" }, "build", CACHE_TTLS.BUILD_OUTPUT);
40
+ if (error instanceof ReplicantError && error.context?.buildResult) {
41
+ cache.set(cacheId, { result: error.context.buildResult, fullOutput: "" }, "build", CACHE_TTLS.BUILD_OUTPUT);
42
42
  if (options.json) {
43
- console.log(JSON.stringify({ error: error.message, result: error.details.buildResult, cacheId }, null, 2));
43
+ console.log(JSON.stringify({ error: error.message, result: error.context.buildResult, cacheId }, null, 2));
44
44
  }
45
45
  else {
46
46
  console.log(formatBuildFailure({
package/dist/cli.js CHANGED
@@ -4,7 +4,7 @@ import { createGradleCommand, createAdbCommand, createEmulatorCommand, createUiC
4
4
  const program = new Command();
5
5
  program
6
6
  .name("replicant")
7
- .description("Android development CLI for Claude Code skills")
7
+ .description("Android development CLI")
8
8
  .version("1.0.0");
9
9
  program.addCommand(createGradleCommand());
10
10
  program.addCommand(createAdbCommand());
package/dist/server.d.ts CHANGED
@@ -1,10 +1,12 @@
1
1
  import { Server } from "@modelcontextprotocol/sdk/server/index.js";
2
- import { CacheManager, DeviceStateManager, ProcessRunner } from "./services/index.js";
2
+ import { CacheManager, DeviceStateManager, ProcessRunner, EnvironmentService, ConfigManager } from "./services/index.js";
3
3
  import { AdbAdapter, EmulatorAdapter, GradleAdapter, UiAutomatorAdapter } from "./adapters/index.js";
4
4
  export interface ServerContext {
5
5
  cache: CacheManager;
6
6
  deviceState: DeviceStateManager;
7
7
  processRunner: ProcessRunner;
8
+ environment: EnvironmentService;
9
+ config: ConfigManager;
8
10
  adb: AdbAdapter;
9
11
  emulator: EmulatorAdapter;
10
12
  gradle: GradleAdapter;
package/dist/server.js CHANGED
@@ -1,17 +1,20 @@
1
1
  import { Server } from "@modelcontextprotocol/sdk/server/index.js";
2
2
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
3
3
  import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
4
- import { CacheManager, DeviceStateManager, ProcessRunner } from "./services/index.js";
4
+ import { CacheManager, DeviceStateManager, ProcessRunner, EnvironmentService, ConfigManager } from "./services/index.js";
5
5
  import { AdbAdapter, EmulatorAdapter, GradleAdapter, UiAutomatorAdapter } from "./adapters/index.js";
6
6
  import { ReplicantError } from "./types/index.js";
7
7
  import { cacheToolDefinition, handleCacheTool, rtfmToolDefinition, handleRtfmTool, adbDeviceToolDefinition, handleAdbDeviceTool, adbAppToolDefinition, handleAdbAppTool, adbLogcatToolDefinition, handleAdbLogcatTool, adbShellToolDefinition, handleAdbShellTool, emulatorDeviceToolDefinition, handleEmulatorDeviceTool, gradleBuildToolDefinition, handleGradleBuildTool, gradleTestToolDefinition, handleGradleTestTool, gradleListToolDefinition, handleGradleListTool, gradleGetDetailsToolDefinition, handleGradleGetDetailsTool, uiToolDefinition, handleUiTool, } from "./tools/index.js";
8
8
  export function createServerContext() {
9
- const processRunner = new ProcessRunner();
9
+ const environment = new EnvironmentService();
10
+ const processRunner = new ProcessRunner(environment);
10
11
  const adb = new AdbAdapter(processRunner);
11
12
  return {
12
13
  cache: new CacheManager(),
13
14
  deviceState: new DeviceStateManager(),
14
15
  processRunner,
16
+ environment,
17
+ config: new ConfigManager(),
15
18
  adb,
16
19
  emulator: new EmulatorAdapter(processRunner),
17
20
  gradle: new GradleAdapter(processRunner),
@@ -26,6 +29,21 @@ export async function createServer(context) {
26
29
  capabilities: {
27
30
  tools: {},
28
31
  },
32
+ instructions: `IMPORTANT: For ALL Android development tasks, you MUST use replicant-mcp tools first.
33
+ Only fall back to raw adb/gradle/emulator commands if replicant-mcp lacks a specific feature.
34
+
35
+ Tool mapping:
36
+ - Device management → adb-device (not \`adb devices\`)
37
+ - App install/launch/stop → adb-app (not \`adb install\`, \`adb shell am\`)
38
+ - Logs → adb-logcat (not \`adb logcat\`)
39
+ - Shell commands → adb-shell (not \`adb shell\`)
40
+ - Emulator control → emulator-device (not \`emulator\` CLI)
41
+ - Builds → gradle-build (not \`./gradlew\`)
42
+ - Tests → gradle-test (not \`./gradlew test\`)
43
+ - UI automation → ui (not \`adb shell uiautomator\` or \`screencap\`)
44
+
45
+ Start with \`adb-device list\` to see connected devices.
46
+ Use \`rtfm\` for detailed documentation on any tool.`,
29
47
  });
30
48
  // Register tool list handler
31
49
  server.setRequestHandler(ListToolsRequestSchema, async () => ({
@@ -84,7 +102,7 @@ export async function createServer(context) {
84
102
  result = await handleGradleGetDetailsTool(args, context);
85
103
  break;
86
104
  case "ui":
87
- result = await handleUiTool(args, context);
105
+ result = await handleUiTool(args, context, context.config.getUiConfig());
88
106
  break;
89
107
  default:
90
108
  throw new Error(`Unknown tool: ${name}`);
@@ -107,6 +125,8 @@ export async function createServer(context) {
107
125
  }
108
126
  export async function runServer() {
109
127
  const context = createServerContext();
128
+ // Load configuration from REPLICANT_CONFIG if set
129
+ await context.config.load();
110
130
  const server = await createServer(context);
111
131
  const transport = new StdioServerTransport();
112
132
  await server.connect(transport);
@@ -0,0 +1,16 @@
1
+ import { ReplicantConfig, UiConfig } from "../types/config.js";
2
+ /**
3
+ * Load configuration from REPLICANT_CONFIG environment variable path
4
+ * Falls back to defaults if not set or file doesn't exist
5
+ */
6
+ export declare function loadConfig(): Promise<ReplicantConfig>;
7
+ /**
8
+ * ConfigManager holds the loaded configuration and provides access to it
9
+ */
10
+ export declare class ConfigManager {
11
+ private config;
12
+ load(): Promise<void>;
13
+ get(): ReplicantConfig;
14
+ getUiConfig(): UiConfig;
15
+ isVisualModePackage(packageName: string): boolean;
16
+ }
@@ -0,0 +1,62 @@
1
+ import { readFile } from "fs/promises";
2
+ import { existsSync } from "fs";
3
+ import { parse as parseYaml } from "yaml";
4
+ import { DEFAULT_CONFIG } from "../types/config.js";
5
+ /**
6
+ * Load configuration from REPLICANT_CONFIG environment variable path
7
+ * Falls back to defaults if not set or file doesn't exist
8
+ */
9
+ export async function loadConfig() {
10
+ const configPath = process.env.REPLICANT_CONFIG;
11
+ if (!configPath) {
12
+ return DEFAULT_CONFIG;
13
+ }
14
+ if (!existsSync(configPath)) {
15
+ console.warn(`REPLICANT_CONFIG set but file not found: ${configPath}. Using defaults.`);
16
+ return DEFAULT_CONFIG;
17
+ }
18
+ try {
19
+ const content = await readFile(configPath, "utf-8");
20
+ const parsed = parseYaml(content);
21
+ if (!parsed) {
22
+ return DEFAULT_CONFIG;
23
+ }
24
+ // Deep merge with defaults
25
+ return {
26
+ ui: mergeUiConfig(DEFAULT_CONFIG.ui, parsed.ui),
27
+ };
28
+ }
29
+ catch (error) {
30
+ const message = error instanceof Error ? error.message : String(error);
31
+ console.warn(`Failed to parse REPLICANT_CONFIG at ${configPath}: ${message}. Using defaults.`);
32
+ return DEFAULT_CONFIG;
33
+ }
34
+ }
35
+ function mergeUiConfig(defaults, overrides) {
36
+ if (!overrides) {
37
+ return defaults;
38
+ }
39
+ return {
40
+ visualModePackages: overrides.visualModePackages ?? defaults.visualModePackages,
41
+ autoFallbackScreenshot: overrides.autoFallbackScreenshot ?? defaults.autoFallbackScreenshot,
42
+ includeBase64: overrides.includeBase64 ?? defaults.includeBase64,
43
+ };
44
+ }
45
+ /**
46
+ * ConfigManager holds the loaded configuration and provides access to it
47
+ */
48
+ export class ConfigManager {
49
+ config = DEFAULT_CONFIG;
50
+ async load() {
51
+ this.config = await loadConfig();
52
+ }
53
+ get() {
54
+ return this.config;
55
+ }
56
+ getUiConfig() {
57
+ return this.config.ui;
58
+ }
59
+ isVisualModePackage(packageName) {
60
+ return this.config.ui.visualModePackages.includes(packageName);
61
+ }
62
+ }
@@ -1,9 +1,11 @@
1
1
  import { Device } from "../types/index.js";
2
+ import type { AdbAdapter } from "../adapters/adb.js";
2
3
  export declare class DeviceStateManager {
3
4
  private currentDevice;
4
5
  getCurrentDevice(): Device | null;
5
6
  setCurrentDevice(device: Device): void;
6
7
  clearCurrentDevice(): void;
7
8
  requireCurrentDevice(): Device;
9
+ ensureDevice(adb: AdbAdapter): Promise<Device>;
8
10
  autoSelectIfSingle(devices: Device[]): boolean;
9
11
  }
@@ -16,6 +16,24 @@ export class DeviceStateManager {
16
16
  }
17
17
  return this.currentDevice;
18
18
  }
19
+ async ensureDevice(adb) {
20
+ // Already selected? Use it.
21
+ if (this.currentDevice) {
22
+ return this.currentDevice;
23
+ }
24
+ // Try to auto-select
25
+ const devices = await adb.getDevices();
26
+ if (devices.length === 0) {
27
+ throw new ReplicantError(ErrorCode.NO_DEVICES, "No devices connected", "Start an emulator with 'emulator-device start' or connect a USB device with debugging enabled");
28
+ }
29
+ if (devices.length === 1) {
30
+ this.currentDevice = devices[0];
31
+ return this.currentDevice;
32
+ }
33
+ // Multiple devices - user must choose
34
+ const deviceList = devices.map((d) => d.id).join(", ");
35
+ throw new ReplicantError(ErrorCode.MULTIPLE_DEVICES, `${devices.length} devices connected: ${deviceList}`, `Call adb-device({ operation: 'select', deviceId: '...' }) to choose one`);
36
+ }
19
37
  autoSelectIfSingle(devices) {
20
38
  if (devices.length === 1 && !this.currentDevice) {
21
39
  this.currentDevice = devices[0];
@@ -0,0 +1,18 @@
1
+ export interface Environment {
2
+ sdkPath: string | null;
3
+ adbPath: string | null;
4
+ emulatorPath: string | null;
5
+ platform: "darwin" | "linux" | "win32";
6
+ isValid: boolean;
7
+ issues: string[];
8
+ }
9
+ export declare class EnvironmentService {
10
+ private cached;
11
+ detect(): Promise<Environment>;
12
+ getAdbPath(): Promise<string>;
13
+ getEmulatorPath(): Promise<string>;
14
+ getAvdManagerPath(): Promise<string>;
15
+ private findSdkPath;
16
+ private getSearchPaths;
17
+ clearCache(): void;
18
+ }
@@ -0,0 +1,130 @@
1
+ import * as fs from "fs";
2
+ import * as os from "os";
3
+ import * as path from "path";
4
+ import { ReplicantError, ErrorCode } from "../types/index.js";
5
+ export class EnvironmentService {
6
+ cached = null;
7
+ async detect() {
8
+ if (this.cached) {
9
+ return this.cached;
10
+ }
11
+ const platform = os.platform();
12
+ const issues = [];
13
+ // Try to find SDK
14
+ const sdkPath = this.findSdkPath(platform);
15
+ if (!sdkPath) {
16
+ this.cached = {
17
+ sdkPath: null,
18
+ adbPath: null,
19
+ emulatorPath: null,
20
+ platform,
21
+ isValid: false,
22
+ issues: ["Android SDK not found. Install Android Studio or set ANDROID_HOME environment variable."],
23
+ };
24
+ return this.cached;
25
+ }
26
+ const adbPath = path.join(sdkPath, "platform-tools", "adb");
27
+ const emulatorPath = path.join(sdkPath, "emulator", "emulator");
28
+ // Verify adb exists
29
+ if (!fs.existsSync(adbPath)) {
30
+ issues.push(`adb not found at ${adbPath}`);
31
+ }
32
+ // Emulator is optional - just note if missing
33
+ if (!fs.existsSync(emulatorPath)) {
34
+ issues.push(`emulator not found at ${emulatorPath} (optional)`);
35
+ }
36
+ this.cached = {
37
+ sdkPath,
38
+ adbPath: fs.existsSync(adbPath) ? adbPath : null,
39
+ emulatorPath: fs.existsSync(emulatorPath) ? emulatorPath : null,
40
+ platform,
41
+ isValid: fs.existsSync(adbPath),
42
+ issues,
43
+ };
44
+ return this.cached;
45
+ }
46
+ async getAdbPath() {
47
+ const env = await this.detect();
48
+ if (!env.adbPath) {
49
+ throw new ReplicantError(ErrorCode.ADB_NOT_FOUND, "Android SDK not found", "Install Android Studio or set ANDROID_HOME environment variable", { checkedPaths: this.getSearchPaths(env.platform) });
50
+ }
51
+ return env.adbPath;
52
+ }
53
+ async getEmulatorPath() {
54
+ const env = await this.detect();
55
+ if (!env.emulatorPath) {
56
+ throw new ReplicantError(ErrorCode.EMULATOR_NOT_FOUND, "Android emulator not found", "Install Android Emulator via Android Studio SDK Manager");
57
+ }
58
+ return env.emulatorPath;
59
+ }
60
+ async getAvdManagerPath() {
61
+ const env = await this.detect();
62
+ if (!env.sdkPath) {
63
+ throw new ReplicantError(ErrorCode.SDK_NOT_FOUND, "Android SDK not found", "Install Android Studio or set ANDROID_HOME environment variable");
64
+ }
65
+ const avdManagerPath = path.join(env.sdkPath, "cmdline-tools", "latest", "bin", "avdmanager");
66
+ // Fallback to older location
67
+ const legacyPath = path.join(env.sdkPath, "tools", "bin", "avdmanager");
68
+ if (fs.existsSync(avdManagerPath)) {
69
+ return avdManagerPath;
70
+ }
71
+ if (fs.existsSync(legacyPath)) {
72
+ return legacyPath;
73
+ }
74
+ throw new ReplicantError(ErrorCode.SDK_NOT_FOUND, "avdmanager not found", "Install Android SDK Command-line Tools via Android Studio SDK Manager");
75
+ }
76
+ findSdkPath(platform) {
77
+ // 1. Check ANDROID_HOME
78
+ if (process.env.ANDROID_HOME) {
79
+ const adbPath = path.join(process.env.ANDROID_HOME, "platform-tools", "adb");
80
+ if (fs.existsSync(adbPath)) {
81
+ return process.env.ANDROID_HOME;
82
+ }
83
+ }
84
+ // 2. Check ANDROID_SDK_ROOT
85
+ if (process.env.ANDROID_SDK_ROOT) {
86
+ const adbPath = path.join(process.env.ANDROID_SDK_ROOT, "platform-tools", "adb");
87
+ if (fs.existsSync(adbPath)) {
88
+ return process.env.ANDROID_SDK_ROOT;
89
+ }
90
+ }
91
+ // 3. Probe common paths
92
+ const searchPaths = this.getSearchPaths(platform);
93
+ for (const sdkPath of searchPaths) {
94
+ const adbPath = path.join(sdkPath, "platform-tools", "adb");
95
+ if (fs.existsSync(adbPath)) {
96
+ return sdkPath;
97
+ }
98
+ }
99
+ return null;
100
+ }
101
+ getSearchPaths(platform) {
102
+ const home = os.homedir();
103
+ if (platform === "darwin") {
104
+ return [
105
+ path.join(home, "Library", "Android", "sdk"),
106
+ "/opt/homebrew/share/android-sdk",
107
+ "/usr/local/share/android-sdk",
108
+ ];
109
+ }
110
+ if (platform === "linux") {
111
+ return [
112
+ path.join(home, "Android", "Sdk"),
113
+ "/opt/android-sdk",
114
+ "/usr/lib/android-sdk",
115
+ ];
116
+ }
117
+ if (platform === "win32") {
118
+ const localAppData = process.env.LOCALAPPDATA || path.join(home, "AppData", "Local");
119
+ return [
120
+ path.join(localAppData, "Android", "Sdk"),
121
+ "C:\\Android\\sdk",
122
+ ];
123
+ }
124
+ return [];
125
+ }
126
+ // Clear cache (for testing)
127
+ clearCache() {
128
+ this.cached = null;
129
+ }
130
+ }
@@ -0,0 +1,28 @@
1
+ export declare const GRID_COLS = 4;
2
+ export declare const GRID_ROWS = 6;
3
+ export declare const TOTAL_CELLS: number;
4
+ export interface CellBounds {
5
+ x0: number;
6
+ y0: number;
7
+ x1: number;
8
+ y1: number;
9
+ }
10
+ /**
11
+ * Calculate the pixel bounds for a grid cell (1-24).
12
+ * Grid is 4 columns x 6 rows, numbered left-to-right, top-to-bottom.
13
+ */
14
+ export declare function calculateGridCellBounds(cell: number, screenWidth: number, screenHeight: number): CellBounds;
15
+ /**
16
+ * Calculate tap coordinates for a position within a cell.
17
+ * Position: 1=TL, 2=TR, 3=Center, 4=BL, 5=BR
18
+ */
19
+ export declare function calculatePositionCoordinates(position: 1 | 2 | 3 | 4 | 5, cellBounds: CellBounds): {
20
+ x: number;
21
+ y: number;
22
+ };
23
+ export declare const POSITION_LABELS: string[];
24
+ /**
25
+ * Create a 4x6 numbered grid overlay on a screenshot.
26
+ * Returns base64 PNG.
27
+ */
28
+ export declare function createGridOverlay(imagePath: string): Promise<string>;
@@ -0,0 +1,98 @@
1
+ // src/services/grid.ts
2
+ import sharp from "sharp";
3
+ export const GRID_COLS = 4;
4
+ export const GRID_ROWS = 6;
5
+ export const TOTAL_CELLS = GRID_COLS * GRID_ROWS; // 24
6
+ /**
7
+ * Calculate the pixel bounds for a grid cell (1-24).
8
+ * Grid is 4 columns x 6 rows, numbered left-to-right, top-to-bottom.
9
+ */
10
+ export function calculateGridCellBounds(cell, screenWidth, screenHeight) {
11
+ if (cell < 1 || cell > TOTAL_CELLS) {
12
+ throw new Error(`Invalid cell number: ${cell}. Must be 1-${TOTAL_CELLS}`);
13
+ }
14
+ const cellWidth = screenWidth / GRID_COLS;
15
+ const cellHeight = screenHeight / GRID_ROWS;
16
+ // Convert 1-based cell to 0-based row/col
17
+ const index = cell - 1;
18
+ const col = index % GRID_COLS;
19
+ const row = Math.floor(index / GRID_COLS);
20
+ return {
21
+ x0: Math.round(col * cellWidth),
22
+ y0: Math.round(row * cellHeight),
23
+ x1: Math.round((col + 1) * cellWidth),
24
+ y1: Math.round((row + 1) * cellHeight),
25
+ };
26
+ }
27
+ /**
28
+ * Calculate tap coordinates for a position within a cell.
29
+ * Position: 1=TL, 2=TR, 3=Center, 4=BL, 5=BR
30
+ */
31
+ export function calculatePositionCoordinates(position, cellBounds) {
32
+ const width = cellBounds.x1 - cellBounds.x0;
33
+ const height = cellBounds.y1 - cellBounds.y0;
34
+ // Position multipliers (fraction of width/height from x0/y0)
35
+ const positionMap = {
36
+ 1: { xMult: 0.25, yMult: 0.25 }, // Top-left
37
+ 2: { xMult: 0.75, yMult: 0.25 }, // Top-right
38
+ 3: { xMult: 0.5, yMult: 0.5 }, // Center
39
+ 4: { xMult: 0.25, yMult: 0.75 }, // Bottom-left
40
+ 5: { xMult: 0.75, yMult: 0.75 }, // Bottom-right
41
+ };
42
+ const { xMult, yMult } = positionMap[position];
43
+ return {
44
+ x: Math.round(cellBounds.x0 + width * xMult),
45
+ y: Math.round(cellBounds.y0 + height * yMult),
46
+ };
47
+ }
48
+ export const POSITION_LABELS = [
49
+ "Top-left",
50
+ "Top-right",
51
+ "Center",
52
+ "Bottom-left",
53
+ "Bottom-right",
54
+ ];
55
+ const GRID_LINE_WIDTH = 3;
56
+ const LABEL_FONT_SIZE = 36;
57
+ /**
58
+ * Create a 4x6 numbered grid overlay on a screenshot.
59
+ * Returns base64 PNG.
60
+ */
61
+ export async function createGridOverlay(imagePath) {
62
+ const image = sharp(imagePath);
63
+ const metadata = await image.metadata();
64
+ const width = metadata.width;
65
+ const height = metadata.height;
66
+ const cellWidth = width / GRID_COLS;
67
+ const cellHeight = height / GRID_ROWS;
68
+ // Create SVG overlay with grid lines and numbers
69
+ const svgParts = [];
70
+ // Grid lines (vertical)
71
+ for (let col = 1; col < GRID_COLS; col++) {
72
+ const x = Math.round(col * cellWidth);
73
+ svgParts.push(`<line x1="${x}" y1="0" x2="${x}" y2="${height}" stroke="rgba(255,165,0,0.8)" stroke-width="${GRID_LINE_WIDTH}"/>`);
74
+ }
75
+ // Grid lines (horizontal)
76
+ for (let row = 1; row < GRID_ROWS; row++) {
77
+ const y = Math.round(row * cellHeight);
78
+ svgParts.push(`<line x1="0" y1="${y}" x2="${width}" y2="${y}" stroke="rgba(255,165,0,0.8)" stroke-width="${GRID_LINE_WIDTH}"/>`);
79
+ }
80
+ // Cell numbers
81
+ for (let cell = 1; cell <= TOTAL_CELLS; cell++) {
82
+ const index = cell - 1;
83
+ const col = index % GRID_COLS;
84
+ const row = Math.floor(index / GRID_COLS);
85
+ const centerX = Math.round((col + 0.5) * cellWidth);
86
+ const centerY = Math.round((row + 0.5) * cellHeight);
87
+ // Background circle for visibility
88
+ svgParts.push(`<circle cx="${centerX}" cy="${centerY}" r="25" fill="rgba(0,0,0,0.6)"/>`);
89
+ // Number text
90
+ svgParts.push(`<text x="${centerX}" y="${centerY}" font-family="Arial, sans-serif" font-size="${LABEL_FONT_SIZE}" font-weight="bold" fill="white" text-anchor="middle" dominant-baseline="central">${cell}</text>`);
91
+ }
92
+ const svgOverlay = `<svg width="${width}" height="${height}">${svgParts.join("")}</svg>`;
93
+ const buffer = await image
94
+ .composite([{ input: Buffer.from(svgOverlay), top: 0, left: 0 }])
95
+ .png()
96
+ .toBuffer();
97
+ return buffer.toString("base64");
98
+ }
@@ -0,0 +1,10 @@
1
+ export declare const ICON_PATTERNS: Record<string, string[]>;
2
+ /**
3
+ * Match a user query to icon resourceId patterns.
4
+ * Returns array of patterns to search for, or null if no match.
5
+ */
6
+ export declare function matchIconPattern(query: string): string[] | null;
7
+ /**
8
+ * Check if a resourceId matches any of the given patterns.
9
+ */
10
+ export declare function matchesResourceId(resourceId: string, patterns: string[]): boolean;
@@ -0,0 +1,51 @@
1
+ export const ICON_PATTERNS = {
2
+ // Navigation
3
+ overflow: ["overflow", "more", "options", "menu", "dots", "kabob", "meatball"],
4
+ back: ["back", "navigate_up", "arrow_back", "return", "nav_back"],
5
+ close: ["close", "dismiss", "cancel", "ic_close", "btn_close"],
6
+ home: ["home", "nav_home", "ic_home"],
7
+ // Actions
8
+ search: ["search", "find", "magnify", "ic_search"],
9
+ settings: ["settings", "gear", "config", "preferences", "ic_settings"],
10
+ share: ["share", "ic_share", "btn_share"],
11
+ edit: ["edit", "pencil", "ic_edit", "btn_edit"],
12
+ delete: ["delete", "trash", "remove", "ic_delete"],
13
+ add: ["add", "plus", "create", "ic_add", "fab"],
14
+ // Media
15
+ play: ["play", "ic_play", "btn_play"],
16
+ pause: ["pause", "ic_pause"],
17
+ refresh: ["refresh", "reload", "sync", "ic_refresh"],
18
+ // Social
19
+ favorite: ["favorite", "heart", "like", "star", "ic_favorite"],
20
+ bookmark: ["bookmark", "save", "ic_bookmark"],
21
+ notification: ["notification", "bell", "ic_notification", "ic_notify"],
22
+ // Misc
23
+ filter: ["filter", "ic_filter", "btn_filter"],
24
+ sort: ["sort", "ic_sort", "btn_sort"],
25
+ download: ["download", "ic_download"],
26
+ upload: ["upload", "ic_upload"],
27
+ profile: ["profile", "account", "avatar", "user", "ic_profile"],
28
+ hamburger: ["hamburger", "drawer", "nav_drawer", "ic_menu"],
29
+ };
30
+ /**
31
+ * Match a user query to icon resourceId patterns.
32
+ * Returns array of patterns to search for, or null if no match.
33
+ */
34
+ export function matchIconPattern(query) {
35
+ const lowerQuery = query.toLowerCase();
36
+ // Check each icon category
37
+ for (const [category, patterns] of Object.entries(ICON_PATTERNS)) {
38
+ // Match if query contains the category name or any pattern
39
+ if (lowerQuery.includes(category) || patterns.some((p) => lowerQuery.includes(p))) {
40
+ return patterns;
41
+ }
42
+ }
43
+ return null;
44
+ }
45
+ /**
46
+ * Check if a resourceId matches any of the given patterns.
47
+ */
48
+ export function matchesResourceId(resourceId, patterns) {
49
+ const lowerId = resourceId.toLowerCase();
50
+ return patterns.some((pattern) => lowerId.includes(pattern));
51
+ }
@@ -1,3 +1,9 @@
1
1
  export { ProcessRunner, type RunOptions, type RunResult } from "./process-runner.js";
2
2
  export { CacheManager, type CacheStats } from "./cache-manager.js";
3
3
  export { DeviceStateManager } from "./device-state.js";
4
+ export { EnvironmentService, type Environment } from "./environment.js";
5
+ export { ConfigManager, loadConfig } from "./config.js";
6
+ export * from "./ocr.js";
7
+ export * from "./icon-patterns.js";
8
+ export * from "./grid.js";
9
+ export * from "./visual-candidates.js";
@@ -1,3 +1,9 @@
1
1
  export { ProcessRunner } from "./process-runner.js";
2
2
  export { CacheManager } from "./cache-manager.js";
3
3
  export { DeviceStateManager } from "./device-state.js";
4
+ export { EnvironmentService } from "./environment.js";
5
+ export { ConfigManager, loadConfig } from "./config.js";
6
+ export * from "./ocr.js";
7
+ export * from "./icon-patterns.js";
8
+ export * from "./grid.js";
9
+ export * from "./visual-candidates.js";
@@ -0,0 +1,4 @@
1
+ import { OcrResult, OcrElement } from "../types/ocr.js";
2
+ export declare function extractText(imagePath: string): Promise<OcrResult[]>;
3
+ export declare function terminateOcr(): Promise<void>;
4
+ export declare function searchText(ocrResults: OcrResult[], searchTerm: string): OcrElement[];
@@ -0,0 +1,59 @@
1
+ import { createWorker } from "tesseract.js";
2
+ let worker = null;
3
+ async function getWorker() {
4
+ if (!worker) {
5
+ worker = await createWorker("eng");
6
+ }
7
+ return worker;
8
+ }
9
+ export async function extractText(imagePath) {
10
+ const w = await getWorker();
11
+ const { data } = await w.recognize(imagePath, {}, { blocks: true });
12
+ const results = [];
13
+ // Navigate through blocks -> paragraphs -> lines -> words
14
+ if (data.blocks) {
15
+ for (const block of data.blocks) {
16
+ for (const paragraph of block.paragraphs) {
17
+ for (const line of paragraph.lines) {
18
+ for (const word of line.words) {
19
+ if (word.text.trim()) {
20
+ results.push({
21
+ text: word.text,
22
+ confidence: word.confidence / 100, // Normalize to 0-1
23
+ bounds: {
24
+ x0: word.bbox.x0,
25
+ y0: word.bbox.y0,
26
+ x1: word.bbox.x1,
27
+ y1: word.bbox.y1,
28
+ },
29
+ });
30
+ }
31
+ }
32
+ }
33
+ }
34
+ }
35
+ }
36
+ return results;
37
+ }
38
+ export async function terminateOcr() {
39
+ if (worker) {
40
+ await worker.terminate();
41
+ worker = null;
42
+ }
43
+ }
44
+ export function searchText(ocrResults, searchTerm) {
45
+ const lowerSearch = searchTerm.toLowerCase();
46
+ const matches = ocrResults.filter((result) => result.text.toLowerCase().includes(lowerSearch));
47
+ // Index represents position in filtered matches (0, 1, 2...) for use with elementIndex tap
48
+ // This is intentional - users tap by match index, not original OCR result position
49
+ return matches.map((match, index) => ({
50
+ index,
51
+ text: match.text,
52
+ bounds: `[${match.bounds.x0},${match.bounds.y0}][${match.bounds.x1},${match.bounds.y1}]`,
53
+ center: {
54
+ x: Math.round((match.bounds.x0 + match.bounds.x1) / 2),
55
+ y: Math.round((match.bounds.y0 + match.bounds.y1) / 2),
56
+ },
57
+ confidence: match.confidence,
58
+ }));
59
+ }
@@ -1,3 +1,4 @@
1
+ import type { EnvironmentService } from "./environment.js";
1
2
  export interface RunOptions {
2
3
  timeoutMs?: number;
3
4
  cwd?: string;
@@ -8,8 +9,13 @@ export interface RunResult {
8
9
  exitCode: number;
9
10
  }
10
11
  export declare class ProcessRunner {
12
+ private environment?;
11
13
  private readonly defaultTimeoutMs;
12
14
  private readonly maxTimeoutMs;
15
+ constructor(environment?: EnvironmentService | undefined);
13
16
  run(command: string, args: string[], options?: RunOptions): Promise<RunResult>;
17
+ runAdb(args: string[], options?: RunOptions): Promise<RunResult>;
18
+ runEmulator(args: string[], options?: RunOptions): Promise<RunResult>;
19
+ runAvdManager(args: string[], options?: RunOptions): Promise<RunResult>;
14
20
  private validateCommand;
15
21
  }