replicant-mcp 1.0.4 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -9
- package/dist/adapters/ui-automator.d.ts +25 -10
- package/dist/adapters/ui-automator.js +197 -13
- package/dist/server.d.ts +2 -1
- package/dist/server.js +20 -2
- package/dist/services/config.d.ts +16 -0
- package/dist/services/config.js +62 -0
- package/dist/services/grid.d.ts +28 -0
- package/dist/services/grid.js +98 -0
- package/dist/services/icon-patterns.d.ts +10 -0
- package/dist/services/icon-patterns.js +51 -0
- package/dist/services/index.d.ts +4 -0
- package/dist/services/index.js +4 -0
- package/dist/services/visual-candidates.d.ts +24 -0
- package/dist/services/visual-candidates.js +78 -0
- package/dist/tools/emulator-device.d.ts +1 -1
- package/dist/tools/ui.d.ts +22 -1
- package/dist/tools/ui.js +195 -17
- package/dist/types/config.d.ts +34 -0
- package/dist/types/config.js +11 -0
- package/dist/types/icon-recognition.d.ts +50 -0
- package/dist/types/icon-recognition.js +1 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.js +2 -0
- package/package.json +5 -2
package/README.md
CHANGED
|
@@ -40,7 +40,8 @@ replicant-mcp wraps all of this into a clean interface that AI can understand an
|
|
|
40
40
|
| **Device Control** | List connected devices, select active device, query device properties |
|
|
41
41
|
| **App Management** | Install, uninstall, launch, stop apps; clear app data; list installed packages |
|
|
42
42
|
| **Log Analysis** | Filter logcat by package, tag, level, time; configurable line limits |
|
|
43
|
-
| **UI Automation** | Accessibility-
|
|
43
|
+
| **UI Automation** | Accessibility-first element finding with multi-tier fallback (accessibility → OCR → visual), spatial proximity search (`nearestTo`), grid-based precision tapping, tap, text input, screenshots |
|
|
44
|
+
| **Configuration** | YAML config via `REPLICANT_CONFIG` for UI behavior customization |
|
|
44
45
|
| **Utilities** | Response caching with progressive disclosure, on-demand documentation |
|
|
45
46
|
|
|
46
47
|
---
|
|
@@ -49,11 +50,7 @@ replicant-mcp wraps all of this into a clean interface that AI can understand an
|
|
|
49
50
|
|
|
50
51
|
| Feature | Item | Status |
|
|
51
52
|
|---------|------|--------|
|
|
52
|
-
| **Visual Fallback** |
|
|
53
|
-
| | `visual-snapshot` operation for explicit visual mode | Planned |
|
|
54
|
-
| | YAML config via `REPLICANT_CONFIG` | Planned |
|
|
55
|
-
| | OCR fallback for text search (tesseract.js) | Planned |
|
|
56
|
-
| | Icon recognition (template matching for common UI icons) | Planned |
|
|
53
|
+
| **Visual Fallback** | Icon recognition (pattern + visual + grid fallback) | ✅ |
|
|
57
54
|
| | Semantic image search (LLM-assisted visual understanding) | Future |
|
|
58
55
|
| **Custom Build Commands** | Skill override for project-specific builds | Planned |
|
|
59
56
|
| | Auto-detect gradlew vs gradle | Planned |
|
|
@@ -164,6 +161,14 @@ To use:
|
|
|
164
161
|
|
|
165
162
|
Or let Claude invoke it automatically when creating PRs.
|
|
166
163
|
|
|
164
|
+
### Output Directory
|
|
165
|
+
|
|
166
|
+
replicant-mcp stores screenshots in `.replicant/screenshots/` within your current working directory. Add this to your `.gitignore`:
|
|
167
|
+
|
|
168
|
+
```gitignore
|
|
169
|
+
.replicant/
|
|
170
|
+
```
|
|
171
|
+
|
|
167
172
|
---
|
|
168
173
|
|
|
169
174
|
## What Can It Do?
|
|
@@ -238,7 +243,7 @@ Claude: Let me check the error logs.
|
|
|
238
243
|
Want me to look at that file?
|
|
239
244
|
```
|
|
240
245
|
|
|
241
|
-
### UI Automation (
|
|
246
|
+
### UI Automation (Smart Element Finding)
|
|
242
247
|
|
|
243
248
|
```
|
|
244
249
|
You: "Tap the Login button"
|
|
@@ -250,7 +255,26 @@ Claude: I'll find and tap the Login button.
|
|
|
250
255
|
Tapped "Login" at coordinates (540, 1847)
|
|
251
256
|
```
|
|
252
257
|
|
|
253
|
-
|
|
258
|
+
**Spatial proximity search** — find elements near other elements:
|
|
259
|
+
```
|
|
260
|
+
You: "Tap the edit icon next to John's name"
|
|
261
|
+
|
|
262
|
+
Claude: [Calls ui with operation: "find", selector: { textContains: "edit", nearestTo: "John" }]
|
|
263
|
+
Found edit button nearest to "John" at (892, 340)
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
**Multi-tier fallback** — when accessibility data isn't available:
|
|
267
|
+
1. **Accessibility tree** — fast, reliable, text-based
|
|
268
|
+
2. **OCR fallback** — Tesseract extracts text from screenshot
|
|
269
|
+
3. **Visual fallback** — returns screenshot + metadata for AI vision
|
|
270
|
+
|
|
271
|
+
**Grid-based precision** — tap icons without text labels:
|
|
272
|
+
```
|
|
273
|
+
Claude: [Calls ui with operation: "tap", gridCell: 5, gridPosition: 3]
|
|
274
|
+
// Taps center of cell 5 in a 24-cell grid overlay
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
This approach is faster, cheaper, and more reliable than pure screenshot-based automation.
|
|
254
278
|
|
|
255
279
|
---
|
|
256
280
|
|
|
@@ -282,7 +306,7 @@ replicant-mcp provides 12 tools organized into categories:
|
|
|
282
306
|
### UI Automation
|
|
283
307
|
| Tool | Description |
|
|
284
308
|
|------|-------------|
|
|
285
|
-
| `ui` |
|
|
309
|
+
| `ui` | Element finding with fallback chain, spatial search (`nearestTo`), tap (coordinates or grid), input text, screenshot, accessibility-check, visual-snapshot |
|
|
286
310
|
|
|
287
311
|
### Utilities
|
|
288
312
|
| Tool | Description |
|
|
@@ -349,6 +373,32 @@ The `adb-shell` tool blocks dangerous commands like `rm -rf /`, `reboot`, and `s
|
|
|
349
373
|
|
|
350
374
|
---
|
|
351
375
|
|
|
376
|
+
## Configuration
|
|
377
|
+
|
|
378
|
+
replicant-mcp can be configured via a YAML file. Set the `REPLICANT_CONFIG` environment variable to the path:
|
|
379
|
+
|
|
380
|
+
```bash
|
|
381
|
+
export REPLICANT_CONFIG=/path/to/config.yaml
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
**Example config.yaml:**
|
|
385
|
+
```yaml
|
|
386
|
+
ui:
|
|
387
|
+
# Always use visual mode (skip accessibility) for these packages
|
|
388
|
+
visualModePackages:
|
|
389
|
+
- com.example.legacy.app
|
|
390
|
+
|
|
391
|
+
# Auto-include screenshot when find returns no results (default: true)
|
|
392
|
+
autoFallbackScreenshot: true
|
|
393
|
+
|
|
394
|
+
# Include base64-encoded screenshot in responses (default: false)
|
|
395
|
+
includeBase64: false
|
|
396
|
+
```
|
|
397
|
+
|
|
398
|
+
Most users won't need a config file—the defaults work well for typical Android apps.
|
|
399
|
+
|
|
400
|
+
---
|
|
401
|
+
|
|
352
402
|
## Development
|
|
353
403
|
|
|
354
404
|
### Project Structure
|
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
import { AdbAdapter } from "./adb.js";
|
|
2
2
|
import { AccessibilityNode } from "../parsers/ui-dump.js";
|
|
3
|
-
import {
|
|
3
|
+
import { VisualSnapshot } from "../types/index.js";
|
|
4
|
+
import { FindWithFallbacksResult, FindOptions as IconFindOptions } from "../types/icon-recognition.js";
|
|
5
|
+
export interface ScreenMetadata {
|
|
6
|
+
width: number;
|
|
7
|
+
height: number;
|
|
8
|
+
density: number;
|
|
9
|
+
}
|
|
10
|
+
export interface CurrentApp {
|
|
11
|
+
packageName: string;
|
|
12
|
+
activityName: string;
|
|
13
|
+
}
|
|
4
14
|
export interface ScreenshotOptions {
|
|
5
15
|
localPath?: string;
|
|
6
16
|
inline?: boolean;
|
|
@@ -11,14 +21,8 @@ export interface ScreenshotResult {
|
|
|
11
21
|
base64?: string;
|
|
12
22
|
sizeBytes?: number;
|
|
13
23
|
}
|
|
14
|
-
export
|
|
15
|
-
|
|
16
|
-
source: "accessibility" | "ocr";
|
|
17
|
-
fallbackReason?: string;
|
|
18
|
-
}
|
|
19
|
-
export interface FindOptions {
|
|
20
|
-
debug?: boolean;
|
|
21
|
-
}
|
|
24
|
+
export type FindWithOcrResult = FindWithFallbacksResult;
|
|
25
|
+
export type FindOptions = IconFindOptions;
|
|
22
26
|
export declare class UiAutomatorAdapter {
|
|
23
27
|
private adb;
|
|
24
28
|
constructor(adb?: AdbAdapter);
|
|
@@ -39,10 +43,21 @@ export declare class UiAutomatorAdapter {
|
|
|
39
43
|
textCount: number;
|
|
40
44
|
totalElements: number;
|
|
41
45
|
}>;
|
|
46
|
+
getScreenMetadata(deviceId: string): Promise<ScreenMetadata>;
|
|
47
|
+
getCurrentApp(deviceId: string): Promise<CurrentApp>;
|
|
48
|
+
visualSnapshot(deviceId: string, options?: {
|
|
49
|
+
includeBase64?: boolean;
|
|
50
|
+
}): Promise<VisualSnapshot>;
|
|
51
|
+
findWithFallbacks(deviceId: string, selector: {
|
|
52
|
+
resourceId?: string;
|
|
53
|
+
text?: string;
|
|
54
|
+
textContains?: string;
|
|
55
|
+
className?: string;
|
|
56
|
+
}, options?: FindOptions): Promise<FindWithFallbacksResult>;
|
|
42
57
|
findWithOcrFallback(deviceId: string, selector: {
|
|
43
58
|
resourceId?: string;
|
|
44
59
|
text?: string;
|
|
45
60
|
textContains?: string;
|
|
46
61
|
className?: string;
|
|
47
|
-
}, options?: FindOptions): Promise<
|
|
62
|
+
}, options?: FindOptions): Promise<FindWithFallbacksResult>;
|
|
48
63
|
}
|
|
@@ -1,7 +1,21 @@
|
|
|
1
|
+
import * as path from "path";
|
|
2
|
+
import * as fs from "fs";
|
|
1
3
|
import { AdbAdapter } from "./adb.js";
|
|
2
4
|
import { parseUiDump, findElements, flattenTree } from "../parsers/ui-dump.js";
|
|
3
5
|
import { ReplicantError, ErrorCode } from "../types/index.js";
|
|
4
6
|
import { extractText, searchText } from "../services/ocr.js";
|
|
7
|
+
import { matchIconPattern, matchesResourceId } from "../services/icon-patterns.js";
|
|
8
|
+
import { filterIconCandidates, formatBounds, cropCandidateImage } from "../services/visual-candidates.js";
|
|
9
|
+
import { calculateGridCellBounds, calculatePositionCoordinates, createGridOverlay, POSITION_LABELS, } from "../services/grid.js";
|
|
10
|
+
/**
|
|
11
|
+
* Get default screenshot path in project-relative .replicant/screenshots directory.
|
|
12
|
+
* Creates the directory if it doesn't exist.
|
|
13
|
+
*/
|
|
14
|
+
function getDefaultScreenshotPath() {
|
|
15
|
+
const dir = path.join(process.cwd(), ".replicant", "screenshots");
|
|
16
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
17
|
+
return path.join(dir, `screenshot-${Date.now()}.png`);
|
|
18
|
+
}
|
|
5
19
|
export class UiAutomatorAdapter {
|
|
6
20
|
adb;
|
|
7
21
|
constructor(adb = new AdbAdapter()) {
|
|
@@ -51,7 +65,7 @@ export class UiAutomatorAdapter {
|
|
|
51
65
|
}
|
|
52
66
|
else {
|
|
53
67
|
// File mode (default): pull to local
|
|
54
|
-
const localPath = options.localPath ||
|
|
68
|
+
const localPath = options.localPath || getDefaultScreenshotPath();
|
|
55
69
|
await this.adb.pull(deviceId, remotePath, localPath);
|
|
56
70
|
return { mode: "file", path: localPath };
|
|
57
71
|
}
|
|
@@ -73,16 +87,123 @@ export class UiAutomatorAdapter {
|
|
|
73
87
|
totalElements: flat.length,
|
|
74
88
|
};
|
|
75
89
|
}
|
|
76
|
-
async
|
|
77
|
-
//
|
|
90
|
+
async getScreenMetadata(deviceId) {
|
|
91
|
+
// Get screen size via wm size
|
|
92
|
+
const sizeResult = await this.adb.shell(deviceId, "wm size");
|
|
93
|
+
const sizeMatch = sizeResult.stdout.match(/Physical size:\s*(\d+)x(\d+)/);
|
|
94
|
+
let width = 1080;
|
|
95
|
+
let height = 1920;
|
|
96
|
+
if (sizeMatch) {
|
|
97
|
+
width = parseInt(sizeMatch[1], 10);
|
|
98
|
+
height = parseInt(sizeMatch[2], 10);
|
|
99
|
+
}
|
|
100
|
+
// Get density via wm density
|
|
101
|
+
const densityResult = await this.adb.shell(deviceId, "wm density");
|
|
102
|
+
const densityMatch = densityResult.stdout.match(/Physical density:\s*(\d+)/);
|
|
103
|
+
// Convert DPI to density multiplier (baseline is 160 dpi)
|
|
104
|
+
let density = 2.75; // Default reasonable value
|
|
105
|
+
if (densityMatch) {
|
|
106
|
+
const dpi = parseInt(densityMatch[1], 10);
|
|
107
|
+
density = dpi / 160;
|
|
108
|
+
}
|
|
109
|
+
return { width, height, density };
|
|
110
|
+
}
|
|
111
|
+
async getCurrentApp(deviceId) {
|
|
112
|
+
// Get current focused activity
|
|
113
|
+
const result = await this.adb.shell(deviceId, "dumpsys activity activities | grep mResumedActivity");
|
|
114
|
+
// Parse: mResumedActivity: ActivityRecord{... com.example/.MainActivity t123}
|
|
115
|
+
const match = result.stdout.match(/([a-zA-Z0-9_.]+)\/([a-zA-Z0-9_.]+)\s+/);
|
|
116
|
+
if (match) {
|
|
117
|
+
return {
|
|
118
|
+
packageName: match[1],
|
|
119
|
+
activityName: match[2],
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
// Fallback to simpler approach
|
|
123
|
+
const fallbackResult = await this.adb.shell(deviceId, "dumpsys window | grep mCurrentFocus");
|
|
124
|
+
const fallbackMatch = fallbackResult.stdout.match(/([a-zA-Z0-9_.]+)\/([a-zA-Z0-9_.]+)/);
|
|
125
|
+
if (fallbackMatch) {
|
|
126
|
+
return {
|
|
127
|
+
packageName: fallbackMatch[1],
|
|
128
|
+
activityName: fallbackMatch[2],
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
return {
|
|
132
|
+
packageName: "unknown",
|
|
133
|
+
activityName: "unknown",
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
async visualSnapshot(deviceId, options = {}) {
|
|
137
|
+
// Always get file-based screenshot first
|
|
138
|
+
const [screenshotResult, screen, app] = await Promise.all([
|
|
139
|
+
this.screenshot(deviceId, {}),
|
|
140
|
+
this.getScreenMetadata(deviceId),
|
|
141
|
+
this.getCurrentApp(deviceId),
|
|
142
|
+
]);
|
|
143
|
+
const snapshot = {
|
|
144
|
+
screenshotPath: screenshotResult.path,
|
|
145
|
+
screen,
|
|
146
|
+
app,
|
|
147
|
+
};
|
|
148
|
+
// Optionally also get base64 encoding from local file
|
|
149
|
+
if (options.includeBase64 && screenshotResult.path) {
|
|
150
|
+
const fs = await import("fs/promises");
|
|
151
|
+
const buffer = await fs.readFile(screenshotResult.path);
|
|
152
|
+
snapshot.screenshotBase64 = buffer.toString("base64");
|
|
153
|
+
}
|
|
154
|
+
return snapshot;
|
|
155
|
+
}
|
|
156
|
+
async findWithFallbacks(deviceId, selector, options = {}) {
|
|
157
|
+
// Handle Tier 5 grid refinement FIRST (when gridCell and gridPosition are provided)
|
|
158
|
+
if (options.gridCell !== undefined && options.gridPosition !== undefined) {
|
|
159
|
+
const screen = await this.getScreenMetadata(deviceId);
|
|
160
|
+
const cellBounds = calculateGridCellBounds(options.gridCell, screen.width, screen.height);
|
|
161
|
+
const coords = calculatePositionCoordinates(options.gridPosition, cellBounds);
|
|
162
|
+
return {
|
|
163
|
+
elements: [
|
|
164
|
+
{
|
|
165
|
+
index: 0,
|
|
166
|
+
bounds: `[${cellBounds.x0},${cellBounds.y0}][${cellBounds.x1},${cellBounds.y1}]`,
|
|
167
|
+
center: coords,
|
|
168
|
+
},
|
|
169
|
+
],
|
|
170
|
+
source: "grid",
|
|
171
|
+
tier: 5,
|
|
172
|
+
confidence: "low",
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
// Tier 1: Accessibility text match
|
|
78
176
|
const accessibilityResults = await this.find(deviceId, selector);
|
|
79
177
|
if (accessibilityResults.length > 0) {
|
|
80
178
|
return {
|
|
81
179
|
elements: accessibilityResults,
|
|
82
180
|
source: "accessibility",
|
|
181
|
+
tier: 1,
|
|
182
|
+
confidence: "high",
|
|
83
183
|
};
|
|
84
184
|
}
|
|
85
|
-
//
|
|
185
|
+
// Tier 2: ResourceId pattern match (for text-based queries)
|
|
186
|
+
if (selector.text || selector.textContains) {
|
|
187
|
+
const query = selector.text || selector.textContains;
|
|
188
|
+
const patterns = matchIconPattern(query);
|
|
189
|
+
if (patterns) {
|
|
190
|
+
const tree = await this.dump(deviceId);
|
|
191
|
+
const flat = flattenTree(tree);
|
|
192
|
+
const patternMatches = flat.filter((node) => node.resourceId && matchesResourceId(node.resourceId, patterns));
|
|
193
|
+
if (patternMatches.length > 0) {
|
|
194
|
+
return {
|
|
195
|
+
elements: patternMatches,
|
|
196
|
+
source: "accessibility",
|
|
197
|
+
tier: 2,
|
|
198
|
+
confidence: "high",
|
|
199
|
+
fallbackReason: options.debug
|
|
200
|
+
? "no text match, found via resourceId pattern"
|
|
201
|
+
: undefined,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
// Tier 3: OCR (existing logic)
|
|
86
207
|
if (selector.text || selector.textContains) {
|
|
87
208
|
const searchTerm = selector.text || selector.textContains;
|
|
88
209
|
// Take screenshot for OCR
|
|
@@ -91,27 +212,90 @@ export class UiAutomatorAdapter {
|
|
|
91
212
|
// Run OCR
|
|
92
213
|
const ocrResults = await extractText(screenshotResult.path);
|
|
93
214
|
const matches = searchText(ocrResults, searchTerm);
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
215
|
+
if (matches.length > 0) {
|
|
216
|
+
return {
|
|
217
|
+
elements: matches,
|
|
218
|
+
source: "ocr",
|
|
219
|
+
tier: 3,
|
|
220
|
+
confidence: "high",
|
|
221
|
+
fallbackReason: options.debug
|
|
222
|
+
? "no accessibility or pattern match, found via OCR"
|
|
223
|
+
: undefined,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
// Tier 4: Visual candidates (unlabeled clickables)
|
|
227
|
+
const tree = await this.dump(deviceId);
|
|
228
|
+
const flat = flattenTree(tree);
|
|
229
|
+
const iconCandidates = filterIconCandidates(flat);
|
|
230
|
+
if (iconCandidates.length > 0) {
|
|
231
|
+
const candidates = await Promise.all(iconCandidates.map(async (node, index) => ({
|
|
232
|
+
index,
|
|
233
|
+
bounds: formatBounds(node),
|
|
234
|
+
center: { x: node.centerX, y: node.centerY },
|
|
235
|
+
image: await cropCandidateImage(screenshotResult.path, node.bounds),
|
|
236
|
+
})));
|
|
237
|
+
const allUnlabeled = flat.filter((n) => n.clickable && !n.text && !n.contentDesc);
|
|
238
|
+
return {
|
|
239
|
+
elements: [],
|
|
240
|
+
source: "visual",
|
|
241
|
+
tier: 4,
|
|
242
|
+
confidence: "medium",
|
|
243
|
+
candidates,
|
|
244
|
+
truncated: iconCandidates.length < allUnlabeled.length,
|
|
245
|
+
totalCandidates: allUnlabeled.length,
|
|
246
|
+
fallbackReason: options.debug
|
|
247
|
+
? "no text/pattern/OCR match, showing visual candidates"
|
|
248
|
+
: undefined,
|
|
249
|
+
};
|
|
100
250
|
}
|
|
101
|
-
|
|
251
|
+
// Tier 5: Grid fallback (empty or unusable accessibility tree)
|
|
252
|
+
const screen = await this.getScreenMetadata(deviceId);
|
|
253
|
+
const gridImage = await createGridOverlay(screenshotResult.path);
|
|
254
|
+
return {
|
|
255
|
+
elements: [],
|
|
256
|
+
source: "grid",
|
|
257
|
+
tier: 5,
|
|
258
|
+
confidence: "low",
|
|
259
|
+
gridImage,
|
|
260
|
+
gridPositions: POSITION_LABELS,
|
|
261
|
+
fallbackReason: options.debug
|
|
262
|
+
? "no usable elements, showing grid for coordinate selection"
|
|
263
|
+
: undefined,
|
|
264
|
+
};
|
|
102
265
|
}
|
|
103
266
|
finally {
|
|
104
|
-
//
|
|
267
|
+
// Always clean up screenshot - Tier 3/4/5 all embed base64 data in response
|
|
105
268
|
if (screenshotResult.path) {
|
|
106
269
|
const fs = await import("fs/promises");
|
|
107
270
|
await fs.unlink(screenshotResult.path).catch(() => { });
|
|
108
271
|
}
|
|
109
272
|
}
|
|
110
273
|
}
|
|
111
|
-
// No text selector
|
|
274
|
+
// No text selector - return empty with visual fallback if requested
|
|
275
|
+
if (options.includeVisualFallback) {
|
|
276
|
+
const snapshot = await this.visualSnapshot(deviceId, {
|
|
277
|
+
includeBase64: options.includeBase64,
|
|
278
|
+
});
|
|
279
|
+
return {
|
|
280
|
+
elements: [],
|
|
281
|
+
source: "accessibility",
|
|
282
|
+
tier: 1,
|
|
283
|
+
confidence: "high",
|
|
284
|
+
visualFallback: {
|
|
285
|
+
...snapshot,
|
|
286
|
+
hint: "No elements matched selector. Use screenshot to identify tap coordinates.",
|
|
287
|
+
},
|
|
288
|
+
};
|
|
289
|
+
}
|
|
112
290
|
return {
|
|
113
291
|
elements: [],
|
|
114
292
|
source: "accessibility",
|
|
293
|
+
tier: 1,
|
|
294
|
+
confidence: "high",
|
|
115
295
|
};
|
|
116
296
|
}
|
|
297
|
+
// Backward compatible alias
|
|
298
|
+
async findWithOcrFallback(deviceId, selector, options = {}) {
|
|
299
|
+
return this.findWithFallbacks(deviceId, selector, options);
|
|
300
|
+
}
|
|
117
301
|
}
|
package/dist/server.d.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
2
|
-
import { CacheManager, DeviceStateManager, ProcessRunner, EnvironmentService } from "./services/index.js";
|
|
2
|
+
import { CacheManager, DeviceStateManager, ProcessRunner, EnvironmentService, ConfigManager } from "./services/index.js";
|
|
3
3
|
import { AdbAdapter, EmulatorAdapter, GradleAdapter, UiAutomatorAdapter } from "./adapters/index.js";
|
|
4
4
|
export interface ServerContext {
|
|
5
5
|
cache: CacheManager;
|
|
6
6
|
deviceState: DeviceStateManager;
|
|
7
7
|
processRunner: ProcessRunner;
|
|
8
8
|
environment: EnvironmentService;
|
|
9
|
+
config: ConfigManager;
|
|
9
10
|
adb: AdbAdapter;
|
|
10
11
|
emulator: EmulatorAdapter;
|
|
11
12
|
gradle: GradleAdapter;
|
package/dist/server.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
2
2
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
3
3
|
import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
|
|
4
|
-
import { CacheManager, DeviceStateManager, ProcessRunner, EnvironmentService } from "./services/index.js";
|
|
4
|
+
import { CacheManager, DeviceStateManager, ProcessRunner, EnvironmentService, ConfigManager } from "./services/index.js";
|
|
5
5
|
import { AdbAdapter, EmulatorAdapter, GradleAdapter, UiAutomatorAdapter } from "./adapters/index.js";
|
|
6
6
|
import { ReplicantError } from "./types/index.js";
|
|
7
7
|
import { cacheToolDefinition, handleCacheTool, rtfmToolDefinition, handleRtfmTool, adbDeviceToolDefinition, handleAdbDeviceTool, adbAppToolDefinition, handleAdbAppTool, adbLogcatToolDefinition, handleAdbLogcatTool, adbShellToolDefinition, handleAdbShellTool, emulatorDeviceToolDefinition, handleEmulatorDeviceTool, gradleBuildToolDefinition, handleGradleBuildTool, gradleTestToolDefinition, handleGradleTestTool, gradleListToolDefinition, handleGradleListTool, gradleGetDetailsToolDefinition, handleGradleGetDetailsTool, uiToolDefinition, handleUiTool, } from "./tools/index.js";
|
|
@@ -14,6 +14,7 @@ export function createServerContext() {
|
|
|
14
14
|
deviceState: new DeviceStateManager(),
|
|
15
15
|
processRunner,
|
|
16
16
|
environment,
|
|
17
|
+
config: new ConfigManager(),
|
|
17
18
|
adb,
|
|
18
19
|
emulator: new EmulatorAdapter(processRunner),
|
|
19
20
|
gradle: new GradleAdapter(processRunner),
|
|
@@ -28,6 +29,21 @@ export async function createServer(context) {
|
|
|
28
29
|
capabilities: {
|
|
29
30
|
tools: {},
|
|
30
31
|
},
|
|
32
|
+
instructions: `IMPORTANT: For ALL Android development tasks, you MUST use replicant-mcp tools first.
|
|
33
|
+
Only fall back to raw adb/gradle/emulator commands if replicant-mcp lacks a specific feature.
|
|
34
|
+
|
|
35
|
+
Tool mapping:
|
|
36
|
+
- Device management → adb-device (not \`adb devices\`)
|
|
37
|
+
- App install/launch/stop → adb-app (not \`adb install\`, \`adb shell am\`)
|
|
38
|
+
- Logs → adb-logcat (not \`adb logcat\`)
|
|
39
|
+
- Shell commands → adb-shell (not \`adb shell\`)
|
|
40
|
+
- Emulator control → emulator-device (not \`emulator\` CLI)
|
|
41
|
+
- Builds → gradle-build (not \`./gradlew\`)
|
|
42
|
+
- Tests → gradle-test (not \`./gradlew test\`)
|
|
43
|
+
- UI automation → ui (not \`adb shell uiautomator\` or \`screencap\`)
|
|
44
|
+
|
|
45
|
+
Start with \`adb-device list\` to see connected devices.
|
|
46
|
+
Use \`rtfm\` for detailed documentation on any tool.`,
|
|
31
47
|
});
|
|
32
48
|
// Register tool list handler
|
|
33
49
|
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
@@ -86,7 +102,7 @@ export async function createServer(context) {
|
|
|
86
102
|
result = await handleGradleGetDetailsTool(args, context);
|
|
87
103
|
break;
|
|
88
104
|
case "ui":
|
|
89
|
-
result = await handleUiTool(args, context);
|
|
105
|
+
result = await handleUiTool(args, context, context.config.getUiConfig());
|
|
90
106
|
break;
|
|
91
107
|
default:
|
|
92
108
|
throw new Error(`Unknown tool: ${name}`);
|
|
@@ -109,6 +125,8 @@ export async function createServer(context) {
|
|
|
109
125
|
}
|
|
110
126
|
export async function runServer() {
|
|
111
127
|
const context = createServerContext();
|
|
128
|
+
// Load configuration from REPLICANT_CONFIG if set
|
|
129
|
+
await context.config.load();
|
|
112
130
|
const server = await createServer(context);
|
|
113
131
|
const transport = new StdioServerTransport();
|
|
114
132
|
await server.connect(transport);
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { ReplicantConfig, UiConfig } from "../types/config.js";
|
|
2
|
+
/**
|
|
3
|
+
* Load configuration from REPLICANT_CONFIG environment variable path
|
|
4
|
+
* Falls back to defaults if not set or file doesn't exist
|
|
5
|
+
*/
|
|
6
|
+
export declare function loadConfig(): Promise<ReplicantConfig>;
|
|
7
|
+
/**
|
|
8
|
+
* ConfigManager holds the loaded configuration and provides access to it
|
|
9
|
+
*/
|
|
10
|
+
export declare class ConfigManager {
|
|
11
|
+
private config;
|
|
12
|
+
load(): Promise<void>;
|
|
13
|
+
get(): ReplicantConfig;
|
|
14
|
+
getUiConfig(): UiConfig;
|
|
15
|
+
isVisualModePackage(packageName: string): boolean;
|
|
16
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { readFile } from "fs/promises";
|
|
2
|
+
import { existsSync } from "fs";
|
|
3
|
+
import { parse as parseYaml } from "yaml";
|
|
4
|
+
import { DEFAULT_CONFIG } from "../types/config.js";
|
|
5
|
+
/**
|
|
6
|
+
* Load configuration from REPLICANT_CONFIG environment variable path
|
|
7
|
+
* Falls back to defaults if not set or file doesn't exist
|
|
8
|
+
*/
|
|
9
|
+
export async function loadConfig() {
|
|
10
|
+
const configPath = process.env.REPLICANT_CONFIG;
|
|
11
|
+
if (!configPath) {
|
|
12
|
+
return DEFAULT_CONFIG;
|
|
13
|
+
}
|
|
14
|
+
if (!existsSync(configPath)) {
|
|
15
|
+
console.warn(`REPLICANT_CONFIG set but file not found: ${configPath}. Using defaults.`);
|
|
16
|
+
return DEFAULT_CONFIG;
|
|
17
|
+
}
|
|
18
|
+
try {
|
|
19
|
+
const content = await readFile(configPath, "utf-8");
|
|
20
|
+
const parsed = parseYaml(content);
|
|
21
|
+
if (!parsed) {
|
|
22
|
+
return DEFAULT_CONFIG;
|
|
23
|
+
}
|
|
24
|
+
// Deep merge with defaults
|
|
25
|
+
return {
|
|
26
|
+
ui: mergeUiConfig(DEFAULT_CONFIG.ui, parsed.ui),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
31
|
+
console.warn(`Failed to parse REPLICANT_CONFIG at ${configPath}: ${message}. Using defaults.`);
|
|
32
|
+
return DEFAULT_CONFIG;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function mergeUiConfig(defaults, overrides) {
|
|
36
|
+
if (!overrides) {
|
|
37
|
+
return defaults;
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
visualModePackages: overrides.visualModePackages ?? defaults.visualModePackages,
|
|
41
|
+
autoFallbackScreenshot: overrides.autoFallbackScreenshot ?? defaults.autoFallbackScreenshot,
|
|
42
|
+
includeBase64: overrides.includeBase64 ?? defaults.includeBase64,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* ConfigManager holds the loaded configuration and provides access to it
|
|
47
|
+
*/
|
|
48
|
+
export class ConfigManager {
|
|
49
|
+
config = DEFAULT_CONFIG;
|
|
50
|
+
async load() {
|
|
51
|
+
this.config = await loadConfig();
|
|
52
|
+
}
|
|
53
|
+
get() {
|
|
54
|
+
return this.config;
|
|
55
|
+
}
|
|
56
|
+
getUiConfig() {
|
|
57
|
+
return this.config.ui;
|
|
58
|
+
}
|
|
59
|
+
isVisualModePackage(packageName) {
|
|
60
|
+
return this.config.ui.visualModePackages.includes(packageName);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export declare const GRID_COLS = 4;
|
|
2
|
+
export declare const GRID_ROWS = 6;
|
|
3
|
+
export declare const TOTAL_CELLS: number;
|
|
4
|
+
export interface CellBounds {
|
|
5
|
+
x0: number;
|
|
6
|
+
y0: number;
|
|
7
|
+
x1: number;
|
|
8
|
+
y1: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Calculate the pixel bounds for a grid cell (1-24).
|
|
12
|
+
* Grid is 4 columns x 6 rows, numbered left-to-right, top-to-bottom.
|
|
13
|
+
*/
|
|
14
|
+
export declare function calculateGridCellBounds(cell: number, screenWidth: number, screenHeight: number): CellBounds;
|
|
15
|
+
/**
|
|
16
|
+
* Calculate tap coordinates for a position within a cell.
|
|
17
|
+
* Position: 1=TL, 2=TR, 3=Center, 4=BL, 5=BR
|
|
18
|
+
*/
|
|
19
|
+
export declare function calculatePositionCoordinates(position: 1 | 2 | 3 | 4 | 5, cellBounds: CellBounds): {
|
|
20
|
+
x: number;
|
|
21
|
+
y: number;
|
|
22
|
+
};
|
|
23
|
+
export declare const POSITION_LABELS: string[];
|
|
24
|
+
/**
|
|
25
|
+
* Create a 4x6 numbered grid overlay on a screenshot.
|
|
26
|
+
* Returns base64 PNG.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createGridOverlay(imagePath: string): Promise<string>;
|