prompttest 1.3.5 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +6 -0
- package/README.md +191 -41
- package/TERMS.md +138 -0
- package/dist/bin/prompttest.js +489 -230
- package/dist/bin/server.d.ts +1 -0
- package/dist/engine.bundle.js +1000 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +292 -154
- package/dist/lib/adaptive-timing.d.ts +55 -0
- package/dist/lib/adb-provisioner.d.ts +55 -0
- package/dist/lib/adb.d.ts +29 -0
- package/dist/lib/baseline.d.ts +52 -4
- package/dist/lib/dfs-engine.d.ts +43 -0
- package/dist/lib/explorer.d.ts +44 -0
- package/dist/lib/feedback.d.ts +35 -0
- package/dist/lib/jail-guard.d.ts +59 -0
- package/dist/lib/live-server.d.ts +5 -0
- package/dist/lib/prompt-runner.d.ts +1 -0
- package/dist/lib/quiescence.d.ts +44 -0
- package/dist/lib/recorder.d.ts +24 -0
- package/dist/lib/reporter.d.ts +2 -0
- package/dist/lib/step-handlers.d.ts +2 -0
- package/dist/lib/triage.d.ts +50 -0
- package/dist/lib/wizard.d.ts +42 -0
- package/dist/lib/zip-util.d.ts +36 -0
- package/docs/ARCHITECTURE.md +4 -4
- package/docs/CLI_CONTRACT.md +131 -0
- package/docs/CLI_STUDIO_CONTRACT.md +123 -0
- package/docs/PERFORMANCE_BASELINE.md +71 -0
- package/docs/PRODUCT_STATUS.md +56 -0
- package/docs/USER_MANUAL.md +6 -7
- package/package.json +12 -3
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module wizard
|
|
3
|
+
* @description
|
|
4
|
+
* Interactive guided CLI wizard for PromptTest.
|
|
5
|
+
* Auto-detects connected devices, identifies foreground apps, discovers existing
|
|
6
|
+
* test specifications, and guides developers through autonomous exploration,
|
|
7
|
+
* test recording, spec execution, and diagnostics with zero required CLI arguments.
|
|
8
|
+
*/
|
|
9
|
+
import { AndroidDriver, AndroidDevice } from './adb.js';
|
|
10
|
+
import { PromptRunner } from './prompt-runner.js';
|
|
11
|
+
export interface EnvironmentInfo {
|
|
12
|
+
device?: AndroidDevice;
|
|
13
|
+
serial?: string;
|
|
14
|
+
activePackage?: string;
|
|
15
|
+
isOnline: boolean;
|
|
16
|
+
}
|
|
17
|
+
export interface WizardOptions {
|
|
18
|
+
input?: NodeJS.ReadableStream;
|
|
19
|
+
output?: NodeJS.WritableStream;
|
|
20
|
+
driver?: AndroidDriver;
|
|
21
|
+
runner?: PromptRunner;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Automatically inspects ADB to detect online devices and active foreground application.
|
|
25
|
+
*
|
|
26
|
+
* @param driver - AndroidDriver instance to query.
|
|
27
|
+
* @returns Detected environment state.
|
|
28
|
+
*/
|
|
29
|
+
export declare function detectActiveEnvironment(driver?: AndroidDriver): Promise<EnvironmentInfo>;
|
|
30
|
+
/**
|
|
31
|
+
* Scans the current workspace for existing plain-English test specs (*.txt, *.spec).
|
|
32
|
+
*
|
|
33
|
+
* @param baseDir - Root directory to search (defaults to process.cwd()).
|
|
34
|
+
* @returns Array of discovered relative spec paths.
|
|
35
|
+
*/
|
|
36
|
+
export declare function findExistingSpecs(baseDir?: string): string[];
|
|
37
|
+
/**
|
|
38
|
+
* Starts the interactive guided CLI wizard.
|
|
39
|
+
*
|
|
40
|
+
* @param options - Configuration options for input/output streams and drivers.
|
|
41
|
+
*/
|
|
42
|
+
export declare function startInteractiveWizard(options?: WizardOptions): Promise<void>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module zip-util
|
|
3
|
+
* @description
|
|
4
|
+
* Lightweight, zero-dependency ZIP archive generator built using Node.js native zlib.
|
|
5
|
+
* Generates standard PKZIP archives containing files with deflation compression.
|
|
6
|
+
*/
|
|
7
|
+
export interface ZipEntry {
|
|
8
|
+
name: string;
|
|
9
|
+
data: Buffer;
|
|
10
|
+
date?: Date;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Calculates a standard CRC32 checksum for a buffer.
|
|
14
|
+
*/
|
|
15
|
+
export declare function crc32(buf: Buffer): number;
|
|
16
|
+
/**
|
|
17
|
+
* Converts a JS Date into standard MS-DOS format (16-bit time, 16-bit date).
|
|
18
|
+
*/
|
|
19
|
+
export declare function dosDateTime(d?: Date): {
|
|
20
|
+
time: number;
|
|
21
|
+
date: number;
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Packs an array of ZipEntry files into a standard PKZIP buffer using Deflate compression.
|
|
25
|
+
*
|
|
26
|
+
* @param entries - Array of file entries with names and data buffers.
|
|
27
|
+
* @returns A Buffer containing the valid ZIP archive.
|
|
28
|
+
*/
|
|
29
|
+
export declare function createZipArchive(entries: ZipEntry[]): Buffer;
|
|
30
|
+
/**
|
|
31
|
+
* Saves a list of entries as a ZIP file to disk.
|
|
32
|
+
*
|
|
33
|
+
* @param filePath - Target file path on disk.
|
|
34
|
+
* @param entries - Array of zip entries.
|
|
35
|
+
*/
|
|
36
|
+
export declare function writeZipFile(filePath: string, entries: ZipEntry[]): Promise<void>;
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -6,7 +6,7 @@ PromptTest is a zero-code, AI-assisted Android test automation and visual QA fra
|
|
|
6
6
|
|
|
7
7
|
## 1. High-Level Architecture Overview
|
|
8
8
|
|
|
9
|
-
PromptTest operates on a modular layered architecture where test scripts written in natural language or AST-driven directives are compiled, bound to live or synthetic data, executed against connected devices via an isolated locking layer, and reported with visual diffs and interactive HTML reports.
|
|
9
|
+
PromptTest operates on a modular layered architecture where test scripts written in natural language or AST-driven directives are compiled, bound to live or synthetic data, executed against connected devices via an isolated locking layer, and reported with visual diffs and interactive HTML reports. Current release boundaries are tracked in [PRODUCT_STATUS.md](PRODUCT_STATUS.md).
|
|
10
10
|
|
|
11
11
|
```
|
|
12
12
|
┌───────────────────────────────────────────────────────────┐
|
|
@@ -35,14 +35,14 @@ PromptTest operates on a modular layered architecture where test scripts written
|
|
|
35
35
|
│ │ │
|
|
36
36
|
┌──────▼──────────────────────▼──────────────────────▼──────┐
|
|
37
37
|
│ Visual Testing & Diagnostics │
|
|
38
|
-
│ lib/baseline.ts (
|
|
38
|
+
│ lib/baseline.ts (pixelmatch / PNG Diff) │
|
|
39
39
|
│ lib/explorer.ts (Autonomous Crawler) │
|
|
40
40
|
└─────────────────────────────┬─────────────────────────────┘
|
|
41
41
|
│
|
|
42
42
|
┌─────────────────────────────▼─────────────────────────────┐
|
|
43
43
|
│ Reporting & Observability │
|
|
44
44
|
│ lib/reporter.ts (HTML, JSON, JUnit XML, Markdown) │
|
|
45
|
-
│ lib/live-server.ts (
|
|
45
|
+
│ lib/live-server.ts (HTTP / SSE Live Monitor) │
|
|
46
46
|
└───────────────────────────────────────────────────────────┘
|
|
47
47
|
```
|
|
48
48
|
|
|
@@ -100,7 +100,7 @@ Instead of an unwieldy switch-case statement, execution is delegated to decouple
|
|
|
100
100
|
|
|
101
101
|
### 2.6 Visual Baselines & Computer Vision (`lib/baseline.ts`, `lib/explorer.ts`)
|
|
102
102
|
|
|
103
|
-
- Pixel-
|
|
103
|
+
- Pixel-level image diffing using `pixelmatch` and PNG decoding via `pngjs`.
|
|
104
104
|
- Configurable failure thresholds (e.g. `diffThreshold: 0.02` for 2% variance).
|
|
105
105
|
- Automatic bounding box masking for dynamic regions (e.g. status bar clocks, live feeds) to eliminate false positives in visual regression testing.
|
|
106
106
|
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# PromptTest CLI Contract
|
|
2
|
+
|
|
3
|
+
**Contract date:** 2026-09-22
|
|
4
|
+
**Package:** `prompttest` 1.3.5
|
|
5
|
+
|
|
6
|
+
This document describes the currently shipped CLI and SDK integration surfaces. It is intended for CI consumers and desktop clients. It does not promise functionality that is only planned or hardware-unvalidated.
|
|
7
|
+
|
|
8
|
+
## Compatibility Rules
|
|
9
|
+
|
|
10
|
+
- Existing public exports and command behavior are additive-compatible within a major version.
|
|
11
|
+
- Consumers should use exported types and documented fields rather than parsing terminal text.
|
|
12
|
+
- Report fields may gain optional properties. Existing required fields will not be removed without a breaking release.
|
|
13
|
+
- Hardware operations require ADB and a reachable Android device or emulator.
|
|
14
|
+
|
|
15
|
+
## Exit Codes
|
|
16
|
+
|
|
17
|
+
| Code | Name | Meaning |
|
|
18
|
+
| ---: | -------------------- | ----------------------------------------------------------------- |
|
|
19
|
+
| 0 | `SUCCESS` | Run completed successfully. |
|
|
20
|
+
| 1 | `TEST_FAILED` | An assertion, locator, or test step failed. |
|
|
21
|
+
| 2 | `DEVICE_UNAVAILABLE` | ADB/device connection or device availability failed. |
|
|
22
|
+
| 3 | `CONFIG_ERROR` | A spec, configuration, argument, or include reference is invalid. |
|
|
23
|
+
| 4 | `LOCKED` | The target device is locked by another PromptTest process. |
|
|
24
|
+
|
|
25
|
+
CI integrations should use these codes instead of matching human-readable output.
|
|
26
|
+
|
|
27
|
+
## Report Model
|
|
28
|
+
|
|
29
|
+
The canonical report is JSON with this shape:
|
|
30
|
+
|
|
31
|
+
```ts
|
|
32
|
+
interface QaReportData {
|
|
33
|
+
packageName: string;
|
|
34
|
+
deviceId: string;
|
|
35
|
+
startTime: string;
|
|
36
|
+
endTime: string;
|
|
37
|
+
durationSeconds: number;
|
|
38
|
+
totalSteps: number;
|
|
39
|
+
passedSteps: number;
|
|
40
|
+
failedSteps: number;
|
|
41
|
+
skippedSteps: number;
|
|
42
|
+
warnSteps: number;
|
|
43
|
+
errorsDetected: string[];
|
|
44
|
+
videoPath?: string;
|
|
45
|
+
steps: AuditStep[];
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
interface AuditStep {
|
|
49
|
+
stepIndex: number;
|
|
50
|
+
timestamp: string;
|
|
51
|
+
phase: string;
|
|
52
|
+
description: string;
|
|
53
|
+
action: string;
|
|
54
|
+
status: 'PASS' | 'WARN' | 'FAIL' | 'SKIPPED';
|
|
55
|
+
target?: string;
|
|
56
|
+
screenshotPath?: string;
|
|
57
|
+
durationMs?: number;
|
|
58
|
+
triageBundle?: string;
|
|
59
|
+
videoOffsetMs?: number;
|
|
60
|
+
details?: Record<string, unknown>;
|
|
61
|
+
}
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Supported serializers are JSON, Markdown, JUnit XML, and HTML. JSON is the machine-readable source of truth; the other formats are views of the same run.
|
|
65
|
+
|
|
66
|
+
## Output Conventions
|
|
67
|
+
|
|
68
|
+
The default output directory is `output/` in the working directory. Common artifacts include:
|
|
69
|
+
|
|
70
|
+
- `<name>-results.json`
|
|
71
|
+
- `<name>-report.md`
|
|
72
|
+
- `<name>-report.html`
|
|
73
|
+
- `<name>-junit.xml`
|
|
74
|
+
- Failure triage folders or ZIPs containing screenshots, hierarchy XML, logcat, and match scores
|
|
75
|
+
- Baselines under `output/baselines/`
|
|
76
|
+
- Checkpoints under `output/checkpoints/`
|
|
77
|
+
- Optional MP4 recordings referenced by `videoPath`
|
|
78
|
+
|
|
79
|
+
Paths in report fields are relative to the report/output directory unless the producer explicitly returns an absolute path.
|
|
80
|
+
|
|
81
|
+
## Live Engine Protocol
|
|
82
|
+
|
|
83
|
+
The live server is started with `prompttest serve [port]` or `startLiveServer()`. Defaults:
|
|
84
|
+
|
|
85
|
+
- Host: `127.0.0.1`
|
|
86
|
+
- Port: `4040`
|
|
87
|
+
- Transport: HTTP plus Server-Sent Events
|
|
88
|
+
- Event stream: `GET /events`
|
|
89
|
+
- Report view: `GET /` or `GET /report`
|
|
90
|
+
|
|
91
|
+
The current server has no authentication and sends permissive CORS headers. It must be treated as a local trusted-process interface. Do not bind it to a non-loopback host or expose it outside a controlled machine without adding authentication and access controls.
|
|
92
|
+
|
|
93
|
+
### Stable read endpoints
|
|
94
|
+
|
|
95
|
+
- `GET /api/devices` returns connected device metadata.
|
|
96
|
+
- `GET /api/screen-state?deviceId=<id>&hierarchy=true|false` returns a screenshot, foreground package, and optional flattened UI elements.
|
|
97
|
+
- `GET /api/specs` lists local `.txt` and `.spec` files.
|
|
98
|
+
- `GET /api/specs/<name>` reads a spec file.
|
|
99
|
+
- `GET /api/reports` lists generated reports.
|
|
100
|
+
- `GET /api/reports/<name>` reads a report detail.
|
|
101
|
+
- `GET /api/baselines` lists available baselines.
|
|
102
|
+
- `GET /api/baselines/masks` lists baseline masks.
|
|
103
|
+
- `GET /api/report` and `GET /results.json` return the current report data.
|
|
104
|
+
|
|
105
|
+
### Mutating endpoints
|
|
106
|
+
|
|
107
|
+
These endpoints control a connected device or write local files and should only be called by an explicit local user action:
|
|
108
|
+
|
|
109
|
+
- `POST /api/tap` with `{ x, y, deviceId? }`
|
|
110
|
+
- `POST /api/type` with `{ text, deviceId? }`
|
|
111
|
+
- `POST /api/key` with `{ keyCode, deviceId? }`
|
|
112
|
+
- `POST /api/wake` with `{ deviceId? }`
|
|
113
|
+
- `POST /api/specs` with `{ name, content }`
|
|
114
|
+
- `POST /api/action` with `{ step, deviceId? }`
|
|
115
|
+
- `POST /api/run` with `{ specName? | spec?, deviceId?, saveBaseline?, compareBaseline? }`
|
|
116
|
+
- Baseline approval, comparison, upload, mask, clear, and delete endpoints under `/api/baselines/*`
|
|
117
|
+
|
|
118
|
+
### SSE events
|
|
119
|
+
|
|
120
|
+
Clients connecting to `/events` may receive:
|
|
121
|
+
|
|
122
|
+
- `step`: step number, instruction, status, duration, and optional error
|
|
123
|
+
- `complete`: final `QaReportData`
|
|
124
|
+
- `error`: an error message payload
|
|
125
|
+
- Heartbeat comments every 15 seconds
|
|
126
|
+
|
|
127
|
+
Clients must reconnect and tolerate duplicate or missing transient events; the final report is authoritative.
|
|
128
|
+
|
|
129
|
+
## Studio Boundary
|
|
130
|
+
|
|
131
|
+
The CLI owns device control, test execution, report generation, and artifact storage. A Studio client owns presentation, browsing, device selection, and desktop packaging. Studio must not duplicate locator resolution or report aggregation logic. Any new endpoint or event should be added here with a compatibility note and an integration test.
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# CLI and Studio Contract
|
|
2
|
+
|
|
3
|
+
**Contract version:** `1`
|
|
4
|
+
**CLI package:** `prompttest` 1.3.5
|
|
5
|
+
**Status:** Current implementation contract, verified 2026-09-22
|
|
6
|
+
|
|
7
|
+
This document defines the integration surface between the PromptTest CLI engine and a desktop or browser Studio client. The CLI remains independently usable; Studio consumes this contract and must not reimplement device control, test execution, report generation, or baseline comparison.
|
|
8
|
+
|
|
9
|
+
## Engine Startup
|
|
10
|
+
|
|
11
|
+
Start the engine with:
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
prompttest serve [port]
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
The default port is `4040`. The server binds to `127.0.0.1` by default. Programmatic consumers can call `startLiveServer({ port, host, outputDir, specsDir })`.
|
|
18
|
+
|
|
19
|
+
The server is not an authenticated remote control plane. It exposes device actions, screenshots, UI hierarchy data, specifications, reports, and baselines. Keep the default loopback binding; do not expose the server on a network interface without adding authentication and authorization first.
|
|
20
|
+
|
|
21
|
+
## Exit Codes
|
|
22
|
+
|
|
23
|
+
| Code | Meaning |
|
|
24
|
+
| ---: | --------------------------------------------- |
|
|
25
|
+
| `0` | Successful execution |
|
|
26
|
+
| `1` | Test assertion or locator failure |
|
|
27
|
+
| `2` | Device unavailable, offline, or unauthorized |
|
|
28
|
+
| `3` | Invalid configuration, spec, or CLI arguments |
|
|
29
|
+
| `4` | Device lock contention |
|
|
30
|
+
|
|
31
|
+
Studio should display the error message and preserve the numeric exit code when it starts the CLI as a child process.
|
|
32
|
+
|
|
33
|
+
## Report Contract
|
|
34
|
+
|
|
35
|
+
The canonical report is JSON. The current `QaReportData` shape is:
|
|
36
|
+
|
|
37
|
+
```json
|
|
38
|
+
{
|
|
39
|
+
"packageName": "com.example.app",
|
|
40
|
+
"deviceId": "emulator-5554",
|
|
41
|
+
"startTime": "2026-09-22T00:00:00.000Z",
|
|
42
|
+
"endTime": "2026-09-22T00:00:03.000Z",
|
|
43
|
+
"durationSeconds": 3,
|
|
44
|
+
"totalSteps": 2,
|
|
45
|
+
"passedSteps": 1,
|
|
46
|
+
"failedSteps": 1,
|
|
47
|
+
"skippedSteps": 0,
|
|
48
|
+
"warnSteps": 0,
|
|
49
|
+
"errorsDetected": [],
|
|
50
|
+
"videoPath": "login-recording.mp4",
|
|
51
|
+
"steps": []
|
|
52
|
+
}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Each step can contain `stepIndex`, `timestamp`, `phase`, `description`, `action`, `target`, `status`, `screenshotPath`, `durationMs`, `triageBundle`, `videoOffsetMs`, and `details`.
|
|
56
|
+
|
|
57
|
+
Supported report serializers are `json`, `md`, `junit`, and `html`. JSON is the integration source of truth. File names use the selected base name:
|
|
58
|
+
|
|
59
|
+
- `<name>-results.json`
|
|
60
|
+
- `<name>-report.md`
|
|
61
|
+
- `<name>-junit.xml`
|
|
62
|
+
- `<name>-report.html`
|
|
63
|
+
|
|
64
|
+
## SSE Events
|
|
65
|
+
|
|
66
|
+
Connect to `GET /events` with an `EventSource`. Events currently emitted are:
|
|
67
|
+
|
|
68
|
+
- `step`: step number, instruction, status, duration, and optional error
|
|
69
|
+
- `complete`: final `QaReportData`
|
|
70
|
+
- `error`: an error message object
|
|
71
|
+
|
|
72
|
+
Clients should tolerate unknown future events and reconnect after a dropped connection. The server sends comment heartbeats approximately every 15 seconds.
|
|
73
|
+
|
|
74
|
+
## HTTP Routes
|
|
75
|
+
|
|
76
|
+
All successful JSON responses include `success: true` where applicable. Errors use `success: false` and an `error` string.
|
|
77
|
+
|
|
78
|
+
### Devices and live device state
|
|
79
|
+
|
|
80
|
+
- `GET /api/devices`: connected device list
|
|
81
|
+
- `GET /api/screen-state?deviceId=<id>&hierarchy=true|false`: screenshot, foreground package, and UI elements
|
|
82
|
+
- `POST /api/tap`: `{ "x": number, "y": number, "deviceId"?: string }`
|
|
83
|
+
- `POST /api/type`: `{ "text": string, "deviceId"?: string }`
|
|
84
|
+
- `POST /api/key`: `{ "keyCode": number|string, "deviceId"?: string }`
|
|
85
|
+
- `POST /api/wake`: `{ "deviceId"?: string }`
|
|
86
|
+
|
|
87
|
+
### Specifications and execution
|
|
88
|
+
|
|
89
|
+
- `GET /api/specs`: list `.txt` and `.spec` files
|
|
90
|
+
- `GET /api/specs/<name>` or `GET /api/spec?name=<name>`: read a specification
|
|
91
|
+
- `POST /api/specs` or `POST /api/spec`: `{ "name": string, "content": string }`
|
|
92
|
+
- `POST /api/action`: `{ "step": string, "deviceId"?: string }`
|
|
93
|
+
- `POST /api/run`: `{ "specName"?: string, "spec"?: string, "deviceId"?: string, "saveBaseline"?: boolean, "compareBaseline"?: boolean }`
|
|
94
|
+
|
|
95
|
+
`/api/run` starts execution asynchronously. Use SSE and report endpoints to observe completion.
|
|
96
|
+
|
|
97
|
+
### Reports and baselines
|
|
98
|
+
|
|
99
|
+
- `GET /api/reports`: list available report artifacts
|
|
100
|
+
- `GET /api/reports/<name>` or `GET /api/report-detail?name=<name>`: retrieve report details
|
|
101
|
+
- `GET /api/report` or `GET /results.json`: retrieve the current JSON report
|
|
102
|
+
- `GET /api/baselines`: list baseline images and metadata
|
|
103
|
+
- `POST /api/baselines/compare`: compare an image against a baseline
|
|
104
|
+
- `POST /api/baselines/approve`: approve a baseline result
|
|
105
|
+
- `POST /api/baselines/clear`: clear baseline data
|
|
106
|
+
- `POST /api/baselines/delete-step`: remove a step baseline
|
|
107
|
+
- `POST /api/baselines/upload`: upload a baseline image
|
|
108
|
+
- `GET /api/baselines/masks`: list baseline masks
|
|
109
|
+
- `POST /api/baselines/masks`: create or update a baseline mask
|
|
110
|
+
|
|
111
|
+
The exact request bodies for baseline management should be generated from the CLI client implementation until a schema is added in a future contract version.
|
|
112
|
+
|
|
113
|
+
## Artifact Rules
|
|
114
|
+
|
|
115
|
+
The CLI owns artifact creation under `output/` by default. Studio should display and download paths returned by reports rather than copying large screenshots or videos into JSON. Triage artifacts may include screenshots, hierarchy XML, logcat, match scores, a manifest, and a ZIP archive. A video path and failure offset do not imply that a clipped video snippet exists.
|
|
116
|
+
|
|
117
|
+
## Compatibility Rules
|
|
118
|
+
|
|
119
|
+
- Existing public SDK exports and CLI flags are additive and semver-protected.
|
|
120
|
+
- Studio must tolerate missing optional fields and unknown JSON fields.
|
|
121
|
+
- Studio must not assume Android hardware is available during engine startup.
|
|
122
|
+
- A failed run is not an engine crash; inspect the report and exit code separately.
|
|
123
|
+
- Contract changes require a version update, documentation update, and package smoke-test coverage.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# PromptTest Performance Baseline
|
|
2
|
+
|
|
3
|
+
**Measured:** 2026-09-22
|
|
4
|
+
**Package:** `prompttest` 1.3.5
|
|
5
|
+
**Environment:** macOS, Node.js 24.13.0, no ADB device required
|
|
6
|
+
|
|
7
|
+
This is a reproducible CI and package-size baseline. Synthetic timings are health signals, not physical-device performance claims.
|
|
8
|
+
|
|
9
|
+
## Current Measurements
|
|
10
|
+
|
|
11
|
+
- Offline test suite: 39 test files, 337 tests passing
|
|
12
|
+
- Test wall time: approximately 14 seconds in the current environment
|
|
13
|
+
- CLI bundle: approximately 332 KB
|
|
14
|
+
- SDK bundle: approximately 316 KB
|
|
15
|
+
- Engine bundle: approximately 316 KB
|
|
16
|
+
- Runtime dependencies: `chalk`, `pixelmatch`, `pngjs`
|
|
17
|
+
- Supported Node.js range: `>=18.0.0`
|
|
18
|
+
|
|
19
|
+
## Existing Benchmark Coverage
|
|
20
|
+
|
|
21
|
+
`tests/performance-benchmark.test.ts` and `lib/benchmark.ts` cover:
|
|
22
|
+
|
|
23
|
+
- Synthetic screenshot allocation timing
|
|
24
|
+
- Synthetic UI hierarchy timing
|
|
25
|
+
- Matching latency across a synthetic 1,000-node hierarchy
|
|
26
|
+
- Percentile calculation and summary metrics
|
|
27
|
+
- Hierarchy hashing and structural diffing
|
|
28
|
+
- Verify-handler cache reuse
|
|
29
|
+
- Reporter screenshot-buffer cleanup
|
|
30
|
+
- PromptRunner benchmark mode
|
|
31
|
+
|
|
32
|
+
Run the offline benchmark tests with:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
npx vitest run tests/performance-benchmark.test.ts
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Run the complete offline suite with:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
npm test
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Required Hardware Measurements
|
|
45
|
+
|
|
46
|
+
The following values must be collected on representative Android emulator and physical-device configurations before making performance promises:
|
|
47
|
+
|
|
48
|
+
- ADB screenshot p50/p95 latency
|
|
49
|
+
- `uiautomator dump` p50/p95 latency
|
|
50
|
+
- Hierarchy dumps per visited screen
|
|
51
|
+
- Screenshots per exploration state
|
|
52
|
+
- Peak RSS for a 50-screen exploration
|
|
53
|
+
- HTML report size with and without embedded screenshots
|
|
54
|
+
- Triage ZIP size
|
|
55
|
+
- Aggregation time for large output directories
|
|
56
|
+
- Video-enabled versus video-disabled run time
|
|
57
|
+
|
|
58
|
+
Hardware results should include device model, Android version, screen size, Node version, iteration count, and whether the device is USB, Wi-Fi, or emulator. Do not compare synthetic benchmark timings directly with ADB measurements.
|
|
59
|
+
|
|
60
|
+
## Release Guardrails
|
|
61
|
+
|
|
62
|
+
Any new runtime dependency or report-embedding behavior should be evaluated against:
|
|
63
|
+
|
|
64
|
+
- Published tarball size
|
|
65
|
+
- Installed unpacked size
|
|
66
|
+
- Cold CLI startup time
|
|
67
|
+
- Peak memory during report generation
|
|
68
|
+
- Existing offline test duration
|
|
69
|
+
- Node 18 compatibility
|
|
70
|
+
|
|
71
|
+
The npm package should remain independently usable without Studio or a running server.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# PromptTest Product Status
|
|
2
|
+
|
|
3
|
+
**Status date:** 2026-09-22
|
|
4
|
+
**Package:** `prompttest` 1.3.5
|
|
5
|
+
|
|
6
|
+
This document is the current status index for the PromptTest CLI. Older implementation plans and the internal SRS are historical context, not release requirements.
|
|
7
|
+
|
|
8
|
+
## Status Definitions
|
|
9
|
+
|
|
10
|
+
- **Released:** Implemented, tested in the repository, and included in the published package.
|
|
11
|
+
- **Implemented:** Present in the CLI, but physical-device or cross-platform validation may still be incomplete.
|
|
12
|
+
- **Planned:** Intended work that is not part of the current release contract.
|
|
13
|
+
- **Studio:** Requires the separate desktop application; that source is not part of this repository.
|
|
14
|
+
|
|
15
|
+
## Current CLI Contract
|
|
16
|
+
|
|
17
|
+
### Released or implemented
|
|
18
|
+
|
|
19
|
+
- Android execution through ADB
|
|
20
|
+
- Natural-language prompt parsing and structured flow control
|
|
21
|
+
- Device locking and device-pool support
|
|
22
|
+
- Data-driven runs and dynamic test data
|
|
23
|
+
- Visual baselines using `pixelmatch` and PNG decoding
|
|
24
|
+
- Markdown, JSON, JUnit, and HTML reporting
|
|
25
|
+
- Failure triage artifacts and optional video recording
|
|
26
|
+
- Autonomous exploration with DFS traversal and safety modes
|
|
27
|
+
- Optional AI providers, disabled unless configured
|
|
28
|
+
- HTTP/SSE live monitoring server
|
|
29
|
+
- CI workflow generation through `init-ci`
|
|
30
|
+
- Committed offline quality CI and scheduled emulator CI workflows
|
|
31
|
+
- npm package and standalone distribution build paths
|
|
32
|
+
|
|
33
|
+
### Implemented with validation limits
|
|
34
|
+
|
|
35
|
+
- iOS driver interfaces require external Apple tooling and need a documented real-device acceptance matrix.
|
|
36
|
+
- Autonomous exploration is unit-tested and fixture-tested; coverage across physical applications is not implied.
|
|
37
|
+
- The emulator workflow is configured but requires a successful GitHub Actions run before hardware support is release-validated.
|
|
38
|
+
- Live-server consumers must use the documented loopback and artifact-access contract.
|
|
39
|
+
- Hardware tests require ADB, an emulator, or a connected device and are not part of the offline default test guarantee.
|
|
40
|
+
|
|
41
|
+
### Planned or deferred
|
|
42
|
+
|
|
43
|
+
- Performance benchmark evidence and output-retention limits
|
|
44
|
+
- Stronger subprocess isolation for untrusted JavaScript specifications
|
|
45
|
+
- Locale packs and broader non-English resolution
|
|
46
|
+
- Spec scaffolding and a dedicated profiler
|
|
47
|
+
- Offline licensing and report watermarking
|
|
48
|
+
- Hosted collaboration, cloud reporting, and multi-device orchestration
|
|
49
|
+
|
|
50
|
+
## CLI and Studio Boundary
|
|
51
|
+
|
|
52
|
+
The CLI owns device control, execution, comparison, report generation, and artifact contracts. A separate Studio application may consume those contracts for device selection, live views, report browsing, packaging, and desktop workflows. Studio release status must be tracked in its own repository and must not be represented as CLI completion.
|
|
53
|
+
|
|
54
|
+
## npm Compatibility Policy
|
|
55
|
+
|
|
56
|
+
Changes to the CLI must preserve existing public exports and behavior unless a versioned breaking change is explicitly planned. Every release should pass typecheck, lint, offline tests, package verification, tarball installation smoke tests, and the supported Node.js compatibility checks before publication.
|
package/docs/USER_MANUAL.md
CHANGED
|
@@ -584,7 +584,7 @@ PromptTest can capture visual baselines and perform perceptual diffing to detect
|
|
|
584
584
|
|
|
585
585
|
- `--save-baseline`: Saves the current screen layout as a visual baseline for future comparisons.
|
|
586
586
|
- `--compare-baseline`: Compares the current screen layout against the saved baseline and fails if differences exceed the threshold.
|
|
587
|
-
- `--baseline-threshold=N`: Adjusts the
|
|
587
|
+
- `--baseline-threshold=N`: Adjusts the diffing tolerance as a ratio from `0` to `1` (e.g., `--baseline-threshold=0.05` for 5%).
|
|
588
588
|
|
|
589
589
|
### Custom Reporters
|
|
590
590
|
|
|
@@ -615,7 +615,7 @@ PromptTest includes a cross-process, file-based mutex lock (`lib/lock.ts`) that
|
|
|
615
615
|
PromptTest exposes a clean programmatic API using factory functions and a unified `DriverInterface`. Both Android and iOS drivers share the identical driver abstraction:
|
|
616
616
|
|
|
617
617
|
```typescript
|
|
618
|
-
import { createPromptRunner, createAndroidDriver,
|
|
618
|
+
import { createPromptRunner, createAndroidDriver, IosDriver } from 'prompttest';
|
|
619
619
|
|
|
620
620
|
async function runCrossPlatformTests() {
|
|
621
621
|
// 🤖 Android Execution
|
|
@@ -624,9 +624,8 @@ async function runCrossPlatformTests() {
|
|
|
624
624
|
const androidReport = await androidRunner.runSpec('specs/login.txt', 'com.example.app');
|
|
625
625
|
|
|
626
626
|
// 🍎 iOS Execution (Simulator or Physical Device)
|
|
627
|
-
const iosDriver =
|
|
627
|
+
const iosDriver = new IosDriver({
|
|
628
628
|
udid: '00008101-001234567890',
|
|
629
|
-
bundleId: 'com.example.app',
|
|
630
629
|
});
|
|
631
630
|
const iosRunner = createPromptRunner(iosDriver);
|
|
632
631
|
const iosReport = await iosRunner.runSpec('specs/login.txt', 'com.example.app');
|
|
@@ -760,6 +759,6 @@ PromptTest is a **fast UI-tree automation engine over ADB**. It "sees" what `uia
|
|
|
760
759
|
|
|
761
760
|
## 💡 Support & Contribution
|
|
762
761
|
|
|
763
|
-
- **Repository**: [github.com/shriramsingh/
|
|
764
|
-
- **Issues & Requests**: [File an Issue](https://github.com/shriramsingh/
|
|
765
|
-
- **License**:
|
|
762
|
+
- **Community Repository**: [github.com/shriramsingh/prompttest-community](https://github.com/shriramsingh/prompttest-community)
|
|
763
|
+
- **Issues & Requests**: [File an Issue](https://github.com/shriramsingh/prompttest-community/issues)
|
|
764
|
+
- **License**: Business Source License 1.1 (BSL 1.1)
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "prompttest",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "Ultra-fast, zero-code autonomous mobile testing & visual QA brain for Android",
|
|
5
|
-
"license": "
|
|
5
|
+
"license": "BUSL-1.1",
|
|
6
6
|
"homepage": "https://github.com/shriramsingh/prompttest-community#readme",
|
|
7
7
|
"bugs": {
|
|
8
8
|
"url": "https://github.com/shriramsingh/prompttest-community/issues"
|
|
@@ -21,11 +21,16 @@
|
|
|
21
21
|
"dist",
|
|
22
22
|
"README.md",
|
|
23
23
|
"LICENSE",
|
|
24
|
+
"TERMS.md",
|
|
24
25
|
"CONTRIBUTING.md",
|
|
25
26
|
"SECURITY.md",
|
|
26
27
|
"LICENSES.md",
|
|
27
28
|
"docs/USER_MANUAL.md",
|
|
28
|
-
"docs/ARCHITECTURE.md"
|
|
29
|
+
"docs/ARCHITECTURE.md",
|
|
30
|
+
"docs/PRODUCT_STATUS.md",
|
|
31
|
+
"docs/CLI_CONTRACT.md",
|
|
32
|
+
"docs/CLI_STUDIO_CONTRACT.md",
|
|
33
|
+
"docs/PERFORMANCE_BASELINE.md"
|
|
29
34
|
],
|
|
30
35
|
"engines": {
|
|
31
36
|
"node": ">=18.0.0"
|
|
@@ -44,6 +49,7 @@
|
|
|
44
49
|
"explore:dev": "tsx bin/prompttest.ts explore --safety-mode=moderate",
|
|
45
50
|
"explore:yollow": "tsx bin/prompttest.ts explore --safety-mode=disabled",
|
|
46
51
|
"run": "tsx bin/prompttest.ts run",
|
|
52
|
+
"serve": "tsx bin/server.ts",
|
|
47
53
|
"watch": "tsx bin/prompttest.ts watch",
|
|
48
54
|
"record": "tsx bin/prompttest.ts record",
|
|
49
55
|
"init-ci": "tsx bin/prompttest.ts init-ci",
|
|
@@ -53,6 +59,9 @@
|
|
|
53
59
|
"test:student": "tsx bin/prompttest.ts run specs/coachconnect_student.txt host.exp.exponent",
|
|
54
60
|
"audit": "npm audit --audit-level=high",
|
|
55
61
|
"build": "node scripts/build.mjs",
|
|
62
|
+
"package:sea": "node scripts/package-sea.mjs",
|
|
63
|
+
"pack:verify": "node scripts/verify-pack.mjs",
|
|
64
|
+
"pack:smoke": "npm run build && node scripts/smoke-pack.mjs",
|
|
56
65
|
"prepare": "node -e \"if (fs.existsSync('scripts/build.mjs')) import('./scripts/build.mjs')\"",
|
|
57
66
|
"prepublishOnly": "npm run typecheck && npm run build",
|
|
58
67
|
"pack:check": "npm pack --dry-run",
|