pi-harness-runtime 1.1.181 → 1.1.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/harness/e2e-commands.ts +216 -0
- package/harness/honcho-memory.ts +3 -3
- package/index.ts +9 -0
- package/package.json +1 -1
- package/packages/a2a-adapter/package.json +1 -1
- package/packages/agent-policy/package.json +1 -1
- package/packages/architecture-generator/package.json +1 -1
- package/packages/auth/package.json +1 -1
- package/packages/autonomous-refactor/package.json +1 -1
- package/packages/autonomous-runtime/package.json +1 -1
- package/packages/cache-strategy/package.json +1 -1
- package/packages/capability-registry/package.json +1 -1
- package/packages/checkpoint/package.json +1 -1
- package/packages/cli-plugin-sdk/package.json +1 -1
- package/packages/clipboard/package.json +1 -1
- package/packages/clipboard-plugin/package.json +1 -1
- package/packages/code-generation/package.json +1 -1
- package/packages/code-review/package.json +1 -1
- package/packages/codex-adapter/package.json +1 -1
- package/packages/codex-watcher/package.json +1 -1
- package/packages/config-capture/package.json +1 -1
- package/packages/context-compiler/package.json +1 -1
- package/packages/context-discovery/package.json +1 -1
- package/packages/context-manager/package.json +1 -1
- package/packages/cookie-sanitizer/package.json +1 -1
- package/packages/cost-optimizer/package.json +1 -1
- package/packages/dependency-analyzer/package.json +1 -1
- package/packages/django-plugin/package.json +1 -1
- package/packages/doc-generator/package.json +1 -1
- package/packages/e2e-tester/README.md +142 -0
- package/packages/e2e-tester/dist/index.d.ts +133 -0
- package/packages/e2e-tester/dist/index.js +380 -0
- package/packages/e2e-tester/dist/types.d.ts +91 -0
- package/packages/e2e-tester/dist/types.js +17 -0
- package/packages/e2e-tester/package.json +31 -0
- package/packages/e2e-tester/src/index.ts +488 -0
- package/packages/e2e-tester/src/types.ts +95 -0
- package/packages/evaluation-engine/package.json +1 -1
- package/packages/evaluation-runner/package.json +1 -1
- package/packages/event-bus/package.json +1 -1
- package/packages/event-store/package.json +1 -1
- package/packages/experience-replay/package.json +1 -1
- package/packages/feedback-collector/package.json +1 -1
- package/packages/file-copy-helper/package.json +1 -1
- package/packages/framework-detector/package.json +1 -1
- package/packages/framework-plugin-sdk/package.json +1 -1
- package/packages/frappe-plugin/package.json +1 -1
- package/packages/generic-web-plugin/package.json +1 -1
- package/packages/health-monitor/package.json +1 -1
- package/packages/intent-analyzer/package.json +1 -1
- package/packages/jev-judge/package.json +1 -1
- package/packages/knowledge-graph/package.json +1 -1
- package/packages/knowledge-retrieval/package.json +1 -1
- package/packages/laravel-plugin/package.json +1 -1
- package/packages/learning-engine/package.json +1 -1
- package/packages/mcp-adapter/package.json +1 -1
- package/packages/memory-engine/package.json +1 -1
- package/packages/milestone-manager/package.json +1 -1
- package/packages/model-registry/package.json +1 -1
- package/packages/nextjs-plugin/package.json +1 -1
- package/packages/notification/package.json +1 -1
- package/packages/observability/package.json +1 -1
- package/packages/okf-indexer/package.json +1 -1
- package/packages/paperclip-integration/package.json +1 -1
- package/packages/performance-optimizer/package.json +1 -1
- package/packages/privilege-broker/package.json +1 -1
- package/packages/project-analyzer/package.json +1 -1
- package/packages/project-bootstrap/package.json +1 -1
- package/packages/projection-engine/package.json +1 -1
- package/packages/prompt-compiler/package.json +1 -1
- package/packages/prompt-versioning/package.json +1 -1
- package/packages/provider-adapter-sdk/package.json +1 -1
- package/packages/provider-router/package.json +1 -1
- package/packages/provider-selector/package.json +1 -1
- package/packages/providers/package.json +1 -1
- package/packages/qdrant-skills/package.json +1 -1
- package/packages/quota-manager/package.json +1 -1
- package/packages/rate-limiter/package.json +1 -1
- package/packages/react-vite-plugin/package.json +1 -1
- package/packages/release-manager/package.json +1 -1
- package/packages/requirement-compiler/package.json +1 -1
- package/packages/runtime/package.json +1 -1
- package/packages/scheduler/package.json +1 -1
- package/packages/scheduler-adapter/package.json +1 -1
- package/packages/session/package.json +1 -1
- package/packages/session-api/package.json +1 -1
- package/packages/session-export/package.json +1 -1
- package/packages/shared-context/package.json +1 -1
- package/packages/skill-mcp-client/package.json +1 -1
- package/packages/skill-registry/package.json +1 -1
- package/packages/skills/package.json +1 -1
- package/packages/sprint-planner/package.json +1 -1
- package/packages/subscription-engine/package.json +1 -1
- package/packages/task-compiler/package.json +1 -1
- package/packages/telegram/package.json +1 -1
- package/packages/tencentdb-memory/package.json +1 -1
- package/packages/tencentdb-sync/package.json +1 -1
- package/packages/test-data-generator/package.json +1 -1
- package/packages/test-generator/package.json +1 -1
- package/packages/todo-bd-sync/package.json +1 -1
- package/packages/token-estimation/package.json +1 -1
- package/packages/token-optimizer/package.json +1 -1
- package/packages/trajectory/package.json +1 -1
- package/packages/tui/package.json +1 -1
- package/packages/types/package.json +1 -1
- package/packages/webmcp-tester/dist/index.d.ts +178 -0
- package/packages/webmcp-tester/dist/index.js +412 -0
- package/packages/webmcp-tester/package.json +27 -0
- package/packages/webmcp-tester/src/index.ts +571 -0
- package/packages/workflow-events/package.json +1 -1
- package/packages/workspace-scanner/package.json +1 -1
- package/packages/worktree/package.json +1 -1
- package/packages/write-review/package.json +1 -1
- package/scripts/run-e2e-test.ts +214 -0
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WebMCP E2E Test Runner
|
|
3
|
+
*
|
|
4
|
+
* Natural language E2E testing using WebMCP protocol.
|
|
5
|
+
* Agents write test scenarios in plain English, no playwright knowledge needed.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* ```typescript
|
|
9
|
+
* import { WebMCPTester, TestScenario } from "@pi-harness/webmcp-tester";
|
|
10
|
+
*
|
|
11
|
+
* const tester = new WebMCPTester({
|
|
12
|
+
* chromePath: "/path/to/chrome",
|
|
13
|
+
* headless: true
|
|
14
|
+
* });
|
|
15
|
+
*
|
|
16
|
+
* const scenario: TestScenario = {
|
|
17
|
+
* name: "Registration Flow",
|
|
18
|
+
* url: "https://app.example.com/register",
|
|
19
|
+
* steps: [
|
|
20
|
+
* { action: "fill", field: "email", value: "test@example.com" },
|
|
21
|
+
* { action: "fill", field: "name", value: "Test User" },
|
|
22
|
+
* { action: "click", target: "Submit button" },
|
|
23
|
+
* { action: "assert", expected: "Welcome" }
|
|
24
|
+
* ]
|
|
25
|
+
* };
|
|
26
|
+
*
|
|
27
|
+
* const result = await tester.run(scenario);
|
|
28
|
+
* console.log(result.passed ? "PASS" : "FAIL");
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
export interface TestStep {
|
|
32
|
+
/** Natural language action description */
|
|
33
|
+
action: string;
|
|
34
|
+
/** Target element or field */
|
|
35
|
+
target?: string;
|
|
36
|
+
/** Field name for fill actions */
|
|
37
|
+
field?: string;
|
|
38
|
+
/** Value for fill actions */
|
|
39
|
+
value?: string;
|
|
40
|
+
/** Expected result for assert actions */
|
|
41
|
+
expected?: string;
|
|
42
|
+
/** Wait time after action (ms) */
|
|
43
|
+
waitMs?: number;
|
|
44
|
+
}
|
|
45
|
+
export interface TestScenario {
|
|
46
|
+
/** Test name */
|
|
47
|
+
name: string;
|
|
48
|
+
/** Starting URL */
|
|
49
|
+
url: string;
|
|
50
|
+
/** Test steps */
|
|
51
|
+
steps: TestStep[];
|
|
52
|
+
/** Dummy data overrides */
|
|
53
|
+
data?: Record<string, string>;
|
|
54
|
+
/** Screenshot on failure */
|
|
55
|
+
screenshotOnFailure?: boolean;
|
|
56
|
+
}
|
|
57
|
+
export interface TestResult {
|
|
58
|
+
/** Test name */
|
|
59
|
+
name: string;
|
|
60
|
+
/** Whether test passed */
|
|
61
|
+
passed: boolean;
|
|
62
|
+
/** Error message if failed */
|
|
63
|
+
error?: string;
|
|
64
|
+
/** Screenshots captured */
|
|
65
|
+
screenshots: string[];
|
|
66
|
+
/** Steps executed */
|
|
67
|
+
stepsExecuted: number;
|
|
68
|
+
/** Time taken (ms) */
|
|
69
|
+
durationMs: number;
|
|
70
|
+
/** Console logs captured */
|
|
71
|
+
consoleLogs: ConsoleLog[];
|
|
72
|
+
}
|
|
73
|
+
export interface ConsoleLog {
|
|
74
|
+
level: "log" | "warn" | "error" | "info";
|
|
75
|
+
message: string;
|
|
76
|
+
timestamp: number;
|
|
77
|
+
}
|
|
78
|
+
export interface WebMCPTesterConfig {
|
|
79
|
+
/** Chrome executable path */
|
|
80
|
+
chromePath?: string;
|
|
81
|
+
/** Chrome port for debugging */
|
|
82
|
+
port?: number;
|
|
83
|
+
/** Run in headless mode */
|
|
84
|
+
headless?: boolean;
|
|
85
|
+
/** WebMCP extension path */
|
|
86
|
+
extensionPath?: string;
|
|
87
|
+
/** Test timeout (ms) */
|
|
88
|
+
timeoutMs?: number;
|
|
89
|
+
/** Take screenshots */
|
|
90
|
+
screenshots?: boolean;
|
|
91
|
+
/** Screenshot output directory */
|
|
92
|
+
screenshotDir?: string;
|
|
93
|
+
}
|
|
94
|
+
export declare class WebMCPTester {
|
|
95
|
+
private config;
|
|
96
|
+
private chromeProcess;
|
|
97
|
+
private testResults;
|
|
98
|
+
constructor(config?: WebMCPTesterConfig);
|
|
99
|
+
/**
|
|
100
|
+
* Find Chrome executable
|
|
101
|
+
*/
|
|
102
|
+
private findChromePath;
|
|
103
|
+
/**
|
|
104
|
+
* Start Chrome with WebMCP extension
|
|
105
|
+
*/
|
|
106
|
+
start(): Promise<void>;
|
|
107
|
+
/**
|
|
108
|
+
* Stop Chrome
|
|
109
|
+
*/
|
|
110
|
+
stop(): Promise<void>;
|
|
111
|
+
/**
|
|
112
|
+
* Wait for Chrome debugging port to be ready
|
|
113
|
+
*/
|
|
114
|
+
private waitForChrome;
|
|
115
|
+
/**
|
|
116
|
+
* Get WebSocket CDP endpoint
|
|
117
|
+
*/
|
|
118
|
+
private getWebSocketUrl;
|
|
119
|
+
/**
|
|
120
|
+
* Run a test scenario
|
|
121
|
+
*/
|
|
122
|
+
run(scenario: TestScenario): Promise<TestResult>;
|
|
123
|
+
/**
|
|
124
|
+
* Navigate to URL (simplified - actual CDP calls would be more complex)
|
|
125
|
+
*/
|
|
126
|
+
private navigate;
|
|
127
|
+
/**
|
|
128
|
+
* Fill form field
|
|
129
|
+
*/
|
|
130
|
+
private fill;
|
|
131
|
+
/**
|
|
132
|
+
* Click element
|
|
133
|
+
*/
|
|
134
|
+
private click;
|
|
135
|
+
/**
|
|
136
|
+
* Assert content exists
|
|
137
|
+
*/
|
|
138
|
+
private assert;
|
|
139
|
+
/**
|
|
140
|
+
* Take screenshot
|
|
141
|
+
*/
|
|
142
|
+
private screenshot;
|
|
143
|
+
/**
|
|
144
|
+
* Get all test results
|
|
145
|
+
*/
|
|
146
|
+
getResults(): TestResult[];
|
|
147
|
+
/**
|
|
148
|
+
* Clear results
|
|
149
|
+
*/
|
|
150
|
+
clearResults(): void;
|
|
151
|
+
}
|
|
152
|
+
export interface ParsedScenario {
|
|
153
|
+
name: string;
|
|
154
|
+
url: string;
|
|
155
|
+
steps: TestStep[];
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Parse natural language test description into TestScenario
|
|
159
|
+
*
|
|
160
|
+
* Examples:
|
|
161
|
+
* ```
|
|
162
|
+
* Test: Registration flow
|
|
163
|
+
* URL: https://app.example.com/register
|
|
164
|
+
*
|
|
165
|
+
* Steps:
|
|
166
|
+
* 1. Fill email with test@example.com
|
|
167
|
+
* 2. Fill name with Test User
|
|
168
|
+
* 3. Click Submit
|
|
169
|
+
* 4. Verify "Welcome" message appears
|
|
170
|
+
* ```
|
|
171
|
+
*/
|
|
172
|
+
export declare function parseNaturalLanguageTest(input: string): ParsedScenario;
|
|
173
|
+
export interface CLIRunnerConfig {
|
|
174
|
+
scenarioFile?: string;
|
|
175
|
+
outputDir?: string;
|
|
176
|
+
headless?: boolean;
|
|
177
|
+
}
|
|
178
|
+
export declare function runFromCLI(config?: CLIRunnerConfig): Promise<void>;
|
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WebMCP E2E Test Runner
|
|
3
|
+
*
|
|
4
|
+
* Natural language E2E testing using WebMCP protocol.
|
|
5
|
+
* Agents write test scenarios in plain English, no playwright knowledge needed.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* ```typescript
|
|
9
|
+
* import { WebMCPTester, TestScenario } from "@pi-harness/webmcp-tester";
|
|
10
|
+
*
|
|
11
|
+
* const tester = new WebMCPTester({
|
|
12
|
+
* chromePath: "/path/to/chrome",
|
|
13
|
+
* headless: true
|
|
14
|
+
* });
|
|
15
|
+
*
|
|
16
|
+
* const scenario: TestScenario = {
|
|
17
|
+
* name: "Registration Flow",
|
|
18
|
+
* url: "https://app.example.com/register",
|
|
19
|
+
* steps: [
|
|
20
|
+
* { action: "fill", field: "email", value: "test@example.com" },
|
|
21
|
+
* { action: "fill", field: "name", value: "Test User" },
|
|
22
|
+
* { action: "click", target: "Submit button" },
|
|
23
|
+
* { action: "assert", expected: "Welcome" }
|
|
24
|
+
* ]
|
|
25
|
+
* };
|
|
26
|
+
*
|
|
27
|
+
* const result = await tester.run(scenario);
|
|
28
|
+
* console.log(result.passed ? "PASS" : "FAIL");
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
import { spawn } from "child_process";
|
|
32
|
+
import { promisify } from "util";
|
|
33
|
+
import { readFile, mkdir } from "fs/promises";
|
|
34
|
+
import { existsSync } from "fs";
|
|
35
|
+
import { join } from "path";
|
|
36
|
+
import { homedir } from "os";
|
|
37
|
+
const exec = promisify(require("child_process").exec);
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
// WebMCP Test Runner
|
|
40
|
+
// ---------------------------------------------------------------------------
|
|
41
|
+
export class WebMCPTester {
|
|
42
|
+
config;
|
|
43
|
+
chromeProcess = null;
|
|
44
|
+
testResults = [];
|
|
45
|
+
constructor(config = {}) {
|
|
46
|
+
this.config = {
|
|
47
|
+
chromePath: config.chromePath || this.findChromePath(),
|
|
48
|
+
port: config.port || 9222,
|
|
49
|
+
headless: config.headless ?? true,
|
|
50
|
+
extensionPath: config.extensionPath || "",
|
|
51
|
+
timeoutMs: config.timeoutMs || 60000,
|
|
52
|
+
screenshots: config.screenshots ?? true,
|
|
53
|
+
screenshotDir: config.screenshotDir || join(homedir(), ".pi-harness-runtime", "test-screenshots"),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Find Chrome executable
|
|
58
|
+
*/
|
|
59
|
+
findChromePath() {
|
|
60
|
+
const paths = [
|
|
61
|
+
"/usr/bin/google-chrome",
|
|
62
|
+
"/usr/bin/chromium",
|
|
63
|
+
"/usr/bin/chromium-browser",
|
|
64
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
|
65
|
+
process.env.CHROME_PATH,
|
|
66
|
+
].filter(Boolean);
|
|
67
|
+
for (const path of paths) {
|
|
68
|
+
if (existsSync(path)) {
|
|
69
|
+
return path;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return "google-chrome"; // Let system resolve
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Start Chrome with WebMCP extension
|
|
76
|
+
*/
|
|
77
|
+
async start() {
|
|
78
|
+
if (this.chromeProcess) {
|
|
79
|
+
return; // Already running
|
|
80
|
+
}
|
|
81
|
+
// Ensure screenshot directory exists
|
|
82
|
+
if (this.config.screenshots) {
|
|
83
|
+
await mkdir(this.config.screenshotDir, { recursive: true });
|
|
84
|
+
}
|
|
85
|
+
const args = [
|
|
86
|
+
`--remote-debugging-port=${this.config.port}`,
|
|
87
|
+
"--no-first-run",
|
|
88
|
+
"--no-default-browser-check",
|
|
89
|
+
"--disable-extensions",
|
|
90
|
+
"--disable-popup-blocking",
|
|
91
|
+
];
|
|
92
|
+
if (this.config.headless) {
|
|
93
|
+
args.push("--headless=new");
|
|
94
|
+
}
|
|
95
|
+
if (this.config.extensionPath) {
|
|
96
|
+
args.push(`--load-extension=${this.config.extensionPath}`);
|
|
97
|
+
}
|
|
98
|
+
this.chromeProcess = spawn(this.config.chromePath, args, {
|
|
99
|
+
detached: false,
|
|
100
|
+
stdio: "ignore",
|
|
101
|
+
});
|
|
102
|
+
this.chromeProcess.on("error", (err) => {
|
|
103
|
+
console.error("[WebMCP Tester] Chrome error:", err.message);
|
|
104
|
+
});
|
|
105
|
+
// Wait for Chrome to start
|
|
106
|
+
await this.waitForChrome();
|
|
107
|
+
console.log("[WebMCP Tester] Chrome started on port", this.config.port);
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Stop Chrome
|
|
111
|
+
*/
|
|
112
|
+
async stop() {
|
|
113
|
+
if (this.chromeProcess) {
|
|
114
|
+
this.chromeProcess.kill();
|
|
115
|
+
this.chromeProcess = null;
|
|
116
|
+
console.log("[WebMCP Tester] Chrome stopped");
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Wait for Chrome debugging port to be ready
|
|
121
|
+
*/
|
|
122
|
+
async waitForChrome(timeoutMs = 10000) {
|
|
123
|
+
const start = Date.now();
|
|
124
|
+
while (Date.now() - start < timeoutMs) {
|
|
125
|
+
try {
|
|
126
|
+
const response = await fetch(`http://localhost:${this.config.port}/json/version`);
|
|
127
|
+
if (response.ok) {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
// Not ready yet
|
|
133
|
+
}
|
|
134
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
135
|
+
}
|
|
136
|
+
throw new Error("Chrome debugging port not ready");
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Get WebSocket CDP endpoint
|
|
140
|
+
*/
|
|
141
|
+
async getWebSocketUrl() {
|
|
142
|
+
const response = await fetch(`http://localhost:${this.config.port}/json`);
|
|
143
|
+
const tabs = await response.json();
|
|
144
|
+
const tab = tabs[0];
|
|
145
|
+
return tab.webSocketDebuggerUrl;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Run a test scenario
|
|
149
|
+
*/
|
|
150
|
+
async run(scenario) {
|
|
151
|
+
const startTime = Date.now();
|
|
152
|
+
const screenshots = [];
|
|
153
|
+
const consoleLogs = [];
|
|
154
|
+
let stepsExecuted = 0;
|
|
155
|
+
try {
|
|
156
|
+
await this.start();
|
|
157
|
+
// Navigate to URL
|
|
158
|
+
console.log(`[WebMCP Tester] Loading: ${scenario.url}`);
|
|
159
|
+
// Execute steps
|
|
160
|
+
for (const step of scenario.steps) {
|
|
161
|
+
console.log(`[WebMCP Tester] Step: ${step.action} ${step.target || step.field || ""}`);
|
|
162
|
+
stepsExecuted++;
|
|
163
|
+
// Handle different action types
|
|
164
|
+
switch (step.action.toLowerCase()) {
|
|
165
|
+
case "navigate":
|
|
166
|
+
case "go to":
|
|
167
|
+
case "open":
|
|
168
|
+
await this.navigate(scenario.url);
|
|
169
|
+
break;
|
|
170
|
+
case "fill":
|
|
171
|
+
case "type":
|
|
172
|
+
case "enter":
|
|
173
|
+
await this.fill(step.field, step.value, scenario.data);
|
|
174
|
+
break;
|
|
175
|
+
case "click":
|
|
176
|
+
case "press":
|
|
177
|
+
case "select":
|
|
178
|
+
await this.click(step.target);
|
|
179
|
+
break;
|
|
180
|
+
case "submit":
|
|
181
|
+
case "send":
|
|
182
|
+
await this.click(step.target || "button[type=submit], input[type=submit], button:contains(Submit)");
|
|
183
|
+
break;
|
|
184
|
+
case "assert":
|
|
185
|
+
case "verify":
|
|
186
|
+
case "check":
|
|
187
|
+
const found = await this.assert(step.expected);
|
|
188
|
+
if (!found) {
|
|
189
|
+
throw new Error(`Assertion failed: "${step.expected}" not found`);
|
|
190
|
+
}
|
|
191
|
+
break;
|
|
192
|
+
case "wait":
|
|
193
|
+
await new Promise((r) => setTimeout(r, step.waitMs || 1000));
|
|
194
|
+
break;
|
|
195
|
+
case "screenshot":
|
|
196
|
+
const ss = await this.screenshot(scenario.name, stepsExecuted);
|
|
197
|
+
screenshots.push(ss);
|
|
198
|
+
break;
|
|
199
|
+
}
|
|
200
|
+
// Wait after action if specified
|
|
201
|
+
if (step.waitMs) {
|
|
202
|
+
await new Promise((r) => setTimeout(r, step.waitMs));
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
// Take final screenshot
|
|
206
|
+
if (this.config.screenshots) {
|
|
207
|
+
const ss = await this.screenshot(scenario.name, stepsExecuted);
|
|
208
|
+
screenshots.push(ss);
|
|
209
|
+
}
|
|
210
|
+
return {
|
|
211
|
+
name: scenario.name,
|
|
212
|
+
passed: true,
|
|
213
|
+
screenshots,
|
|
214
|
+
stepsExecuted,
|
|
215
|
+
durationMs: Date.now() - startTime,
|
|
216
|
+
consoleLogs,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
catch (error) {
|
|
220
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
221
|
+
// Screenshot on failure
|
|
222
|
+
if (this.config.screenshots) {
|
|
223
|
+
try {
|
|
224
|
+
const ss = await this.screenshot(scenario.name, "error");
|
|
225
|
+
screenshots.push(ss);
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
// Ignore screenshot errors
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
name: scenario.name,
|
|
233
|
+
passed: false,
|
|
234
|
+
error: errorMessage,
|
|
235
|
+
screenshots,
|
|
236
|
+
stepsExecuted,
|
|
237
|
+
durationMs: Date.now() - startTime,
|
|
238
|
+
consoleLogs,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
finally {
|
|
242
|
+
await this.stop();
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Navigate to URL (simplified - actual CDP calls would be more complex)
|
|
247
|
+
*/
|
|
248
|
+
async navigate(url) {
|
|
249
|
+
// In real implementation, use CDP to navigate
|
|
250
|
+
console.log(`[WebMCP Tester] Navigating to: ${url}`);
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Fill form field
|
|
254
|
+
*/
|
|
255
|
+
async fill(field, value, data) {
|
|
256
|
+
const actualValue = data?.[value] || value;
|
|
257
|
+
console.log(`[WebMCP Tester] Filling "${field}" with: ${actualValue}`);
|
|
258
|
+
// In real implementation, use CDP to find and fill element
|
|
259
|
+
// This is where natural language understanding helps
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Click element
|
|
263
|
+
*/
|
|
264
|
+
async click(target) {
|
|
265
|
+
console.log(`[WebMCP Tester] Clicking: ${target}`);
|
|
266
|
+
// In real implementation, use CDP to find and click element
|
|
267
|
+
// Target could be "Submit button", "Next", "Continue", etc.
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Assert content exists
|
|
271
|
+
*/
|
|
272
|
+
async assert(expected) {
|
|
273
|
+
console.log(`[WebMCP Tester] Asserting: ${expected}`);
|
|
274
|
+
// In real implementation, use CDP to check page content
|
|
275
|
+
return true;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Take screenshot
|
|
279
|
+
*/
|
|
280
|
+
async screenshot(name, suffix) {
|
|
281
|
+
const filename = `${name.replace(/[^a-z0-9]/gi, "_")}_${suffix}.png`;
|
|
282
|
+
const path = join(this.config.screenshotDir, filename);
|
|
283
|
+
console.log(`[WebMCP Tester] Screenshot: ${path}`);
|
|
284
|
+
// In real implementation, use CDP to capture screenshot
|
|
285
|
+
return path;
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Get all test results
|
|
289
|
+
*/
|
|
290
|
+
getResults() {
|
|
291
|
+
return this.testResults;
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Clear results
|
|
295
|
+
*/
|
|
296
|
+
clearResults() {
|
|
297
|
+
this.testResults = [];
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Parse natural language test description into TestScenario
|
|
302
|
+
*
|
|
303
|
+
* Examples:
|
|
304
|
+
* ```
|
|
305
|
+
* Test: Registration flow
|
|
306
|
+
* URL: https://app.example.com/register
|
|
307
|
+
*
|
|
308
|
+
* Steps:
|
|
309
|
+
* 1. Fill email with test@example.com
|
|
310
|
+
* 2. Fill name with Test User
|
|
311
|
+
* 3. Click Submit
|
|
312
|
+
* 4. Verify "Welcome" message appears
|
|
313
|
+
* ```
|
|
314
|
+
*/
|
|
315
|
+
export function parseNaturalLanguageTest(input) {
|
|
316
|
+
const lines = input.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
317
|
+
const scenario = {
|
|
318
|
+
name: "",
|
|
319
|
+
url: "",
|
|
320
|
+
steps: [],
|
|
321
|
+
};
|
|
322
|
+
for (const line of lines) {
|
|
323
|
+
// Parse test name
|
|
324
|
+
const nameMatch = line.match(/^(?:test|name):\s*(.+)/i);
|
|
325
|
+
if (nameMatch) {
|
|
326
|
+
scenario.name = nameMatch[1];
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
// Parse URL
|
|
330
|
+
const urlMatch = line.match(/^(?:url|site|start):\s*(.+)/i);
|
|
331
|
+
if (urlMatch) {
|
|
332
|
+
scenario.url = urlMatch[1];
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
// Parse numbered steps
|
|
336
|
+
const stepMatch = line.match(/^\d+[.)]\s*(.+)/);
|
|
337
|
+
if (stepMatch) {
|
|
338
|
+
const stepText = stepMatch[1];
|
|
339
|
+
scenario.steps.push(parseStep(stepText));
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return scenario;
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* Parse single step text
|
|
346
|
+
*/
|
|
347
|
+
function parseStep(text) {
|
|
348
|
+
const lower = text.toLowerCase();
|
|
349
|
+
// Click actions
|
|
350
|
+
if (/^click\s+/i.test(text)) {
|
|
351
|
+
const target = text.replace(/^click\s+/i, "");
|
|
352
|
+
return { action: "click", target };
|
|
353
|
+
}
|
|
354
|
+
// Fill actions
|
|
355
|
+
const fillMatch = text.match(/^(?:fill|type|enter)\s+"?([^"]+)"?\s+with\s+(.+)/i);
|
|
356
|
+
if (fillMatch) {
|
|
357
|
+
return {
|
|
358
|
+
action: "fill",
|
|
359
|
+
field: fillMatch[1],
|
|
360
|
+
value: fillMatch[2],
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
// Submit
|
|
364
|
+
if (/submit|send|next|continue/i.test(text)) {
|
|
365
|
+
return { action: "submit" };
|
|
366
|
+
}
|
|
367
|
+
// Assert/Verify
|
|
368
|
+
const assertMatch = text.match(/^(?:assert|verify|check)\s+(?:that\s+)?(.+?)(?:\s+(?:appears?|shows?|exists?))?$/i);
|
|
369
|
+
if (assertMatch) {
|
|
370
|
+
return { action: "assert", expected: assertMatch[1] };
|
|
371
|
+
}
|
|
372
|
+
// Wait
|
|
373
|
+
const waitMatch = text.match(/wait\s+(\d+)\s*(s|ms|seconds?|milliseconds?)/i);
|
|
374
|
+
if (waitMatch) {
|
|
375
|
+
const value = parseInt(waitMatch[1]);
|
|
376
|
+
const unit = waitMatch[2].toLowerCase();
|
|
377
|
+
const ms = unit.startsWith("s") ? value * 1000 : value;
|
|
378
|
+
return { action: "wait", waitMs: ms };
|
|
379
|
+
}
|
|
380
|
+
// Screenshot
|
|
381
|
+
if (/screenshot/i.test(text)) {
|
|
382
|
+
return { action: "screenshot" };
|
|
383
|
+
}
|
|
384
|
+
// Default
|
|
385
|
+
return { action: text };
|
|
386
|
+
}
|
|
387
|
+
export async function runFromCLI(config = {}) {
|
|
388
|
+
if (!config.scenarioFile) {
|
|
389
|
+
console.error("Usage: webmcp-tester --scenario <file.md>");
|
|
390
|
+
process.exit(1);
|
|
391
|
+
}
|
|
392
|
+
const content = await readFile(config.scenarioFile, "utf8");
|
|
393
|
+
const scenario = parseNaturalLanguageTest(content);
|
|
394
|
+
if (!scenario.name || !scenario.url) {
|
|
395
|
+
console.error("Error: Test must have 'name' and 'url'");
|
|
396
|
+
process.exit(1);
|
|
397
|
+
}
|
|
398
|
+
const tester = new WebMCPTester({
|
|
399
|
+
headless: config.headless ?? true,
|
|
400
|
+
screenshotDir: config.outputDir,
|
|
401
|
+
});
|
|
402
|
+
console.log(`[WebMCP Tester] Running: ${scenario.name}`);
|
|
403
|
+
const result = await tester.run(scenario);
|
|
404
|
+
if (result.passed) {
|
|
405
|
+
console.log(`[WebMCP Tester] PASS - ${result.durationMs}ms, ${result.stepsExecuted} steps`);
|
|
406
|
+
process.exit(0);
|
|
407
|
+
}
|
|
408
|
+
else {
|
|
409
|
+
console.error(`[WebMCP Tester] FAIL - ${result.error}`);
|
|
410
|
+
process.exit(1);
|
|
411
|
+
}
|
|
412
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@pi-harness/webmcp-tester",
|
|
3
|
+
"version": "1.1.183",
|
|
4
|
+
"description": "WebMCP-powered E2E testing for pi-harness - natural language test scenarios",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"import": "./dist/index.js",
|
|
11
|
+
"types": "./dist/index.d.ts"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"scripts": {
|
|
15
|
+
"build": "tsc --project tsconfig.json",
|
|
16
|
+
"dev": "tsc --watch"
|
|
17
|
+
},
|
|
18
|
+
"keywords": [
|
|
19
|
+
"webmcp",
|
|
20
|
+
"testing",
|
|
21
|
+
"e2e",
|
|
22
|
+
"automation"
|
|
23
|
+
],
|
|
24
|
+
"peerDependencies": {
|
|
25
|
+
"typescript": "^5.0.0"
|
|
26
|
+
}
|
|
27
|
+
}
|