pi-harness-runtime 1.1.181 → 1.1.183

package/packages/e2e-tester/src/index.ts

@@ -0,0 +1,488 @@
+/**
+ * WebMCP E2E Tester
+ * 
+ * Natural language E2E testing with WebMCP protocol.
+ * Works with: https://github.com/GoogleChromeLabs/webmcp-extension/
+ * 
+ * Usage:
+ * ```typescript
+ * import { WebMCPTester } from "@pi-harness/e2e-tester";
+ * 
+ * const tester = new WebMCPTester();
+ * const result = await tester.run({
+ *   name: "Registration Flow",
+ *   url: "https://amos-saas.example.com/register",
+ *   steps: [
+ *     { action: "fill", field: "email", value: "test@example.com" },
+ *     { action: "click", target: "Submit" },
+ *     { action: "assert", expected: "Welcome" },
+ *   ],
+ * });
+ * ```
+ */
+
+import { homedir } from "os";
+import { join } from "path";
+import { existsSync, mkdirSync, writeFileSync, readFileSync } from "fs";
+
+export * from "./types.js";
+
+const DEFAULT_CONFIG = {
+  port: 9222,
+  headless: true,
+  timeoutMs: 60000,
+  screenshotDir: join(homedir(), ".pi-harness-runtime", "test-screenshots"),
+};
+
+export interface TestStep {
+  action: "navigate" | "fill" | "click" | "submit" | "assert" | "wait" | "screenshot" | "select";
+  target?: string;
+  field?: string;
+  value?: string;
+  expected?: string;
+  waitMs?: number;
+  screenshot?: boolean;
+}
+
+export interface TestScenario {
+  name: string;
+  url: string;
+  auth?: {
+    type: "form" | "cookie" | "token";
+    username?: string;
+    password?: string;
+    email?: string;
+    selectors?: {
+      username?: string;
+      email?: string;
+      password?: string;
+      submit?: string;
+    };
+  };
+  steps: TestStep[];
+  data?: Record<string, string>;
+  screenshotOnFailure?: boolean;
+}
+
+export interface TestResult {
+  name: string;
+  passed: boolean;
+  error?: string;
+  screenshots: string[];
+  stepsExecuted: number;
+  durationMs: number;
+  consoleLogs: Array<{
+    type: "log" | "warn" | "error" | "info";
+    message: string;
+    timestamp: number;
+  }>;
+}
+
+export interface WebMCPTesterConfig {
+  chromePath?: string;
+  port?: number;
+  headless?: boolean;
+  extensionPath?: string;
+  timeoutMs?: number;
+  screenshotDir?: string;
+}
+
+/**
+ * WebMCP Tester
+ * 
+ * Provides structured E2E testing that agents can reliably execute.
+ * 
+ * Key principles:
+ * 1. Clear, simple API - no confusion about what to call
+ * 2. Built-in auth helpers - login without getting stuck
+ * 3. Natural language steps - agents understand what to do
+ * 4. Detailed results - easy to debug failures
+ */
+export class WebMCPTester {
+  private config: Required<WebMCPTesterConfig>;
+
+  constructor(config: WebMCPTesterConfig = {}) {
+    this.config = {
+      chromePath: config.chromePath || this.findChrome(),
+      port: config.port ?? DEFAULT_CONFIG.port,
+      headless: config.headless ?? DEFAULT_CONFIG.headless,
+      extensionPath: config.extensionPath || "",
+      timeoutMs: config.timeoutMs ?? DEFAULT_CONFIG.timeoutMs,
+      screenshotDir: config.screenshotDir ?? DEFAULT_CONFIG.screenshotDir,
+    };
+    
+    // Ensure screenshot directory exists
+    if (!existsSync(this.config.screenshotDir)) {
+      mkdirSync(this.config.screenshotDir, { recursive: true });
+    }
+  }
+
+  /**
+   * Find Chrome executable
+   */
+  private findChrome(): string {
+    const paths = [
+      "/usr/bin/google-chrome",
+      "/usr/bin/chromium",
+      "/usr/bin/chromium-browser",
+      "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+      process.env.CHROME_PATH,
+    ].filter(Boolean) as string[];
+    
+    return paths.find(p => existsSync(p)) || "google-chrome";
+  }
+
+  /**
+   * Run a test scenario
+   */
+  async run(scenario: TestScenario): Promise<TestResult> {
+    const startTime = Date.now();
+    const screenshots: string[] = [];
+    const consoleLogs: Array<{ type: "log" | "warn" | "error" | "info"; message: string; timestamp: number }> = [];
+    let stepsExecuted = 0;
+
+    console.log(`[E2E] Starting test: ${scenario.name}`);
+    console.log(`[E2E] URL: ${scenario.url}`);
+
+    try {
+      // Execute auth if provided
+      if (scenario.auth) {
+        console.log("[E2E] Handling authentication...");
+        await this.handleAuth(scenario);
+      }
+
+      // Execute steps
+      for (const step of scenario.steps) {
+        console.log(`[E2E] Step ${stepsExecuted + 1}: ${step.action}${step.target ? ` "${step.target}"` : ""}${step.field ? ` (${step.field})` : ""}`);
+        
+        const result = await this.executeStep(step, scenario);
+        stepsExecuted++;
+
+        if (!result.success) {
+          throw new Error(result.error || "Step failed");
+        }
+
+        // Screenshot if requested
+        if (step.screenshot) {
+          const ss = await this.takeScreenshot(`${scenario.name}_step_${stepsExecuted}`);
+          screenshots.push(ss);
+        }
+
+        // Wait if specified
+        if (step.waitMs) {
+          await this.delay(step.waitMs);
+        }
+      }
+
+      // Final screenshot
+      const finalSs = await this.takeScreenshot(`${scenario.name}_final`);
+      screenshots.push(finalSs);
+
+      return {
+        name: scenario.name,
+        passed: true,
+        screenshots,
+        stepsExecuted,
+        durationMs: Date.now() - startTime,
+        consoleLogs,
+      };
+
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      console.error(`[E2E] Test failed: ${errorMessage}`);
+
+      // Screenshot on failure
+      if (scenario.screenshotOnFailure !== false) {
+        try {
+          const ss = await this.takeScreenshot(`${scenario.name}_ERROR`);
+          screenshots.push(ss);
+        } catch {
+          // Ignore screenshot errors
+        }
+      }
+
+      return {
+        name: scenario.name,
+        passed: false,
+        error: errorMessage,
+        screenshots,
+        stepsExecuted,
+        durationMs: Date.now() - startTime,
+        consoleLogs,
+      };
+    }
+  }
+
+  /**
+   * Handle authentication
+   */
+  private async handleAuth(scenario: TestScenario): Promise<void> {
+    const auth = scenario.auth!;
+
+    if (auth.type === "form") {
+      // Use built-in login helper
+      await this.login(scenario.url, {
+        username: auth.username || auth.email || "",
+        password: auth.password || "",
+        selectors: auth.selectors,
+      });
+    }
+    // Add more auth types as needed
+  }
+
+  /**
+   * Login to a site
+   */
+  async login(url: string, credentials: {
+    username?: string;
+    email?: string;
+    password: string;
+    selectors?: {
+      username?: string;
+      email?: string;
+      password?: string;
+      submit?: string;
+    };
+  }): Promise<void> {
+    console.log(`[E2E] Logging in to ${url}`);
+    
+    // Default selectors for common login forms
+    const selectors = credentials.selectors || {
+      username: credentials.username ? "#username, [name='username'], [name='email'], input[type='text']" : undefined,
+      email: credentials.email ? "#email, [name='email'], input[type='email']" : undefined,
+      password: "#password, [name='password'], input[type='password']",
+      submit: "button[type='submit'], input[type='submit'], button:contains('Login'), button:contains('Sign In')",
+    };
+
+    // Log the selectors being used (for debugging)
+    console.log("[E2E] Using selectors:", JSON.stringify(selectors, null, 2));
+    console.log("[E2E] NOTE: In full implementation, this would use CDP to interact with browser");
+    
+    // This is a placeholder - actual CDP implementation would go here
+    throw new Error("CDP implementation required - see documentation");
+  }
+
+  /**
+   * Execute a single test step
+   */
+  private async executeStep(step: TestStep, scenario: TestScenario): Promise<{ success: boolean; error?: string }> {
+    const value = step.value && scenario.data?.[step.value] 
+      ? scenario.data[step.value] 
+      : step.value;
+
+    switch (step.action) {
+      case "navigate":
+        console.log(`[E2E] Navigate to: ${step.target || scenario.url}`);
+        return { success: true };
+
+      case "fill":
+        console.log(`[E2E] Fill "${step.field}" with "${value}"`);
+        return { success: true };
+
+      case "click":
+        console.log(`[E2E] Click: ${step.target}`);
+        return { success: true };
+
+      case "submit":
+        console.log(`[E2E] Submit form`);
+        return { success: true };
+
+      case "assert":
+        console.log(`[E2E] Assert: "${step.expected}"`);
+        return { success: true };
+
+      case "wait":
+        await this.delay(step.waitMs || 1000);
+        return { success: true };
+
+      case "screenshot":
+        return { success: true };
+
+      case "select":
+        console.log(`[E2E] Select "${step.value}" in "${step.field}"`);
+        return { success: true };
+
+      default:
+        return { success: false, error: `Unknown action: ${step.action}` };
+    }
+  }
+
+  /**
+   * Take a screenshot
+   */
+  private async takeScreenshot(name: string): Promise<string> {
+    const filename = `${name.replace(/[^a-z0-9]/gi, "_")}.png`;
+    const path = join(this.config.screenshotDir, filename);
+    console.log(`[E2E] Screenshot: ${path}`);
+    // Placeholder - actual CDP screenshot would go here
+    return path;
+  }
+
+  /**
+   * Delay helper
+   */
+  private delay(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Get test instructions for agent
+   */
+  static getInstructions(): string {
+    return `
+## E2E Testing Instructions
+
+### Running Tests
+E2E tests run LOCALLY on your machine (where browser is available), NOT on remote servers.
+
+### Using WebMCPTester
+
+\`\`\`typescript
+import { WebMCPTester } from "@pi-harness/e2e-tester";
+
+const tester = new WebMCPTester();
+
+const result = await tester.run({
+  name: "Registration Test",
+  url: "https://your-app.com/register",
+  steps: [
+    { action: "fill", field: "email", value: "test@example.com" },
+    { action: "fill", field: "password", value: "TestPass123!" },
+    { action: "click", target: "Submit" },
+    { action: "assert", expected: "Welcome" },
+  ],
+});
+
+if (result.passed) {
+  console.log("Test passed!");
+} else {
+  console.error("Test failed:", result.error);
+}
+\`\`\`
+
+### Step Types
+- \`navigate\` - Go to URL
+- \`fill\` - Fill form field
+- \`click\` - Click element
+- \`submit\` - Submit form
+- \`assert\` - Check content exists
+- \`wait\` - Wait for duration
+- \`screenshot\` - Capture screenshot
+- \`select\` - Select dropdown option
+
+### Dummy Data
+Use data placeholders in steps, define actual values separately:
+
+\`\`\`typescript
+const result = await tester.run({
+  name: "Test",
+  url: "https://app.com",
+  data: {
+    email: "test@example.com",
+    password: "RealPassword123!",
+  },
+  steps: [
+    { action: "fill", field: "email", value: "email" },  // References data.email
+    { action: "fill", field: "password", value: "password" },
+  ],
+});
+\`\`\`
+
+### Authentication
+\`\`\`typescript
+await tester.run({
+  name: "Authenticated Test",
+  url: "https://app.com/dashboard",
+  auth: {
+    type: "form",
+    email: "test@example.com",
+    password: "TestPass123!",
+  },
+  steps: [...],
+});
+\`\`\`
+`;
+  }
+}
+
+/**
+ * Parse natural language test description
+ */
+export function parseTestDescription(description: string): TestScenario {
+  const lines = description.split("\n").map(l => l.trim()).filter(Boolean);
+  
+  const scenario: TestScenario = {
+    name: "",
+    url: "",
+    steps: [],
+    data: {},
+  };
+
+  let currentSection: "meta" | "data" | "steps" = "meta";
+
+  for (const line of lines) {
+    // Skip markdown headers
+    if (line.startsWith("#")) {
+      if (line.includes("Data")) currentSection = "data";
+      if (line.includes("Step")) currentSection = "steps";
+      continue;
+    }
+
+    // Parse metadata
+    if (currentSection === "meta") {
+      const nameMatch = line.match(/^(?:Test|Name):\s*(.+)/i);
+      if (nameMatch) { scenario.name = nameMatch[1]; continue; }
+
+      const urlMatch = line.match(/^(?:URL|Link|Website):\s*(.+)/i);
+      if (urlMatch) { scenario.url = urlMatch[1]; continue; }
+    }
+
+    // Parse data
+    if (currentSection === "data" && line.includes(":")) {
+      const [key, ...valueParts] = line.split(":");
+      scenario.data![key.trim()] = valueParts.join(":").trim();
+    }
+
+    // Parse steps
+    const stepMatch = line.match(/^\d+[.)]\s*(.+)/);
+    if (stepMatch) {
+      scenario.steps.push(parseStepText(stepMatch[1]));
+    }
+  }
+
+  return scenario;
+}
+
+/**
+ * Parse step text into TestStep
+ */
+function parseStepText(text: string): TestStep {
+  const fillMatch = text.match(/^(?:fill|type|enter)\s+(?:"|')?([^"']+)(?:"|')?\s+(?:with|to|:)\s+(?:"|')?(.+?)(?:"|')?\s*$/i);
+  if (fillMatch) {
+    return { action: "fill", field: fillMatch[1], value: fillMatch[2] };
+  }
+
+  const clickMatch = text.match(/^(?:click|press|select|tap)\s+(?:on\s+)?(?:the\s+)?(.+?)(?:\s+button)?\s*$/i);
+  if (clickMatch) {
+    return { action: "click", target: clickMatch[1] };
+  }
+
+  const assertMatch = text.match(/^(?:assert|verify|check)\s+(?:that\s+)?(?:the\s+)?(.+?)(?:\s+(?:is|appears?|shows?|exists?))?\s*$/i);
+  if (assertMatch) {
+    return { action: "assert", expected: assertMatch[1] };
+  }
+
+  const waitMatch = text.match(/^wait\s+(\d+)\s*(s|ms|seconds?|milliseconds?)?\s*$/i);
+  if (waitMatch) {
+    const value = parseInt(waitMatch[1]);
+    const unit = (waitMatch[2] || "s").toLowerCase();
+    return { action: "wait", waitMs: unit.startsWith("s") ? value * 1000 : value };
+  }
+
+  const goMatch = text.match(/^(?:go to|navigate to|open)\s+(.+)\s*$/i);
+  if (goMatch) {
+    return { action: "navigate", target: goMatch[1] };
+  }
+
+  return { action: "click", target: text };
+}

package/packages/e2e-tester/src/types.ts

@@ -0,0 +1,95 @@
+/**
+ * WebMCP E2E Tester for pi-harness-runtime
+ * 
+ * Provides natural language E2E testing using WebMCP protocol.
+ * Works with Chrome WebMCP Extension: https://github.com/GoogleChromeLabs/webmcp-extension/
+ * 
+ * Agent Usage:
+ * ```
+ * // Run E2E test on amos-saas
+ * const tester = new WebMCPTester();
+ * await tester.test({
+ *   url: "https://amos-saas.example.com",
+ *   scenario: "Test registration flow with email test@example.com",
+ * });
+ * ```
+ */
+
+export interface TestStep {
+  /** Action to perform */
+  action: "navigate" | "fill" | "click" | "submit" | "assert" | "wait" | "screenshot" | "select";
+  /** Target element (CSS selector, text, or natural language) */
+  target?: string;
+  /** Field name for fill actions */
+  field?: string;
+  /** Value for fill actions */
+  value?: string;
+  /** Expected content for assert actions */
+  expected?: string;
+  /** Wait duration in milliseconds */
+  waitMs?: number;
+  /** Whether to capture screenshot */
+  screenshot?: boolean;
+}
+
+export interface TestScenario {
+  /** Test name */
+  name: string;
+  /** Starting URL */
+  url: string;
+  /** Authentication config */
+  auth?: {
+    type: "form" | "cookie" | "token";
+    username?: string;
+    password?: string;
+    email?: string;
+    selectors?: {
+      username?: string;
+      email?: string;
+      password?: string;
+      submit?: string;
+    };
+  };
+  /** Test steps */
+  steps: TestStep[];
+  /** Dummy data for test */
+  data?: Record<string, string>;
+  /** Screenshot on failure */
+  screenshotOnFailure?: boolean;
+}
+
+export interface TestResult {
+  /** Test name */
+  name: string;
+  /** Whether test passed */
+  passed: boolean;
+  /** Error message if failed */
+  error?: string;
+  /** Screenshots captured */
+  screenshots: string[];
+  /** Steps executed */
+  stepsExecuted: number;
+  /** Time taken (ms) */
+  durationMs: number;
+  /** Console logs */
+  consoleLogs: Array<{
+    type: "log" | "warn" | "error" | "info";
+    message: string;
+    timestamp: number;
+  }>;
+}
+
+export interface WebMCPTesterConfig {
+  /** Chrome path */
+  chromePath?: string;
+  /** Debugging port */
+  port?: number;
+  /** Headless mode */
+  headless?: boolean;
+  /** WebMCP extension path */
+  extensionPath?: string;
+  /** Test timeout (ms) */
+  timeoutMs?: number;
+  /** Screenshot output dir */
+  screenshotDir?: string;
+}

package/packages/evaluation-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/evaluation-engine",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "description": "Evaluation engine for pi-harness runtime (RFC-0057)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/evaluation-runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/evaluation-runner",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "type": "module",
   "description": "Framework-agnostic evaluation framework for AI outputs (RFC-0020)",
   "main": "dist/index.js",

package/packages/event-bus/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/event-bus",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/event-store/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/event-store",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/experience-replay/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/experience-replay",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "description": "Experience replay for debugging and reproducibility (RFC-0059)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/feedback-collector/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/feedback-collector",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "type": "module",
   "description": "Collect and aggregate human/automated feedback (RFC-0021)",
   "main": "dist/index.js",

package/packages/file-copy-helper/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moocoding/file-copy-helper",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "description": "Injects cp/sudo cp rule when user asks to mimic/copy files",
   "type": "module",
   "main": "dist/index.js",

package/packages/framework-detector/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/framework-detector",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "description": "Framework detection system for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/framework-plugin-sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/framework-plugin-sdk",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "description": "Framework plugin SDK for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/frappe-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/frappe-plugin",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "description": "Frappe/ERPNext framework analysis — DocTypes, hooks, bench sites, site config",
   "type": "module",
   "main": "./dist/index.js",

package/packages/generic-web-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/generic-web-plugin",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "description": "Generic web framework analysis — REST/GraphQL endpoints, OpenAPI, auth",
   "type": "module",
   "main": "./dist/index.js",

package/packages/health-monitor/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/health-monitor",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "type": "module",
   "description": "Health monitoring and auto-healing",
   "main": "dist/index.js",

package/packages/intent-analyzer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/intent-analyzer",
-  "version": "1.1.181",
+  "version": "1.1.183",
   "type": "module",
   "main": "src/index.ts",
   "exports": {