supipowers 0.4.0 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "supipowers",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "OMP-native workflow extension inspired by Superpowers.",
   "type": "module",
   "scripts": {

package/skills/qa-strategy/SKILL.md

@@ -1,32 +1,114 @@
 ---
 name: qa-strategy
-description: QA test planning for comprehensive coverage
+description: E2E product testing strategy using Playwright — flow-based, autonomous, close to human interaction
 ---
 
-# QA Strategy Skill
+# E2E Product Testing Strategy
 
-## Test Pyramid
+## Core Principle
 
-1. **Unit tests**: Fast, isolated, cover individual functions
-2. **Integration tests**: Test component interactions
-3. **E2E tests**: Test user-facing flows end-to-end
+Test the product the way a user uses it. Every test simulates a real user flow — navigating, clicking, filling forms, waiting for responses. If a human wouldn't do it, don't test it here.
 
-## When to Write What
+**This is NOT for unit or integration tests.** This pipeline tests complete user journeys through the running application.
 
-- New function → unit test
-- New API endpoint → integration test
-- New user flow → E2E test
-- Bug fix → regression test at the appropriate level
+## Flow Discovery
 
-## Coverage Priorities
+Before writing tests, understand what the product does:
 
-Focus testing effort on:
-1. Business logic (highest value)
-2. Error handling paths
-3. Edge cases in input validation
-4. Integration points (API boundaries, DB queries)
+1. **Scan routes and pages** — every URL a user can visit is a potential flow entry point
+2. **Identify forms** — login, signup, search, create, edit — these are high-value interaction points
+3. **Map navigation** — how does a user get from page A to page B? What's the happy path?
+4. **Find auth boundaries** — what's public vs protected? Test both sides
+5. **Check CRUD operations** — can you create, read, update, delete the core entities?
 
-Don't test:
-- Framework boilerplate
-- Simple getters/setters
-- Third-party library behavior
+## Flow Prioritization
+
+| Priority | Description | Examples |
+|----------|-------------|---------|
+| **Critical** | Revenue or access blocking | Login, checkout, payment |
+| **High** | Core product value | Create/edit main entities, dashboard |
+| **Medium** | Secondary features | Settings, profile, search |
+| **Low** | Nice-to-have | Theme toggle, tooltips |
+
+Test critical and high flows first. Skip low flows if hitting the token budget.
+
+## Playwright Best Practices
+
+### Locators (prefer resilient selectors)
+
+```typescript
+// GOOD — role-based, resilient to styling changes
+page.getByRole('button', { name: 'Submit' })
+page.getByLabel('Email')
+page.getByText('Welcome back')
+page.getByTestId('user-avatar')
+
+// BAD — fragile, breaks on refactoring
+page.locator('.btn-primary')
+page.locator('#submit-btn')
+page.locator('div > form > button:nth-child(2)')
+```
+
+### Assertions
+
+```typescript
+// Wait for navigation
+await expect(page).toHaveURL('/dashboard');
+
+// Wait for element visibility
+await expect(page.getByText('Success')).toBeVisible();
+
+// Wait for element to disappear (loading states)
+await expect(page.getByText('Loading...')).not.toBeVisible();
+```
+
+### Waiting
+
+```typescript
+// GOOD — wait for specific condition
+await page.waitForResponse(resp => resp.url().includes('/api/users'));
+await page.waitForLoadState('networkidle');
+
+// BAD — arbitrary delays
+await page.waitForTimeout(3000);
+```
+
+### Test Structure
+
+One flow per file. Each test in the flow tests a step or variant:
+
+```typescript
+test.describe('Checkout flow', () => {
+  test('adds item to cart', async ({ page }) => { ... });
+  test('fills shipping info', async ({ page }) => { ... });
+  test('completes payment', async ({ page }) => { ... });
+  test('shows confirmation', async ({ page }) => { ... });
+});
+```
+
+## What Makes a Good E2E Test
+
+| Quality | Good | Bad |
+|---------|------|-----|
+| **User-centric** | Tests what a user would do | Tests implementation details |
+| **Independent** | Each test can run alone | Tests depend on previous test state |
+| **Resilient** | Uses role/label selectors | Uses CSS classes or DOM structure |
+| **Fast-failing** | Fails clearly on the broken step | Fails on a timeout with no context |
+| **Readable** | Test name describes the user action | Test name is a technical description |
+
+## Common Pitfalls
+
+1. **Testing internal state** — don't check Redux store, localStorage, or cookies directly. Test what the user sees.
+2. **Flaky waits** — use `waitForResponse` or `waitForSelector`, never `waitForTimeout`.
+3. **Shared state** — each test should set up its own state. Don't rely on test execution order.
+4. **Over-testing** — one flow per critical path. Don't test every permutation of a form.
+5. **Ignoring error states** — test what happens when the API returns an error, the network is slow, or the user enters invalid data.
+
+## Regression Analysis
+
+When a previously-passing test fails:
+
+1. **Read the error** — what element wasn't found? What URL didn't match?
+2. **Check if the app changed** — did a route move? Did a button get renamed?
+3. **Distinguish bug from change** — if the app intentionally changed, the test needs updating. If not, it's a regression.
+4. **Record the finding** — update the flow matrix with the new status and reasoning.

package/src/commands/qa.ts

@@ -1,184 +1,268 @@
 import type { ExtensionAPI } from "@oh-my-pi/pi-coding-agent";
-import { detectAndCache } from "../qa/detector.js";
-import { notifyInfo, notifyError } from "../notifications/renderer.js";
-import { findActiveSession, findSessionWithFailures } from "../storage/qa-sessions.js";
-import {
-  createNewSession,
-  advancePhase,
-  getFailedTests,
-  getNextPhase,
-  getPhaseStatusLine,
-} from "../qa/session.js";
-import { buildDiscoveryPrompt } from "../qa/phases/discovery.js";
-import { buildMatrixPrompt } from "../qa/phases/matrix.js";
-import { buildExecutionPrompt } from "../qa/phases/execution.js";
-import { buildReportingPrompt } from "../qa/phases/reporting.js";
-import type { QaPhase, QaSessionLedger } from "../types.js";
-
-const PHASE_LABELS: Record<QaPhase, string> = {
-  discovery: "Discovery — Scan for test cases",
-  matrix: "Matrix — Build traceability matrix",
-  execution: "Execution — Run tests",
-  reporting: "Reporting — Generate summary",
-};
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { notifyInfo, notifyError, notifyWarning } from "../notifications/renderer.js";
+import { loadE2eQaConfig, saveE2eQaConfig, DEFAULT_E2E_QA_CONFIG } from "../qa/config.js";
+import { loadE2eMatrix } from "../qa/matrix.js";
+import { createNewE2eSession } from "../qa/session.js";
+import { buildE2eOrchestratorPrompt } from "../qa/prompt-builder.js";
+import { findActiveSession, getSessionDir } from "../storage/qa-sessions.js";
+import type { E2eQaConfig, AppType, E2eRegression } from "../qa/types.js";
 
-export function registerQaCommand(pi: ExtensionAPI): void {
-  pi.registerCommand("supi:qa", {
-    description: "Run QA pipeline with session management (discovery → matrix → execution → reporting)",
-    async handler(args, ctx) {
-      const framework = detectAndCache(ctx.cwd);
-
-      if (!framework) {
-        notifyError(
-          ctx,
-          "No test framework detected",
-          "Configure manually via /supi:config"
-        );
-        return;
-      }
+function getScriptsDir(): string {
+  return path.join(path.dirname(new URL(import.meta.url).pathname), "..", "qa", "scripts");
+}
 
-      // ── Step 1: Session selection ──────────────────────────────────
-      let ledger: QaSessionLedger | null = null;
+function findSkillPath(skillName: string): string | null {
+  const candidates = [
+    path.join(process.cwd(), "skills", skillName, "SKILL.md"),
+    path.join(path.dirname(new URL(import.meta.url).pathname), "..", "..", "skills", skillName, "SKILL.md"),
+  ];
+  for (const p of candidates) {
+    if (fs.existsSync(p)) return p;
+  }
+  return null;
+}
 
-      const activeSession = findActiveSession(ctx.cwd);
-      const failedSession = findSessionWithFailures(ctx.cwd);
+const APP_TYPE_OPTIONS = [
+  "nextjs-app — Next.js App Router",
+  "nextjs-pages — Next.js Pages Router",
+  "react-router — React with React Router",
+  "vite — Vite-based app",
+  "express — Express.js server",
+  "generic — Other web app",
+];
 
-      if (ctx.hasUI && !args?.trim()) {
-        const sessionOptions: string[] = [];
+const BROWSER_OPTIONS = [
+  "chromium (recommended)",
+  "firefox",
+  "webkit",
+];
 
-        if (failedSession) {
-          const failCount = failedSession.results.filter((r) => r.status === "fail").length;
-          sessionOptions.push(`Resume ${failedSession.id} (${failCount} failed test${failCount !== 1 ? "s" : ""})`);
-        } else if (activeSession) {
-          const next = getNextPhase(activeSession);
-          sessionOptions.push(`Resume ${activeSession.id} (${next ?? "all phases done"} pending)`);
-        }
+const RETRY_OPTIONS = [
+  "1",
+  "2 (recommended)",
+  "3",
+];
 
-        sessionOptions.push("Start new session");
+async function runSetupWizard(
+  ctx: any,
+  detectedAppType: string | null,
+  detectedDevCommand: string | null,
+  detectedPort: number | null,
+): Promise<E2eQaConfig | null> {
+  // 1. App type
+  const appTypeChoice = await ctx.ui.select(
+    "App type",
+    APP_TYPE_OPTIONS,
+    { helpText: detectedAppType ? `Auto-detected: ${detectedAppType}` : "Select your web app framework" },
+  );
+  if (!appTypeChoice) return null;
+  const appType = appTypeChoice.split(" ")[0] as AppType;
 
-        if (sessionOptions.length > 1) {
-          const choice = await ctx.ui.select(
-            "QA Session",
-            sessionOptions,
-            { helpText: "Select session · Esc to cancel" },
-          );
-          if (!choice) return;
+  // 2. Dev command
+  const defaultDev = detectedDevCommand || "npm run dev";
+  const devCommand = await ctx.ui.input(
+    "Dev server command",
+    defaultDev,
+    { helpText: "Command to start your development server" },
+  );
+  if (devCommand === undefined) return null;
 
-          if (choice.startsWith("Resume")) {
-            ledger = failedSession ?? activeSession;
-          }
-        }
-      }
+  // 3. Port
+  const defaultPort = String(detectedPort || 3000);
+  const portStr = await ctx.ui.input(
+    "Dev server port",
+    defaultPort,
+    { helpText: "Port your dev server runs on" },
+  );
+  if (portStr === undefined) return null;
+  const port = parseInt(portStr, 10) || 3000;
 
-      // Create new session if none selected
-      if (!ledger) {
-        ledger = createNewSession(ctx.cwd, framework.name);
-        notifyInfo(ctx, "QA session created", ledger.id);
-      }
+  // 4. Browser
+  const browserChoice = await ctx.ui.select(
+    "Browser for E2E tests",
+    BROWSER_OPTIONS,
+    { helpText: "Playwright browser to use" },
+  );
+  if (!browserChoice) return null;
+  const browser = browserChoice.split(" ")[0] as "chromium" | "firefox" | "webkit";
 
-      // ── Step 2: Phase selection ────────────────────────────────────
-      type PhaseAction =
-        | { type: "run-phase"; phase: QaPhase }
-        | { type: "rerun-failed" };
+  // 5. Max retries
+  const retryChoice = await ctx.ui.select(
+    "Max test retries",
+    RETRY_OPTIONS,
+    { helpText: "How many times to retry failing tests" },
+  );
+  if (!retryChoice) return null;
+  const maxRetries = parseInt(retryChoice, 10);
 
-      let action: PhaseAction | null = null;
-      const nextPhase = getNextPhase(ledger);
-      const failedTests = getFailedTests(ledger);
+  return {
+    app: {
+      type: appType,
+      devCommand: devCommand || defaultDev,
+      port,
+      baseUrl: `http://localhost:${port}`,
+    },
+    playwright: {
+      browser,
+      headless: true,
+      timeout: 30000,
+    },
+    execution: {
+      maxRetries,
+      maxFlows: 20,
+    },
+  };
+}
 
-      if (ctx.hasUI && !args?.trim()) {
-        const phaseOptions: string[] = [];
+export function registerQaCommand(pi: ExtensionAPI): void {
+  pi.registerCommand("supi:qa", {
+    description: "Run autonomous E2E product testing pipeline with playwright",
+    async handler(args, ctx) {
+      const scriptsDir = getScriptsDir();
 
-        // Offer re-run failed if there are failures
-        if (failedTests.length > 0) {
-          phaseOptions.push(`Re-run ${failedTests.length} failed test${failedTests.length !== 1 ? "s" : ""} only`);
+      // ── Step 1: Detect app type ─────────────────────────────────────
+      let detectedType: string | null = null;
+      let detectedDevCommand: string | null = null;
+      let detectedPort: number | null = null;
+
+      try {
+        const detectResult = await pi.exec("bash", [
+          path.join(scriptsDir, "detect-app-type.sh"),
+          ctx.cwd,
+        ], { cwd: ctx.cwd });
+
+        if (detectResult.code === 0) {
+          const detected = JSON.parse(detectResult.stdout.trim());
+          detectedType = detected.type;
+          detectedDevCommand = detected.devCommand;
+          detectedPort = detected.port;
         }
+      } catch { /* proceed without detection */ }
+
+      // ── Step 2: Ensure playwright ────────────────────────────────────
+      try {
+        const pwResult = await pi.exec("bash", [
+          path.join(scriptsDir, "ensure-playwright.sh"),
+          ctx.cwd,
+        ], { cwd: ctx.cwd });
 
-        // Offer starting from next pending phase
-        if (nextPhase) {
-          phaseOptions.push(PHASE_LABELS[nextPhase]);
+        if (pwResult.code !== 0) {
+          notifyWarning(ctx, "Playwright setup issue", "Could not verify playwright installation. The agent will handle it.");
         }
+      } catch {
+        notifyWarning(ctx, "Playwright check skipped", "Will be handled during execution");
+      }
+
+      // ── Step 3: Load or create config ────────────────────────────────
+      let config = loadE2eQaConfig(ctx.cwd);
+
+      if (!config && ctx.hasUI) {
+        config = await runSetupWizard(ctx, detectedType, detectedDevCommand, detectedPort);
+        if (!config) return; // user cancelled
+        saveE2eQaConfig(ctx.cwd, config);
+        ctx.ui.notify("E2E QA config saved to .omp/supipowers/e2e-qa.json", "info");
+      }
+
+      if (!config) {
+        // Use defaults with detected values
+        config = {
+          ...DEFAULT_E2E_QA_CONFIG,
+          app: {
+            type: (detectedType as AppType) || "generic",
+            devCommand: detectedDevCommand || "npm run dev",
+            port: detectedPort || 3000,
+            baseUrl: `http://localhost:${detectedPort || 3000}`,
+          },
+        };
+      }
 
-        if (phaseOptions.length > 1) {
-          const statusLine = getPhaseStatusLine(ledger);
-          const choice = await ctx.ui.select(
-            `QA Phase · ${statusLine}`,
-            phaseOptions,
-            { helpText: "Select action · Esc to cancel" },
-          );
-          if (!choice) return;
-
-          if (choice.startsWith("Re-run")) {
-            action = { type: "rerun-failed" };
-          } else {
-            // Extract phase from label
-            const selectedPhase = (Object.entries(PHASE_LABELS) as [QaPhase, string][])
-              .find(([, label]) => label === choice)?.[0];
-            if (selectedPhase) {
-              action = { type: "run-phase", phase: selectedPhase };
-            }
+      // ── Step 4: Check for unresolved regressions ─────────────────────
+      const activeSession = findActiveSession(ctx.cwd);
+      if (activeSession && activeSession.regressions.length > 0 && ctx.hasUI) {
+        const unresolvedRegressions = activeSession.regressions.filter((r: E2eRegression) => !r.resolution);
+        if (unresolvedRegressions.length > 0) {
+          for (const regression of unresolvedRegressions) {
+            const choice = await ctx.ui.select(
+              `Regression: ${regression.flowName}`,
+              [
+                "This is a bug — needs fixing",
+                "Behavior changed intentionally — update the test",
+                "Skip for now",
+              ],
+              { helpText: `Was passing, now fails: ${regression.error}` },
+            );
+            if (!choice) continue;
+            if (choice.startsWith("This is a bug")) regression.resolution = "bug";
+            else if (choice.startsWith("Behavior changed")) regression.resolution = "intentional-change";
+            else regression.resolution = "skipped";
           }
-        } else if (nextPhase) {
-          // Only one option — just run the next phase
-          action = { type: "run-phase", phase: nextPhase };
         }
-      } else if (nextPhase) {
-        action = { type: "run-phase", phase: nextPhase };
       }
 
-      if (!action) {
-        notifyInfo(ctx, "QA pipeline complete", getPhaseStatusLine(ledger));
-        return;
-      }
+      // ── Step 5: Route discovery ──────────────────────────────────────
+      let discoveredRoutes = "";
+      try {
+        const routeResult = await pi.exec("bash", [
+          path.join(scriptsDir, "discover-routes.sh"),
+          ctx.cwd,
+          config.app.type,
+        ], { cwd: ctx.cwd });
 
-      // ── Step 3: Execute ────────────────────────────────────────────
-      let prompt: string;
-
-      if (action.type === "rerun-failed") {
-        ledger = advancePhase(ctx.cwd, ledger, "execution", "running");
-        prompt = buildExecutionPrompt(ledger, { failedOnly: true, failedTests });
-        notifyInfo(ctx, "QA re-running failed tests", `${failedTests.length} test(s)`);
-      } else {
-        const phase = action.phase;
-        ledger = advancePhase(ctx.cwd, ledger, phase, "running");
-
-        switch (phase) {
-          case "discovery":
-            prompt = buildDiscoveryPrompt(framework, ctx.cwd);
-            break;
-          case "matrix":
-            prompt = buildMatrixPrompt(ledger);
-            break;
-          case "execution":
-            prompt = buildExecutionPrompt(ledger);
-            break;
-          case "reporting":
-            prompt = buildReportingPrompt(ledger);
-            break;
+        if (routeResult.code === 0 && routeResult.stdout.trim()) {
+          discoveredRoutes = routeResult.stdout.trim();
         }
+      } catch { /* agent will discover routes manually */ }
 
-        notifyInfo(ctx, `QA phase: ${phase}`, `session: ${ledger.id}`);
+      if (!discoveredRoutes) {
+        discoveredRoutes = '{"path": "/", "file": "unknown", "type": "page", "hasForm": false}';
       }
 
-      // Include session context for the sub-agent
-      const sessionContext = [
-        `\n\n## QA Session Context`,
-        ``,
-        `Session ID: ${ledger.id}`,
-        `Session ledger path: .omp/supipowers/qa-sessions/${ledger.id}/ledger.json`,
-        ``,
-        `Current ledger state:`,
-        "```json",
-        JSON.stringify(ledger, null, 2),
-        "```",
-      ].join("\n");
+      // ── Step 6: Load previous matrix ─────────────────────────────────
+      const previousMatrix = loadE2eMatrix(ctx.cwd);
+      const matrixJson = previousMatrix ? JSON.stringify(previousMatrix, null, 2) : null;
+
+      // ── Step 7: Create session ───────────────────────────────────────
+      const ledger = createNewE2eSession(ctx.cwd, config);
+      const sessionDir = getSessionDir(ctx.cwd, ledger.id);
+
+      // ── Step 8: Load skill ───────────────────────────────────────────
+      let skillContent = "";
+      const skillPath = findSkillPath("qa-strategy");
+      if (skillPath) {
+        try {
+          skillContent = fs.readFileSync(skillPath, "utf-8");
+        } catch { /* proceed without */ }
+      }
+
+      // ── Step 9: Build and send prompt ────────────────────────────────
+      const routeCount = discoveredRoutes.split("\n").filter(Boolean).length;
+
+      const prompt = buildE2eOrchestratorPrompt({
+        cwd: ctx.cwd,
+        appType: config.app,
+        sessionDir,
+        scriptsDir,
+        config,
+        discoveredRoutes,
+        previousMatrix: matrixJson,
+        skillContent,
+      });
 
       pi.sendMessage(
         {
           customType: "supi-qa",
-          content: [{ type: "text", text: prompt + sessionContext }],
+          content: [{ type: "text", text: prompt }],
           display: "none",
         },
-        { deliverAs: "steer" }
+        { deliverAs: "steer" },
+      );
+
+      notifyInfo(
+        ctx,
+        `E2E QA started: ${config.app.type}`,
+        `${routeCount} routes discovered | session ${ledger.id}`,
       );
     },
   });

package/src/commands/supi.ts

@@ -15,7 +15,7 @@ export function handleSupi(pi: ExtensionAPI, ctx: ExtensionContext): void {
       "/supi:plan     — Start collaborative planning",
       "/supi:run      — Execute a plan with sub-agents",
       "/supi:review   — Run quality gates",
-      "/supi:qa       — Run QA pipeline",
+      "/supi:qa       — E2E product testing with Playwright",
       "/supi:fix-pr   — Fix PR review comments",
       "/supi:release  — Release automation",
       "/supi:config   — Manage configuration",

package/src/config/defaults.ts

@@ -19,6 +19,7 @@ export const DEFAULT_CONFIG: SupipowersConfig = {
   qa: {
     framework: null,
     command: null,
+    e2e: false,
   },
   release: {
     pipeline: null,

package/src/config/schema.ts

@@ -25,6 +25,7 @@ const ConfigSchema = Type.Object({
   qa: Type.Object({
     framework: Type.Union([Type.String(), Type.Null()]),
     command: Type.Union([Type.String(), Type.Null()]),
+    e2e: Type.Boolean(),
   }),
   release: Type.Object({
     pipeline: Type.Union([Type.String(), Type.Null()]),

package/src/qa/config.ts

@@ -0,0 +1,43 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { E2eQaConfig } from "./types.js";
+
+const CONFIG_FILENAME = "e2e-qa.json";
+
+function getConfigPath(cwd: string): string {
+  return path.join(cwd, ".omp", "supipowers", CONFIG_FILENAME);
+}
+
+export const DEFAULT_E2E_QA_CONFIG: E2eQaConfig = {
+  app: {
+    type: "generic",
+    devCommand: "npm run dev",
+    port: 3000,
+    baseUrl: "http://localhost:3000",
+  },
+  playwright: {
+    browser: "chromium",
+    headless: true,
+    timeout: 30000,
+  },
+  execution: {
+    maxRetries: 2,
+    maxFlows: 20,
+  },
+};
+
+export function loadE2eQaConfig(cwd: string): E2eQaConfig | null {
+  const configPath = getConfigPath(cwd);
+  if (!fs.existsSync(configPath)) return null;
+  try {
+    return JSON.parse(fs.readFileSync(configPath, "utf-8")) as E2eQaConfig;
+  } catch {
+    return null;
+  }
+}
+
+export function saveE2eQaConfig(cwd: string, config: E2eQaConfig): void {
+  const configPath = getConfigPath(cwd);
+  fs.mkdirSync(path.dirname(configPath), { recursive: true });
+  fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
+}