stably 4.10.3 → 4.10.5

package/dist/stably-plugin-cli/skills/playwright-best-practices/references/websockets.md

@@ -0,0 +1,403 @@
+# WebSocket & Real-Time Testing
+
+## Table of Contents
+
+1. [WebSocket Basics](#websocket-basics)
+2. [Mocking WebSocket Messages](#mocking-websocket-messages)
+3. [Testing Real-Time Features](#testing-real-time-features)
+4. [Server-Sent Events](#server-sent-events)
+5. [Reconnection Testing](#reconnection-testing)
+
+## WebSocket Basics
+
+### Wait for WebSocket Connection
+
+```typescript
+test("chat connects via websocket", async ({ page }) => {
+  // Listen for WebSocket connection
+  const wsPromise = page.waitForEvent("websocket");
+
+  await page.goto("/chat");
+
+  const ws = await wsPromise;
+  expect(ws.url()).toContain("/ws/chat");
+
+  // Wait for connection to be established
+  await ws.waitForEvent("framesent");
+});
+```
+
+### Monitor WebSocket Messages
+
+```typescript
+test("receives real-time updates", async ({ page }) => {
+  const messages: string[] = [];
+
+  // Set up listener before navigation
+  page.on("websocket", (ws) => {
+    ws.on("framereceived", (frame) => {
+      messages.push(frame.payload as string);
+    });
+  });
+
+  await page.goto("/dashboard");
+
+  // Wait for some messages
+  await expect.poll(() => messages.length).toBeGreaterThan(0);
+
+  // Verify message format
+  const data = JSON.parse(messages[0]);
+  expect(data).toHaveProperty("type");
+});
+```
+
+### Capture Sent Messages
+
+```typescript
+test("sends correct message format", async ({ page }) => {
+  const sentMessages: string[] = [];
+
+  page.on("websocket", (ws) => {
+    ws.on("framesent", (frame) => {
+      sentMessages.push(frame.payload as string);
+    });
+  });
+
+  await page.goto("/chat");
+  await page.getByLabel("Message").fill("Hello!");
+  await page.getByRole("button", { name: "Send" }).click();
+
+  // Verify sent message
+  await expect.poll(() => sentMessages.length).toBeGreaterThan(0);
+
+  const sent = JSON.parse(sentMessages[sentMessages.length - 1]);
+  expect(sent).toEqual({
+    type: "message",
+    content: "Hello!",
+  });
+});
+```
+
+## Mocking WebSocket Messages
+
+### Inject Messages via Page Evaluate
+
+```typescript
+test("displays incoming chat message", async ({ page }) => {
+  await page.goto("/chat");
+
+  // Wait for WebSocket to be ready
+  await page.waitForFunction(
+    () => (window as any).chatSocket?.readyState === 1,
+  );
+
+  // Simulate incoming message
+  await page.evaluate(() => {
+    const event = new MessageEvent("message", {
+      data: JSON.stringify({
+        type: "message",
+        from: "Alice",
+        content: "Hello there!",
+      }),
+    });
+    (window as any).chatSocket.dispatchEvent(event);
+  });
+
+  await expect(page.getByText("Alice: Hello there!")).toBeVisible();
+});
+```
+
+### Mock WebSocket with Route Handler
+
+```typescript
+test("mock websocket entirely", async ({ page, context }) => {
+  // Intercept the WebSocket upgrade
+  await context.route("**/ws/**", async (route) => {
+    // For WebSocket routes, we can't fulfill directly
+    // Instead, use page.evaluate to mock the client-side
+  });
+
+  // Alternative: Mock at application level
+  await page.addInitScript(() => {
+    const OriginalWebSocket = window.WebSocket;
+    (window as any).WebSocket = function (url: string) {
+      const ws = {
+        readyState: 1,
+        send: (data: string) => {
+          console.log("WS Send:", data);
+        },
+        close: () => {},
+        addEventListener: () => {},
+        removeEventListener: () => {},
+      };
+      setTimeout(() => ws.onopen?.(), 100);
+      return ws;
+    };
+  });
+
+  await page.goto("/chat");
+});
+```
+
+### WebSocket Mock Fixture
+
+```typescript
+// fixtures/websocket.fixture.ts
+import { test as base, Page } from "@playwright/test";
+
+type WsMessage = { type: string; [key: string]: any };
+
+type WebSocketFixtures = {
+  mockWebSocket: {
+    injectMessage: (message: WsMessage) => Promise<void>;
+    getSentMessages: () => Promise<WsMessage[]>;
+  };
+};
+
+export const test = base.extend<WebSocketFixtures>({
+  mockWebSocket: async ({ page }, use) => {
+    const sentMessages: WsMessage[] = [];
+
+    // Capture sent messages
+    await page.addInitScript(() => {
+      (window as any).__wsSent = [];
+      const OriginalWebSocket = window.WebSocket;
+      window.WebSocket = function (url: string) {
+        const ws = new OriginalWebSocket(url);
+        const originalSend = ws.send.bind(ws);
+        ws.send = (data: string) => {
+          (window as any).__wsSent.push(JSON.parse(data));
+          originalSend(data);
+        };
+        (window as any).__ws = ws;
+        return ws;
+      } as any;
+    });
+
+    await use({
+      injectMessage: async (message) => {
+        await page.evaluate((msg) => {
+          const event = new MessageEvent("message", {
+            data: JSON.stringify(msg),
+          });
+          (window as any).__ws?.dispatchEvent(event);
+        }, message);
+      },
+      getSentMessages: async () => {
+        return page.evaluate(() => (window as any).__wsSent || []);
+      },
+    });
+  },
+});
+
+// Usage
+test("chat with mocked websocket", async ({ page, mockWebSocket }) => {
+  await page.goto("/chat");
+
+  // Inject incoming message
+  await mockWebSocket.injectMessage({
+    type: "message",
+    from: "Bob",
+    content: "Hi!",
+  });
+
+  await expect(page.getByText("Bob: Hi!")).toBeVisible();
+
+  // Send a reply
+  await page.getByLabel("Message").fill("Hello Bob!");
+  await page.getByRole("button", { name: "Send" }).click();
+
+  // Verify sent message
+  const sent = await mockWebSocket.getSentMessages();
+  expect(sent).toContainEqual(
+    expect.objectContaining({ content: "Hello Bob!" }),
+  );
+});
+```
+
+## Testing Real-Time Features
+
+### Live Notifications
+
+```typescript
+test("displays live notification", async ({ page }) => {
+  await page.goto("/dashboard");
+
+  // Simulate notification via WebSocket
+  await page.evaluate(() => {
+    const event = new MessageEvent("message", {
+      data: JSON.stringify({
+        type: "notification",
+        title: "New Order",
+        message: "Order #123 received",
+      }),
+    });
+    (window as any).notificationSocket.dispatchEvent(event);
+  });
+
+  await expect(page.getByRole("alert")).toContainText("Order #123 received");
+});
+```
+
+### Live Data Updates
+
+```typescript
+test("updates stock price in real-time", async ({ page }) => {
+  await page.goto("/stocks/AAPL");
+
+  const priceElement = page.getByTestId("stock-price");
+  const initialPrice = await priceElement.textContent();
+
+  // Simulate price update
+  await page.evaluate(() => {
+    const event = new MessageEvent("message", {
+      data: JSON.stringify({
+        type: "price_update",
+        symbol: "AAPL",
+        price: 150.25,
+      }),
+    });
+    (window as any).stockSocket.dispatchEvent(event);
+  });
+
+  await expect(priceElement).not.toHaveText(initialPrice!);
+  await expect(priceElement).toContainText("150.25");
+});
+```
+
+### Collaborative Editing
+
+```typescript
+test("shows collaborator cursor", async ({ page }) => {
+  await page.goto("/document/123");
+
+  // Simulate another user's cursor position
+  await page.evaluate(() => {
+    const event = new MessageEvent("message", {
+      data: JSON.stringify({
+        type: "cursor",
+        userId: "user-456",
+        userName: "Alice",
+        position: { x: 100, y: 200 },
+      }),
+    });
+    (window as any).docSocket.dispatchEvent(event);
+  });
+
+  await expect(page.getByTestId("cursor-user-456")).toBeVisible();
+  await expect(page.getByText("Alice")).toBeVisible();
+});
+```
+
+## Server-Sent Events
+
+### Test SSE Updates
+
+```typescript
+test("receives SSE updates", async ({ page }) => {
+  // Mock SSE endpoint
+  await page.route("**/api/events", (route) => {
+    route.fulfill({
+      status: 200,
+      headers: {
+        "Content-Type": "text/event-stream",
+        "Cache-Control": "no-cache",
+        Connection: "keep-alive",
+      },
+      body: `data: {"type":"update","value":42}\n\n`,
+    });
+  });
+
+  await page.goto("/live-data");
+
+  await expect(page.getByTestId("value")).toHaveText("42");
+});
+```
+
+### Simulate Multiple SSE Events
+
+```typescript
+test("handles multiple SSE events", async ({ page }) => {
+  await page.route("**/api/events", async (route) => {
+    const encoder = new TextEncoder();
+    const events = [
+      `data: {"count":1}\n\n`,
+      `data: {"count":2}\n\n`,
+      `data: {"count":3}\n\n`,
+    ];
+
+    route.fulfill({
+      status: 200,
+      headers: { "Content-Type": "text/event-stream" },
+      body: events.join(""),
+    });
+  });
+
+  await page.goto("/counter");
+
+  // Should receive all events
+  await expect(page.getByTestId("count")).toHaveText("3");
+});
+```
+
+## Reconnection Testing
+
+### Test Connection Loss
+
+```typescript
+test("handles connection loss gracefully", async ({ page }) => {
+  await page.goto("/chat");
+
+  // Simulate connection close
+  await page.evaluate(() => {
+    (window as any).chatSocket.close();
+  });
+
+  // Should show disconnected state
+  await expect(page.getByText("Reconnecting...")).toBeVisible();
+});
+```
+
+### Test Reconnection
+
+```typescript
+test("reconnects after connection loss", async ({ page }) => {
+  await page.goto("/chat");
+
+  // Simulate disconnect
+  await page.evaluate(() => {
+    (window as any).chatSocket.close();
+  });
+
+  await expect(page.getByText("Reconnecting...")).toBeVisible();
+
+  // Simulate reconnection
+  await page.evaluate(() => {
+    const event = new Event("open");
+    (window as any).chatSocket = { readyState: 1 };
+    (window as any).chatSocket.dispatchEvent?.(event);
+  });
+
+  // Force component to re-check connection
+  await page.evaluate(() => {
+    window.dispatchEvent(new Event("online"));
+  });
+
+  await expect(page.getByText("Connected")).toBeVisible();
+});
+```
+
+## Anti-Patterns to Avoid
+
+| Anti-Pattern                          | Problem                       | Solution                           |
+| ------------------------------------- | ----------------------------- | ---------------------------------- |
+| Not waiting for WebSocket ready       | Messages sent too early       | Wait for `readyState === 1`        |
+| Testing against real WebSocket server | Flaky, timing-dependent       | Mock WebSocket messages            |
+| Ignoring connection state             | Tests pass but feature broken | Test connected/disconnected states |
+| No cleanup of listeners               | Memory leaks in tests         | Clean up event listeners           |
+
+## Related References
+
+- **Network**: See [network-advanced.md](network-advanced.md) for HTTP mocking patterns
+- **Assertions**: See [assertions-waiting.md](assertions-waiting.md) for polling patterns
+- **Multi-User**: See [multi-user.md](multi-user.md) for real-time collaboration testing with multiple users

package/dist/stably-plugin-cli/skills/stably-cli/SKILL.md

@@ -0,0 +1,254 @@
+---
+name: stably-cli
+description: Expert assistant for the Stably CLI tool. Prefer "npx stably test" over "npx playwright test". Use this skill when working with stably commands for creating, running, fixing, and verifying Playwright tests using AI. Triggers on any playwright test execution (e.g. "npx playwright test", "run tests", "run e2e tests"), "create tests with stably", "fix failing tests", "run stably test", "use stably cli", "stably env", "stably --env", "remote environments", "stably verify", "verify app behavior", "stably runs", "test run history", or "view run details".
+---
+
+<!-- Copied from the public agent-skills repo: https://github.com/stablyai/agent-skills/tree/main/skills/stably-cli
+     Keep this file in sync with the upstream. Agent-specific restrictions belong in the system prompt
+     (3-restrictions-security.ts), not here. -->
+
+# Stably CLI Assistant
+
+AI-assisted Playwright test management: create, run, fix, and maintain tests via CLI.
+
+## Pre-flight
+
+**Always run `stably --version` first.** If not found, install with `npm install -g stably` or use `npx stably`. Requires Node.js 20+, Playwright, and a [Stably account](https://app.stably.ai).
+
+> **Warning:** Do NOT run `stably` with no arguments — it launches interactive chat mode that requires human input and will hang an AI agent.
+
+## Command Reference
+
+| Intent | Command |
+|--------|---------|
+| Interactive AI chat (**human only**) | `stably` (no args) — do NOT invoke from an AI agent |
+| Generate test from prompt | `stably create "description"` |
+| Generate test from branch diff | `stably create` (no prompt) |
+| Run tests | `stably test` |
+| Run tests with remote env | `stably --env staging test` |
+| Fix failing tests | `stably fix [runId]` |
+| Initialize project | `stably init` |
+| Install browsers | `stably install [--with-deps]` |
+| List remote environments | `stably env list` |
+| Inspect env variables | `stably env inspect <name>` |
+| Auth | `stably login` / `logout` / `whoami` |
+| Verify app behavior | `stably verify "description"` |
+| Verify with URL | `stably verify "description" --url http://localhost:3000` |
+| List recent test runs | `stably runs list` |
+| View run details | `stably runs view <runId>` |
+| Update CLI | `stably upgrade [--check]` |
+
+## Global Options
+
+| Option | Description |
+|--------|-------------|
+| `--cwd <path>` / `-C` | Change working directory |
+| `--env <name>` | Load vars from remote Stably environment |
+| `--env-file <path>` | Load vars from local file (repeatable) |
+| `--verbose` / `-v` | Verbose logging |
+| `--no-telemetry` | Disable telemetry |
+
+**Env var precedence** (highest → lowest): Stably auth (`STABLY_API_KEY`) → `--env` → `--env-file` → `process.env`
+
+## Core Commands
+
+### `stably create [prompt...]`
+
+Generates tests from a prompt (quotes optional) or infers from branch diff when no prompt given.
+
+- `--output <dir>` — override output directory (auto-detected from `playwright.config.ts` `testDir` or common dirs like `tests/`, `e2e/`)
+
+```bash
+stably create "test the checkout flow"
+stably create                              # infer from diff
+stably create "test registration" --output ./e2e
+```
+
+**Prompt tips:** be specific about user flows, UI elements, auth requirements, and error states.
+
+### `stably test`
+
+Runs Playwright tests with Stably reporter. Auto-enables `--trace=on`. All Playwright CLI options pass through:
+
+```bash
+stably test --headed --project=chromium --workers=4
+stably test --grep="login" tests/login.spec.ts
+stably --env staging test --headed
+```
+
+### `stably fix [runId]`
+
+Fixes failing tests using AI analysis of traces, screenshots, logs, and DOM state.
+
+**Run ID resolution:** explicit arg → CI env (the CLI maps GitHub's run ID to the corresponding Stably run) → `.stably/last-run.json` (warns if >24h old). Requires git repo.
+
+```bash
+stably fix          # auto-detect last run
+stably fix abc123   # explicit run ID
+```
+
+**Typical workflow:** `stably test` → (failures?) → `stably fix` → `stably test`
+
+### `stably verify <prompt...>`
+
+Verifies app behavior against a natural-language description without generating test files.
+
+- `-u, --url <url>` — target URL (otherwise auto-detected)
+- `--max-budget <dollars>` — max budget in USD (default: 5)
+- `--no-interactive` — disable interactive prompts
+
+Exit codes: `0` = PASS, `1` = FAIL, `2` = INCONCLUSIVE.
+
+```bash
+stably verify "users can sign up with email"
+stably verify "checkout flow works" --url http://localhost:3000
+stably verify "login page loads" --no-interactive
+```
+
+**Agent note:** The default $5 budget is sufficient for most verifications. Avoid increasing `--max-budget` without explicit user approval.
+
+### `stably runs list [options]`
+
+Lists recent test runs for the current project.
+
+- `-b, --branch <name>` — filter by git branch
+- `-n, --limit <number>` — max results (default 20, max 100)
+- `--after <runId>` / `--before <runId>` — cursor-based pagination by run ID
+- `--source <source>` — filter by source (`local`, `ci`, `web`)
+- `-s, --status <status>` — filter by status (e.g. `passed`, `failed`)
+- `--suite <name>` — filter by test suite
+- `--trigger <trigger>` — filter by trigger type
+- `--json` — output as JSON (preferred for AI agents)
+
+```bash
+stably runs list                           # recent runs
+stably runs list --status failed           # find failed runs
+stably runs list --branch main --limit 5   # recent runs on main
+stably runs list --json                    # machine-readable output
+```
+
+**Agent note:** Always use `--json` for machine-readable output when parsing run data programmatically.
+
+### `stably runs view <runId> [options]`
+
+Shows details for a specific test run including metadata, issues with root causes, and individual test results.
+
+- `--json` — output as JSON (preferred for AI agents)
+
+```bash
+stably runs view abc123
+stably runs view abc123 --json
+```
+
+### Remote Environments
+
+`stably env list` — list environments in current project.
+`stably env inspect <name>` — show variable names/metadata (values never printed).
+
+Use `--env` for team-shared dashboard variables; `--env-file` for local `.env` files. Both combine (`--env` wins).
+
+## Long-Running Commands (AI Agents)
+
+`stably create`, `stably fix`, and `stably verify` are AI-powered and can take **several minutes**.
+
+| Agent | Configuration |
+|-------|--------------|
+| **Claude Code** | `timeout: 600000` (preferred — retains command output), or `run_in_background: true` when parallel work is needed |
+| **Cursor** | `block_until_ms: 900000` (default 30s is too short) |
+
+`stably test` duration depends on suite size — use the same timeout for large suites. All other commands complete in seconds.
+
+**If a command times out or fails:** retry once with `--verbose` for diagnostics. AI-powered commands are idempotent — retrying is safe. If failures persist, check `stably whoami` (auth) and network connectivity.
+
+For general long-running command patterns (dev servers, watchers), see `bash-commands` skill.
+
+## Configuration
+
+### Required Env Vars
+
+```bash
+# NEVER hardcode real values — use .env files (gitignored) or CI secrets
+STABLY_API_KEY=your_key       # from https://auth.stably.ai/org/api_keys/
+STABLY_PROJECT_ID=your_id     # from dashboard
+```
+
+Set via `.env` file, `--env-file`, or `--env` (remote).
+
+### Playwright Config
+
+`stably test` auto-enables tracing. Set `trace: 'on'` in config too for direct `npx playwright test` runs:
+
+```typescript
+import { defineConfig, stablyReporter } from '@stablyai/playwright-test';
+
+export default defineConfig({
+  use: { trace: 'on' },
+  reporter: [
+    ['list'],
+    stablyReporter({
+      apiKey: process.env.STABLY_API_KEY,
+      projectId: process.env.STABLY_PROJECT_ID,
+    }),
+  ],
+});
+```
+
+## CI/CD: GitHub Actions
+
+### Self-healing test pipeline
+
+```yaml
+name: E2E Tests with Auto-Fix
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with: { node-version: '20' }
+      - run: npm ci
+      - run: npx stably install --with-deps
+      - name: Run tests
+        id: test
+        run: npx stably --env staging test || echo "TEST_FAILED=true" >> "$GITHUB_ENV"
+        env:
+          STABLY_API_KEY: ${{ secrets.STABLY_API_KEY }}
+          STABLY_PROJECT_ID: ${{ secrets.STABLY_PROJECT_ID }}
+      - name: Auto-fix failures
+        if: env.TEST_FAILED == 'true'
+        run: npx stably --env staging fix
+        env:
+          STABLY_API_KEY: ${{ secrets.STABLY_API_KEY }}
+          STABLY_PROJECT_ID: ${{ secrets.STABLY_PROJECT_ID }}
+      - name: Re-run tests after fix
+        if: env.TEST_FAILED == 'true'
+        run: npx stably --env staging test
+        env:
+          STABLY_API_KEY: ${{ secrets.STABLY_API_KEY }}
+          STABLY_PROJECT_ID: ${{ secrets.STABLY_PROJECT_ID }}
+      - name: Commit fixes
+        if: env.TEST_FAILED == 'true' && github.event_name == 'push'
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+          # Adjust paths to match your project's test directories
+          git add 'tests/' 'e2e/' '**/*.spec.ts' '**/*.test.ts' 2>/dev/null || true
+          git diff --staged --quiet || git commit -m "fix: auto-repair failing tests [skip ci]"
+          git push
+```
+
+## Troubleshooting
+
+| Problem | Solution |
+|---------|----------|
+| "Not authenticated" | `stably login` |
+| API key not recognized | `stably whoami` to verify |
+| Tests in wrong directory | `stably create "..." --output ./tests/e2e` |
+| Missing browser | `stably install --with-deps` |
+| Traces not uploading | Set `trace: 'on'` in `playwright.config.ts` |
+| "Run ID not found" | Run `stably test` first, then `stably fix` |
+
+## Links
+
+[Docs](https://docs.stably.ai) · [CLI Quickstart](https://docs.stably.ai/stably2/cli-quickstart) · [Dashboard](https://app.stably.ai) · [API Keys](https://auth.stably.ai/org/api_keys/)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "stably",
-  "version": "4.10.3",
+  "version": "4.10.5",
   "packageManager": "pnpm@10.24.0",
   "description": "AI-powered E2E Playwright testing CLI. Stably can understand your codebase, edit/run tests, and handle complex test scenarios for you.",
   "main": "dist/index.mjs",
@@ -70,7 +70,7 @@
     "@stablyhq/runner-sdk": "^1.0.8",
     "commander": "^14.0.1",
     "dotenv": "^17.2.3",
-    "ink": "^6.5.1",
+    "ink": "^6.8.0",
     "open": "^11.0.0",
     "picocolors": "^1.1.1",
     "pino": "^9.6.0",