start-vibing 4.4.0 → 4.4.1

package/template/.claude/skills/e2e-audit/scripts/inventory-existing-tests.sh

@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+# inventory-existing-tests.sh — catalog every E2E test the project already has,
+# so the audit can reuse setup and detect DRIFT between audit runs.
+#
+# Why: (1) If the project has tests/e2e/, we must not reinvent the fixtures,
+# auth storage state, or page objects. (2) Between audit runs the test layout
+# may change (test deleted, fixture renamed, new storageState file), and the
+# skill should warn the user when that happens instead of silently producing
+# stale findings.
+#
+# Output: JSON object on stdout:
+# {
+#   "runner":          "playwright" | "cypress" | "vitest-browser" | "none",
+#   "config_file":     "playwright.config.ts" | "cypress.config.ts" | null,
+#   "test_dirs":       ["tests/e2e", "e2e"],
+#   "test_files":      [{ "file": "tests/e2e/login.spec.ts", "test_count": 5, "describe_count": 1 }],
+#   "fixtures":        [{ "file": "...", "fixtures_defined": ["authenticatedPage", "apiErrors"] }],
+#   "page_objects":    [{ "file": "...", "class": "LoginPage" }],
+#   "storage_states":  ["tests/e2e/.auth/owner.json", ...],
+#   "has_global_setup": true | false,
+#   "hash":            "<sha256 of file-list + sizes>"   // for drift detection
+# }
+set -euo pipefail
+
+command -v jq >/dev/null || { echo "jq required" >&2; exit 2; }
+
+RUNNER="none"
+CFG="null"
+TDIRS='[]'
+TFILES='[]'
+FIXTURES='[]'
+PAGES='[]'
+STATES='[]'
+HAS_GLOBAL_SETUP=false
+
+# --- detect runner + config -------------------------------------------------
+for c in playwright.config.ts playwright.config.js playwright.config.mjs; do
+  if [[ -f "$c" ]]; then RUNNER="playwright"; CFG="\"$c\""; break; fi
+done
+if [[ "$RUNNER" == "none" ]]; then
+  for c in cypress.config.ts cypress.config.js; do
+    [[ -f "$c" ]] && RUNNER="cypress" && CFG="\"$c\"" && break
+  done
+fi
+if [[ "$RUNNER" == "none" ]] && jq -e '.dependencies["@vitest/browser"] // .devDependencies["@vitest/browser"]' package.json >/dev/null 2>&1; then
+  RUNNER="vitest-browser"
+fi
+
+# --- detect test dirs -------------------------------------------------------
+CANDIDATE_DIRS=(tests/e2e tests/playwright tests/integration e2e playwright-tests cypress/e2e cypress/integration)
+for d in "${CANDIDATE_DIRS[@]}"; do
+  if [[ -d "$d" ]]; then
+    TDIRS="$(jq --arg d "$d" '. + [$d]' <<<"$TDIRS")"
+  fi
+done
+
+# --- list test files + count tests -----------------------------------------
+SPEC_GLOBS=()
+if [[ "$RUNNER" == "playwright" ]]; then
+  SPEC_GLOBS=('*.spec.ts' '*.spec.tsx' '*.spec.js' '*.test.ts' '*.test.tsx')
+elif [[ "$RUNNER" == "cypress" ]]; then
+  SPEC_GLOBS=('*.cy.ts' '*.cy.js' '*.spec.ts' '*.spec.js')
+elif [[ "$RUNNER" == "vitest-browser" ]]; then
+  SPEC_GLOBS=('*.test.ts' '*.test.tsx')
+fi
+
+if [[ ${#SPEC_GLOBS[@]} -gt 0 ]]; then
+  while IFS= read -r d; do
+    [[ -z "$d" ]] && continue
+    for g in "${SPEC_GLOBS[@]}"; do
+      while IFS= read -r f; do
+        [[ -z "$f" ]] && continue
+        tc=$(grep -cE "^[[:space:]]*(test|it)\\(" "$f" 2>/dev/null || echo 0)
+        dc=$(grep -cE "^[[:space:]]*describe\\(" "$f" 2>/dev/null || echo 0)
+        TFILES="$(jq --arg f "$f" --argjson t "$tc" --argjson dc "$dc" \
+                    '. + [{file:$f, test_count:$t, describe_count:$dc}]' <<<"$TFILES")"
+      done < <(find "$d" -type f -name "$g" 2>/dev/null)
+    done
+  done < <(jq -r '.[]' <<<"$TDIRS")
+fi
+
+# --- detect fixtures --------------------------------------------------------
+# Playwright: files that use `test.extend<>`, or export a test with extend.
+while IFS= read -r d; do
+  [[ -z "$d" ]] && continue
+  while IFS= read -r f; do
+    [[ -z "$f" ]] && continue
+    defined='[]'
+    # Extract fixture names from `extend<{ name1: ..., name2: ... }>` blocks.
+    while IFS= read -r name; do
+      [[ -z "$name" ]] && continue
+      defined="$(jq --arg n "$name" '. + [$n]' <<<"$defined")"
+    done < <(grep -oE "([A-Za-z_][A-Za-z0-9_]*)[[:space:]]*:" "$f" 2>/dev/null \
+             | awk '{sub(":",""); print $1}' | sort -u | head -30)
+    if [[ "$(jq 'length' <<<"$defined")" -gt 0 ]]; then
+      FIXTURES="$(jq --arg fi "$f" --argjson df "$defined" '. + [{file:$fi, fixtures_defined:$df}]' <<<"$FIXTURES")"
+    fi
+  done < <(grep -rl "test\\.extend\\|base\\.extend\\|defineConfig" "$d" --include='*.ts' --include='*.tsx' 2>/dev/null | head -40)
+done < <(jq -r '.[]' <<<"$TDIRS")
+
+# --- detect page objects ---------------------------------------------------
+while IFS= read -r d; do
+  [[ -z "$d" ]] && continue
+  while IFS= read -r hit; do
+    [[ -z "$hit" ]] && continue
+    f="${hit%%:*}"; rest="${hit#*:}"
+    cls="$(echo "$rest" | grep -oE "class[[:space:]]+[A-Z][A-Za-z0-9_]*Page" | awk '{print $2}' | head -1)"
+    [[ -n "$cls" ]] && PAGES="$(jq --arg f "$f" --arg c "$cls" '. + [{file:$f, class:$c}]' <<<"$PAGES")"
+  done < <(grep -rHn "class[[:space:]]\\+[A-Z][A-Za-z0-9_]*Page" "$d" --include='*.ts' --include='*.tsx' 2>/dev/null)
+done < <(jq -r '.[]' <<<"$TDIRS")
+
+# --- detect storage states --------------------------------------------------
+while IFS= read -r d; do
+  [[ -z "$d" ]] && continue
+  while IFS= read -r f; do
+    [[ -z "$f" ]] && continue
+    STATES="$(jq --arg f "$f" '. + [$f]' <<<"$STATES")"
+  done < <(find "$d" -type f \( -path '*/.auth/*.json' -o -name 'storageState*.json' -o -path '*/storage/*.json' \) 2>/dev/null | head -40)
+done < <(jq -r '.[]' <<<"$TDIRS")
+
+# --- global setup ----------------------------------------------------------
+if [[ "$CFG" != "null" ]]; then
+  cfg_file="$(jq -r . <<<"$CFG")"
+  if grep -Eq "globalSetup|globalTeardown" "$cfg_file" 2>/dev/null; then
+    HAS_GLOBAL_SETUP=true
+  fi
+fi
+
+# --- drift hash -------------------------------------------------------------
+# Hash the sorted list of test files + their byte sizes. Changing any test or
+# adding/removing one flips this hash; the skill uses this to alert the user.
+HASH=""
+if command -v sha256sum >/dev/null 2>&1; then
+  HASH="$(jq -r '.[] | .file' <<<"$TFILES" 2>/dev/null | sort | while IFS= read -r f; do
+    sz=$(wc -c <"$f" 2>/dev/null | awk '{print $1}'); printf '%s|%s\n' "$f" "${sz:-0}";
+  done | sha256sum | awk '{print $1}')"
+fi
+[[ -z "$HASH" ]] && HASH="unknown"
+
+# --- assemble ---------------------------------------------------------------
+jq -n \
+  --arg runner "$RUNNER" \
+  --argjson config_file "$CFG" \
+  --argjson test_dirs "$TDIRS" \
+  --argjson test_files "$TFILES" \
+  --argjson fixtures "$FIXTURES" \
+  --argjson page_objects "$PAGES" \
+  --argjson storage_states "$STATES" \
+  --argjson has_global_setup "$HAS_GLOBAL_SETUP" \
+  --arg hash "$HASH" \
+  '{
+    runner: $runner,
+    config_file: $config_file,
+    test_dirs: $test_dirs,
+    test_files: $test_files,
+    fixtures: $fixtures,
+    page_objects: $page_objects,
+    storage_states: $storage_states,
+    has_global_setup: $has_global_setup,
+    hash: $hash
+  }'

package/template/.claude/skills/e2e-audit/scripts/verify-audit.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Usage: verify-audit.sh [--strict] <session_dir>
+#
+# Verifies artifacts produced by an e2e-audit session:
+#   1. stack.json, routes.json, api-surface.json, existing-tests.json, uncovered.json exist and parse.
+#   2. findings.json exists, is a JSON array, and every item has a SHOT+TRACE+ASSERT+SOURCE quad.
+#   3. Every referenced screenshot / trace / source file resolves to a non-empty file.
+#   4. If post-run-feedback.json exists, its problems[] reference valid finding IDs.
+#   5. Warn when uncovered surfaces exist but no finding of rule=coverage-gap was emitted.
+#
+# Exit codes:
+#   0  OK
+#   1  verification failure (or warnings in --strict)
+#   2  missing prerequisites
+set -euo pipefail
+
+STRICT=0
+if [ "${1:-}" = "--strict" ]; then STRICT=1; shift; fi
+
+SESSION_DIR="${1:?usage: verify-audit.sh [--strict] <session_dir>}"
+
+for f in stack.json routes.json api-surface.json existing-tests.json uncovered.json findings.json; do
+  [ -f "$SESSION_DIR/$f" ] || { echo "FATAL: missing $SESSION_DIR/$f" >&2; exit 2; }
+  jq -e . "$SESSION_DIR/$f" >/dev/null 2>&1 || { echo "FATAL: $SESSION_DIR/$f is not valid JSON" >&2; exit 1; }
+done
+
+FINDINGS="$SESSION_DIR/findings.json"
+if ! jq -e 'type == "array"' "$FINDINGS" >/dev/null; then
+  echo "FATAL: findings.json must be a JSON array" >&2; exit 1
+fi
+
+WARNINGS=0
+warn() { echo "WARN: $*" >&2; WARNINGS=$((WARNINGS + 1)); }
+
+# Every finding has id + rule + severity + files_affected + the evidence quad.
+jq -c '.[]' "$FINDINGS" | while read -r f; do
+  id="$(echo "$f" | jq -r '.id // empty')"
+  rule="$(echo "$f" | jq -r '.rule // empty')"
+  [ -z "$id" ] && { echo "FAIL: finding missing id: $f" >&2; exit 1; }
+  [ -z "$rule" ] && { echo "FAIL: $id missing rule" >&2; exit 1; }
+
+  # Meta findings (coverage-gap, test-drift, stack-detect, post-run-feedback)
+  # are aggregate and do not require SHOT/TRACE.
+  case "$rule" in
+    coverage-gap-*|test-drift|stack-detect|post-run-feedback|uncovered-*)
+      continue
+      ;;
+  esac
+
+  shot="$(echo "$f" | jq -r '.screenshot_path // empty')"
+  trace="$(echo "$f" | jq -r '.trace_path // empty')"
+  source_file="$(echo "$f" | jq -r '.source_file // empty')"
+  assert="$(echo "$f" | jq -r '.assertion // empty')"
+
+  [ -n "$shot" ]        || { echo "FAIL: $id missing screenshot_path" >&2; exit 1; }
+  [ -n "$trace" ]       || { echo "FAIL: $id missing trace_path" >&2; exit 1; }
+  [ -n "$source_file" ] || { echo "FAIL: $id missing source_file" >&2; exit 1; }
+  [ -n "$assert" ]      || { echo "FAIL: $id missing assertion" >&2; exit 1; }
+
+  [ -s "$shot" ]        || { echo "FAIL: $id screenshot_path does not resolve: $shot" >&2; exit 1; }
+  [ -s "$trace" ]       || { echo "FAIL: $id trace_path does not resolve: $trace" >&2; exit 1; }
+  [ -f "$source_file" ] || { echo "FAIL: $id source_file does not exist: $source_file" >&2; exit 1; }
+done
+
+# coverage-gap meta findings should exist if uncovered.json has non-empty arrays.
+UNC_TOTAL=$(jq '
+  (.uncovered_routes  | length) +
+  (.uncovered_http    | length) +
+  (.uncovered_trpc    | length) +
+  (.uncovered_actions | length)
+' "$SESSION_DIR/uncovered.json")
+COV_FINDINGS=$(jq '[.[] | select(.rule | startswith("coverage-gap"))] | length' "$FINDINGS")
+if [ "$UNC_TOTAL" -gt 0 ] && [ "$COV_FINDINGS" -eq 0 ]; then
+  warn "uncovered.json has $UNC_TOTAL uncovered surfaces but no coverage-gap finding was emitted"
+fi
+
+# post-run-feedback sanity
+if [ -f "$SESSION_DIR/post-run-feedback.json" ]; then
+  jq -e 'type == "object" and has("problems")' "$SESSION_DIR/post-run-feedback.json" >/dev/null \
+    || { echo "FAIL: post-run-feedback.json malformed" >&2; exit 1; }
+fi
+
+COUNT=$(jq 'length' "$FINDINGS")
+if [ "$STRICT" -eq 1 ] && [ "$WARNINGS" -gt 0 ]; then
+  echo "STRICT: $COUNT findings verified, $WARNINGS warning(s)" >&2
+  exit 1
+fi
+echo "OK: $COUNT findings verified${WARNINGS:+ ($WARNINGS warning(s))}"

package/template/.claude/skills/e2e-audit/templates/auth-setup.ts.tpl

@@ -0,0 +1,24 @@
+// {{SESSION_DIR}}/auth.setup.ts — one storageState per role
+// Run once via Playwright `projects[{ name: 'setup', testMatch: /.*\.setup\.ts/ }]`.
+// NEVER reads .env* directly; credentials come from process.env set at invoke time.
+import { test as setup } from '@playwright/test';
+
+const ROLES = ['owner', 'admin', 'member'] as const;
+
+for (const role of ROLES) {
+  setup(`authenticate ${role}`, async ({ page }) => {
+    const email    = process.env[`E2E_${role.toUpperCase()}_EMAIL`];
+    const password = process.env[`E2E_${role.toUpperCase()}_PASSWORD`];
+    if (!email || !password) {
+      setup.skip(true, `missing E2E_${role.toUpperCase()}_EMAIL / _PASSWORD`);
+      return;
+    }
+    await page.goto('/signin');
+    await page.getByLabel(/email/i).fill(email);
+    await page.getByLabel(/password/i).fill(password);
+    await page.getByRole('button', { name: /sign in|log in|continue/i }).click();
+    // Adjust the wait URL to your post-login destination.
+    await page.waitForURL(/\/(dashboard|home|app|overview)/, { timeout: 30_000 });
+    await page.context().storageState({ path: `.e2e-audit/current/auth/${role}.json` });
+  });
+}

package/template/.claude/skills/e2e-audit/templates/base-fixture.ts.tpl

@@ -0,0 +1,75 @@
+// {{SESSION_DIR}}/fixtures/base.ts — e2e-audit base fixture
+// Renders a Playwright test with three fixtures:
+//   1. apiErrors: asserts no unexpected 4xx/5xx on API paths during the test
+//   2. consoleErrors: fails when any console.error fires (with ignore list)
+//   3. authenticatedPage: uses storageState from $STORAGE_STATE
+//
+// Use: import { test, expect } from './fixtures/base';
+import { test as base, expect, type Page } from '@playwright/test';
+
+const IGNORED_API_PATTERNS: (string | RegExp)[] = [
+  /\/api\/auth\//,       // next-auth heartbeats
+  /\/_next\//,
+  /\/__nextjs_/,
+  /hot-update/,
+  /\.well-known\//,
+];
+
+const IGNORED_CONSOLE_PATTERNS: (string | RegExp)[] = [
+  /React DevTools/,
+  /Download the React DevTools/,
+  /\[Fast Refresh\]/,
+];
+
+type ApiError = { url: string; method: string; status: number; body: string };
+
+export const test = base.extend<{
+  apiErrors: ApiError[];
+  consoleErrors: string[];
+  authenticatedPage: Page;
+}>({
+  apiErrors: async ({ page }, use) => {
+    const errors: ApiError[] = [];
+    page.on('response', async (res) => {
+      const url = res.url();
+      const isApi = /\/api\/|\/v1\/|\/trpc\//.test(url);
+      if (!isApi) return;
+      if (IGNORED_API_PATTERNS.some((p) => (typeof p === 'string' ? url.includes(p) : p.test(url)))) return;
+      if (res.status() < 400) return;
+      let body = '';
+      try { body = (await res.text()).slice(0, 400); } catch {}
+      errors.push({ url, method: res.request().method(), status: res.status(), body });
+    });
+    await use(errors);
+    if (errors.length > 0) {
+      const lines = errors.map((e) => `  ${e.method} ${e.url} → ${e.status}  ${e.body.replace(/\s+/g, ' ')}`);
+      throw new Error(`API errors during test:\n${lines.join('\n')}`);
+    }
+  },
+
+  consoleErrors: async ({ page }, use) => {
+    const errors: string[] = [];
+    page.on('console', (msg) => {
+      if (msg.type() !== 'error') return;
+      const text = msg.text();
+      if (IGNORED_CONSOLE_PATTERNS.some((p) => (typeof p === 'string' ? text.includes(p) : p.test(text)))) return;
+      errors.push(text);
+    });
+    page.on('pageerror', (err) => errors.push(`pageerror: ${err.message}`));
+    await use(errors);
+    if (errors.length > 0) {
+      throw new Error(`Console errors during test:\n${errors.map((e) => `  ${e}`).join('\n')}`);
+    }
+  },
+
+  authenticatedPage: async ({ browser }, use) => {
+    const context = await browser.newContext({
+      storageState: process.env.E2E_STORAGE_STATE || '.e2e-audit/current/auth/owner.json',
+    });
+    const page = await context.newPage();
+    await use(page);
+    await context.close();
+  },
+});
+
+export { expect };

package/template/.claude/skills/e2e-audit/templates/findings-report.md.tpl

@@ -0,0 +1,54 @@
+# e2e-audit findings — {{SESSION_ID}}
+
+Generated: {{GENERATED_AT}}
+Session dir: `{{SESSION_DIR}}`
+Base ref: `{{BASE_REF}}`
+
+## Summary
+
+- Total findings: **{{TOTAL}}**
+- Critical / High / Medium / Low / Info: **{{CRIT}} / {{HIGH}} / {{MED}} / {{LOW}} / {{INFO}}**
+- Coverage gaps (meta): **{{COVERAGE_GAPS}}**
+- Existing specs: **{{SPECS_TOTAL}}** · drift status: **{{DRIFT_STATUS}}**
+
+## Critical + High
+
+{{#each critical_high}}
+### {{id}} — {{summary}}
+
+- Rule: `{{rule}}` · Severity: **{{severity}}** · Category: `{{category}}`
+- Files: {{files_affected}}
+- Assertion: `{{assertion}}`
+- Evidence: [screenshot]({{screenshot_path}}) · [trace]({{trace_path}}) · source: `{{source_file}}`
+{{#if http}}
+- HTTP: `{{http.method}} {{http.path}} → {{http.status}}`
+  ```
+  {{http.response_snippet}}
+  ```
+{{/if}}
+{{#if detail}}
+- Detail: {{detail}}
+{{/if}}
+
+{{/each}}
+
+## Coverage gaps
+
+{{#each coverage_gaps}}
+### {{id}} — {{summary}}
+
+{{detail}}
+
+Suggested next step: {{#if suggested_fix}}`{{suggested_fix.kind}}` → {{suggested_fix.files}}{{else}}add specs covering the items above{{/if}}
+
+{{/each}}
+
+## Other findings
+
+{{#each others}}
+- **{{id}}** [{{severity}}] `{{rule}}` — {{summary}}
+{{/each}}
+
+---
+
+_Generated by `e2e-audit` v0.2.0. Verify with `bash .claude/skills/e2e-audit/scripts/verify-audit.sh {{SESSION_DIR}}`._

package/template/.claude/skills/e2e-audit/templates/post-run-feedback.md.tpl

@@ -0,0 +1,36 @@
+# Post-run feedback — {{SESSION_ID}}
+
+Duration: **{{DURATION_S}}s** · Tests: **{{TESTS_PASSED}}/{{TESTS_TOTAL}}** passed · **{{TESTS_FLAKY}}** flaky · **{{TESTS_FAILED}}** failed
+
+## Verdict
+
+{{VERDICT}}
+
+## Top problems
+
+{{#each problems}}
+- **[{{severity}}] {{kind}}** — {{where}} (count: {{count}})
+{{#if sample_trace}}
+  - Trace: `{{sample_trace}}`
+{{/if}}
+{{#if sample_log_tail}}
+  - Log tail:
+    ```
+    {{sample_log_tail}}
+    ```
+{{/if}}
+{{#if sample}}
+  - Sample: `{{sample}}`
+{{/if}}
+{{/each}}
+
+## Coverage still uncovered after this run
+
+- Routes: **{{UNCOVERED.routes}}**
+- HTTP handlers: **{{UNCOVERED.http}}**
+- tRPC procedures: **{{UNCOVERED.trpc}}**
+- Server actions: **{{UNCOVERED.actions}}**
+
+## Suggested next actions
+
+{{NEXT_ACTIONS}}