start-vibing 4.4.0 → 4.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/template/.claude/commands/e2e-audit.md +16 -0
- package/template/.claude/hooks/e2e-audit-session-start.sh +4 -0
- package/template/.claude/settings.json +4 -0
- package/template/.claude/skills/e2e-audit/SKILL.md +216 -660
- package/template/.claude/skills/e2e-audit/findings.schema.json +98 -0
- package/template/.claude/skills/e2e-audit/references/api-contract-playbook.md +66 -0
- package/template/.claude/skills/e2e-audit/references/auth-setup-playbook.md +78 -0
- package/template/.claude/skills/e2e-audit/references/coverage-gap-playbook.md +95 -0
- package/template/.claude/skills/e2e-audit/references/post-run-feedback-playbook.md +80 -0
- package/template/.claude/skills/e2e-audit/scripts/detect-stack.sh +205 -0
- package/template/.claude/skills/e2e-audit/scripts/detect-uncovered.sh +137 -0
- package/template/.claude/skills/e2e-audit/scripts/discover-api-surface.sh +242 -0
- package/template/.claude/skills/e2e-audit/scripts/discover-routes.sh +163 -0
- package/template/.claude/skills/e2e-audit/scripts/inventory-existing-tests.sh +161 -0
- package/template/.claude/skills/e2e-audit/scripts/verify-audit.sh +88 -0
- package/template/.claude/skills/e2e-audit/templates/auth-setup.ts.tpl +24 -0
- package/template/.claude/skills/e2e-audit/templates/base-fixture.ts.tpl +75 -0
- package/template/.claude/skills/e2e-audit/templates/findings-report.md.tpl +54 -0
- package/template/.claude/skills/e2e-audit/templates/post-run-feedback.md.tpl +36 -0
- package/template/.claude/skills/e2e-audit/DESIGN.md +0 -294
- package/template/.claude/skills/e2e-audit/e2e/fixtures/auth.setup.ts +0 -70
- package/template/.claude/skills/e2e-audit/e2e/fixtures/auth.ts +0 -21
- package/template/.claude/skills/e2e-audit/e2e/fixtures/base.ts +0 -90
- package/template/.claude/skills/e2e-audit/e2e/fixtures/storage/.gitkeep +0 -0
- package/template/.claude/skills/e2e-audit/e2e/fixtures/storage/admin.json +0 -50
- package/template/.claude/skills/e2e-audit/e2e/fixtures/storage/manager.json +0 -50
- package/template/.claude/skills/e2e-audit/e2e/fixtures/storage/member.json +0 -50
- package/template/.claude/skills/e2e-audit/e2e/fixtures/storage/owner.json +0 -50
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-admin.page.ts +0 -141
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-billing.page.ts +0 -47
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-chat.page.ts +0 -35
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-home.page.ts +0 -134
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-integrations.page.ts +0 -334
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-knowledge.page.ts +0 -30
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-ontology.page.ts +0 -71
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-profile.page.ts +0 -38
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-teams.page.ts +0 -123
- package/template/.claude/skills/e2e-audit/e2e/pages/dashboard-transcripts.page.ts +0 -109
- package/template/.claude/skills/e2e-audit/e2e/specs/auth/login.spec.ts +0 -59
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-admin.spec.ts +0 -233
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-billing.spec.ts +0 -44
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-chat.spec.ts +0 -50
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-home.spec.ts +0 -243
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-integrations.spec.ts +0 -472
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-knowledge.spec.ts +0 -57
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-ontology.spec.ts +0 -72
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-profile.spec.ts +0 -48
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-teams.spec.ts +0 -247
- package/template/.claude/skills/e2e-audit/e2e/specs/dashboard-transcripts.spec.ts +0 -122
- package/template/.claude/skills/e2e-audit/e2e/specs/security/headers.spec.ts +0 -39
- package/template/.claude/skills/e2e-audit/e2e/specs/security/rbac.spec.ts +0 -92
- package/template/.claude/skills/e2e-audit/e2e/specs/security/xss.spec.ts +0 -74
- package/template/.claude/skills/e2e-audit/e2e/utils/console-collector.ts +0 -89
- package/template/.claude/skills/e2e-audit/e2e/utils/security-helpers.ts +0 -114
- package/template/.claude/skills/e2e-audit/e2e/utils/test-data.ts +0 -64
- package/template/.claude/skills/e2e-audit/runbook.md +0 -115
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
3
|
+
"$id": "https://start-vibing.dev/schema/e2e-audit-findings-0.2.0.json",
|
|
4
|
+
"title": "e2e-audit findings v0.2.0",
|
|
5
|
+
"description": "Output contract for e2e-audit. Every finding carries an SHOT+TRACE+ASSERT+SOURCE quad; aggregate (meta) findings are exempt.",
|
|
6
|
+
"type": "array",
|
|
7
|
+
"items": {
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["id", "rule", "severity", "summary", "files_affected"],
|
|
10
|
+
"additionalProperties": true,
|
|
11
|
+
"properties": {
|
|
12
|
+
"id": {
|
|
13
|
+
"type": "string",
|
|
14
|
+
"pattern": "^E2E-\\d{4}$",
|
|
15
|
+
"description": "Stable finding id. Format E2E-NNNN. Allocate sequentially per session."
|
|
16
|
+
},
|
|
17
|
+
"rule": {
|
|
18
|
+
"type": "string",
|
|
19
|
+
"description": "Machine-matchable rule slug (kebab-case). Examples: auth-flow-broken, api-500, rbac-bypass, console-error, uncovered-route, coverage-gap-trpc, test-drift, post-run-feedback."
|
|
20
|
+
},
|
|
21
|
+
"category": {
|
|
22
|
+
"type": "string",
|
|
23
|
+
"enum": [
|
|
24
|
+
"auth",
|
|
25
|
+
"api-contract",
|
|
26
|
+
"rbac",
|
|
27
|
+
"console",
|
|
28
|
+
"server-crash",
|
|
29
|
+
"coverage-gap",
|
|
30
|
+
"test-drift",
|
|
31
|
+
"flake",
|
|
32
|
+
"ui-regression",
|
|
33
|
+
"a11y",
|
|
34
|
+
"meta"
|
|
35
|
+
]
|
|
36
|
+
},
|
|
37
|
+
"severity": {
|
|
38
|
+
"type": "string",
|
|
39
|
+
"enum": ["critical", "high", "medium", "low", "info"]
|
|
40
|
+
},
|
|
41
|
+
"summary": { "type": "string", "minLength": 3 },
|
|
42
|
+
"detail": { "type": "string" },
|
|
43
|
+
"files_affected": {
|
|
44
|
+
"type": "array",
|
|
45
|
+
"items": { "type": "string" },
|
|
46
|
+
"description": "Source files this finding implicates. sd-fix-style guards MUST prevent edits outside this list unless risk is reclassified."
|
|
47
|
+
},
|
|
48
|
+
"viewport": {
|
|
49
|
+
"type": "string",
|
|
50
|
+
"enum": ["mobile", "tablet", "desktop"],
|
|
51
|
+
"description": "Optional — only set when a viewport-specific flow reproduced the issue."
|
|
52
|
+
},
|
|
53
|
+
"test_file": { "type": "string", "description": "Path to the spec that reproduced this (if any)." },
|
|
54
|
+
"test_title": { "type": "string", "description": "Title of the failing/passing test." },
|
|
55
|
+
|
|
56
|
+
"screenshot_path": { "type": "string", "description": "SHOT — non-empty PNG captured at the assertion moment." },
|
|
57
|
+
"trace_path": { "type": "string", "description": "TRACE — Playwright trace.zip or equivalent recording." },
|
|
58
|
+
"assertion": { "type": "string", "description": "ASSERT — the literal assertion that fired, e.g. `expect(response.status()).toBe(200)` observed 500." },
|
|
59
|
+
"source_file": { "type": "string", "description": "SOURCE — the source file the assertion implicates (router.ts, page.tsx, action.ts, ...)." },
|
|
60
|
+
"source_quote": { "type": "string", "description": "Optional verbatim quote from source_file explaining the implicated code." },
|
|
61
|
+
|
|
62
|
+
"http": {
|
|
63
|
+
"type": "object",
|
|
64
|
+
"description": "Optional — populated for api-contract / server-crash rules.",
|
|
65
|
+
"properties": {
|
|
66
|
+
"method": { "type": "string" },
|
|
67
|
+
"path": { "type": "string" },
|
|
68
|
+
"status": { "type": "integer" },
|
|
69
|
+
"response_snippet": { "type": "string", "maxLength": 512 },
|
|
70
|
+
"content_type": { "type": "string" }
|
|
71
|
+
}
|
|
72
|
+
},
|
|
73
|
+
"console_messages": {
|
|
74
|
+
"type": "array",
|
|
75
|
+
"items": {
|
|
76
|
+
"type": "object",
|
|
77
|
+
"required": ["level", "text"],
|
|
78
|
+
"properties": {
|
|
79
|
+
"level": { "type": "string", "enum": ["log", "warning", "error", "debug", "info"] },
|
|
80
|
+
"text": { "type": "string" },
|
|
81
|
+
"url": { "type": "string" },
|
|
82
|
+
"line": { "type": "integer" }
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
},
|
|
86
|
+
|
|
87
|
+
"suggested_fix": {
|
|
88
|
+
"type": "object",
|
|
89
|
+
"description": "Advisory only. sd-fix-style agents may consume this.",
|
|
90
|
+
"properties": {
|
|
91
|
+
"kind": { "type": "string", "enum": ["add-test", "fix-route", "fix-zod", "fix-rbac", "fix-auth", "add-fixture", "advisory"] },
|
|
92
|
+
"notes": { "type": "string" },
|
|
93
|
+
"files": { "type": "array", "items": { "type": "string" } }
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# api-contract-playbook (e2e-audit 0.2.0)
|
|
2
|
+
|
|
3
|
+
> How to turn Playwright's network observations into contract findings.
|
|
4
|
+
|
|
5
|
+
## Observations to capture per test
|
|
6
|
+
|
|
7
|
+
Wire these fixtures once (see `templates/base-fixture.ts.tpl`):
|
|
8
|
+
|
|
9
|
+
- `page.on('response')` — filter by `url` prefix matching `/api/`, `/v1/`, `/trpc/`.
|
|
10
|
+
- `page.on('console')` — levels `error` and `warning`.
|
|
11
|
+
- `page.on('pageerror')` — unhandled runtime errors in the SPA.
|
|
12
|
+
- `request.response()` — for explicit `page.request` calls in specs.
|
|
13
|
+
|
|
14
|
+
## Detection rules
|
|
15
|
+
|
|
16
|
+
### HTTP 4xx on a flow that should succeed
|
|
17
|
+
|
|
18
|
+
- **Rule:** `api-4xx`
|
|
19
|
+
- **Severity:** `high` (user-blocking) or `medium` (inconsistent UX)
|
|
20
|
+
- **Signal:** response status in [400, 499] on a request the test expected to succeed.
|
|
21
|
+
- **Evidence:** trace zip + screenshot at the assertion moment + response snippet (first 400 chars) + source_file = the route handler or tRPC procedure file.
|
|
22
|
+
|
|
23
|
+
### HTTP 5xx anywhere
|
|
24
|
+
|
|
25
|
+
- **Rule:** `api-5xx`
|
|
26
|
+
- **Severity:** `critical`
|
|
27
|
+
- **Always fails the test.** Trace zip is mandatory; Playwright's trace viewer will show the request/response payloads.
|
|
28
|
+
|
|
29
|
+
### HTML-instead-of-JSON (server crash signal)
|
|
30
|
+
|
|
31
|
+
Next.js and most frameworks render an HTML error page when the server throws. Detection:
|
|
32
|
+
|
|
33
|
+
```ts
|
|
34
|
+
if (
|
|
35
|
+
res.status() >= 500 &&
|
|
36
|
+
(res.headers()['content-type'] || '').includes('text/html') &&
|
|
37
|
+
/\/api\/|\/trpc\//.test(res.url())
|
|
38
|
+
) emit({ rule: 'server-crash', ... });
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Surface the last ~40 lines of `dev-server.log` in `detail` so the user can see the stack trace without opening the trace.
|
|
42
|
+
|
|
43
|
+
### Zod validation gap
|
|
44
|
+
|
|
45
|
+
- **Rule:** `zod-validation-missing`
|
|
46
|
+
- **Severity:** `medium`
|
|
47
|
+
- **Signal:** `api-surface.http_routes[].zod_schema_found == false` AND the route accepts `POST`/`PUT`/`PATCH`.
|
|
48
|
+
- **Evidence:** SHOT is waived (meta-ish), but `source_file` is required.
|
|
49
|
+
|
|
50
|
+
### RBAC bypass
|
|
51
|
+
|
|
52
|
+
- **Rule:** `rbac-bypass`
|
|
53
|
+
- **Severity:** `critical`
|
|
54
|
+
- **Signal:** an endpoint tagged `auth: "protected"` in `api-surface.json` returned 200 for a role that should be rejected.
|
|
55
|
+
- **Evidence:** trace showing the call under the "wrong" storageState + source_file (the middleware or procedure).
|
|
56
|
+
|
|
57
|
+
## What NOT to flag as a contract problem
|
|
58
|
+
|
|
59
|
+
- 404 on a page navigation the user typed manually — that's a route-missing finding, not a contract one.
|
|
60
|
+
- 401 on an endpoint protected BEFORE login — that's expected behavior; only flag after valid login.
|
|
61
|
+
- Third-party hosts (analytics, stripe, intercom) — ignore by prefix match on `stack.base_url`.
|
|
62
|
+
- `/api/auth/*` during login form submit — transient 4xx (invalid-credentials) is expected; only flag if the happy-path login produced it.
|
|
63
|
+
|
|
64
|
+
## Sampling
|
|
65
|
+
|
|
66
|
+
To avoid flooding findings, de-dupe by `(method, path, status)` and keep the first occurrence's trace. Add `http.count` for repeats.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# auth-setup-playbook (e2e-audit 0.2.0)
|
|
2
|
+
|
|
3
|
+
> How to obtain and reuse authenticated browser state, per auth provider. Goal: one `storageState.json` per role × per audit session, produced exactly once.
|
|
4
|
+
|
|
5
|
+
## Principles
|
|
6
|
+
|
|
7
|
+
1. **Never read `.env*`** inside this skill. If credentials are needed, ask the user inline. State files live under `$SESSION_DIR/auth/` and are gitignored (caller's responsibility).
|
|
8
|
+
2. **Reuse before synthesize.** If `existing-tests.storage_states` contains files, use them. Check freshness: any file older than 7 days is considered stale; regenerate.
|
|
9
|
+
3. **One role per file.** `owner.json`, `admin.json`, `member.json` — do not collapse roles.
|
|
10
|
+
4. **State files contain secrets.** They must not enter `findings.json`, `map.md`, or any output that ships to the user beyond the session dir.
|
|
11
|
+
|
|
12
|
+
## Playwright baseline
|
|
13
|
+
|
|
14
|
+
```ts
|
|
15
|
+
// playwright.config.ts
|
|
16
|
+
import { defineConfig, devices } from '@playwright/test';
|
|
17
|
+
|
|
18
|
+
export default defineConfig({
|
|
19
|
+
projects: [
|
|
20
|
+
{
|
|
21
|
+
name: 'setup',
|
|
22
|
+
testMatch: /.*\.setup\.ts/,
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
name: 'authed',
|
|
26
|
+
dependencies: ['setup'],
|
|
27
|
+
use: {
|
|
28
|
+
...devices['Desktop Chrome'],
|
|
29
|
+
storageState: '.e2e-audit/current/auth/owner.json',
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
],
|
|
33
|
+
});
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Per-provider recipes
|
|
37
|
+
|
|
38
|
+
### next-auth / Auth.js (credentials provider)
|
|
39
|
+
|
|
40
|
+
```ts
|
|
41
|
+
// tests/e2e/auth.setup.ts
|
|
42
|
+
import { test as setup } from '@playwright/test';
|
|
43
|
+
setup('authenticate owner', async ({ page }) => {
|
|
44
|
+
await page.goto('/signin');
|
|
45
|
+
await page.getByLabel('Email').fill(process.env.E2E_OWNER_EMAIL!);
|
|
46
|
+
await page.getByLabel('Password').fill(process.env.E2E_OWNER_PASSWORD!);
|
|
47
|
+
await page.getByRole('button', { name: /sign in/i }).click();
|
|
48
|
+
await page.waitForURL(/\/(dashboard|home)/);
|
|
49
|
+
await page.context().storageState({ path: '.e2e-audit/current/auth/owner.json' });
|
|
50
|
+
});
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Clerk
|
|
54
|
+
|
|
55
|
+
Use Clerk's `@clerk/testing` helper. It writes a session via `setupClerkTestingToken()` and then calls `storageState`.
|
|
56
|
+
|
|
57
|
+
### better-auth / Lucia
|
|
58
|
+
|
|
59
|
+
Same pattern as next-auth: drive the login form, then persist storage state. Both libraries store a session cookie which storageState captures.
|
|
60
|
+
|
|
61
|
+
### Supabase (JWT)
|
|
62
|
+
|
|
63
|
+
After login, `localStorage` holds the session. `storageState` serializes localStorage so no extra work is needed. If the dev project uses PKCE, ensure the setup runs in a chromium context.
|
|
64
|
+
|
|
65
|
+
### Custom (cookie-session / JWT header)
|
|
66
|
+
|
|
67
|
+
If the app does not have a login form (API-only auth), seed storage via `page.context().addCookies([...])` using a short-lived token the user pastes in. Never store long-lived tokens in `auth/*.json`.
|
|
68
|
+
|
|
69
|
+
## RBAC coverage
|
|
70
|
+
|
|
71
|
+
For every role declared in `stack.auth`:
|
|
72
|
+
|
|
73
|
+
1. Drive one happy-path login per role.
|
|
74
|
+
2. For each `trpc_procedures[]` with `auth == "protected"`, attempt the call with a role that should be forbidden. Expect 401 or 403. If 200, emit an `rbac-bypass` finding.
|
|
75
|
+
|
|
76
|
+
## Global teardown
|
|
77
|
+
|
|
78
|
+
Do NOT teardown or delete storageState files at the end of a run — the skill keeps them inside `$SESSION_DIR`, which is the audit's own sandbox.
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# coverage-gap-playbook (e2e-audit 0.2.0)
|
|
2
|
+
|
|
3
|
+
> How to translate `uncovered.json` into actionable meta findings + spec suggestions.
|
|
4
|
+
|
|
5
|
+
## Input
|
|
6
|
+
|
|
7
|
+
`detect-uncovered.sh` produces four uncovered arrays:
|
|
8
|
+
|
|
9
|
+
- `uncovered_routes` — user-facing pages the branch changed with no test referencing their URL.
|
|
10
|
+
- `uncovered_http` — REST handlers the branch changed with no test referencing their path.
|
|
11
|
+
- `uncovered_trpc` — tRPC procedures the branch changed with no test referencing their name.
|
|
12
|
+
- `uncovered_actions` — server actions the branch changed with no test referencing their name.
|
|
13
|
+
|
|
14
|
+
## One finding per category
|
|
15
|
+
|
|
16
|
+
Emit at most four meta findings:
|
|
17
|
+
|
|
18
|
+
```json
|
|
19
|
+
{
|
|
20
|
+
"id": "E2E-00XX",
|
|
21
|
+
"rule": "coverage-gap-routes",
|
|
22
|
+
"category": "coverage-gap",
|
|
23
|
+
"severity": "medium",
|
|
24
|
+
"summary": "N routes changed on branch without E2E coverage",
|
|
25
|
+
"detail": "<bulleted list of paths + files>",
|
|
26
|
+
"files_affected": ["<every file from the uncovered entries>"],
|
|
27
|
+
"suggested_fix": { "kind": "add-test", "files": ["tests/e2e/<slug>.spec.ts"] }
|
|
28
|
+
}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Severities:
|
|
32
|
+
|
|
33
|
+
- `coverage-gap-trpc` — `medium` (contract risk)
|
|
34
|
+
- `coverage-gap-http` — `medium` (contract risk)
|
|
35
|
+
- `coverage-gap-routes` — `low` if diff is cosmetic, `medium` otherwise
|
|
36
|
+
- `coverage-gap-actions` — `medium`
|
|
37
|
+
|
|
38
|
+
## Spec suggestions per surface type
|
|
39
|
+
|
|
40
|
+
**Page (route)**
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
test('GET /users/[id] renders user dashboard', async ({ authenticatedPage }) => {
|
|
44
|
+
const res = await authenticatedPage.goto('/users/1');
|
|
45
|
+
await expect(res!.status()).toBeLessThan(400);
|
|
46
|
+
await expect(authenticatedPage.getByRole('heading')).toBeVisible();
|
|
47
|
+
});
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
**HTTP route handler**
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
test('POST /api/users creates user (200)', async ({ request }) => {
|
|
54
|
+
const res = await request.post('/api/users', { data: { email: 'u@test', name: 't' } });
|
|
55
|
+
expect(res.status()).toBe(200);
|
|
56
|
+
expect((await res.json()).id).toBeTruthy();
|
|
57
|
+
});
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
**tRPC procedure**
|
|
61
|
+
|
|
62
|
+
```ts
|
|
63
|
+
test('users.create rejects invalid input (400)', async ({ request }) => {
|
|
64
|
+
const res = await request.post('/api/trpc/users.create?batch=1', {
|
|
65
|
+
data: { 0: { json: { email: 'not-an-email' } } },
|
|
66
|
+
});
|
|
67
|
+
expect(res.status()).toBeGreaterThanOrEqual(400);
|
|
68
|
+
});
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
**Server action**
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
test('createUser action returns redirect', async ({ page }) => {
|
|
75
|
+
await page.goto('/users/new');
|
|
76
|
+
await page.getByLabel('Email').fill('u@test');
|
|
77
|
+
await page.getByRole('button', { name: /create/i }).click();
|
|
78
|
+
await page.waitForURL(/\/users\/\d+/);
|
|
79
|
+
});
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## What to NOT flag
|
|
83
|
+
|
|
84
|
+
- Unchanged surfaces. If a file is not in `uncovered.diff_files`, it did not change on this branch — no finding, even if tests are missing. Coverage auditing of the whole app is out-of-scope for branch-diff mode.
|
|
85
|
+
- `loading.tsx`, `error.tsx`, `layout.tsx` in Next.js — these are treated separately. A coverage gap finding should not fire for them; they are covered transitively by any page that renders through them.
|
|
86
|
+
|
|
87
|
+
## suggested_fix files
|
|
88
|
+
|
|
89
|
+
For each uncovered entry, suggest a plausible spec filename:
|
|
90
|
+
|
|
91
|
+
- `/users/[id]` → `tests/e2e/users-id.spec.ts`
|
|
92
|
+
- `POST /api/users` → `tests/e2e/api-users.spec.ts`
|
|
93
|
+
- `users.create` → `tests/e2e/trpc-users.spec.ts`
|
|
94
|
+
|
|
95
|
+
Do not CREATE the files. This skill stops at reporting.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# post-run-feedback-playbook (e2e-audit 0.2.0)
|
|
2
|
+
|
|
3
|
+
> How to consolidate every signal emitted during a Playwright run into one feedback document that precedes findings.json.
|
|
4
|
+
|
|
5
|
+
## Why a separate document?
|
|
6
|
+
|
|
7
|
+
Findings are atomic and per-problem; the user needs a 30-second summary of "what broke during this run" before drilling into specifics. `post-run-feedback.md` answers that. `findings.json` answers "what do I fix."
|
|
8
|
+
|
|
9
|
+
## Signals to consolidate
|
|
10
|
+
|
|
11
|
+
Pull from these inputs:
|
|
12
|
+
|
|
13
|
+
1. Playwright JSON reporter output (`--reporter=json`).
|
|
14
|
+
2. `$SESSION_DIR/logs/dev-server.log` (last-N-lines per crash).
|
|
15
|
+
3. Console + pageerror logs captured by fixtures.
|
|
16
|
+
4. Network responses matching 4xx/5xx on API paths.
|
|
17
|
+
5. Dev-server PID status at run end (exited unexpectedly?).
|
|
18
|
+
|
|
19
|
+
## Classification
|
|
20
|
+
|
|
21
|
+
| kind | trigger | severity |
|
|
22
|
+
| ------------------ | ------------------------------------------------------------ | -------- |
|
|
23
|
+
| `api-4xx` | unexpected 4xx on a test's expected-success request | high |
|
|
24
|
+
| `api-5xx` | any 5xx response on API path | critical |
|
|
25
|
+
| `server-crash` | 5xx AND `Content-Type: text/html` on API path | critical |
|
|
26
|
+
| `console-error` | `page.on('console')` level=error, de-duped by message stem | medium |
|
|
27
|
+
| `pageerror` | `page.on('pageerror')` raised | high |
|
|
28
|
+
| `rbac-bypass` | protected endpoint returned 200 to wrong role | critical |
|
|
29
|
+
| `auth-flow-broken` | redirect loop or 401 on happy-path login | critical |
|
|
30
|
+
| `dev-server-log` | unhandledRejection / uncaughtException in dev-server.log | high |
|
|
31
|
+
| `test-timeout` | test exceeded Playwright timeout | medium |
|
|
32
|
+
| `flake` | retried test passed on retry (suggests flake) | low |
|
|
33
|
+
|
|
34
|
+
## Output shape
|
|
35
|
+
|
|
36
|
+
`post-run-feedback.json`:
|
|
37
|
+
|
|
38
|
+
```jsonc
|
|
39
|
+
{
|
|
40
|
+
"session": "<abs path to session dir>",
|
|
41
|
+
"duration_s": 128,
|
|
42
|
+
"tests_total": 42,
|
|
43
|
+
"tests_passed": 39,
|
|
44
|
+
"tests_failed": 3,
|
|
45
|
+
"tests_flaky": 1,
|
|
46
|
+
"problems": [
|
|
47
|
+
{
|
|
48
|
+
"kind": "server-crash",
|
|
49
|
+
"where": "POST /api/users",
|
|
50
|
+
"count": 2,
|
|
51
|
+
"severity": "critical",
|
|
52
|
+
"sample_trace": "traces/users-create-1.zip",
|
|
53
|
+
"sample_log_tail": "TypeError: Cannot read properties of undefined (reading 'id')\n at ..."
|
|
54
|
+
}
|
|
55
|
+
],
|
|
56
|
+
"uncovered_carried_forward": {
|
|
57
|
+
"routes": 4, "http": 2, "trpc": 9, "actions": 1
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`post-run-feedback.md` renders the same data as a human document with:
|
|
63
|
+
|
|
64
|
+
1. A one-line verdict ("3 tests failed, 2 critical problems, 16 uncovered surfaces — requires attention").
|
|
65
|
+
2. Top 5 problems by severity.
|
|
66
|
+
3. Link to the trace zip for each.
|
|
67
|
+
4. Next-step recommendations (add spec / fix handler / update RBAC middleware).
|
|
68
|
+
|
|
69
|
+
## De-duplication
|
|
70
|
+
|
|
71
|
+
- `console-error`: group by the first 120 chars of `text` (to collapse stack variation across retries).
|
|
72
|
+
- `api-4xx`/`api-5xx`: group by `(method, path, status)`.
|
|
73
|
+
- `pageerror`: group by the error message line.
|
|
74
|
+
|
|
75
|
+
## What NOT to include
|
|
76
|
+
|
|
77
|
+
- Full trace bytes — only file paths.
|
|
78
|
+
- Full stack traces for every occurrence — only a single `sample_log_tail` per group.
|
|
79
|
+
- Source code excerpts — that belongs in a finding's `source_quote`, not the feedback.
|
|
80
|
+
- Secret values (tokens, session cookies) — actively scrub any `Authorization:` header fragments out of `sample_log_tail`.
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# detect-stack.sh — identify framework, test runner, package manager, dev server,
|
|
3
|
+
# and environment files so downstream scripts + the skill prompt can adapt.
|
|
4
|
+
#
|
|
5
|
+
# Output: JSON object on stdout:
|
|
6
|
+
# {
|
|
7
|
+
# "framework": "next" | "remix" | "sveltekit" | "nuxt" | "astro" | "express" | "hono" | "fastify" | "unknown",
|
|
8
|
+
# "router_style": "app-router" | "pages-router" | "mixed" | "file-routes" | "n/a",
|
|
9
|
+
# "trpc": true | false,
|
|
10
|
+
# "trpc_version": "10" | "11" | "unknown" | null,
|
|
11
|
+
# "graphql": true | false,
|
|
12
|
+
# "orm": ["prisma" | "drizzle" | "mongoose" | "typeorm" | "kysely", ...],
|
|
13
|
+
# "auth": ["next-auth" | "authjs" | "clerk" | "auth0" | "lucia" | "better-auth" | "supabase" | "custom", ...],
|
|
14
|
+
# "test_runner": "playwright" | "cypress" | "vitest-browser" | "jest-puppeteer" | "none",
|
|
15
|
+
# "playwright_config": "playwright.config.ts" | null,
|
|
16
|
+
# "package_manager": "bun" | "pnpm" | "yarn" | "npm",
|
|
17
|
+
# "dev_command": "bun run dev" | "pnpm dev" | ...,
|
|
18
|
+
# "dev_port": <number> | null,
|
|
19
|
+
# "base_url": "http://localhost:<port>",
|
|
20
|
+
# "env_files": [".env.local", ".env.test", ...],
|
|
21
|
+
# "src_root": "src" | "app" | "." ,
|
|
22
|
+
# "has_middleware": true | false,
|
|
23
|
+
# "middleware_file": "middleware.ts" | "src/middleware.ts" | null
|
|
24
|
+
# }
|
|
25
|
+
#
|
|
26
|
+
# Best-effort. Absence of jq is fatal (downstream scripts require it).
|
|
27
|
+
set -euo pipefail
|
|
28
|
+
|
|
29
|
+
command -v jq >/dev/null || { echo "jq required" >&2; exit 2; }
|
|
30
|
+
|
|
31
|
+
read_pkg() {
|
|
32
|
+
[[ -f package.json ]] || { echo "null"; return; }
|
|
33
|
+
jq -r "${1:-.}" package.json 2>/dev/null || echo "null"
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
has_dep() {
|
|
37
|
+
local name="$1"
|
|
38
|
+
[[ -f package.json ]] || return 1
|
|
39
|
+
jq -e --arg n "$name" '(.dependencies[$n]? // .devDependencies[$n]? // empty)' package.json >/dev/null 2>&1
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
dep_version() {
|
|
43
|
+
local name="$1"
|
|
44
|
+
[[ -f package.json ]] || { echo ""; return; }
|
|
45
|
+
jq -r --arg n "$name" '.dependencies[$n]? // .devDependencies[$n]? // ""' package.json 2>/dev/null | sed 's/^[^0-9]*//'
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
# --- framework --------------------------------------------------------------
|
|
49
|
+
FRAMEWORK="unknown"
|
|
50
|
+
ROUTER_STYLE="n/a"
|
|
51
|
+
if has_dep next; then
|
|
52
|
+
FRAMEWORK="next"
|
|
53
|
+
if [[ -d app || -d src/app ]] && [[ -d pages || -d src/pages ]]; then ROUTER_STYLE="mixed"
|
|
54
|
+
elif [[ -d app || -d src/app ]]; then ROUTER_STYLE="app-router"
|
|
55
|
+
elif [[ -d pages || -d src/pages ]]; then ROUTER_STYLE="pages-router"
|
|
56
|
+
fi
|
|
57
|
+
elif has_dep @remix-run/react || has_dep @remix-run/node; then FRAMEWORK="remix"; ROUTER_STYLE="file-routes"
|
|
58
|
+
elif has_dep @sveltejs/kit; then FRAMEWORK="sveltekit"; ROUTER_STYLE="file-routes"
|
|
59
|
+
elif has_dep nuxt; then FRAMEWORK="nuxt"; ROUTER_STYLE="file-routes"
|
|
60
|
+
elif has_dep astro; then FRAMEWORK="astro"; ROUTER_STYLE="file-routes"
|
|
61
|
+
elif has_dep hono; then FRAMEWORK="hono"
|
|
62
|
+
elif has_dep fastify; then FRAMEWORK="fastify"
|
|
63
|
+
elif has_dep express; then FRAMEWORK="express"
|
|
64
|
+
fi
|
|
65
|
+
|
|
66
|
+
# --- trpc / graphql ---------------------------------------------------------
|
|
67
|
+
TRPC=false; TRPC_VER="null"
|
|
68
|
+
if has_dep @trpc/server; then
|
|
69
|
+
TRPC=true
|
|
70
|
+
v="$(dep_version @trpc/server)"
|
|
71
|
+
if [[ "$v" =~ ^10\. ]]; then TRPC_VER='"10"'
|
|
72
|
+
elif [[ "$v" =~ ^11\. ]]; then TRPC_VER='"11"'
|
|
73
|
+
else TRPC_VER='"unknown"'
|
|
74
|
+
fi
|
|
75
|
+
fi
|
|
76
|
+
GRAPHQL=false
|
|
77
|
+
if has_dep graphql || has_dep @apollo/server || has_dep @apollo/client || has_dep graphql-yoga; then
|
|
78
|
+
GRAPHQL=true
|
|
79
|
+
fi
|
|
80
|
+
|
|
81
|
+
# --- ORMs -------------------------------------------------------------------
|
|
82
|
+
orm_arr='[]'
|
|
83
|
+
for candidate in @prisma/client drizzle-orm mongoose typeorm kysely; do
|
|
84
|
+
if has_dep "$candidate"; then
|
|
85
|
+
short="${candidate##*/}"; short="${short%-orm}"
|
|
86
|
+
orm_arr="$(jq --arg o "$short" '. + [$o]' <<<"$orm_arr")"
|
|
87
|
+
fi
|
|
88
|
+
done
|
|
89
|
+
|
|
90
|
+
# --- auth providers ---------------------------------------------------------
|
|
91
|
+
auth_arr='[]'
|
|
92
|
+
add_auth() { auth_arr="$(jq --arg o "$1" '. + [$o]' <<<"$auth_arr")"; }
|
|
93
|
+
has_dep next-auth && add_auth next-auth
|
|
94
|
+
has_dep @auth/core && add_auth authjs
|
|
95
|
+
has_dep @clerk/nextjs && add_auth clerk
|
|
96
|
+
has_dep @clerk/clerk-sdk-node && add_auth clerk
|
|
97
|
+
has_dep @auth0/nextjs-auth0 && add_auth auth0
|
|
98
|
+
has_dep lucia && add_auth lucia
|
|
99
|
+
has_dep better-auth && add_auth better-auth
|
|
100
|
+
has_dep @supabase/supabase-js && add_auth supabase
|
|
101
|
+
|
|
102
|
+
# --- test runner ------------------------------------------------------------
|
|
103
|
+
TEST_RUNNER="none"
|
|
104
|
+
PLAYWRIGHT_CFG="null"
|
|
105
|
+
if has_dep @playwright/test; then
|
|
106
|
+
TEST_RUNNER="playwright"
|
|
107
|
+
for c in playwright.config.ts playwright.config.js playwright.config.mjs; do
|
|
108
|
+
[[ -f "$c" ]] && PLAYWRIGHT_CFG="\"$c\"" && break
|
|
109
|
+
done
|
|
110
|
+
elif has_dep cypress; then TEST_RUNNER="cypress"
|
|
111
|
+
elif has_dep @vitest/browser; then TEST_RUNNER="vitest-browser"
|
|
112
|
+
elif has_dep jest-puppeteer; then TEST_RUNNER="jest-puppeteer"
|
|
113
|
+
fi
|
|
114
|
+
|
|
115
|
+
# --- package manager + dev command -----------------------------------------
|
|
116
|
+
if [[ -f bun.lockb || -f bun.lock ]]; then PM="bun"; DEV_CMD="bun run dev"
|
|
117
|
+
elif [[ -f pnpm-lock.yaml ]]; then PM="pnpm"; DEV_CMD="pnpm dev"
|
|
118
|
+
elif [[ -f yarn.lock ]]; then PM="yarn"; DEV_CMD="yarn dev"
|
|
119
|
+
else PM="npm"; DEV_CMD="npm run dev"
|
|
120
|
+
fi
|
|
121
|
+
|
|
122
|
+
# Check `scripts.dev` exists; fall back to start script if not.
|
|
123
|
+
if [[ -f package.json ]]; then
|
|
124
|
+
HAS_DEV="$(jq -r '.scripts.dev // empty' package.json)"
|
|
125
|
+
[[ -z "$HAS_DEV" ]] && {
|
|
126
|
+
if jq -re '.scripts.start' package.json >/dev/null 2>&1; then
|
|
127
|
+
DEV_CMD="${DEV_CMD% dev} start"
|
|
128
|
+
fi
|
|
129
|
+
}
|
|
130
|
+
fi
|
|
131
|
+
|
|
132
|
+
# --- dev port ---------------------------------------------------------------
|
|
133
|
+
DEV_PORT="null"
|
|
134
|
+
# Common places: package.json scripts.dev "-p 3000" or "--port 3000", next.config, remix config
|
|
135
|
+
if [[ -f package.json ]]; then
|
|
136
|
+
if script="$(jq -r '.scripts.dev // ""' package.json)"; then
|
|
137
|
+
p="$(echo "$script" | grep -oE -- '--port[= ]+[0-9]+|-p[= ]+[0-9]+|PORT=[0-9]+' | grep -oE '[0-9]+' | head -1 || true)"
|
|
138
|
+
[[ -n "$p" ]] && DEV_PORT="$p"
|
|
139
|
+
fi
|
|
140
|
+
fi
|
|
141
|
+
# Fallback defaults per framework
|
|
142
|
+
if [[ "$DEV_PORT" == "null" ]]; then
|
|
143
|
+
case "$FRAMEWORK" in
|
|
144
|
+
next|express|hono|fastify) DEV_PORT=3000 ;;
|
|
145
|
+
remix) DEV_PORT=3000 ;;
|
|
146
|
+
sveltekit) DEV_PORT=5173 ;;
|
|
147
|
+
nuxt) DEV_PORT=3000 ;;
|
|
148
|
+
astro) DEV_PORT=4321 ;;
|
|
149
|
+
esac
|
|
150
|
+
fi
|
|
151
|
+
BASE_URL="http://localhost:${DEV_PORT:-3000}"
|
|
152
|
+
|
|
153
|
+
# --- env files --------------------------------------------------------------
|
|
154
|
+
env_arr='[]'
|
|
155
|
+
for f in .env .env.local .env.development .env.development.local .env.test .env.test.local; do
|
|
156
|
+
[[ -f "$f" ]] && env_arr="$(jq --arg n "$f" '. + [$n]' <<<"$env_arr")"
|
|
157
|
+
done
|
|
158
|
+
|
|
159
|
+
# --- src root + middleware --------------------------------------------------
|
|
160
|
+
SRC_ROOT="."
|
|
161
|
+
[[ -d src ]] && SRC_ROOT="src"
|
|
162
|
+
[[ -d app && ! -d src/app ]] && SRC_ROOT="."
|
|
163
|
+
HAS_MW=false; MW_FILE="null"
|
|
164
|
+
for f in middleware.ts middleware.js src/middleware.ts src/middleware.js; do
|
|
165
|
+
if [[ -f "$f" ]]; then HAS_MW=true; MW_FILE="\"$f\""; break; fi
|
|
166
|
+
done
|
|
167
|
+
|
|
168
|
+
# --- assemble ---------------------------------------------------------------
|
|
169
|
+
jq -n \
|
|
170
|
+
--arg framework "$FRAMEWORK" \
|
|
171
|
+
--arg router_style "$ROUTER_STYLE" \
|
|
172
|
+
--argjson trpc "$TRPC" \
|
|
173
|
+
--argjson trpc_version "$TRPC_VER" \
|
|
174
|
+
--argjson graphql "$GRAPHQL" \
|
|
175
|
+
--argjson orm "$orm_arr" \
|
|
176
|
+
--argjson auth "$auth_arr" \
|
|
177
|
+
--arg test_runner "$TEST_RUNNER" \
|
|
178
|
+
--argjson playwright_config "$PLAYWRIGHT_CFG" \
|
|
179
|
+
--arg package_manager "$PM" \
|
|
180
|
+
--arg dev_command "$DEV_CMD" \
|
|
181
|
+
--argjson dev_port "${DEV_PORT:-null}" \
|
|
182
|
+
--arg base_url "$BASE_URL" \
|
|
183
|
+
--argjson env_files "$env_arr" \
|
|
184
|
+
--arg src_root "$SRC_ROOT" \
|
|
185
|
+
--argjson has_middleware "$HAS_MW" \
|
|
186
|
+
--argjson middleware_file "$MW_FILE" \
|
|
187
|
+
'{
|
|
188
|
+
framework: $framework,
|
|
189
|
+
router_style: $router_style,
|
|
190
|
+
trpc: $trpc,
|
|
191
|
+
trpc_version: $trpc_version,
|
|
192
|
+
graphql: $graphql,
|
|
193
|
+
orm: $orm,
|
|
194
|
+
auth: $auth,
|
|
195
|
+
test_runner: $test_runner,
|
|
196
|
+
playwright_config: $playwright_config,
|
|
197
|
+
package_manager: $package_manager,
|
|
198
|
+
dev_command: $dev_command,
|
|
199
|
+
dev_port: $dev_port,
|
|
200
|
+
base_url: $base_url,
|
|
201
|
+
env_files: $env_files,
|
|
202
|
+
src_root: $src_root,
|
|
203
|
+
has_middleware: $has_middleware,
|
|
204
|
+
middleware_file: $middleware_file
|
|
205
|
+
}'
|