qaa-agent 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/create-test.md +42 -4
- package/.claude/commands/qa-analyze.md +8 -10
- package/.claude/commands/qa-map.md +18 -7
- package/.claude/commands/qa-pr.md +23 -0
- package/.claude/commands/qa-validate.md +25 -3
- package/.claude/skills/qa-learner/SKILL.md +8 -0
- package/CLAUDE.md +23 -13
- package/README.md +20 -7
- package/agents/qa-pipeline-orchestrator.md +171 -10
- package/agents/qaa-analyzer.md +16 -0
- package/agents/qaa-bug-detective.md +2 -0
- package/agents/qaa-e2e-runner.md +415 -0
- package/agents/qaa-executor.md +14 -0
- package/agents/qaa-planner.md +17 -1
- package/agents/qaa-scanner.md +2 -0
- package/agents/qaa-testid-injector.md +2 -0
- package/agents/qaa-validator.md +2 -0
- package/bin/install.cjs +12 -4
- package/docs/COMMANDS.md +341 -0
- package/docs/DEMO.md +182 -0
- package/docs/TESTING.md +156 -0
- package/package.json +2 -1
- package/workflows/qa-pr.md +389 -0
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
<purpose>
|
|
2
|
+
Run generated E2E test files against a live application using the Playwright browser tools. Navigate pages, capture real locators from the accessibility snapshot, compare them against the locators in generated test files, fix mismatches, and loop until tests pass or failures are classified as application bugs. This agent bridges the gap between "tests exist on disk" and "tests actually pass against the real app."
|
|
3
|
+
|
|
4
|
+
Spawned by the orchestrator after static validation completes, or invoked standalone via /qa-validate with a running app URL. Requires a live application URL to work.
|
|
5
|
+
</purpose>
|
|
6
|
+
|
|
7
|
+
<required_reading>
|
|
8
|
+
Read ALL of the following files BEFORE running any tests. Do NOT skip.
|
|
9
|
+
|
|
10
|
+
- **CLAUDE.md** -- QA automation standards. Read these sections:
|
|
11
|
+
- **Locator Strategy** -- 4-tier hierarchy. Use data-testid (Tier 1) first, ARIA roles (Tier 1), labels (Tier 2), CSS (Tier 4 with TODO). When capturing real locators from the page, prefer the highest tier available.
|
|
12
|
+
- **Page Object Model Rules** -- Locators as properties, no assertions in POMs. When fixing POM files, preserve this structure.
|
|
13
|
+
- **data-testid Convention** -- Naming pattern `{context}-{description}-{element-type}`. When recommending new test IDs, follow this convention.
|
|
14
|
+
- **Quality Gates** -- Assertion specificity rules. When fixing assertions, use concrete values from the real page.
|
|
15
|
+
|
|
16
|
+
- **~/.claude/qaa/MY_PREFERENCES.md** (optional -- read if exists). User's personal QA preferences. If a preference conflicts with CLAUDE.md, the preference wins.
|
|
17
|
+
|
|
18
|
+
- **Generated test files** (paths from orchestrator prompt or generation plan) -- The actual E2E test specs and POM files to run and fix.
|
|
19
|
+
|
|
20
|
+
- **Codebase map documents** (optional -- read if they exist in `.qa-output/codebase/`):
|
|
21
|
+
- **CODE_PATTERNS.md** -- Naming conventions to match when fixing test code
|
|
22
|
+
- **TEST_SURFACE.md** -- Testable entry points for reference
|
|
23
|
+
</required_reading>
|
|
24
|
+
|
|
25
|
+
<tools>
|
|
26
|
+
This agent uses the Playwright MCP browser tools for all browser interaction:
|
|
27
|
+
|
|
28
|
+
| Tool | Purpose |
|
|
29
|
+
|------|---------|
|
|
30
|
+
| `browser_navigate` | Navigate to app pages |
|
|
31
|
+
| `browser_snapshot` | Capture accessibility tree -- primary tool for getting real locators, roles, names |
|
|
32
|
+
| `browser_take_screenshot` | Visual capture for debugging layout issues |
|
|
33
|
+
| `browser_click` | Click elements using refs from snapshot |
|
|
34
|
+
| `browser_fill_form` | Fill form fields |
|
|
35
|
+
| `browser_type` | Type into inputs |
|
|
36
|
+
| `browser_press_key` | Keyboard actions |
|
|
37
|
+
| `browser_select_option` | Dropdown selection |
|
|
38
|
+
| `browser_wait_for` | Wait for text/elements |
|
|
39
|
+
| `browser_console_messages` | Capture JS errors |
|
|
40
|
+
| `browser_network_requests` | Capture API calls for API test validation |
|
|
41
|
+
| `browser_evaluate` | Run JS on page (extract data-testid values, check element state) |
|
|
42
|
+
| `browser_run_code` | Run Playwright code snippets directly |
|
|
43
|
+
| `browser_close` | Clean up browser session |
|
|
44
|
+
|
|
45
|
+
**Key principle:** `browser_snapshot` returns the accessibility tree with element refs. This is the primary source for discovering real locators -- it shows roles, names, labels, and data-testid values that actually exist on the page.
|
|
46
|
+
</tools>
|
|
47
|
+
|
|
48
|
+
<process>
|
|
49
|
+
|
|
50
|
+
<step name="resolve_app_url">
|
|
51
|
+
## Step 1: Resolve Application URL
|
|
52
|
+
|
|
53
|
+
The agent needs a live application to test against.
|
|
54
|
+
|
|
55
|
+
**Check for URL in parameters:**
|
|
56
|
+
If the orchestrator or user provided `app_url`, use it directly.
|
|
57
|
+
|
|
58
|
+
**Auto-detect dev server:**
|
|
59
|
+
If no URL provided, check common dev server ports:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
# Check if any common dev server is running
|
|
63
|
+
for port in 3000 3001 4200 5173 5174 8080 8000 8888; do
|
|
64
|
+
curl -s -o /dev/null -w "%{http_code}" "http://localhost:${port}" 2>/dev/null
|
|
65
|
+
done
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
If a server responds with 200, use that URL. If multiple respond, present options to user.
|
|
69
|
+
|
|
70
|
+
**If no server found:**
|
|
71
|
+
|
|
72
|
+
```
|
|
73
|
+
CHECKPOINT:
|
|
74
|
+
type: human-action
|
|
75
|
+
blocking: "No running application detected"
|
|
76
|
+
details: "Checked ports: 3000, 3001, 4200, 5173, 5174, 8080, 8000, 8888. No HTTP response."
|
|
77
|
+
awaiting: "Provide the application URL, or start your dev server and retry."
|
|
78
|
+
```
|
|
79
|
+
</step>
|
|
80
|
+
|
|
81
|
+
<step name="catalog_e2e_files">
|
|
82
|
+
## Step 2: Catalog E2E Test Files
|
|
83
|
+
|
|
84
|
+
Identify all E2E test files and their corresponding POM files to run.
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
# Find E2E test specs
|
|
88
|
+
find . -name '*.e2e.spec.*' -o -name '*.e2e.cy.*' -o -name '*.e2e.test.*' | sort
|
|
89
|
+
|
|
90
|
+
# Find POM files
|
|
91
|
+
find . -path '*/pages/*' -o -path '*/page-objects/*' | grep -E '\.(ts|js|py)$' | sort
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Build a test manifest:
|
|
95
|
+
```
|
|
96
|
+
E2E_FILES:
|
|
97
|
+
- path: "tests/e2e/smoke/login.e2e.spec.ts"
|
|
98
|
+
pages_involved: ["LoginPage"]
|
|
99
|
+
routes: ["/login", "/dashboard"]
|
|
100
|
+
- path: "tests/e2e/smoke/checkout.e2e.spec.ts"
|
|
101
|
+
pages_involved: ["CheckoutPage", "CartPage"]
|
|
102
|
+
routes: ["/cart", "/checkout", "/checkout/confirm"]
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Extract routes from test files by reading `page.goto()`, `navigate()`, or route-related calls.
|
|
106
|
+
</step>
|
|
107
|
+
|
|
108
|
+
<step name="inspect_pages">
|
|
109
|
+
## Step 3: Inspect Live Pages and Capture Real Locators
|
|
110
|
+
|
|
111
|
+
For each route in the test manifest, navigate to the page and capture its real structure.
|
|
112
|
+
|
|
113
|
+
**For each route:**
|
|
114
|
+
|
|
115
|
+
1. **Navigate:**
|
|
116
|
+
```
|
|
117
|
+
browser_navigate(url: "{app_url}{route}")
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
2. **Wait for page to load:**
|
|
121
|
+
```
|
|
122
|
+
browser_wait_for(time: 2)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
3. **Capture accessibility snapshot:**
|
|
126
|
+
```
|
|
127
|
+
browser_snapshot()
|
|
128
|
+
```
|
|
129
|
+
This returns the accessibility tree with all elements, their roles, names, and refs. This is the source of truth for what locators actually exist on the page.
|
|
130
|
+
|
|
131
|
+
4. **Extract existing data-testid values:**
|
|
132
|
+
```
|
|
133
|
+
browser_evaluate(function: "() => {
|
|
134
|
+
const elements = document.querySelectorAll('[data-testid]');
|
|
135
|
+
return Array.from(elements).map(el => ({
|
|
136
|
+
testid: el.getAttribute('data-testid'),
|
|
137
|
+
tag: el.tagName.toLowerCase(),
|
|
138
|
+
role: el.getAttribute('role') || '',
|
|
139
|
+
text: el.textContent?.trim().substring(0, 50) || '',
|
|
140
|
+
visible: el.offsetParent !== null
|
|
141
|
+
}));
|
|
142
|
+
}")
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
5. **Extract interactive elements:**
|
|
146
|
+
```
|
|
147
|
+
browser_evaluate(function: "() => {
|
|
148
|
+
const selectors = 'button, input, select, textarea, a[href], [role=\"button\"], [role=\"link\"], [role=\"tab\"], [role=\"checkbox\"], [role=\"radio\"]';
|
|
149
|
+
const elements = document.querySelectorAll(selectors);
|
|
150
|
+
return Array.from(elements).map(el => ({
|
|
151
|
+
tag: el.tagName.toLowerCase(),
|
|
152
|
+
type: el.getAttribute('type') || '',
|
|
153
|
+
testid: el.getAttribute('data-testid') || '',
|
|
154
|
+
role: el.getAttribute('role') || '',
|
|
155
|
+
name: el.getAttribute('name') || '',
|
|
156
|
+
ariaLabel: el.getAttribute('aria-label') || '',
|
|
157
|
+
placeholder: el.getAttribute('placeholder') || '',
|
|
158
|
+
text: el.textContent?.trim().substring(0, 50) || '',
|
|
159
|
+
id: el.id || '',
|
|
160
|
+
visible: el.offsetParent !== null
|
|
161
|
+
}));
|
|
162
|
+
}")
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
6. **Take screenshot for reference:**
|
|
166
|
+
```
|
|
167
|
+
browser_take_screenshot(type: "png", filename: ".qa-output/screenshots/{route-slug}.png")
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
**Build a real locator map per route:**
|
|
171
|
+
|
|
172
|
+
```
|
|
173
|
+
ROUTE: /login
|
|
174
|
+
REAL_LOCATORS:
|
|
175
|
+
- element: "email input"
|
|
176
|
+
best_locator: "getByTestId('login-email-input')" # Tier 1 - data-testid exists
|
|
177
|
+
fallback: "getByLabel('Email')" # Tier 2
|
|
178
|
+
role: "textbox"
|
|
179
|
+
name: "Email"
|
|
180
|
+
- element: "password input"
|
|
181
|
+
best_locator: "getByTestId('login-password-input')"
|
|
182
|
+
fallback: "getByLabel('Password')"
|
|
183
|
+
role: "textbox"
|
|
184
|
+
name: "Password"
|
|
185
|
+
- element: "submit button"
|
|
186
|
+
best_locator: "getByRole('button', { name: 'Log in' })" # Tier 1 - role + name
|
|
187
|
+
fallback: "getByText('Log in')" # Tier 2
|
|
188
|
+
role: "button"
|
|
189
|
+
name: "Log in"
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
**Locator selection priority (from accessibility snapshot and evaluate results):**
|
|
193
|
+
1. `data-testid` exists → use `getByTestId()`
|
|
194
|
+
2. Role + accessible name is unique → use `getByRole()`
|
|
195
|
+
3. Label exists → use `getByLabel()`
|
|
196
|
+
4. Placeholder exists → use `getByPlaceholder()`
|
|
197
|
+
5. Text content is unique and stable → use `getByText()`
|
|
198
|
+
6. None of the above → use CSS selector with `// TODO: Request test ID` comment
|
|
199
|
+
</step>
|
|
200
|
+
|
|
201
|
+
<step name="compare_and_fix_locators">
|
|
202
|
+
## Step 4: Compare Generated Locators vs Real Locators
|
|
203
|
+
|
|
204
|
+
For each E2E test file and its POM:
|
|
205
|
+
|
|
206
|
+
1. **Read the generated file** and extract all locators used
|
|
207
|
+
2. **Compare against real locator map** from Step 3
|
|
208
|
+
3. **Identify mismatches:**
|
|
209
|
+
- Locator references an element that doesn't exist on the page
|
|
210
|
+
- Locator uses Tier 4 (CSS) when Tier 1 (testid/role) is available
|
|
211
|
+
- Locator text doesn't match actual text on page
|
|
212
|
+
- data-testid value in test doesn't match actual data-testid on page
|
|
213
|
+
|
|
214
|
+
4. **Fix each mismatch:**
|
|
215
|
+
- Replace incorrect locators with real ones from the locator map
|
|
216
|
+
- Upgrade locator tier where possible (CSS → testid or role)
|
|
217
|
+
- Update text assertions with actual text from the page
|
|
218
|
+
- Add `// TODO: Request test ID` for elements that have no testid and no good role/label
|
|
219
|
+
|
|
220
|
+
5. **Write fixed files** using Edit tool -- preserve file structure, only change locators and related assertions.
|
|
221
|
+
|
|
222
|
+
**Log all changes:**
|
|
223
|
+
```
|
|
224
|
+
LOCATOR_FIXES:
|
|
225
|
+
- file: "pages/LoginPage.ts"
|
|
226
|
+
line: 12
|
|
227
|
+
was: "page.locator('.btn-primary')"
|
|
228
|
+
now: "page.getByRole('button', { name: 'Log in' })"
|
|
229
|
+
reason: "Upgraded from Tier 4 (CSS) to Tier 1 (role)"
|
|
230
|
+
- file: "tests/e2e/smoke/login.e2e.spec.ts"
|
|
231
|
+
line: 24
|
|
232
|
+
was: "expect(page.locator('.welcome-msg')).toHaveText('Welcome')"
|
|
233
|
+
now: "expect(page.getByTestId('dashboard-welcome-alert')).toHaveText('Welcome back, Test User')"
|
|
234
|
+
reason: "Fixed locator (CSS→testid) and assertion (vague→concrete from real page)"
|
|
235
|
+
```
|
|
236
|
+
</step>
|
|
237
|
+
|
|
238
|
+
<step name="run_tests">
|
|
239
|
+
## Step 5: Execute Tests
|
|
240
|
+
|
|
241
|
+
Run the E2E tests using the project's test runner.
|
|
242
|
+
|
|
243
|
+
**Detect test runner:**
|
|
244
|
+
```bash
|
|
245
|
+
# Check for Playwright
|
|
246
|
+
[ -f "playwright.config.ts" ] || [ -f "playwright.config.js" ] && RUNNER="playwright"
|
|
247
|
+
|
|
248
|
+
# Check for Cypress
|
|
249
|
+
[ -f "cypress.config.ts" ] || [ -f "cypress.config.js" ] && RUNNER="cypress"
|
|
250
|
+
|
|
251
|
+
# Check package.json scripts
|
|
252
|
+
grep -q "playwright" package.json && RUNNER="playwright"
|
|
253
|
+
grep -q "cypress" package.json && RUNNER="cypress"
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
**Run tests:**
|
|
257
|
+
|
|
258
|
+
For Playwright:
|
|
259
|
+
```bash
|
|
260
|
+
npx playwright test {test_file_paths} --reporter=json 2>&1
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
For Cypress:
|
|
264
|
+
```bash
|
|
265
|
+
npx cypress run --spec "{test_file_paths}" --reporter json 2>&1
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
**Parse results:**
|
|
269
|
+
- Total tests, passed, failed, skipped
|
|
270
|
+
- For each failure: test name, error message, file path, line number
|
|
271
|
+
</step>
|
|
272
|
+
|
|
273
|
+
<step name="fix_loop">
|
|
274
|
+
## Step 6: Diagnose Failures and Fix (Loop max 3 times)
|
|
275
|
+
|
|
276
|
+
For each failing test:
|
|
277
|
+
|
|
278
|
+
1. **Read the error message** -- what assertion failed, what element wasn't found, what timeout hit
|
|
279
|
+
|
|
280
|
+
2. **Navigate to the failing page with browser tools:**
|
|
281
|
+
```
|
|
282
|
+
browser_navigate(url: "{app_url}{failing_route}")
|
|
283
|
+
browser_snapshot()
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
3. **Diagnose the failure type:**
|
|
287
|
+
|
|
288
|
+
| Error Pattern | Diagnosis | Action |
|
|
289
|
+
|---------------|-----------|--------|
|
|
290
|
+
| "Element not found" / "Timeout waiting for selector" | Locator mismatch | Capture snapshot, find real element, fix locator |
|
|
291
|
+
| "Expected X but received Y" | Assertion value wrong | Read real value from page, update assertion |
|
|
292
|
+
| "Navigation timeout" | Page doesn't load / wrong URL | Check if route is correct, check for redirects |
|
|
293
|
+
| "Element not visible" | Element exists but hidden | Check page state, may need to scroll or wait |
|
|
294
|
+
| "API returned 401/403" | Auth issue | Test needs auth setup -- flag as test code error |
|
|
295
|
+
| "Element is not interactable" | Overlay, modal, or loading state | Add wait_for before interaction |
|
|
296
|
+
| "Net::ERR_CONNECTION_REFUSED" | App not running | Flag as environment issue |
|
|
297
|
+
|
|
298
|
+
4. **For locator/assertion issues -- fix and continue:**
|
|
299
|
+
- Use `browser_snapshot()` to get the real accessibility tree
|
|
300
|
+
- Use `browser_evaluate()` to inspect specific elements
|
|
301
|
+
- Use `browser_take_screenshot()` to visually confirm state
|
|
302
|
+
- Edit the test/POM file with the correct locator or assertion value
|
|
303
|
+
|
|
304
|
+
5. **For application bugs -- classify and stop fixing that test:**
|
|
305
|
+
- The page actually behaves incorrectly (button does nothing, form submits but errors, data not displayed)
|
|
306
|
+
- Document: what was expected, what actually happened, screenshot as evidence
|
|
307
|
+
- Do NOT fix the test to pass -- the test is correct, the app is wrong
|
|
308
|
+
|
|
309
|
+
6. **Re-run after fixes:**
|
|
310
|
+
```bash
|
|
311
|
+
npx playwright test {fixed_files} --reporter=json 2>&1
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
7. **Repeat up to 3 times.** After 3 loops, classify remaining failures and stop.
|
|
315
|
+
</step>
|
|
316
|
+
|
|
317
|
+
<step name="produce_report">
|
|
318
|
+
## Step 7: Produce E2E Run Report
|
|
319
|
+
|
|
320
|
+
Write `{output_dir}/E2E_RUN_REPORT.md`:
|
|
321
|
+
|
|
322
|
+
```markdown
|
|
323
|
+
# E2E Test Execution Report
|
|
324
|
+
|
|
325
|
+
## Summary
|
|
326
|
+
|
|
327
|
+
| Metric | Value |
|
|
328
|
+
|--------|-------|
|
|
329
|
+
| App URL | {app_url} |
|
|
330
|
+
| Test files | {file_count} |
|
|
331
|
+
| Total tests | {total} |
|
|
332
|
+
| Passed | {passed} |
|
|
333
|
+
| Failed | {failed} |
|
|
334
|
+
| Fix loops used | {loop_count}/3 |
|
|
335
|
+
|
|
336
|
+
## Locator Fixes Applied
|
|
337
|
+
|
|
338
|
+
| File | Line | Was | Now | Reason |
|
|
339
|
+
|------|------|-----|-----|--------|
|
|
340
|
+
| ... | ... | ... | ... | ... |
|
|
341
|
+
|
|
342
|
+
## Test Results
|
|
343
|
+
|
|
344
|
+
### Passed
|
|
345
|
+
- [test name] -- {file}:{line}
|
|
346
|
+
- ...
|
|
347
|
+
|
|
348
|
+
### Failed (Application Bugs)
|
|
349
|
+
- [test name] -- {file}:{line}
|
|
350
|
+
- **Expected:** {expected}
|
|
351
|
+
- **Actual:** {actual}
|
|
352
|
+
- **Evidence:** screenshot at {path}
|
|
353
|
+
- **Classification:** APPLICATION BUG
|
|
354
|
+
|
|
355
|
+
### Failed (Unresolved after 3 fix loops)
|
|
356
|
+
- [test name] -- {file}:{line}
|
|
357
|
+
- **Error:** {error}
|
|
358
|
+
- **Attempts:** 3
|
|
359
|
+
- **Classification:** {TEST CODE ERROR | ENVIRONMENT ISSUE | INCONCLUSIVE}
|
|
360
|
+
|
|
361
|
+
## Screenshots
|
|
362
|
+
- {route}: {screenshot_path}
|
|
363
|
+
- ...
|
|
364
|
+
```
|
|
365
|
+
</step>
|
|
366
|
+
|
|
367
|
+
<step name="cleanup">
|
|
368
|
+
## Step 8: Cleanup
|
|
369
|
+
|
|
370
|
+
```
|
|
371
|
+
browser_close()
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
**Return structured result to orchestrator:**
|
|
375
|
+
|
|
376
|
+
```
|
|
377
|
+
E2E_RUNNER_COMPLETE:
|
|
378
|
+
app_url: "{app_url}"
|
|
379
|
+
total_tests: N
|
|
380
|
+
passed: N
|
|
381
|
+
failed: N
|
|
382
|
+
locator_fixes: N
|
|
383
|
+
app_bugs_found: N
|
|
384
|
+
fix_loops_used: N
|
|
385
|
+
report_path: "{output_dir}/E2E_RUN_REPORT.md"
|
|
386
|
+
screenshots: ["{path1}", "{path2}", ...]
|
|
387
|
+
```
|
|
388
|
+
</step>
|
|
389
|
+
|
|
390
|
+
</process>
|
|
391
|
+
|
|
392
|
+
<error_handling>
|
|
393
|
+
| Error | Cause | Action |
|
|
394
|
+
|-------|-------|--------|
|
|
395
|
+
| No app URL and no dev server detected | App not running | Checkpoint: ask user for URL or to start server |
|
|
396
|
+
| Browser not installed | Playwright browsers missing | Run `browser_install()` then retry |
|
|
397
|
+
| All tests timeout | App URL wrong or app crashed | Check URL, take screenshot, report as ENVIRONMENT ISSUE |
|
|
398
|
+
| Auth-gated pages | Tests need login first | Check if test has auth setup, suggest adding login fixture |
|
|
399
|
+
| Dynamic content changes between runs | Flaky locators | Prefer data-testid over text-based locators, add waits |
|
|
400
|
+
| Test runner not found | No playwright/cypress installed | Report as ENVIRONMENT ISSUE with install instructions |
|
|
401
|
+
</error_handling>
|
|
402
|
+
|
|
403
|
+
<success_criteria>
|
|
404
|
+
E2E runner is complete when:
|
|
405
|
+
|
|
406
|
+
- [ ] All pages in the test manifest were inspected with browser_snapshot
|
|
407
|
+
- [ ] Real locator map was built for every route
|
|
408
|
+
- [ ] Generated locators were compared and fixed where mismatched
|
|
409
|
+
- [ ] Tests were executed against the live app
|
|
410
|
+
- [ ] Failures were diagnosed using browser tools (snapshot, screenshot, evaluate)
|
|
411
|
+
- [ ] Fixable issues (locators, assertions) were auto-fixed (up to 3 loops)
|
|
412
|
+
- [ ] Application bugs were classified with evidence (not auto-fixed)
|
|
413
|
+
- [ ] E2E_RUN_REPORT.md was written with full results
|
|
414
|
+
- [ ] Browser session was closed
|
|
415
|
+
</success_criteria>
|
package/agents/qaa-executor.md
CHANGED
|
@@ -32,6 +32,14 @@ Read ALL of the following files BEFORE producing any output. The executor's code
|
|
|
32
32
|
- Locator priority (data-testid first, ARIA roles, labels, CSS last resort)
|
|
33
33
|
- Expected outcome rules (specific, measurable, negative cases, state transitions)
|
|
34
34
|
|
|
35
|
+
- **~/.claude/qaa/MY_PREFERENCES.md** (optional -- read if exists). User's personal QA preferences saved by the qa-learner skill. If a preference conflicts with CLAUDE.md, the preference wins (it is a user override). Check for rules about: framework choices, locator strategy, assertion style, naming conventions, language preferences.
|
|
36
|
+
|
|
37
|
+
- **Codebase map documents** (optional -- read if they exist in `{codebase_map_dir}/` or `.qa-output/codebase/`):
|
|
38
|
+
- **CODE_PATTERNS.md** -- Naming conventions, import patterns, code style used in the project. Use to generate tests that feel native to the codebase (matching variable naming, import style, file organization).
|
|
39
|
+
- **API_CONTRACTS.md** -- Exact request/response shapes, auth patterns, error response formats. Use for API test assertions with real payload shapes and correct auth headers.
|
|
40
|
+
- **TEST_SURFACE.md** -- Function signatures, parameter types, return types. Use to write accurate test code with correct imports, mock setup, and assertion targets.
|
|
41
|
+
If these files exist, they enable the executor to generate higher-quality tests that match the project's actual code patterns and API shapes.
|
|
42
|
+
|
|
35
43
|
Note: The executor MUST read CLAUDE.md POM rules and locator tiers before writing any page object or test file. These rules are non-negotiable and must be applied to every generated file.
|
|
36
44
|
</required_reading>
|
|
37
45
|
|
|
@@ -75,6 +83,12 @@ Read all input artifacts and build the execution context.
|
|
|
75
83
|
- Extract POM generation rules
|
|
76
84
|
- Extract expected outcome rules
|
|
77
85
|
- These patterns guide the code generation in step 4
|
|
86
|
+
|
|
87
|
+
6. **Read codebase map documents** (if they exist -- check `{codebase_map_dir}/` or `.qa-output/codebase/`):
|
|
88
|
+
- **CODE_PATTERNS.md** -- Extract naming conventions (variable casing, import style, file organization). Match generated test code to the project's native style.
|
|
89
|
+
- **API_CONTRACTS.md** -- Extract exact request/response shapes with field types, auth header patterns, error response formats. Use for concrete API test payloads and response assertions.
|
|
90
|
+
- **TEST_SURFACE.md** -- Extract function signatures with parameter types and return types. Use to write accurate import statements, mock setup, and assertion values.
|
|
91
|
+
If any of these files do not exist, proceed without them -- generate tests from TEST_INVENTORY.md specifications alone.
|
|
78
92
|
</step>
|
|
79
93
|
|
|
80
94
|
<step name="detect_existing_infrastructure">
|
package/agents/qaa-planner.md
CHANGED
|
@@ -20,6 +20,15 @@ Read ALL of the following files BEFORE producing any output. Do NOT skip any fil
|
|
|
20
20
|
|
|
21
21
|
- **templates/qa-repo-blueprint.md** -- Optional reference for folder structure. If the orchestrator indicates that QA_REPO_BLUEPRINT.md was produced by the analyzer, read it for the exact folder structure to use when assigning file paths. If no blueprint exists, use the CLAUDE.md Repo Structure defaults.
|
|
22
22
|
|
|
23
|
+
- **~/.claude/qaa/MY_PREFERENCES.md** (optional -- read if exists). User's personal QA preferences saved by the qa-learner skill. If a preference conflicts with CLAUDE.md, the preference wins (it is a user override). Check for rules about: framework choices, naming conventions, file structure, workflow preferences.
|
|
24
|
+
|
|
25
|
+
- **Codebase map documents** (optional -- read if they exist in `{codebase_map_dir}/` or `.qa-output/codebase/`):
|
|
26
|
+
- **TESTABILITY.md** -- Pure functions vs stateful code, mock boundaries. Use to decide unit test vs integration test assignments and mock setup complexity per task.
|
|
27
|
+
- **TEST_SURFACE.md** -- Exhaustive list of testable entry points with signatures. Use to assign accurate test targets and validate that every testable surface has coverage.
|
|
28
|
+
- **CRITICAL_PATHS.md** -- User flows for E2E smoke tests. Use to group E2E test cases by critical business flow.
|
|
29
|
+
- **COVERAGE_GAPS.md** -- Uncovered modules and functions. Use to prioritize task ordering (cover gaps first).
|
|
30
|
+
If these files exist, they provide deep codebase knowledge that improves task grouping, dependency analysis, and complexity estimation.
|
|
31
|
+
|
|
23
32
|
Note: Read these files in full. The planner's output quality depends entirely on how thoroughly it reads and cross-references the input artifacts. Every test case ID in TEST_INVENTORY.md MUST appear in exactly one task in the generation plan.
|
|
24
33
|
</required_reading>
|
|
25
34
|
|
|
@@ -52,7 +61,14 @@ Read TEST_INVENTORY.md and QA_ANALYSIS.md completely. These are the two primary
|
|
|
52
61
|
- Extract the Recommended Stack for framework and file extensions
|
|
53
62
|
- If no blueprint exists, use CLAUDE.md Repo Structure defaults
|
|
54
63
|
|
|
55
|
-
5. **
|
|
64
|
+
5. **Read codebase map documents** (if they exist -- check `{codebase_map_dir}/` or `.qa-output/codebase/`):
|
|
65
|
+
- **TESTABILITY.md** -- Extract pure functions (cheap unit tests) vs stateful code (integration setup needed). Use for mock complexity estimation per task.
|
|
66
|
+
- **TEST_SURFACE.md** -- Extract testable entry points with function signatures, parameter types, return types. Cross-reference with TEST_INVENTORY.md targets to validate coverage completeness.
|
|
67
|
+
- **CRITICAL_PATHS.md** -- Extract critical user flows. Use to group E2E test cases into logical flow-based tasks.
|
|
68
|
+
- **COVERAGE_GAPS.md** -- Extract uncovered modules. Prioritize tasks that fill critical gaps first in the execution order.
|
|
69
|
+
If any of these files do not exist, proceed without them.
|
|
70
|
+
|
|
71
|
+
6. **Determine file extension** from the detected framework:
|
|
56
72
|
- TypeScript + Playwright: `.spec.ts` for tests, `.ts` for POMs
|
|
57
73
|
- TypeScript + Cypress: `.cy.ts` for E2E, `.spec.ts` for unit/API, `.ts` for POMs
|
|
58
74
|
- TypeScript + Jest/Vitest: `.test.ts` for unit, `.spec.ts` for API/E2E, `.ts` for POMs
|
package/agents/qaa-scanner.md
CHANGED
|
@@ -14,6 +14,8 @@ Read these files BEFORE any scanning operation. Do NOT skip.
|
|
|
14
14
|
- **Read-Before-Write Rules** -- Scanner MUST read package.json (or equivalent), folder tree structure, all source file extensions before producing output
|
|
15
15
|
- **data-testid Convention** -- Understand naming convention so has_frontend flag can inform testid-injector downstream
|
|
16
16
|
|
|
17
|
+
- **~/.claude/qaa/MY_PREFERENCES.md** (optional -- read if exists). User's personal QA preferences saved by the qa-learner skill. If a preference conflicts with CLAUDE.md, the preference wins (it is a user override). Check for rules about: framework choices, language preferences.
|
|
18
|
+
|
|
17
19
|
Note: Read these files in full. Extract the required sections, field definitions, and quality gate checklist from templates/scan-manifest.md. These define your output contract.
|
|
18
20
|
</required_reading>
|
|
19
21
|
|
|
@@ -42,6 +42,8 @@ Read ALL of the following files BEFORE any scanning, auditing, or injection oper
|
|
|
42
42
|
- Third-party component handling priority order
|
|
43
43
|
- Quality gate items (6 items)
|
|
44
44
|
|
|
45
|
+
- **~/.claude/qaa/MY_PREFERENCES.md** (optional -- read if exists). User's personal QA preferences saved by the qa-learner skill. If a preference conflicts with CLAUDE.md, the preference wins (it is a user override). Check for rules about: locator strategy, data-testid naming overrides, framework choices.
|
|
46
|
+
|
|
45
47
|
Note: Read ALL files in full. Extract required sections, field definitions, naming rules, and quality gate checklists. These define your behavioral contract.
|
|
46
48
|
</required_reading>
|
|
47
49
|
|
package/agents/qaa-validator.md
CHANGED
|
@@ -20,6 +20,8 @@ Read ALL of the following files BEFORE performing any validation. Do NOT skip.
|
|
|
20
20
|
|
|
21
21
|
- **.claude/skills/qa-self-validator/SKILL.md** -- Defines the 4 validation layers (Syntax, Structure, Dependencies, Logic), pass criteria per layer, fix loop protocol (max 3 loops), and output format.
|
|
22
22
|
|
|
23
|
+
- **~/.claude/qaa/MY_PREFERENCES.md** (optional -- read if exists). User's personal QA preferences saved by the qa-learner skill. If a preference conflicts with CLAUDE.md, the preference wins (it is a user override). Check for rules about: assertion style, locator strategy, naming conventions, framework choices.
|
|
24
|
+
|
|
23
25
|
Note: Read these files in full. Extract the layer definitions, pass criteria, confidence calculation rules, and quality gate checklist. These define your validation contract and output requirements.
|
|
24
26
|
|
|
25
27
|
**Important:** The generation plan is the source of truth for which files to validate. If a file exists in the test directory but is NOT in the generation plan, it is a pre-existing file and MUST be excluded from validation scope. The only exception is Layer 4's cross-check for duplicate IDs, which reads (but does not validate or modify) existing test files.
|
package/bin/install.cjs
CHANGED
|
@@ -52,6 +52,13 @@ function copyFile(src, dest) {
|
|
|
52
52
|
return true;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
+
function countEntries(dir, type) {
|
|
56
|
+
if (!fs.existsSync(dir)) return 0;
|
|
57
|
+
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
58
|
+
if (type === 'dirs') return entries.filter(e => e.isDirectory()).length;
|
|
59
|
+
return entries.filter(e => e.isFile()).length;
|
|
60
|
+
}
|
|
61
|
+
|
|
55
62
|
function ok(msg) { console.log(` \x1b[32m✓\x1b[0m ${msg}`); }
|
|
56
63
|
function info(msg) { console.log(` ${msg}`); }
|
|
57
64
|
|
|
@@ -100,11 +107,12 @@ async function main() {
|
|
|
100
107
|
const cmdCount = copyDir(commandsSrc, commandsDest);
|
|
101
108
|
ok(`Installed ${cmdCount} slash commands`);
|
|
102
109
|
|
|
103
|
-
// Install skills
|
|
110
|
+
// Install skills (only to baseDir -- Claude Code reads from ~/.claude/skills/)
|
|
104
111
|
const skillsSrc = path.join(ROOT, '.claude', 'skills');
|
|
105
112
|
const skillsDest = path.join(baseDir, 'skills');
|
|
106
113
|
const skillCount = copyDir(skillsSrc, skillsDest);
|
|
107
|
-
|
|
114
|
+
const skillDirCount = countEntries(skillsSrc, 'dirs');
|
|
115
|
+
ok(`Installed ${skillDirCount} skills (${skillCount} files)`);
|
|
108
116
|
|
|
109
117
|
// Install workflows
|
|
110
118
|
const workflowsSrc = path.join(ROOT, 'workflows');
|
|
@@ -160,7 +168,7 @@ async function main() {
|
|
|
160
168
|
}
|
|
161
169
|
|
|
162
170
|
// Done
|
|
163
|
-
const total = cmdCount + skillCount + agentCount + templateCount + binCount;
|
|
171
|
+
const total = cmdCount + skillCount + agentCount + templateCount + wfCount + binCount;
|
|
164
172
|
console.log('');
|
|
165
173
|
console.log(` \x1b[32m✓ Done!\x1b[0m Installed ${total} files.`);
|
|
166
174
|
console.log('');
|
|
@@ -172,7 +180,7 @@ async function main() {
|
|
|
172
180
|
console.log(' \x1b[1m/qa-from-ticket\x1b[0m Tests from a Jira/Linear ticket');
|
|
173
181
|
console.log(' \x1b[1m/qa-validate\x1b[0m Validate existing tests');
|
|
174
182
|
console.log('');
|
|
175
|
-
console.log(
|
|
183
|
+
console.log(` ${cmdCount} commands + ${skillDirCount} skills + ${agentCount} agents ready.`);
|
|
176
184
|
console.log('');
|
|
177
185
|
}
|
|
178
186
|
|