synapse-hub 2.4.0 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/agents/orchestrator.md +3 -0
- package/.agents/agents/pom-crafter.md +3 -0
- package/.agents/agents/quality-gatekeeper.md +1 -0
- package/.agents/agents/test-runner.md +1 -0
- package/.agents/rules/graphify.md +14 -14
- package/.agents/rules/mandatory-playwright-live.md +29 -29
- package/.agents/skills/bats-testing-patterns/resources/implementation-playbook.md +614 -614
- package/.agents/skills/clean-pom-crafter/examples/standard_test_case.groovy +28 -28
- package/.agents/skills/clean-pom-crafter/references/pom_patterns.groovy +138 -138
- package/.agents/skills/e2e-testing-patterns/resources/implementation-playbook.md +522 -522
- package/.agents/skills/failure-autopsy-and-healing/references/healing_matrix.md +64 -64
- package/.agents/skills/failure-autopsy-and-healing/references/qa_investigation_protocol.md +56 -56
- package/.agents/skills/failure-autopsy-and-healing/scripts/triage_failures.sh +98 -98
- package/.agents/skills/katalon-cli-runner/references/groovy-patterns.md +209 -209
- package/.agents/skills/katalon-cli-runner/references/katalon-api.md +155 -155
- package/.agents/skills/katalon-cli-runner/references/runner_commands.md +67 -67
- package/.agents/skills/playwright-live-interaction/references/advanced_superpowers.md +160 -160
- package/.agents/skills/playwright-live-interaction/references/compact_accessibility_snapshots.md +75 -75
- package/.agents/skills/playwright-live-interaction/references/recipes.md +388 -388
- package/.agents/skills/playwright-live-interaction/references/swarm_orchestration.md +134 -134
- package/.agents/skills/wcag-audit-patterns/resources/implementation-playbook.md +541 -541
- package/.agents/templates/completion_report.md +57 -57
- package/.agents/workflows/graphify.md +10 -10
- package/.cursor/rules/language.mdc +7 -7
- package/README.en.md +9 -3
- package/README.md +9 -3
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-06_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-07_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-08_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-09_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-10_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-21_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-22_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-23_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-24_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-25_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-01_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-02_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-03_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-04_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-05_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-16_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-17_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-18_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-19_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-20_stale_and_mutation.json +22 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-11_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-12_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-13_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-14_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-15_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/reports/BENCHMARK_REPORT.md +94 -69
- package/benchmarks/qa_eval/reports/eval_results.json +460 -104
- package/bin/agent-qa.js +6 -1
- package/docs/ARCHITECTURE.md +13 -2
- package/package.json +6 -6
- package/scripts/install.ps1 +112 -112
- package/scripts/install_global_plugin.ps1 +1 -1
- package/scripts/install_global_plugin.sh +1 -1
- package/scripts/lint_skills.ps1 +57 -57
- package/skills_manifest.json +1 -1
- package/synapse.config.json +1 -1
- package/tools/TOOLS_RUNBOOK.md +6 -1
- package/tools/__init__.py +1 -1
- package/tools/agent_canary.py +501 -501
- package/tools/api_fuzzer.py +11 -2
- package/tools/browser_helpers.js +2 -2
- package/tools/bump_version.py +8 -0
- package/tools/check_version_sync.py +18 -0
- package/tools/cli.py +210 -48
- package/tools/count_tokens.py +4 -4
- package/tools/dashboard_generator.py +10 -10
- package/tools/e2e_flow_prober.sh +0 -0
- package/tools/generate_allure_report.sh +0 -0
- package/tools/generate_test_reports.sh +0 -0
- package/tools/mcp_excel_server.py +3 -1
- package/tools/mcp_excel_server.sh +0 -0
- package/tools/mcp_server.py +1 -1
- package/tools/micro_mutator.py +4 -2
- package/tools/pinpoint_failure.py +88 -88
- package/tools/setup_project.py +1 -1
- package/tools/skillspector.py +31 -0
- package/tools/update_sentinel.py +3 -3
- package/tools/verify_all.py +16 -6
- package/tools/verify_all.sh +0 -0
- package/tools/visual_som_helper.py +14 -3
- package/tools/welcome_page.py +1 -1
- package/benchmarks/qa_eval/reports/AB_METHODOLOGY_REPORT.md +0 -83
|
@@ -1,64 +1,64 @@
|
|
|
1
|
-
# Enterprise Automation Failure Diagnostic & Healing Matrix
|
|
2
|
-
|
|
3
|
-
Reference documentation for the `failure-autopsy-and-healing` skill.
|
|
4
|
-
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## 1. Diagnostic & Healing Matrix
|
|
8
|
-
|
|
9
|
-
| Visual / Exception Symptom | Root Cause | Groovy POM Healing Patch |
|
|
10
|
-
| :--- | :--- | :--- |
|
|
11
|
-
| **Save button `[ ✓ ]` disabled / dimmed** | Angular Reactive Form missed change detection or missing required field (`*`). | `WebElement el = findElement(By.cssSelector("input[formcontrolname='code']"))`<br>`jsSetInput(el, uniqueCode)` |
|
|
12
|
-
| **Alert: *"Phạm vi giữa các khu/cụm không hợp lệ"*** | OpenLayers polygon overlaps previously saved boundary data. | `Actions actions = new Actions(driver)`<br>`actions.moveToElement(viewport, -180, -80).click().perform()` |
|
|
13
|
-
| **`ElementClickInterceptedException`** | Modal overlay `.p-component-overlay` fading out but still blocking clicks. | `((JavascriptExecutor) driver).executeScript("arguments[0].click();", element)` |
|
|
14
|
-
| **Red Toast: *"Mã đã tồn tại"*** | Non-unique test data colliding with existing records. | `String code = BasePage.uniqueCode("CCN_")` |
|
|
15
|
-
| **`StaleElementReferenceException`** | Angular DOM re-rendered after API response, detaching element. | `for (int i = 0; i < 3; i++) { try { findElement(by).click(); break; } catch (StaleElementReferenceException e) { sleep(500) } }` |
|
|
16
|
-
| **`TimeoutException` (Waiting for table/button)** | Loading overlay `.p-blockui` or spinner `.p-progress-spinner` blocking UI. | `waitForElementNotVisible(By.cssSelector(".p-blockui, .p-progress-spinner"), 10)` |
|
|
17
|
-
| **PrimeNG Dropdown value not applied** | Clicked `p-dropdown` but overlay panel `.p-dropdown-panel` has not rendered. | `waitForElementVisible(By.cssSelector(".p-dropdown-panel"), 5)`<br>`findElement(By.xpath("//li[contains(., '" + text + "')]")).click()` |
|
|
18
|
-
| **Blank Screenshot** | Browser crashed before capturing UI (OOM or lost WebDriver session). | Inspect `console0.log`, kill zombie drivers (`pkill -f chromedriver`), restart session. |
|
|
19
|
-
|
|
20
|
-
---
|
|
21
|
-
|
|
22
|
-
## 2. Four Flaky Test Root Causes & Cures
|
|
23
|
-
|
|
24
|
-
| Flaky Root Cause | Symptom | Curing Strategy (Root Fix) |
|
|
25
|
-
|---|---|---|
|
|
26
|
-
| **1. Unstable Locator** | Random failures due to CSS refactoring or dynamic hashes | Enforce Master Priority Map: switch to `[formcontrolname]`, `[data-testid]`; forbid positional XPaths. |
|
|
27
|
-
| **2. Timing & Race Condition** | Fails under network latency; clicks before Angular finishes render | Replace all hard sleeps with Smart Wait: `waitForElementVisible`, wait for `.p-blockui` / `.p-progress-spinner` to hide. |
|
|
28
|
-
| **3. Test Data Conflict** | Fails on parallel or repeated runs due to duplicate codes | Enforce `BasePage.uniqueCode(prefix)` with epoch timestamps. |
|
|
29
|
-
| **4. Environment Dependency** | Fails due to expired session or stale browser cache | Use `LoginPage.ensureLoggedIn()` before every script. |
|
|
30
|
-
|
|
31
|
-
---
|
|
32
|
-
|
|
33
|
-
## 3. Fast-Fix Triad (Zero-Token Diagnostic & Targeted Healing Workflow)
|
|
34
|
-
|
|
35
|
-
To maximize fix speed and minimize token consumption without degrading test fidelity, follow this 3-step loop:
|
|
36
|
-
|
|
37
|
-
### Step 1: Zero-Token Pinpoint (CLI Log Extraction)
|
|
38
|
-
Never read entire JUnit XML reports (>50KB). Extract failed test cases, exception reasons, and file line numbers in <50 tokens:
|
|
39
|
-
```bash
|
|
40
|
-
python3 tools/pinpoint_failure.py Reports/<timestamp>/<suite>/JUnit_Report.xml
|
|
41
|
-
```
|
|
42
|
-
|
|
43
|
-
### Step 2: Instant Form & DOM Audit (Playwright MCP Recipe 15)
|
|
44
|
-
When a test fails because a button is disabled (e.g. Save `[ ✓ ]`), never guess which field is invalid. Execute Recipe 15 via `browser_run_code_unsafe` on the active live page:
|
|
45
|
-
```javascript
|
|
46
|
-
() => {
|
|
47
|
-
const invalids = Array.from(document.querySelectorAll('form .ng-invalid, input.ng-invalid, p-dropdown.ng-invalid'));
|
|
48
|
-
return invalids.map(el => ({
|
|
49
|
-
tag: el.tagName,
|
|
50
|
-
name: el.getAttribute('name') || el.getAttribute('formcontrolname') || el.id,
|
|
51
|
-
classes: el.className
|
|
52
|
-
}));
|
|
53
|
-
}
|
|
54
|
-
```
|
|
55
|
-
This instantly highlights missing required inputs, pattern mismatches, or invalid dropdowns.
|
|
56
|
-
|
|
57
|
-
### Step 3: Targeted Single-TC Re-test (Zero Waste)
|
|
58
|
-
Never rerun an entire Test Suite to verify a 1-line locator or timing fix. Rerun only the targeted test case:
|
|
59
|
-
```bash
|
|
60
|
-
# Via KRE CLI (License mode):
|
|
61
|
-
katalonc -noSplash -runMode=console -projectPath="." -retry=0 -testCasePath="Test Cases/TC-xxx" -browserType="Chrome (headless)"
|
|
62
|
-
|
|
63
|
-
# Or verify live flow in 2 seconds via Playwright MCP script.
|
|
64
|
-
```
|
|
1
|
+
# Enterprise Automation Failure Diagnostic & Healing Matrix
|
|
2
|
+
|
|
3
|
+
Reference documentation for the `failure-autopsy-and-healing` skill.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. Diagnostic & Healing Matrix
|
|
8
|
+
|
|
9
|
+
| Visual / Exception Symptom | Root Cause | Groovy POM Healing Patch |
|
|
10
|
+
| :--- | :--- | :--- |
|
|
11
|
+
| **Save button `[ ✓ ]` disabled / dimmed** | Angular Reactive Form missed change detection or missing required field (`*`). | `WebElement el = findElement(By.cssSelector("input[formcontrolname='code']"))`<br>`jsSetInput(el, uniqueCode)` |
|
|
12
|
+
| **Alert: *"Phạm vi giữa các khu/cụm không hợp lệ"*** | OpenLayers polygon overlaps previously saved boundary data. | `Actions actions = new Actions(driver)`<br>`actions.moveToElement(viewport, -180, -80).click().perform()` |
|
|
13
|
+
| **`ElementClickInterceptedException`** | Modal overlay `.p-component-overlay` fading out but still blocking clicks. | `((JavascriptExecutor) driver).executeScript("arguments[0].click();", element)` |
|
|
14
|
+
| **Red Toast: *"Mã đã tồn tại"*** | Non-unique test data colliding with existing records. | `String code = BasePage.uniqueCode("CCN_")` |
|
|
15
|
+
| **`StaleElementReferenceException`** | Angular DOM re-rendered after API response, detaching element. | `for (int i = 0; i < 3; i++) { try { findElement(by).click(); break; } catch (StaleElementReferenceException e) { sleep(500) } }` |
|
|
16
|
+
| **`TimeoutException` (Waiting for table/button)** | Loading overlay `.p-blockui` or spinner `.p-progress-spinner` blocking UI. | `waitForElementNotVisible(By.cssSelector(".p-blockui, .p-progress-spinner"), 10)` |
|
|
17
|
+
| **PrimeNG Dropdown value not applied** | Clicked `p-dropdown` but overlay panel `.p-dropdown-panel` has not rendered. | `waitForElementVisible(By.cssSelector(".p-dropdown-panel"), 5)`<br>`findElement(By.xpath("//li[contains(., '" + text + "')]")).click()` |
|
|
18
|
+
| **Blank Screenshot** | Browser crashed before capturing UI (OOM or lost WebDriver session). | Inspect `console0.log`, kill zombie drivers (`pkill -f chromedriver`), restart session. |
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## 2. Four Flaky Test Root Causes & Cures
|
|
23
|
+
|
|
24
|
+
| Flaky Root Cause | Symptom | Curing Strategy (Root Fix) |
|
|
25
|
+
|---|---|---|
|
|
26
|
+
| **1. Unstable Locator** | Random failures due to CSS refactoring or dynamic hashes | Enforce Master Priority Map: switch to `[formcontrolname]`, `[data-testid]`; forbid positional XPaths. |
|
|
27
|
+
| **2. Timing & Race Condition** | Fails under network latency; clicks before Angular finishes render | Replace all hard sleeps with Smart Wait: `waitForElementVisible`, wait for `.p-blockui` / `.p-progress-spinner` to hide. |
|
|
28
|
+
| **3. Test Data Conflict** | Fails on parallel or repeated runs due to duplicate codes | Enforce `BasePage.uniqueCode(prefix)` with epoch timestamps. |
|
|
29
|
+
| **4. Environment Dependency** | Fails due to expired session or stale browser cache | Use `LoginPage.ensureLoggedIn()` before every script. |
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## 3. Fast-Fix Triad (Zero-Token Diagnostic & Targeted Healing Workflow)
|
|
34
|
+
|
|
35
|
+
To maximize fix speed and minimize token consumption without degrading test fidelity, follow this 3-step loop:
|
|
36
|
+
|
|
37
|
+
### Step 1: Zero-Token Pinpoint (CLI Log Extraction)
|
|
38
|
+
Never read entire JUnit XML reports (>50KB). Extract failed test cases, exception reasons, and file line numbers in <50 tokens:
|
|
39
|
+
```bash
|
|
40
|
+
python3 tools/pinpoint_failure.py Reports/<timestamp>/<suite>/JUnit_Report.xml
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### Step 2: Instant Form & DOM Audit (Playwright MCP Recipe 15)
|
|
44
|
+
When a test fails because a button is disabled (e.g. Save `[ ✓ ]`), never guess which field is invalid. Execute Recipe 15 via `browser_run_code_unsafe` on the active live page:
|
|
45
|
+
```javascript
|
|
46
|
+
() => {
|
|
47
|
+
const invalids = Array.from(document.querySelectorAll('form .ng-invalid, input.ng-invalid, p-dropdown.ng-invalid'));
|
|
48
|
+
return invalids.map(el => ({
|
|
49
|
+
tag: el.tagName,
|
|
50
|
+
name: el.getAttribute('name') || el.getAttribute('formcontrolname') || el.id,
|
|
51
|
+
classes: el.className
|
|
52
|
+
}));
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
This instantly highlights missing required inputs, pattern mismatches, or invalid dropdowns.
|
|
56
|
+
|
|
57
|
+
### Step 3: Targeted Single-TC Re-test (Zero Waste)
|
|
58
|
+
Never rerun an entire Test Suite to verify a 1-line locator or timing fix. Rerun only the targeted test case:
|
|
59
|
+
```bash
|
|
60
|
+
# Via KRE CLI (License mode):
|
|
61
|
+
katalonc -noSplash -runMode=console -projectPath="." -retry=0 -testCasePath="Test Cases/TC-xxx" -browserType="Chrome (headless)"
|
|
62
|
+
|
|
63
|
+
# Or verify live flow in 2 seconds via Playwright MCP script.
|
|
64
|
+
```
|
|
@@ -1,56 +1,56 @@
|
|
|
1
|
-
# QA Investigation & Root-Cause Analysis Protocol
|
|
2
|
-
|
|
3
|
-
*(Inherited from `fugazi/test-automation-skills-agents`)*
|
|
4
|
-
|
|
5
|
-
A file-backed, evidence-first investigation journal protocol for diagnosing failing and flaky automation test cases in enterprise automation projects.
|
|
6
|
-
|
|
7
|
-
---
|
|
8
|
-
|
|
9
|
-
## 1. Core Principles
|
|
10
|
-
|
|
11
|
-
1. **Context Window is Volatile RAM; Filesystem is Persistent Disk:**
|
|
12
|
-
- Long triage sessions cause context drift. All goals, hypotheses, and evidence must be logged in markdown.
|
|
13
|
-
2. **Discovered Classification, Not Presumed:**
|
|
14
|
-
- Never assume upfront whether a failure is a Flaky test, an AUT Defect, or a Test Infrastructure issue. The classification is a finding discovered through evidence.
|
|
15
|
-
3. **No Guesswork / No Blind Fixes:**
|
|
16
|
-
- Every fix must address a proven root cause, not a symptom.
|
|
17
|
-
|
|
18
|
-
---
|
|
19
|
-
|
|
20
|
-
## 2. Four-Phase Investigation Cycle
|
|
21
|
-
|
|
22
|
-
```mermaid
|
|
23
|
-
flowchart TD
|
|
24
|
-
P1["<b>Phase 1: Reproduction & Triage</b><br/>• Run single-TC isolated re-test<br/>• Classify: Intermittent (Flaky) vs Deterministic (Bug)"]
|
|
25
|
-
P2["<b>Phase 2: Evidence Collection</b><br/>• Extract CLI logs & stack trace<br/>• Inspect Screenshots/FAILED_*.png<br/>• Capture network & timing data"]
|
|
26
|
-
P3["<b>Phase 3: Hypothesis & Root Cause</b><br/>• Form testable hypothesis<br/>• Isolate: Timing vs Stale DOM vs AUT defect"]
|
|
27
|
-
P4["<b>Phase 4: Targeted Fix & Validation</b><br/>• Apply Clean POM Page Class fix<br/>• Validate 3x consecutive runs<br/>• Run verify_all.py DoD check"]
|
|
28
|
-
|
|
29
|
-
P1 --> P2 --> P3 --> P4
|
|
30
|
-
```
|
|
31
|
-
|
|
32
|
-
### Phase 1: Reproduction & Triage
|
|
33
|
-
- Run the failing test in isolation using the targeted single-TC runner.
|
|
34
|
-
- Determine failure pattern:
|
|
35
|
-
- **Deterministic Bug:** Fails 100% of the time with identical stack trace.
|
|
36
|
-
- **Intermittent Flaky:** Passes sometimes, fails under load/timing variations.
|
|
37
|
-
- **Environment Blocker:** Fails on 401/403 session expiration or network timeout.
|
|
38
|
-
|
|
39
|
-
### Phase 2: Evidence Collection
|
|
40
|
-
- Collect ground-truth artifacts:
|
|
41
|
-
1. CLI log excerpt around the failure point.
|
|
42
|
-
2. Failure screenshot in `Screenshots/FAILED_<TestCaseId>_*.png`.
|
|
43
|
-
3. Form audit state (e.g. disabled Save button, red validation borders).
|
|
44
|
-
4. Redact any sensitive credentials or personal tokens.
|
|
45
|
-
|
|
46
|
-
### Phase 3: Hypothesis Formulation
|
|
47
|
-
- Check the 4 most common failure archetypes:
|
|
48
|
-
- **Archetype A (PrimeNG Toast Collision):** Previous toast covers the button. Cure: `dismissToast()`.
|
|
49
|
-
- **Archetype B (Stale Element / Angular Re-render):** DOM re-created mid-click. Cure: Re-fetch locator with `BasePage.x()`.
|
|
50
|
-
- **Archetype C (Dialog Backdrop Overlay):** `cdk-overlay-backdrop` blocks click. Cure: Wait for backdrop fade-out.
|
|
51
|
-
- **Archetype D (Real AUT Defect):** Backend HTTP 500 or constraint violation. Cure: Log to `KNOWN_ISSUES.md`.
|
|
52
|
-
|
|
53
|
-
### Phase 4: Targeted Fix & Validation
|
|
54
|
-
- Implement fix exclusively inside Page Classes in `Keywords/<namespace>/pages/` or `pages/` (never in `Scripts/`).
|
|
55
|
-
- Validate stability across multiple runs.
|
|
56
|
-
- Run `python3 tools/verify_all.py` to confirm zero regressions.
|
|
1
|
+
# QA Investigation & Root-Cause Analysis Protocol
|
|
2
|
+
|
|
3
|
+
*(Inherited from `fugazi/test-automation-skills-agents`)*
|
|
4
|
+
|
|
5
|
+
A file-backed, evidence-first investigation journal protocol for diagnosing failing and flaky automation test cases in enterprise automation projects.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 1. Core Principles
|
|
10
|
+
|
|
11
|
+
1. **Context Window is Volatile RAM; Filesystem is Persistent Disk:**
|
|
12
|
+
- Long triage sessions cause context drift. All goals, hypotheses, and evidence must be logged in markdown.
|
|
13
|
+
2. **Discovered Classification, Not Presumed:**
|
|
14
|
+
- Never assume upfront whether a failure is a Flaky test, an AUT Defect, or a Test Infrastructure issue. The classification is a finding discovered through evidence.
|
|
15
|
+
3. **No Guesswork / No Blind Fixes:**
|
|
16
|
+
- Every fix must address a proven root cause, not a symptom.
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## 2. Four-Phase Investigation Cycle
|
|
21
|
+
|
|
22
|
+
```mermaid
|
|
23
|
+
flowchart TD
|
|
24
|
+
P1["<b>Phase 1: Reproduction & Triage</b><br/>• Run single-TC isolated re-test<br/>• Classify: Intermittent (Flaky) vs Deterministic (Bug)"]
|
|
25
|
+
P2["<b>Phase 2: Evidence Collection</b><br/>• Extract CLI logs & stack trace<br/>• Inspect Screenshots/FAILED_*.png<br/>• Capture network & timing data"]
|
|
26
|
+
P3["<b>Phase 3: Hypothesis & Root Cause</b><br/>• Form testable hypothesis<br/>• Isolate: Timing vs Stale DOM vs AUT defect"]
|
|
27
|
+
P4["<b>Phase 4: Targeted Fix & Validation</b><br/>• Apply Clean POM Page Class fix<br/>• Validate 3x consecutive runs<br/>• Run verify_all.py DoD check"]
|
|
28
|
+
|
|
29
|
+
P1 --> P2 --> P3 --> P4
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
### Phase 1: Reproduction & Triage
|
|
33
|
+
- Run the failing test in isolation using the targeted single-TC runner.
|
|
34
|
+
- Determine failure pattern:
|
|
35
|
+
- **Deterministic Bug:** Fails 100% of the time with identical stack trace.
|
|
36
|
+
- **Intermittent Flaky:** Passes sometimes, fails under load/timing variations.
|
|
37
|
+
- **Environment Blocker:** Fails on 401/403 session expiration or network timeout.
|
|
38
|
+
|
|
39
|
+
### Phase 2: Evidence Collection
|
|
40
|
+
- Collect ground-truth artifacts:
|
|
41
|
+
1. CLI log excerpt around the failure point.
|
|
42
|
+
2. Failure screenshot in `Screenshots/FAILED_<TestCaseId>_*.png`.
|
|
43
|
+
3. Form audit state (e.g. disabled Save button, red validation borders).
|
|
44
|
+
4. Redact any sensitive credentials or personal tokens.
|
|
45
|
+
|
|
46
|
+
### Phase 3: Hypothesis Formulation
|
|
47
|
+
- Check the 4 most common failure archetypes:
|
|
48
|
+
- **Archetype A (PrimeNG Toast Collision):** Previous toast covers the button. Cure: `dismissToast()`.
|
|
49
|
+
- **Archetype B (Stale Element / Angular Re-render):** DOM re-created mid-click. Cure: Re-fetch locator with `BasePage.x()`.
|
|
50
|
+
- **Archetype C (Dialog Backdrop Overlay):** `cdk-overlay-backdrop` blocks click. Cure: Wait for backdrop fade-out.
|
|
51
|
+
- **Archetype D (Real AUT Defect):** Backend HTTP 500 or constraint violation. Cure: Log to `KNOWN_ISSUES.md`.
|
|
52
|
+
|
|
53
|
+
### Phase 4: Targeted Fix & Validation
|
|
54
|
+
- Implement fix exclusively inside Page Classes in `Keywords/<namespace>/pages/` or `pages/` (never in `Scripts/`).
|
|
55
|
+
- Validate stability across multiple runs.
|
|
56
|
+
- Run `python3 tools/verify_all.py` to confirm zero regressions.
|
|
@@ -1,98 +1,98 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# Triage Failures Script for Enterprise Automation
|
|
3
|
-
# Cross-platform: Compatible with macOS, Linux CI, and Windows Git Bash
|
|
4
|
-
|
|
5
|
-
WORKSPACE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
|
6
|
-
SCREENSHOT_DIR="$WORKSPACE_DIR/Screenshots"
|
|
7
|
-
REPORTS_DIR="$WORKSPACE_DIR/Reports"
|
|
8
|
-
|
|
9
|
-
echo "=========================================================="
|
|
10
|
-
echo " 🔬 ENTERPRISE FAILURE AUTOPSY & TRIAGE SCANNER (CROSS-PLATFORM)"
|
|
11
|
-
echo " Workspace: $WORKSPACE_DIR"
|
|
12
|
-
echo "=========================================================="
|
|
13
|
-
|
|
14
|
-
# 1. Quét báo cáo JUnit XML gần nhất
|
|
15
|
-
LATEST_JUNIT=$(find "$REPORTS_DIR" -type f -name "JUnit_Report.xml" 2>/dev/null | sort -r | head -n 1)
|
|
16
|
-
|
|
17
|
-
if [ -n "$LATEST_JUNIT" ]; then
|
|
18
|
-
echo "📊 Báo cáo JUnit gần nhất: $(basename "$(dirname "$LATEST_JUNIT")")/JUnit_Report.xml"
|
|
19
|
-
FAILED_JUNIT_TESTS=$(grep -B 1 '<failure' "$LATEST_JUNIT" 2>/dev/null | grep '<testcase' | sed -n 's/.*name="\([^"]*\)".*/\1/p')
|
|
20
|
-
if [ -n "$FAILED_JUNIT_TESTS" ]; then
|
|
21
|
-
echo "🚨 Các Test Cases báo FAILED trong JUnit XML:"
|
|
22
|
-
echo "$FAILED_JUNIT_TESTS" | while read -r tc; do
|
|
23
|
-
echo " ❌ $tc"
|
|
24
|
-
done
|
|
25
|
-
echo "----------------------------------------------------------"
|
|
26
|
-
else
|
|
27
|
-
echo "✅ Không phát hiện ca thất bại nào trong JUnit XML gần nhất."
|
|
28
|
-
echo "----------------------------------------------------------"
|
|
29
|
-
fi
|
|
30
|
-
fi
|
|
31
|
-
|
|
32
|
-
# 2. Quét ảnh chụp FAILED_*.png
|
|
33
|
-
if [ ! -d "$SCREENSHOT_DIR" ]; then
|
|
34
|
-
echo "ℹ️ Không tìm thấy thư mục Screenshots. Dự án đang sạch lỗi!"
|
|
35
|
-
exit 0
|
|
36
|
-
fi
|
|
37
|
-
|
|
38
|
-
FAILED_COUNT=$(find "$SCREENSHOT_DIR" -maxdepth 1 -name "FAILED_*.png" 2>/dev/null | wc -l | tr -d ' ')
|
|
39
|
-
|
|
40
|
-
if [ "$FAILED_COUNT" -eq "0" ]; then
|
|
41
|
-
echo "✅ Tuyệt vời! Không có ảnh lỗi nào trong $SCREENSHOT_DIR."
|
|
42
|
-
exit 0
|
|
43
|
-
fi
|
|
44
|
-
|
|
45
|
-
echo "⚠️ Phát hiện $FAILED_COUNT ảnh lỗi kịch bản gần nhất:"
|
|
46
|
-
echo "----------------------------------------------------------"
|
|
47
|
-
|
|
48
|
-
# Hàm lấy thông tin file đa nền tảng (hỗ trợ Linux GNU stat, macOS BSD stat và fallback ls)
|
|
49
|
-
list_recent_screenshots() {
|
|
50
|
-
if stat --version 2>&1 | grep -q "GNU"; then
|
|
51
|
-
# Linux (GNU stat)
|
|
52
|
-
find "$SCREENSHOT_DIR" -maxdepth 1 -name "FAILED_*.png" -type f -exec stat -c "%Y %y %n" {} + 2>/dev/null | \
|
|
53
|
-
sort -rn | head -n 10 | awk '{print $1, $2, $3, $NF}'
|
|
54
|
-
elif stat -f "%m" . >/dev/null 2>&1; then
|
|
55
|
-
# macOS / BSD stat
|
|
56
|
-
find "$SCREENSHOT_DIR" -maxdepth 1 -name "FAILED_*.png" -type f -exec stat -f "%m %Sm %N" -t "%Y-%m-%d %H:%M:%S" {} + 2>/dev/null | \
|
|
57
|
-
sort -rn | head -n 10
|
|
58
|
-
else
|
|
59
|
-
# Fallback (POSIX ls)
|
|
60
|
-
ls -lt "$SCREENSHOT_DIR"/FAILED_*.png 2>/dev/null | head -n 10 | awk '{print "0", $6, $7, $NF}'
|
|
61
|
-
fi
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
list_recent_screenshots | while read -r epoch date time filepath; do
|
|
65
|
-
filename=$(basename "$filepath")
|
|
66
|
-
|
|
67
|
-
# Ánh xạ Page Class gợi ý dựa trên mã Test Case (Dynamic Pattern Matching)
|
|
68
|
-
suggested_page="BasePage.groovy"
|
|
69
|
-
if [[ "$filename" =~ TC-([0-9]+) ]]; then
|
|
70
|
-
tc_num="${BASH_REMATCH[1]}"
|
|
71
|
-
if [ "$tc_num" -ge 1 ] && [ "$tc_num" -le 7 ]; then
|
|
72
|
-
suggested_page="TargetPlanPage.groovy"
|
|
73
|
-
elif [ "$tc_num" -ge 8 ] && [ "$tc_num" -le 12 ]; then
|
|
74
|
-
suggested_page="LoginPage.groovy"
|
|
75
|
-
elif [ "$tc_num" -ge 13 ] && [ "$tc_num" -le 16 ]; then
|
|
76
|
-
suggested_page="UserAdminPage.groovy"
|
|
77
|
-
elif [ "$tc_num" -ge 17 ] && [ "$tc_num" -le 28 ]; then
|
|
78
|
-
suggested_page="CommonCategoryPage.groovy"
|
|
79
|
-
elif [ "$tc_num" -ge 29 ] && [ "$tc_num" -le 92 ]; then
|
|
80
|
-
suggested_page="IndustryCategoryPage.groovy"
|
|
81
|
-
elif [ "$tc_num" -ge 93 ] && [ "$tc_num" -le 122 ]; then
|
|
82
|
-
suggested_page="PlanningPage.groovy"
|
|
83
|
-
elif [ "$tc_num" -ge 123 ] && [ "$tc_num" -le 247 ]; then
|
|
84
|
-
suggested_page="DataMiningPage.groovy"
|
|
85
|
-
elif [ "$tc_num" -ge 248 ] && [ "$tc_num" -le 249 ]; then
|
|
86
|
-
suggested_page="ReportPage.groovy"
|
|
87
|
-
elif [ "$tc_num" -ge 250 ] && [ "$tc_num" -le 259 ]; then
|
|
88
|
-
suggested_page="TokenMgmtPage.groovy"
|
|
89
|
-
fi
|
|
90
|
-
fi
|
|
91
|
-
|
|
92
|
-
echo "📸 [$date $time] $filename"
|
|
93
|
-
echo " -> Đường dẫn ảnh : $filepath"
|
|
94
|
-
echo " -> Page Class cần vá : Keywords/pages/$suggested_page"
|
|
95
|
-
done
|
|
96
|
-
|
|
97
|
-
echo "----------------------------------------------------------"
|
|
98
|
-
echo "💡 Gợi ý AI: Dùng tool read_file mở ảnh hoặc dùng Playwright Live giữ nguyên browser để xác thực và vá Page Class."
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Triage Failures Script for Enterprise Automation
|
|
3
|
+
# Cross-platform: Compatible with macOS, Linux CI, and Windows Git Bash
|
|
4
|
+
|
|
5
|
+
WORKSPACE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
|
6
|
+
SCREENSHOT_DIR="$WORKSPACE_DIR/Screenshots"
|
|
7
|
+
REPORTS_DIR="$WORKSPACE_DIR/Reports"
|
|
8
|
+
|
|
9
|
+
echo "=========================================================="
|
|
10
|
+
echo " 🔬 ENTERPRISE FAILURE AUTOPSY & TRIAGE SCANNER (CROSS-PLATFORM)"
|
|
11
|
+
echo " Workspace: $WORKSPACE_DIR"
|
|
12
|
+
echo "=========================================================="
|
|
13
|
+
|
|
14
|
+
# 1. Quét báo cáo JUnit XML gần nhất
|
|
15
|
+
LATEST_JUNIT=$(find "$REPORTS_DIR" -type f -name "JUnit_Report.xml" 2>/dev/null | sort -r | head -n 1)
|
|
16
|
+
|
|
17
|
+
if [ -n "$LATEST_JUNIT" ]; then
|
|
18
|
+
echo "📊 Báo cáo JUnit gần nhất: $(basename "$(dirname "$LATEST_JUNIT")")/JUnit_Report.xml"
|
|
19
|
+
FAILED_JUNIT_TESTS=$(grep -B 1 '<failure' "$LATEST_JUNIT" 2>/dev/null | grep '<testcase' | sed -n 's/.*name="\([^"]*\)".*/\1/p')
|
|
20
|
+
if [ -n "$FAILED_JUNIT_TESTS" ]; then
|
|
21
|
+
echo "🚨 Các Test Cases báo FAILED trong JUnit XML:"
|
|
22
|
+
echo "$FAILED_JUNIT_TESTS" | while read -r tc; do
|
|
23
|
+
echo " ❌ $tc"
|
|
24
|
+
done
|
|
25
|
+
echo "----------------------------------------------------------"
|
|
26
|
+
else
|
|
27
|
+
echo "✅ Không phát hiện ca thất bại nào trong JUnit XML gần nhất."
|
|
28
|
+
echo "----------------------------------------------------------"
|
|
29
|
+
fi
|
|
30
|
+
fi
|
|
31
|
+
|
|
32
|
+
# 2. Quét ảnh chụp FAILED_*.png
|
|
33
|
+
if [ ! -d "$SCREENSHOT_DIR" ]; then
|
|
34
|
+
echo "ℹ️ Không tìm thấy thư mục Screenshots. Dự án đang sạch lỗi!"
|
|
35
|
+
exit 0
|
|
36
|
+
fi
|
|
37
|
+
|
|
38
|
+
FAILED_COUNT=$(find "$SCREENSHOT_DIR" -maxdepth 1 -name "FAILED_*.png" 2>/dev/null | wc -l | tr -d ' ')
|
|
39
|
+
|
|
40
|
+
if [ "$FAILED_COUNT" -eq "0" ]; then
|
|
41
|
+
echo "✅ Tuyệt vời! Không có ảnh lỗi nào trong $SCREENSHOT_DIR."
|
|
42
|
+
exit 0
|
|
43
|
+
fi
|
|
44
|
+
|
|
45
|
+
echo "⚠️ Phát hiện $FAILED_COUNT ảnh lỗi kịch bản gần nhất:"
|
|
46
|
+
echo "----------------------------------------------------------"
|
|
47
|
+
|
|
48
|
+
# Hàm lấy thông tin file đa nền tảng (hỗ trợ Linux GNU stat, macOS BSD stat và fallback ls)
|
|
49
|
+
list_recent_screenshots() {
|
|
50
|
+
if stat --version 2>&1 | grep -q "GNU"; then
|
|
51
|
+
# Linux (GNU stat)
|
|
52
|
+
find "$SCREENSHOT_DIR" -maxdepth 1 -name "FAILED_*.png" -type f -exec stat -c "%Y %y %n" {} + 2>/dev/null | \
|
|
53
|
+
sort -rn | head -n 10 | awk '{print $1, $2, $3, $NF}'
|
|
54
|
+
elif stat -f "%m" . >/dev/null 2>&1; then
|
|
55
|
+
# macOS / BSD stat
|
|
56
|
+
find "$SCREENSHOT_DIR" -maxdepth 1 -name "FAILED_*.png" -type f -exec stat -f "%m %Sm %N" -t "%Y-%m-%d %H:%M:%S" {} + 2>/dev/null | \
|
|
57
|
+
sort -rn | head -n 10
|
|
58
|
+
else
|
|
59
|
+
# Fallback (POSIX ls)
|
|
60
|
+
ls -lt "$SCREENSHOT_DIR"/FAILED_*.png 2>/dev/null | head -n 10 | awk '{print "0", $6, $7, $NF}'
|
|
61
|
+
fi
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
list_recent_screenshots | while read -r epoch date time filepath; do
|
|
65
|
+
filename=$(basename "$filepath")
|
|
66
|
+
|
|
67
|
+
# Ánh xạ Page Class gợi ý dựa trên mã Test Case (Dynamic Pattern Matching)
|
|
68
|
+
suggested_page="BasePage.groovy"
|
|
69
|
+
if [[ "$filename" =~ TC-([0-9]+) ]]; then
|
|
70
|
+
tc_num="${BASH_REMATCH[1]}"
|
|
71
|
+
if [ "$tc_num" -ge 1 ] && [ "$tc_num" -le 7 ]; then
|
|
72
|
+
suggested_page="TargetPlanPage.groovy"
|
|
73
|
+
elif [ "$tc_num" -ge 8 ] && [ "$tc_num" -le 12 ]; then
|
|
74
|
+
suggested_page="LoginPage.groovy"
|
|
75
|
+
elif [ "$tc_num" -ge 13 ] && [ "$tc_num" -le 16 ]; then
|
|
76
|
+
suggested_page="UserAdminPage.groovy"
|
|
77
|
+
elif [ "$tc_num" -ge 17 ] && [ "$tc_num" -le 28 ]; then
|
|
78
|
+
suggested_page="CommonCategoryPage.groovy"
|
|
79
|
+
elif [ "$tc_num" -ge 29 ] && [ "$tc_num" -le 92 ]; then
|
|
80
|
+
suggested_page="IndustryCategoryPage.groovy"
|
|
81
|
+
elif [ "$tc_num" -ge 93 ] && [ "$tc_num" -le 122 ]; then
|
|
82
|
+
suggested_page="PlanningPage.groovy"
|
|
83
|
+
elif [ "$tc_num" -ge 123 ] && [ "$tc_num" -le 247 ]; then
|
|
84
|
+
suggested_page="DataMiningPage.groovy"
|
|
85
|
+
elif [ "$tc_num" -ge 248 ] && [ "$tc_num" -le 249 ]; then
|
|
86
|
+
suggested_page="ReportPage.groovy"
|
|
87
|
+
elif [ "$tc_num" -ge 250 ] && [ "$tc_num" -le 259 ]; then
|
|
88
|
+
suggested_page="TokenMgmtPage.groovy"
|
|
89
|
+
fi
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
echo "📸 [$date $time] $filename"
|
|
93
|
+
echo " -> Đường dẫn ảnh : $filepath"
|
|
94
|
+
echo " -> Page Class cần vá : Keywords/pages/$suggested_page"
|
|
95
|
+
done
|
|
96
|
+
|
|
97
|
+
echo "----------------------------------------------------------"
|
|
98
|
+
echo "💡 Gợi ý AI: Dùng tool read_file mở ảnh hoặc dùng Playwright Live giữ nguyên browser để xác thực và vá Page Class."
|