synapse-hub 2.4.1 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/rules/graphify.md +14 -14
- package/.agents/rules/mandatory-playwright-live.md +29 -29
- package/.agents/skills/bats-testing-patterns/resources/implementation-playbook.md +614 -614
- package/.agents/skills/clean-pom-crafter/examples/standard_test_case.groovy +28 -28
- package/.agents/skills/clean-pom-crafter/references/pom_patterns.groovy +138 -138
- package/.agents/skills/e2e-testing-patterns/resources/implementation-playbook.md +522 -522
- package/.agents/skills/failure-autopsy-and-healing/references/healing_matrix.md +64 -64
- package/.agents/skills/failure-autopsy-and-healing/references/qa_investigation_protocol.md +56 -56
- package/.agents/skills/failure-autopsy-and-healing/scripts/triage_failures.sh +98 -98
- package/.agents/skills/katalon-cli-runner/references/groovy-patterns.md +209 -209
- package/.agents/skills/katalon-cli-runner/references/katalon-api.md +155 -155
- package/.agents/skills/katalon-cli-runner/references/runner_commands.md +67 -67
- package/.agents/skills/playwright-live-interaction/references/advanced_superpowers.md +160 -160
- package/.agents/skills/playwright-live-interaction/references/compact_accessibility_snapshots.md +75 -75
- package/.agents/skills/playwright-live-interaction/references/recipes.md +388 -388
- package/.agents/skills/playwright-live-interaction/references/swarm_orchestration.md +134 -134
- package/.agents/skills/wcag-audit-patterns/resources/implementation-playbook.md +541 -541
- package/.agents/templates/completion_report.md +57 -57
- package/.agents/workflows/graphify.md +10 -10
- package/.cursor/rules/language.mdc +7 -7
- package/README.en.md +1 -1
- package/README.md +1 -1
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-06_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-07_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-08_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-09_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-10_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-21_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-22_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-23_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-24_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-25_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-01_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-02_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-03_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-04_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-05_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-16_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-17_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-18_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-19_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-20_stale_and_mutation.json +22 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-11_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-12_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-13_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-14_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-15_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/reports/BENCHMARK_REPORT.md +94 -69
- package/benchmarks/qa_eval/reports/eval_results.json +460 -104
- package/bin/agent-qa.js +0 -0
- package/docs/ARCHITECTURE.md +12 -1
- package/package.json +1 -1
- package/scripts/install.ps1 +112 -112
- package/scripts/lint_skills.ps1 +57 -57
- package/skills_manifest.json +1 -1
- package/synapse.config.json +1 -1
- package/tools/__init__.py +1 -1
- package/tools/agent_canary.py +501 -501
- package/tools/api_fuzzer.py +11 -2
- package/tools/browser_helpers.js +2 -2
- package/tools/bump_version.py +1 -0
- package/tools/cli.py +2 -2
- package/tools/dashboard_generator.py +2 -2
- package/tools/e2e_flow_prober.sh +0 -0
- package/tools/generate_allure_report.sh +0 -0
- package/tools/generate_test_reports.sh +0 -0
- package/tools/mcp_excel_server.py +3 -1
- package/tools/mcp_excel_server.sh +0 -0
- package/tools/mcp_server.py +1 -1
- package/tools/pinpoint_failure.py +88 -88
- package/tools/update_sentinel.py +3 -3
- package/tools/verify_all.sh +0 -0
- package/tools/visual_som_helper.py +14 -3
- package/tools/welcome_page.py +1 -1
- package/benchmarks/qa_eval/reports/AB_METHODOLOGY_REPORT.md +0 -83
|
@@ -1,57 +1,57 @@
|
|
|
1
|
-
# AI SESSION COMPLETION REPORT TEMPLATE
|
|
2
|
-
|
|
3
|
-
> **Instructions:** Use this standardized schema at the conclusion of every AI working session to ensure auditability, cross-session comparability, and historical accuracy.
|
|
4
|
-
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## 1. Session Metadata
|
|
8
|
-
- **Date & Time:** `YYYY-MM-DD HH:mm`
|
|
9
|
-
- **Agent / Engineer:** `Antigravity / Claude Code / Cursor / QA Engineer`
|
|
10
|
-
- **Target Functional Module:** `[e.g., User Administration & RBAC]`
|
|
11
|
-
- **Master Test Suite / Test Cases:** `[e.g., Group_07_01_CumCongNghiep.ts / TC-123 to TC-135]`
|
|
12
|
-
|
|
13
|
-
---
|
|
14
|
-
|
|
15
|
-
## 2. Test Execution Evidence
|
|
16
|
-
- **Exit Code:** `0 (PASS) / 1 (FAIL)`
|
|
17
|
-
- **Total Test Cases:** `[Count]`
|
|
18
|
-
- ✅ **Passed:** `[Count]`
|
|
19
|
-
- ❌ **Failed:** `[Count]`
|
|
20
|
-
- **Detailed Execution Artifacts:**
|
|
21
|
-
- Allure / Katalon Report: `Reports/<Timestamp>/...`
|
|
22
|
-
- Verified UI Assertions & Toasts: `[e.g., "Created successfully" (NFC Normalized)]`
|
|
23
|
-
|
|
24
|
-
---
|
|
25
|
-
|
|
26
|
-
## 3. Truth Anchor & Defect Findings
|
|
27
|
-
*(Select exactly one option below)*
|
|
28
|
-
- [ ] **No Defects Discovered:** All scripts passed cleanly; business workflows operate strictly as specified.
|
|
29
|
-
- [ ] **Real AUT Defect Identified:**
|
|
30
|
-
- Defect Description / Test Case: `[e.g., TC-123 - GIS polygon coordinate collision on save]`
|
|
31
|
-
- Logged into: `KNOWN_ISSUES.md` (Issue ID: `ISSUE-AUT-xx`)
|
|
32
|
-
- Commitment: **Assertions remained strict; no fake pass applied.**
|
|
33
|
-
|
|
34
|
-
---
|
|
35
|
-
|
|
36
|
-
## 4. File Changes Breakdown
|
|
37
|
-
| Action | File Path | Purpose & Technical Summary |
|
|
38
|
-
|---|---|---|
|
|
39
|
-
| **Added** | `[File Path]` | `[Reason for addition]` |
|
|
40
|
-
| **Modified** | `[File Path]` | `[Summary of modification]` |
|
|
41
|
-
| **Deleted** | `[File Path]` | `[Reason for removal]` |
|
|
42
|
-
|
|
43
|
-
---
|
|
44
|
-
|
|
45
|
-
## 5. Quality Gate Enforcement (Definition of Done)
|
|
46
|
-
- [x] **100% UTF-8 No BOM:** Verified (`tools/git_pre_commit_guard.py`)
|
|
47
|
-
- [x] **Zero Raw XPaths in Scripts:** Verified (Strict Clean POM adherence)
|
|
48
|
-
- [x] **Path Portability:** Verified (Zero local absolute machine paths)
|
|
49
|
-
- [x] **AI Docs Anti-Drift Linter:** Verified (`tools/lint_ai_docs.py` v2.0 passed)
|
|
50
|
-
- [x] **Temporary File Blacklist:** Verified (Temporary `.properties` cleaned)
|
|
51
|
-
- [x] **Credential Security:** Verified (No plaintext passwords; `APP_PASSWORD` injected)
|
|
52
|
-
|
|
53
|
-
---
|
|
54
|
-
|
|
55
|
-
## 6. Handoff & Memory Synchronization
|
|
56
|
-
- [x] Updated `PROJECT_STATE.md` with latest operational state.
|
|
57
|
-
- [ ] **Next Immediate Actions:** `[Concise instructions for the next agent session]`
|
|
1
|
+
# AI SESSION COMPLETION REPORT TEMPLATE
|
|
2
|
+
|
|
3
|
+
> **Instructions:** Use this standardized schema at the conclusion of every AI working session to ensure auditability, cross-session comparability, and historical accuracy.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. Session Metadata
|
|
8
|
+
- **Date & Time:** `YYYY-MM-DD HH:mm`
|
|
9
|
+
- **Agent / Engineer:** `Antigravity / Claude Code / Cursor / QA Engineer`
|
|
10
|
+
- **Target Functional Module:** `[e.g., User Administration & RBAC]`
|
|
11
|
+
- **Master Test Suite / Test Cases:** `[e.g., Group_07_01_CumCongNghiep.ts / TC-123 to TC-135]`
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## 2. Test Execution Evidence
|
|
16
|
+
- **Exit Code:** `0 (PASS) / 1 (FAIL)`
|
|
17
|
+
- **Total Test Cases:** `[Count]`
|
|
18
|
+
- ✅ **Passed:** `[Count]`
|
|
19
|
+
- ❌ **Failed:** `[Count]`
|
|
20
|
+
- **Detailed Execution Artifacts:**
|
|
21
|
+
- Allure / Katalon Report: `Reports/<Timestamp>/...`
|
|
22
|
+
- Verified UI Assertions & Toasts: `[e.g., "Created successfully" (NFC Normalized)]`
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## 3. Truth Anchor & Defect Findings
|
|
27
|
+
*(Select exactly one option below)*
|
|
28
|
+
- [ ] **No Defects Discovered:** All scripts passed cleanly; business workflows operate strictly as specified.
|
|
29
|
+
- [ ] **Real AUT Defect Identified:**
|
|
30
|
+
- Defect Description / Test Case: `[e.g., TC-123 - GIS polygon coordinate collision on save]`
|
|
31
|
+
- Logged into: `KNOWN_ISSUES.md` (Issue ID: `ISSUE-AUT-xx`)
|
|
32
|
+
- Commitment: **Assertions remained strict; no fake pass applied.**
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## 4. File Changes Breakdown
|
|
37
|
+
| Action | File Path | Purpose & Technical Summary |
|
|
38
|
+
|---|---|---|
|
|
39
|
+
| **Added** | `[File Path]` | `[Reason for addition]` |
|
|
40
|
+
| **Modified** | `[File Path]` | `[Summary of modification]` |
|
|
41
|
+
| **Deleted** | `[File Path]` | `[Reason for removal]` |
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## 5. Quality Gate Enforcement (Definition of Done)
|
|
46
|
+
- [x] **100% UTF-8 No BOM:** Verified (`tools/git_pre_commit_guard.py`)
|
|
47
|
+
- [x] **Zero Raw XPaths in Scripts:** Verified (Strict Clean POM adherence)
|
|
48
|
+
- [x] **Path Portability:** Verified (Zero local absolute machine paths)
|
|
49
|
+
- [x] **AI Docs Anti-Drift Linter:** Verified (`tools/lint_ai_docs.py` v2.0 passed)
|
|
50
|
+
- [x] **Temporary File Blacklist:** Verified (Temporary `.properties` cleaned)
|
|
51
|
+
- [x] **Credential Security:** Verified (No plaintext passwords; `APP_PASSWORD` injected)
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## 6. Handoff & Memory Synchronization
|
|
56
|
+
- [x] Updated `PROJECT_STATE.md` with latest operational state.
|
|
57
|
+
- [ ] **Next Immediate Actions:** `[Concise instructions for the next agent session]`
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: graphify
|
|
3
|
-
description: Turn any folder of files into a navigable knowledge graph
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# Workflow: graphify
|
|
7
|
-
|
|
8
|
-
Follow the graphify skill to run the full pipeline.
|
|
9
|
-
|
|
10
|
-
If no path argument is given, use `.` (current directory).
|
|
1
|
+
---
|
|
2
|
+
name: graphify
|
|
3
|
+
description: Turn any folder of files into a navigable knowledge graph
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Workflow: graphify
|
|
7
|
+
|
|
8
|
+
Follow the graphify skill to run the full pipeline.
|
|
9
|
+
|
|
10
|
+
If no path argument is given, use `.` (current directory).
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
---
|
|
2
|
-
description: Always respond in Vietnamese.
|
|
3
|
-
globs:
|
|
4
|
-
alwaysApply: true
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
Please always respond in Vietnamese.
|
|
1
|
+
---
|
|
2
|
+
description: Always respond in Vietnamese.
|
|
3
|
+
globs:
|
|
4
|
+
alwaysApply: true
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
Please always respond in Vietnamese.
|
package/README.en.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
> **Core Mission:** A standalone, zero-dependency Linter & Guardrail engine (zero external dependencies for core engine & DoD gates) protecting test suite integrity in the era of AI code generation — Catching tautological "fake-pass" tests, eliminating hard sleeps, distilling 95% CI log noise, and enforcing Clean POM architecture. Specialized tools (Excel export, Canvas visual diff) leverage standard optional extras (`openpyxl`, `Pillow`).
|
|
7
7
|
> **Extended Ecosystem:** Bundled with 38 production QA skills, 6 specialized AI roles, and 9 DoD Quality Gates supporting Playwright, Cypress, Pytest, and Katalon.
|
|
8
8
|
> **Author:** Thế Vị ([@TeeVee998](https://github.com/TeeVee998))
|
|
9
|
-
> **Hub Version:** `v2.4.
|
|
9
|
+
> **Hub Version:** `v2.4.2` | **Scope:** 38 Standardized Skills, 6 Specialized AI Agents, 58 CLI Commands & 60+ Test Engineering Tools, 9 DoD Quality Gates.
|
|
10
10
|
|
|
11
11
|
[](https://www.npmjs.com/package/synapse-hub)
|
|
12
12
|
[](https://nodejs.org)
|
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
> **Định vị cốt lõi:** Bộ Linter & Guardrails độc lập (zero external dependencies cho core engine & DoD gates) bảo vệ độ tin cậy của test suite khi AI viết code — Phát hiện & ngăn chặn test "pass ảo" (0 assert, tautology), loại bỏ hoàn toàn sleep cố định, nén 95% log CI/CD, và chuẩn hóa cấu trúc Clean POM. Các tính năng nâng cao (xuất bảng Excel chuẩn doanh nghiệp, so sánh ảnh Canvas) sử dụng thư viện mở rộng tùy chọn (`openpyxl`, `Pillow`).
|
|
7
7
|
> **Module mở rộng:** Sẵn sàng kết nối 38 kỹ năng QA chuyên sâu, 6 vai trò AI Swarm và 9 Cổng kiểm định DoD hỗ trợ Playwright, Cypress, Pytest và Katalon.
|
|
8
8
|
> **Tác giả:** Thế Vị ([@TeeVee998](https://github.com/TeeVee998))
|
|
9
|
-
> **Phiên bản Hub:** `v2.4.
|
|
9
|
+
> **Phiên bản Hub:** `v2.4.2` | **Quy mô:** 38 Kỹ năng chuẩn hóa, 6 Vai trò AI Agent, 58 Lệnh CLI & 60+ Công cụ kiểm thử chuyên sâu, 9 Cổng kiểm định DoD.
|
|
10
10
|
|
|
11
11
|
[](https://www.npmjs.com/package/synapse-hub)
|
|
12
12
|
[](https://nodejs.org)
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-06_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-06",
|
|
3
|
-
"name": "Numeric ID Serialized as String (Type Mutation)",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Backend response changed 'invoice_id: 10423' (int) to 'invoice_id: \"10423\"' (string).",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "TYPE_MUTATION",
|
|
9
|
-
"broken_assertion": "assert response.data.invoice_id == 10423",
|
|
10
|
-
"error_log": "AssertionError: Expected integer 10423 but received string '10423'",
|
|
11
|
-
"api_payload": "{\"status\": \"SUCCESS\", \"invoice_id\": \"10423\", \"amount\": 250.0}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "assert str(response.data.invoice_id) == '10423'",
|
|
15
|
-
"strategy": "TYPE_COERCION_ASSERTION"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 280,
|
|
20
|
-
"min_confidence_score": 0.9
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-06",
|
|
3
|
+
"name": "Numeric ID Serialized as String (Type Mutation)",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "MEDIUM",
|
|
6
|
+
"description": "Backend response changed 'invoice_id: 10423' (int) to 'invoice_id: \"10423\"' (string).",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "TYPE_MUTATION",
|
|
9
|
+
"broken_assertion": "assert response.data.invoice_id == 10423",
|
|
10
|
+
"error_log": "AssertionError: Expected integer 10423 but received string '10423'",
|
|
11
|
+
"api_payload": "{\"status\": \"SUCCESS\", \"invoice_id\": \"10423\", \"amount\": 250.0}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_assertion": "assert str(response.data.invoice_id) == '10423'",
|
|
15
|
+
"strategy": "TYPE_COERCION_ASSERTION"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 2.0,
|
|
19
|
+
"max_token_cost": 280,
|
|
20
|
+
"min_confidence_score": 0.9
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-07_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-07",
|
|
3
|
-
"name": "Payload Response Field Rename",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "User profile endpoint renamed 'user_email' to 'email_address'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "FIELD_RENAME",
|
|
9
|
-
"broken_assertion": "assert response.json()['user_email'] == 'qa@example.com'",
|
|
10
|
-
"error_log": "KeyError: 'user_email' not found in response dictionary",
|
|
11
|
-
"api_payload": "{\"id\": 42, \"email_address\": \"qa@example.com\", \"role\": \"QA\"}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "assert (response.json().get('email_address') or response.json().get('user_email')) == 'qa@example.com'",
|
|
15
|
-
"strategy": "RESILIENT_FIELD_LOOKUP"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 1.5,
|
|
19
|
-
"max_token_cost": 260,
|
|
20
|
-
"min_confidence_score": 0.95
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-07",
|
|
3
|
+
"name": "Payload Response Field Rename",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "User profile endpoint renamed 'user_email' to 'email_address'.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "FIELD_RENAME",
|
|
9
|
+
"broken_assertion": "assert response.json()['user_email'] == 'qa@example.com'",
|
|
10
|
+
"error_log": "KeyError: 'user_email' not found in response dictionary",
|
|
11
|
+
"api_payload": "{\"id\": 42, \"email_address\": \"qa@example.com\", \"role\": \"QA\"}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_assertion": "assert (response.json().get('email_address') or response.json().get('user_email')) == 'qa@example.com'",
|
|
15
|
+
"strategy": "RESILIENT_FIELD_LOOKUP"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 1.5,
|
|
19
|
+
"max_token_cost": 260,
|
|
20
|
+
"min_confidence_score": 0.95
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-08_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-08",
|
|
3
|
-
"name": "Nested Response Object Flattened",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Backend flattened nested profile: 'data.profile.tier' changed to 'data.tier'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "STRUCTURE_FLATTEN",
|
|
9
|
-
"broken_assertion": "assert response.json()['profile']['tier'] == 'GOLD'",
|
|
10
|
-
"error_log": "TypeError: 'NoneType' or KeyError: 'profile'",
|
|
11
|
-
"api_payload": "{\"tier\": \"GOLD\", \"points\": 1200}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "tier = response.json().get('tier') or response.json().get('profile', {}).get('tier'); assert tier == 'GOLD'",
|
|
15
|
-
"strategy": "SCHEMA_PATH_FALLBACK"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 320,
|
|
20
|
-
"min_confidence_score": 0.9
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-08",
|
|
3
|
+
"name": "Nested Response Object Flattened",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "MEDIUM",
|
|
6
|
+
"description": "Backend flattened nested profile: 'data.profile.tier' changed to 'data.tier'.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "STRUCTURE_FLATTEN",
|
|
9
|
+
"broken_assertion": "assert response.json()['profile']['tier'] == 'GOLD'",
|
|
10
|
+
"error_log": "TypeError: 'NoneType' or KeyError: 'profile'",
|
|
11
|
+
"api_payload": "{\"tier\": \"GOLD\", \"points\": 1200}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_assertion": "tier = response.json().get('tier') or response.json().get('profile', {}).get('tier'); assert tier == 'GOLD'",
|
|
15
|
+
"strategy": "SCHEMA_PATH_FALLBACK"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 2.0,
|
|
19
|
+
"max_token_cost": 320,
|
|
20
|
+
"min_confidence_score": 0.9
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-09_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-09",
|
|
3
|
-
"name": "HTTP Creation Status Code Shift (200 OK -> 201 Created)",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "POST endpoint updated to RFC-compliant 201 Created instead of legacy 200 OK.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "STATUS_CODE_SHIFT",
|
|
9
|
-
"broken_assertion": "assert response.status_code == 200",
|
|
10
|
-
"error_log": "AssertionError: Expected status code 200 but received 201",
|
|
11
|
-
"api_payload": "{\"status\": 201, \"created\": true}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "assert response.status_code in (200, 201)",
|
|
15
|
-
"strategy": "SUCCESS_STATUS_RANGE"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 1.0,
|
|
19
|
-
"max_token_cost": 220,
|
|
20
|
-
"min_confidence_score": 0.98
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-09",
|
|
3
|
+
"name": "HTTP Creation Status Code Shift (200 OK -> 201 Created)",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "POST endpoint updated to RFC-compliant 201 Created instead of legacy 200 OK.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "STATUS_CODE_SHIFT",
|
|
9
|
+
"broken_assertion": "assert response.status_code == 200",
|
|
10
|
+
"error_log": "AssertionError: Expected status code 200 but received 201",
|
|
11
|
+
"api_payload": "{\"status\": 201, \"created\": true}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_assertion": "assert response.status_code in (200, 201)",
|
|
15
|
+
"strategy": "SUCCESS_STATUS_RANGE"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 1.0,
|
|
19
|
+
"max_token_cost": 220,
|
|
20
|
+
"min_confidence_score": 0.98
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-10_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-10",
|
|
3
|
-
"name": "Authentication Header Prefix Upgrade",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "API gateway changed auth format from 'Authorization: Token <token>' to 'Bearer <token>'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "AUTH_HEADER_DRIFT",
|
|
9
|
-
"broken_code": "headers = {'Authorization': f'Token {auth_token}'}",
|
|
10
|
-
"error_log": "HTTP 401 Unauthorized: Invalid authorization scheme. Expected 'Bearer'.",
|
|
11
|
-
"api_payload": "{\"error\": \"Unauthorized\", \"code\": 401}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "headers = {'Authorization': f'Bearer {auth_token}'}",
|
|
15
|
-
"strategy": "STANDARD_BEARER_TOKEN"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 1.5,
|
|
19
|
-
"max_token_cost": 250,
|
|
20
|
-
"min_confidence_score": 0.95
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-10",
|
|
3
|
+
"name": "Authentication Header Prefix Upgrade",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "API gateway changed auth format from 'Authorization: Token <token>' to 'Bearer <token>'.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "AUTH_HEADER_DRIFT",
|
|
9
|
+
"broken_code": "headers = {'Authorization': f'Token {auth_token}'}",
|
|
10
|
+
"error_log": "HTTP 401 Unauthorized: Invalid authorization scheme. Expected 'Bearer'.",
|
|
11
|
+
"api_payload": "{\"error\": \"Unauthorized\", \"code\": 401}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_code": "headers = {'Authorization': f'Bearer {auth_token}'}",
|
|
15
|
+
"strategy": "STANDARD_BEARER_TOKEN"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 1.5,
|
|
19
|
+
"max_token_cost": 250,
|
|
20
|
+
"min_confidence_score": 0.95
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-21_fake_pass_tautology.json
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-21",
|
|
3
|
-
"name": "Swallowed HTTP 500 Error in Empty Catch Block",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Test silently catches runtime exceptions in an empty catch, masking severe backend 500 crash.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "SWALLOWED_EXCEPTION",
|
|
9
|
-
"code_snippet": "try { page.click('#submit-payment') } catch (Exception e) {} // Test passes even on 500",
|
|
10
|
-
"detection_rule": "EMPTY_CATCH_PATTERN"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "page.click('#submit-payment'); assert page.locator('.success-toast').isVisible();",
|
|
14
|
-
"strategy": "REMOVE_SWALLOWED_CATCH_AND_ASSERT"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.0,
|
|
18
|
-
"max_token_cost": 200,
|
|
19
|
-
"min_confidence_score": 0.99
|
|
20
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-21",
|
|
3
|
+
"name": "Swallowed HTTP 500 Error in Empty Catch Block",
|
|
4
|
+
"category": "fake_pass_tautology",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "Test silently catches runtime exceptions in an empty catch, masking severe backend 500 crash.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "SWALLOWED_EXCEPTION",
|
|
9
|
+
"code_snippet": "try { page.click('#submit-payment') } catch (Exception e) {} // Test passes even on 500",
|
|
10
|
+
"detection_rule": "EMPTY_CATCH_PATTERN"
|
|
11
|
+
},
|
|
12
|
+
"expected_resolution": {
|
|
13
|
+
"healed_code": "page.click('#submit-payment'); assert page.locator('.success-toast').isVisible();",
|
|
14
|
+
"strategy": "REMOVE_SWALLOWED_CATCH_AND_ASSERT"
|
|
15
|
+
},
|
|
16
|
+
"benchmark_targets": {
|
|
17
|
+
"max_heal_time_sec": 1.0,
|
|
18
|
+
"max_token_cost": 200,
|
|
19
|
+
"min_confidence_score": 0.99
|
|
20
|
+
}
|
|
21
21
|
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-22_fake_pass_tautology.json
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-22",
|
|
3
|
-
"name": "Zero-Assertion Smoke Test (Tautological Pass)",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Test executes form inputs and clicks submit but contains exactly 0 assertion statements.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "ZERO_ASSERTION_PASS",
|
|
9
|
-
"code_snippet": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn');",
|
|
10
|
-
"detection_rule": "FAKE_PASS_NO_ASSERTION"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn'); expect(page.locator('#dashboard-header')).toBeVisible();",
|
|
14
|
-
"strategy": "INJECT_POST_CONDITION_ASSERTION"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.5,
|
|
18
|
-
"max_token_cost": 240,
|
|
19
|
-
"min_confidence_score": 0.97
|
|
20
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-22",
|
|
3
|
+
"name": "Zero-Assertion Smoke Test (Tautological Pass)",
|
|
4
|
+
"category": "fake_pass_tautology",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "Test executes form inputs and clicks submit but contains exactly 0 assertion statements.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "ZERO_ASSERTION_PASS",
|
|
9
|
+
"code_snippet": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn');",
|
|
10
|
+
"detection_rule": "FAKE_PASS_NO_ASSERTION"
|
|
11
|
+
},
|
|
12
|
+
"expected_resolution": {
|
|
13
|
+
"healed_code": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn'); expect(page.locator('#dashboard-header')).toBeVisible();",
|
|
14
|
+
"strategy": "INJECT_POST_CONDITION_ASSERTION"
|
|
15
|
+
},
|
|
16
|
+
"benchmark_targets": {
|
|
17
|
+
"max_heal_time_sec": 1.5,
|
|
18
|
+
"max_token_cost": 240,
|
|
19
|
+
"min_confidence_score": 0.97
|
|
20
|
+
}
|
|
21
21
|
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-23_fake_pass_tautology.json
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-23",
|
|
3
|
-
"name": "Tautological assert true == true",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Test asserts trivial identity 'assert true == true' instead of verifying business state.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "TAUTOLOGY_ASSERTION",
|
|
9
|
-
"code_snippet": "def response = api.createInvoice(); assert true == true;",
|
|
10
|
-
"detection_rule": "TAUTOLOGY_PATTERN"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "def response = api.createInvoice(); assert response.status == 201 && response.id != null;",
|
|
14
|
-
"strategy": "MUTATION_TESTING_KILLS_TAUTOLOGY"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.0,
|
|
18
|
-
"max_token_cost": 220,
|
|
19
|
-
"min_confidence_score": 0.98
|
|
20
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-23",
|
|
3
|
+
"name": "Tautological assert true == true",
|
|
4
|
+
"category": "fake_pass_tautology",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "Test asserts trivial identity 'assert true == true' instead of verifying business state.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "TAUTOLOGY_ASSERTION",
|
|
9
|
+
"code_snippet": "def response = api.createInvoice(); assert true == true;",
|
|
10
|
+
"detection_rule": "TAUTOLOGY_PATTERN"
|
|
11
|
+
},
|
|
12
|
+
"expected_resolution": {
|
|
13
|
+
"healed_code": "def response = api.createInvoice(); assert response.status == 201 && response.id != null;",
|
|
14
|
+
"strategy": "MUTATION_TESTING_KILLS_TAUTOLOGY"
|
|
15
|
+
},
|
|
16
|
+
"benchmark_targets": {
|
|
17
|
+
"max_heal_time_sec": 1.0,
|
|
18
|
+
"max_token_cost": 220,
|
|
19
|
+
"min_confidence_score": 0.98
|
|
20
|
+
}
|
|
21
21
|
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-24_fake_pass_tautology.json
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-24",
|
|
3
|
-
"name": "Unawaited Async Assertion (Silent Green Trap)",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Missing 'await' on expect(locator).toBeVisible(); Promise resolves ignored after test exit.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "UNAWAITED_ASYNC_ASSERTION",
|
|
9
|
-
"code_snippet": "page.click('#save'); expect(page.locator('.alert')).toBeVisible(); // Missing await!",
|
|
10
|
-
"detection_rule": "UNAWAITED_ASYNC_EXPECT"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "page.click('#save'); await expect(page.locator('.alert')).toBeVisible();",
|
|
14
|
-
"strategy": "AST_ASYNC_AWAIT_ENFORCEMENT"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.5,
|
|
18
|
-
"max_token_cost": 250,
|
|
19
|
-
"min_confidence_score": 0.96
|
|
20
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-24",
|
|
3
|
+
"name": "Unawaited Async Assertion (Silent Green Trap)",
|
|
4
|
+
"category": "fake_pass_tautology",
|
|
5
|
+
"difficulty": "MEDIUM",
|
|
6
|
+
"description": "Missing 'await' on expect(locator).toBeVisible(); Promise resolves ignored after test exit.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "UNAWAITED_ASYNC_ASSERTION",
|
|
9
|
+
"code_snippet": "page.click('#save'); expect(page.locator('.alert')).toBeVisible(); // Missing await!",
|
|
10
|
+
"detection_rule": "UNAWAITED_ASYNC_EXPECT"
|
|
11
|
+
},
|
|
12
|
+
"expected_resolution": {
|
|
13
|
+
"healed_code": "page.click('#save'); await expect(page.locator('.alert')).toBeVisible();",
|
|
14
|
+
"strategy": "AST_ASYNC_AWAIT_ENFORCEMENT"
|
|
15
|
+
},
|
|
16
|
+
"benchmark_targets": {
|
|
17
|
+
"max_heal_time_sec": 1.5,
|
|
18
|
+
"max_token_cost": 250,
|
|
19
|
+
"min_confidence_score": 0.96
|
|
20
|
+
}
|
|
21
21
|
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-25_fake_pass_tautology.json
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-25",
|
|
3
|
-
"name": "Unbounded Relational Assertion (status >= 200)",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Assertion checks 'status >= 200' without ceiling, so HTTP 500 error causes false pass.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "UNBOUNDED_STATUS_ASSERTION",
|
|
9
|
-
"code_snippet": "def resp = client.send(); assert resp.statusCode >= 200; // Passes on 500!",
|
|
10
|
-
"detection_rule": "MUTATION_BOUNDARY_SHIFT"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "def resp = client.send(); assert resp.statusCode >= 200 && resp.statusCode < 300;",
|
|
14
|
-
"strategy": "BOUNDED_INTERVAL_ASSERTION"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.2,
|
|
18
|
-
"max_token_cost": 230,
|
|
19
|
-
"min_confidence_score": 0.98
|
|
20
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-25",
|
|
3
|
+
"name": "Unbounded Relational Assertion (status >= 200)",
|
|
4
|
+
"category": "fake_pass_tautology",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "Assertion checks 'status >= 200' without ceiling, so HTTP 500 error causes false pass.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "UNBOUNDED_STATUS_ASSERTION",
|
|
9
|
+
"code_snippet": "def resp = client.send(); assert resp.statusCode >= 200; // Passes on 500!",
|
|
10
|
+
"detection_rule": "MUTATION_BOUNDARY_SHIFT"
|
|
11
|
+
},
|
|
12
|
+
"expected_resolution": {
|
|
13
|
+
"healed_code": "def resp = client.send(); assert resp.statusCode >= 200 && resp.statusCode < 300;",
|
|
14
|
+
"strategy": "BOUNDED_INTERVAL_ASSERTION"
|
|
15
|
+
},
|
|
16
|
+
"benchmark_targets": {
|
|
17
|
+
"max_heal_time_sec": 1.2,
|
|
18
|
+
"max_token_cost": 230,
|
|
19
|
+
"min_confidence_score": 0.98
|
|
20
|
+
}
|
|
21
21
|
}
|