synapse-hub 2.4.0 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/agents/orchestrator.md +3 -0
- package/.agents/agents/pom-crafter.md +3 -0
- package/.agents/agents/quality-gatekeeper.md +1 -0
- package/.agents/agents/test-runner.md +1 -0
- package/.agents/rules/graphify.md +14 -14
- package/.agents/rules/mandatory-playwright-live.md +29 -29
- package/.agents/skills/bats-testing-patterns/resources/implementation-playbook.md +614 -614
- package/.agents/skills/clean-pom-crafter/examples/standard_test_case.groovy +28 -28
- package/.agents/skills/clean-pom-crafter/references/pom_patterns.groovy +138 -138
- package/.agents/skills/e2e-testing-patterns/resources/implementation-playbook.md +522 -522
- package/.agents/skills/failure-autopsy-and-healing/references/healing_matrix.md +64 -64
- package/.agents/skills/failure-autopsy-and-healing/references/qa_investigation_protocol.md +56 -56
- package/.agents/skills/failure-autopsy-and-healing/scripts/triage_failures.sh +98 -98
- package/.agents/skills/katalon-cli-runner/references/groovy-patterns.md +209 -209
- package/.agents/skills/katalon-cli-runner/references/katalon-api.md +155 -155
- package/.agents/skills/katalon-cli-runner/references/runner_commands.md +67 -67
- package/.agents/skills/playwright-live-interaction/references/advanced_superpowers.md +160 -160
- package/.agents/skills/playwright-live-interaction/references/compact_accessibility_snapshots.md +75 -75
- package/.agents/skills/playwright-live-interaction/references/recipes.md +388 -388
- package/.agents/skills/playwright-live-interaction/references/swarm_orchestration.md +134 -134
- package/.agents/skills/wcag-audit-patterns/resources/implementation-playbook.md +541 -541
- package/.agents/templates/completion_report.md +57 -57
- package/.agents/workflows/graphify.md +10 -10
- package/.cursor/rules/language.mdc +7 -7
- package/README.en.md +9 -3
- package/README.md +9 -3
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-06_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-07_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-08_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-09_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-10_api_contract_drift.json +21 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-21_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-22_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-23_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-24_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-25_fake_pass_tautology.json +20 -20
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-01_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-02_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-03_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-04_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-05_locator_drift.json +22 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-16_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-17_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-18_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-19_stale_and_mutation.json +21 -21
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-20_stale_and_mutation.json +22 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-11_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-12_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-13_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-14_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-15_timing_and_race.json +21 -21
- package/benchmarks/qa_eval/reports/BENCHMARK_REPORT.md +94 -69
- package/benchmarks/qa_eval/reports/eval_results.json +460 -104
- package/bin/agent-qa.js +6 -1
- package/docs/ARCHITECTURE.md +13 -2
- package/package.json +6 -6
- package/scripts/install.ps1 +112 -112
- package/scripts/install_global_plugin.ps1 +1 -1
- package/scripts/install_global_plugin.sh +1 -1
- package/scripts/lint_skills.ps1 +57 -57
- package/skills_manifest.json +1 -1
- package/synapse.config.json +1 -1
- package/tools/TOOLS_RUNBOOK.md +6 -1
- package/tools/__init__.py +1 -1
- package/tools/agent_canary.py +501 -501
- package/tools/api_fuzzer.py +11 -2
- package/tools/browser_helpers.js +2 -2
- package/tools/bump_version.py +8 -0
- package/tools/check_version_sync.py +18 -0
- package/tools/cli.py +210 -48
- package/tools/count_tokens.py +4 -4
- package/tools/dashboard_generator.py +10 -10
- package/tools/e2e_flow_prober.sh +0 -0
- package/tools/generate_allure_report.sh +0 -0
- package/tools/generate_test_reports.sh +0 -0
- package/tools/mcp_excel_server.py +3 -1
- package/tools/mcp_excel_server.sh +0 -0
- package/tools/mcp_server.py +1 -1
- package/tools/micro_mutator.py +4 -2
- package/tools/pinpoint_failure.py +88 -88
- package/tools/setup_project.py +1 -1
- package/tools/skillspector.py +31 -0
- package/tools/update_sentinel.py +3 -3
- package/tools/verify_all.py +16 -6
- package/tools/verify_all.sh +0 -0
- package/tools/visual_som_helper.py +14 -3
- package/tools/welcome_page.py +1 -1
- package/benchmarks/qa_eval/reports/AB_METHODOLOGY_REPORT.md +0 -83
|
@@ -1,57 +1,57 @@
|
|
|
1
|
-
# AI SESSION COMPLETION REPORT TEMPLATE
|
|
2
|
-
|
|
3
|
-
> **Instructions:** Use this standardized schema at the conclusion of every AI working session to ensure auditability, cross-session comparability, and historical accuracy.
|
|
4
|
-
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## 1. Session Metadata
|
|
8
|
-
- **Date & Time:** `YYYY-MM-DD HH:mm`
|
|
9
|
-
- **Agent / Engineer:** `Antigravity / Claude Code / Cursor / QA Engineer`
|
|
10
|
-
- **Target Functional Module:** `[e.g., User Administration & RBAC]`
|
|
11
|
-
- **Master Test Suite / Test Cases:** `[e.g., Group_07_01_CumCongNghiep.ts / TC-123 to TC-135]`
|
|
12
|
-
|
|
13
|
-
---
|
|
14
|
-
|
|
15
|
-
## 2. Test Execution Evidence
|
|
16
|
-
- **Exit Code:** `0 (PASS) / 1 (FAIL)`
|
|
17
|
-
- **Total Test Cases:** `[Count]`
|
|
18
|
-
- ✅ **Passed:** `[Count]`
|
|
19
|
-
- ❌ **Failed:** `[Count]`
|
|
20
|
-
- **Detailed Execution Artifacts:**
|
|
21
|
-
- Allure / Katalon Report: `Reports/<Timestamp>/...`
|
|
22
|
-
- Verified UI Assertions & Toasts: `[e.g., "Created successfully" (NFC Normalized)]`
|
|
23
|
-
|
|
24
|
-
---
|
|
25
|
-
|
|
26
|
-
## 3. Truth Anchor & Defect Findings
|
|
27
|
-
*(Select exactly one option below)*
|
|
28
|
-
- [ ] **No Defects Discovered:** All scripts passed cleanly; business workflows operate strictly as specified.
|
|
29
|
-
- [ ] **Real AUT Defect Identified:**
|
|
30
|
-
- Defect Description / Test Case: `[e.g., TC-123 - GIS polygon coordinate collision on save]`
|
|
31
|
-
- Logged into: `KNOWN_ISSUES.md` (Issue ID: `ISSUE-AUT-xx`)
|
|
32
|
-
- Commitment: **Assertions remained strict; no fake pass applied.**
|
|
33
|
-
|
|
34
|
-
---
|
|
35
|
-
|
|
36
|
-
## 4. File Changes Breakdown
|
|
37
|
-
| Action | File Path | Purpose & Technical Summary |
|
|
38
|
-
|---|---|---|
|
|
39
|
-
| **Added** | `[File Path]` | `[Reason for addition]` |
|
|
40
|
-
| **Modified** | `[File Path]` | `[Summary of modification]` |
|
|
41
|
-
| **Deleted** | `[File Path]` | `[Reason for removal]` |
|
|
42
|
-
|
|
43
|
-
---
|
|
44
|
-
|
|
45
|
-
## 5. Quality Gate Enforcement (Definition of Done)
|
|
46
|
-
- [x] **100% UTF-8 No BOM:** Verified (`tools/git_pre_commit_guard.py`)
|
|
47
|
-
- [x] **Zero Raw XPaths in Scripts:** Verified (Strict Clean POM adherence)
|
|
48
|
-
- [x] **Path Portability:** Verified (Zero local absolute machine paths)
|
|
49
|
-
- [x] **AI Docs Anti-Drift Linter:** Verified (`tools/lint_ai_docs.py` v2.0 passed)
|
|
50
|
-
- [x] **Temporary File Blacklist:** Verified (Temporary `.properties` cleaned)
|
|
51
|
-
- [x] **Credential Security:** Verified (No plaintext passwords; `APP_PASSWORD` injected)
|
|
52
|
-
|
|
53
|
-
---
|
|
54
|
-
|
|
55
|
-
## 6. Handoff & Memory Synchronization
|
|
56
|
-
- [x] Updated `PROJECT_STATE.md` with latest operational state.
|
|
57
|
-
- [ ] **Next Immediate Actions:** `[Concise instructions for the next agent session]`
|
|
1
|
+
# AI SESSION COMPLETION REPORT TEMPLATE
|
|
2
|
+
|
|
3
|
+
> **Instructions:** Use this standardized schema at the conclusion of every AI working session to ensure auditability, cross-session comparability, and historical accuracy.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## 1. Session Metadata
|
|
8
|
+
- **Date & Time:** `YYYY-MM-DD HH:mm`
|
|
9
|
+
- **Agent / Engineer:** `Antigravity / Claude Code / Cursor / QA Engineer`
|
|
10
|
+
- **Target Functional Module:** `[e.g., User Administration & RBAC]`
|
|
11
|
+
- **Master Test Suite / Test Cases:** `[e.g., Group_07_01_CumCongNghiep.ts / TC-123 to TC-135]`
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## 2. Test Execution Evidence
|
|
16
|
+
- **Exit Code:** `0 (PASS) / 1 (FAIL)`
|
|
17
|
+
- **Total Test Cases:** `[Count]`
|
|
18
|
+
- ✅ **Passed:** `[Count]`
|
|
19
|
+
- ❌ **Failed:** `[Count]`
|
|
20
|
+
- **Detailed Execution Artifacts:**
|
|
21
|
+
- Allure / Katalon Report: `Reports/<Timestamp>/...`
|
|
22
|
+
- Verified UI Assertions & Toasts: `[e.g., "Created successfully" (NFC Normalized)]`
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## 3. Truth Anchor & Defect Findings
|
|
27
|
+
*(Select exactly one option below)*
|
|
28
|
+
- [ ] **No Defects Discovered:** All scripts passed cleanly; business workflows operate strictly as specified.
|
|
29
|
+
- [ ] **Real AUT Defect Identified:**
|
|
30
|
+
- Defect Description / Test Case: `[e.g., TC-123 - GIS polygon coordinate collision on save]`
|
|
31
|
+
- Logged into: `KNOWN_ISSUES.md` (Issue ID: `ISSUE-AUT-xx`)
|
|
32
|
+
- Commitment: **Assertions remained strict; no fake pass applied.**
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## 4. File Changes Breakdown
|
|
37
|
+
| Action | File Path | Purpose & Technical Summary |
|
|
38
|
+
|---|---|---|
|
|
39
|
+
| **Added** | `[File Path]` | `[Reason for addition]` |
|
|
40
|
+
| **Modified** | `[File Path]` | `[Summary of modification]` |
|
|
41
|
+
| **Deleted** | `[File Path]` | `[Reason for removal]` |
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## 5. Quality Gate Enforcement (Definition of Done)
|
|
46
|
+
- [x] **100% UTF-8 No BOM:** Verified (`tools/git_pre_commit_guard.py`)
|
|
47
|
+
- [x] **Zero Raw XPaths in Scripts:** Verified (Strict Clean POM adherence)
|
|
48
|
+
- [x] **Path Portability:** Verified (Zero local absolute machine paths)
|
|
49
|
+
- [x] **AI Docs Anti-Drift Linter:** Verified (`tools/lint_ai_docs.py` v2.0 passed)
|
|
50
|
+
- [x] **Temporary File Blacklist:** Verified (Temporary `.properties` cleaned)
|
|
51
|
+
- [x] **Credential Security:** Verified (No plaintext passwords; `APP_PASSWORD` injected)
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## 6. Handoff & Memory Synchronization
|
|
56
|
+
- [x] Updated `PROJECT_STATE.md` with latest operational state.
|
|
57
|
+
- [ ] **Next Immediate Actions:** `[Concise instructions for the next agent session]`
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: graphify
|
|
3
|
-
description: Turn any folder of files into a navigable knowledge graph
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# Workflow: graphify
|
|
7
|
-
|
|
8
|
-
Follow the graphify skill to run the full pipeline.
|
|
9
|
-
|
|
10
|
-
If no path argument is given, use `.` (current directory).
|
|
1
|
+
---
|
|
2
|
+
name: graphify
|
|
3
|
+
description: Turn any folder of files into a navigable knowledge graph
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Workflow: graphify
|
|
7
|
+
|
|
8
|
+
Follow the graphify skill to run the full pipeline.
|
|
9
|
+
|
|
10
|
+
If no path argument is given, use `.` (current directory).
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
---
|
|
2
|
-
description: Always respond in Vietnamese.
|
|
3
|
-
globs:
|
|
4
|
-
alwaysApply: true
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
Please always respond in Vietnamese.
|
|
1
|
+
---
|
|
2
|
+
description: Always respond in Vietnamese.
|
|
3
|
+
globs:
|
|
4
|
+
alwaysApply: true
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
Please always respond in Vietnamese.
|
package/README.en.md
CHANGED
|
@@ -6,11 +6,11 @@
|
|
|
6
6
|
> **Core Mission:** A standalone, zero-dependency Linter & Guardrail engine (zero external dependencies for core engine & DoD gates) protecting test suite integrity in the era of AI code generation — Catching tautological "fake-pass" tests, eliminating hard sleeps, distilling 95% CI log noise, and enforcing Clean POM architecture. Specialized tools (Excel export, Canvas visual diff) leverage standard optional extras (`openpyxl`, `Pillow`).
|
|
7
7
|
> **Extended Ecosystem:** Bundled with 38 production QA skills, 6 specialized AI roles, and 9 DoD Quality Gates supporting Playwright, Cypress, Pytest, and Katalon.
|
|
8
8
|
> **Author:** Thế Vị ([@TeeVee998](https://github.com/TeeVee998))
|
|
9
|
-
> **Hub Version:** `v2.4.
|
|
9
|
+
> **Hub Version:** `v2.4.2` | **Scope:** 38 Standardized Skills, 6 Specialized AI Agents, 58 CLI Commands & 60+ Test Engineering Tools, 9 DoD Quality Gates.
|
|
10
10
|
|
|
11
11
|
[](https://www.npmjs.com/package/synapse-hub)
|
|
12
12
|
[](https://nodejs.org)
|
|
13
|
-
[](https://python.org)
|
|
14
14
|
[](https://github.com/TeeVee998/Synapse-Hub/blob/main/LICENSE)
|
|
15
15
|
[](#)
|
|
16
16
|
|
|
@@ -18,6 +18,12 @@
|
|
|
18
18
|
|
|
19
19
|
### 💡 What is Synapse Hub? (In 30 Seconds)
|
|
20
20
|
|
|
21
|
+
> 📌 **Hub & Spoke Architecture Notice (Avoid Cloner Confusion):**
|
|
22
|
+
> • **Synapse-Hub (This Repo):** The **"Central Brain"** hosting 38 skills, linters, and QA tools — **not** your test suite.
|
|
23
|
+
> • **Target Project (Spoke):** Your actual automation test repository (Playwright, Katalon, Pytest, Cypress, etc.).
|
|
24
|
+
> 👉 To bootstrap Synapse Hub protection for your test project, open a terminal inside that project and run:
|
|
25
|
+
> `npx synapse-hub setup` *(or from here with a target flag: `npx synapse-hub setup --target ../your-target-project`)*.
|
|
26
|
+
|
|
21
27
|
Think of Synapse Hub as a versatile, dual-mode engine:
|
|
22
28
|
1. 🛠️ **Standalone CLI Tool (Zero AI needed):** Simply run `npx synapse-hub lint` to scan your test suite, distill bloated CI logs, or export test cases to standard Excel spreadsheets without any complex setup.
|
|
23
29
|
2. 🧠 **Guardrail "Brain" for AI Coding Assistants:** When using Google Antigravity, Cursor, or Claude Code, Synapse Hub acts as an automated quality guardrail — preventing AI assistants from generating tautological fake-passes, swallowing exceptions, or inserting flaky hard sleeps.
|
|
@@ -145,7 +151,7 @@ repos:
|
|
|
145
151
|
| **Gate 5** | **Temp File Blacklist** | Prevents temporary build debris (`*.tmp`, memory dumps, stale artifacts) from leaking. |
|
|
146
152
|
| **Gate 6** | **Credential Security** | Blocks hardcoded plaintext passwords, private keys, and secret tokens in code. |
|
|
147
153
|
| **Gate 7** | **Static Defect & Fake-Pass Sentinel** | Static heuristic analysis catching 0-assertion tests, empty catch blocks, hardcoded/variable sleeps (≥ 3000ms), and unawaited promises. |
|
|
148
|
-
| **Gate 8** | **Assertion Sensitivity Sentinel** | Injects syntactic micro-mutations linked via dataflow (requires Score ≥ 80%). |
|
|
154
|
+
| **Gate 8** | **Assertion Sensitivity Sentinel** | Injects syntactic micro-mutations linked via dataflow (Ultra-fast Heuristic Static mode ~0.1s; pass `--test-cmd` for runtime re-test; requires Score ≥ 80%). |
|
|
149
155
|
|
|
150
156
|
---
|
|
151
157
|
|
package/README.md
CHANGED
|
@@ -6,11 +6,11 @@
|
|
|
6
6
|
> **Định vị cốt lõi:** Bộ Linter & Guardrails độc lập (zero external dependencies cho core engine & DoD gates) bảo vệ độ tin cậy của test suite khi AI viết code — Phát hiện & ngăn chặn test "pass ảo" (0 assert, tautology), loại bỏ hoàn toàn sleep cố định, nén 95% log CI/CD, và chuẩn hóa cấu trúc Clean POM. Các tính năng nâng cao (xuất bảng Excel chuẩn doanh nghiệp, so sánh ảnh Canvas) sử dụng thư viện mở rộng tùy chọn (`openpyxl`, `Pillow`).
|
|
7
7
|
> **Module mở rộng:** Sẵn sàng kết nối 38 kỹ năng QA chuyên sâu, 6 vai trò AI Swarm và 9 Cổng kiểm định DoD hỗ trợ Playwright, Cypress, Pytest và Katalon.
|
|
8
8
|
> **Tác giả:** Thế Vị ([@TeeVee998](https://github.com/TeeVee998))
|
|
9
|
-
> **Phiên bản Hub:** `v2.4.
|
|
9
|
+
> **Phiên bản Hub:** `v2.4.2` | **Quy mô:** 38 Kỹ năng chuẩn hóa, 6 Vai trò AI Agent, 58 Lệnh CLI & 60+ Công cụ kiểm thử chuyên sâu, 9 Cổng kiểm định DoD.
|
|
10
10
|
|
|
11
11
|
[](https://www.npmjs.com/package/synapse-hub)
|
|
12
12
|
[](https://nodejs.org)
|
|
13
|
-
[](https://python.org)
|
|
14
14
|
[](https://github.com/TeeVee998/Synapse-Hub/blob/main/LICENSE)
|
|
15
15
|
[](#)
|
|
16
16
|
|
|
@@ -18,6 +18,12 @@
|
|
|
18
18
|
|
|
19
19
|
### 💡 Synapse Hub là gì? (Hiểu ngay trong 30 giây)
|
|
20
20
|
|
|
21
|
+
> 📌 **Lưu ý kiến trúc Hub & Spoke (Tránh nhầm lẫn khi mới clone):**
|
|
22
|
+
> • **Synapse-Hub (Kho này):** Là **"Bộ Não Trung Tâm"** chứa 38 kỹ năng, linter và công cụ, **không phải** là dự án kiểm thử của bạn.
|
|
23
|
+
> • **Dự án mục tiêu (Target Project / Spoke):** Là mã nguồn test thực tế của bạn (Playwright, Katalon, Pytest, Cypress...).
|
|
24
|
+
> 👉 Để kích hoạt Synapse Hub bảo vệ dự án của bạn, hãy mở terminal ngay tại dự án đó và chạy:
|
|
25
|
+
> `npx synapse-hub setup` *(hoặc đứng từ đây trỏ đường dẫn: `npx synapse-hub setup --target ../du-an-cua-ban`)*.
|
|
26
|
+
|
|
21
27
|
Bạn có thể sử dụng Synapse Hub theo **2 cách cực kỳ linh hoạt**:
|
|
22
28
|
1. 🛠️ **Dùng như CLI độc lập (Không cần AI):** Bạn chỉ cần gõ `npx synapse-hub lint` để kiểm tra test, nén log CI hoặc xuất kịch bản sang Excel mà không cần cài đặt phức tạp.
|
|
23
29
|
2. 🧠 **Dùng như "Bộ Não Bảo Vệ" cho AI Coding:** Khi bạn dùng Cursor, Google Antigravity hay Claude Code, Synapse Hub đóng vai trò làm cẩm nang hướng dẫn ngầm — ngăn AI sinh ra test "pass ảo", bỏ qua ngoại lệ ngầm hay chèn sleep cố định.
|
|
@@ -145,7 +151,7 @@ repos:
|
|
|
145
151
|
| **Gate 5** | **Danh Sách Đen File Tạm** | Chặn rò rỉ file rác (`*.tmp`, memory dump, cache thừa) làm bẩn workspace. |
|
|
146
152
|
| **Gate 6** | **Bảo Vệ Bí Mật & Mật Khẩu** | Quét chặn plaintext password, private keys hoặc API token lưu cứng trong code. |
|
|
147
153
|
| **Gate 7** | **Quét Lỗi Tĩnh & Bắt Test Pass Ảo** | Tự động phát hiện test 0 assertion, try-catch bỏ qua ngoại lệ ngầm, sleep cố định (≥ 3000ms) và unawaited expect. |
|
|
148
|
-
| **Gate 8** | **Đo Độ Nhạy Assert (Micro-Mutation)** | Đảm bảo các câu lệnh assert bắt được lỗi thật khi code bị biến đổi (
|
|
154
|
+
| **Gate 8** | **Đo Độ Nhạy Assert (Micro-Mutation)** | Đảm bảo các câu lệnh assert bắt được lỗi thật khi code bị biến đổi (Chế độ Heuristic Static ~0.1s; truyền `--test-cmd` cho runtime re-test; yêu cầu điểm ≥ 80%). |
|
|
149
155
|
|
|
150
156
|
---
|
|
151
157
|
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-06_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-06",
|
|
3
|
-
"name": "Numeric ID Serialized as String (Type Mutation)",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Backend response changed 'invoice_id: 10423' (int) to 'invoice_id: \"10423\"' (string).",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "TYPE_MUTATION",
|
|
9
|
-
"broken_assertion": "assert response.data.invoice_id == 10423",
|
|
10
|
-
"error_log": "AssertionError: Expected integer 10423 but received string '10423'",
|
|
11
|
-
"api_payload": "{\"status\": \"SUCCESS\", \"invoice_id\": \"10423\", \"amount\": 250.0}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "assert str(response.data.invoice_id) == '10423'",
|
|
15
|
-
"strategy": "TYPE_COERCION_ASSERTION"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 280,
|
|
20
|
-
"min_confidence_score": 0.9
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-06",
|
|
3
|
+
"name": "Numeric ID Serialized as String (Type Mutation)",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "MEDIUM",
|
|
6
|
+
"description": "Backend response changed 'invoice_id: 10423' (int) to 'invoice_id: \"10423\"' (string).",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "TYPE_MUTATION",
|
|
9
|
+
"broken_assertion": "assert response.data.invoice_id == 10423",
|
|
10
|
+
"error_log": "AssertionError: Expected integer 10423 but received string '10423'",
|
|
11
|
+
"api_payload": "{\"status\": \"SUCCESS\", \"invoice_id\": \"10423\", \"amount\": 250.0}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_assertion": "assert str(response.data.invoice_id) == '10423'",
|
|
15
|
+
"strategy": "TYPE_COERCION_ASSERTION"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 2.0,
|
|
19
|
+
"max_token_cost": 280,
|
|
20
|
+
"min_confidence_score": 0.9
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-07_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-07",
|
|
3
|
-
"name": "Payload Response Field Rename",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "User profile endpoint renamed 'user_email' to 'email_address'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "FIELD_RENAME",
|
|
9
|
-
"broken_assertion": "assert response.json()['user_email'] == 'qa@example.com'",
|
|
10
|
-
"error_log": "KeyError: 'user_email' not found in response dictionary",
|
|
11
|
-
"api_payload": "{\"id\": 42, \"email_address\": \"qa@example.com\", \"role\": \"QA\"}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "assert (response.json().get('email_address') or response.json().get('user_email')) == 'qa@example.com'",
|
|
15
|
-
"strategy": "RESILIENT_FIELD_LOOKUP"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 1.5,
|
|
19
|
-
"max_token_cost": 260,
|
|
20
|
-
"min_confidence_score": 0.95
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-07",
|
|
3
|
+
"name": "Payload Response Field Rename",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "User profile endpoint renamed 'user_email' to 'email_address'.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "FIELD_RENAME",
|
|
9
|
+
"broken_assertion": "assert response.json()['user_email'] == 'qa@example.com'",
|
|
10
|
+
"error_log": "KeyError: 'user_email' not found in response dictionary",
|
|
11
|
+
"api_payload": "{\"id\": 42, \"email_address\": \"qa@example.com\", \"role\": \"QA\"}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_assertion": "assert (response.json().get('email_address') or response.json().get('user_email')) == 'qa@example.com'",
|
|
15
|
+
"strategy": "RESILIENT_FIELD_LOOKUP"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 1.5,
|
|
19
|
+
"max_token_cost": 260,
|
|
20
|
+
"min_confidence_score": 0.95
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-08_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-08",
|
|
3
|
-
"name": "Nested Response Object Flattened",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Backend flattened nested profile: 'data.profile.tier' changed to 'data.tier'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "STRUCTURE_FLATTEN",
|
|
9
|
-
"broken_assertion": "assert response.json()['profile']['tier'] == 'GOLD'",
|
|
10
|
-
"error_log": "TypeError: 'NoneType' or KeyError: 'profile'",
|
|
11
|
-
"api_payload": "{\"tier\": \"GOLD\", \"points\": 1200}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "tier = response.json().get('tier') or response.json().get('profile', {}).get('tier'); assert tier == 'GOLD'",
|
|
15
|
-
"strategy": "SCHEMA_PATH_FALLBACK"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 320,
|
|
20
|
-
"min_confidence_score": 0.9
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-08",
|
|
3
|
+
"name": "Nested Response Object Flattened",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "MEDIUM",
|
|
6
|
+
"description": "Backend flattened nested profile: 'data.profile.tier' changed to 'data.tier'.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "STRUCTURE_FLATTEN",
|
|
9
|
+
"broken_assertion": "assert response.json()['profile']['tier'] == 'GOLD'",
|
|
10
|
+
"error_log": "TypeError: 'NoneType' or KeyError: 'profile'",
|
|
11
|
+
"api_payload": "{\"tier\": \"GOLD\", \"points\": 1200}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_assertion": "tier = response.json().get('tier') or response.json().get('profile', {}).get('tier'); assert tier == 'GOLD'",
|
|
15
|
+
"strategy": "SCHEMA_PATH_FALLBACK"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 2.0,
|
|
19
|
+
"max_token_cost": 320,
|
|
20
|
+
"min_confidence_score": 0.9
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-09_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-09",
|
|
3
|
-
"name": "HTTP Creation Status Code Shift (200 OK -> 201 Created)",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "POST endpoint updated to RFC-compliant 201 Created instead of legacy 200 OK.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "STATUS_CODE_SHIFT",
|
|
9
|
-
"broken_assertion": "assert response.status_code == 200",
|
|
10
|
-
"error_log": "AssertionError: Expected status code 200 but received 201",
|
|
11
|
-
"api_payload": "{\"status\": 201, \"created\": true}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "assert response.status_code in (200, 201)",
|
|
15
|
-
"strategy": "SUCCESS_STATUS_RANGE"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 1.0,
|
|
19
|
-
"max_token_cost": 220,
|
|
20
|
-
"min_confidence_score": 0.98
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-09",
|
|
3
|
+
"name": "HTTP Creation Status Code Shift (200 OK -> 201 Created)",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "POST endpoint updated to RFC-compliant 201 Created instead of legacy 200 OK.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "STATUS_CODE_SHIFT",
|
|
9
|
+
"broken_assertion": "assert response.status_code == 200",
|
|
10
|
+
"error_log": "AssertionError: Expected status code 200 but received 201",
|
|
11
|
+
"api_payload": "{\"status\": 201, \"created\": true}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_assertion": "assert response.status_code in (200, 201)",
|
|
15
|
+
"strategy": "SUCCESS_STATUS_RANGE"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 1.0,
|
|
19
|
+
"max_token_cost": 220,
|
|
20
|
+
"min_confidence_score": 0.98
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-10_api_contract_drift.json
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-10",
|
|
3
|
-
"name": "Authentication Header Prefix Upgrade",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "API gateway changed auth format from 'Authorization: Token <token>' to 'Bearer <token>'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "AUTH_HEADER_DRIFT",
|
|
9
|
-
"broken_code": "headers = {'Authorization': f'Token {auth_token}'}",
|
|
10
|
-
"error_log": "HTTP 401 Unauthorized: Invalid authorization scheme. Expected 'Bearer'.",
|
|
11
|
-
"api_payload": "{\"error\": \"Unauthorized\", \"code\": 401}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "headers = {'Authorization': f'Bearer {auth_token}'}",
|
|
15
|
-
"strategy": "STANDARD_BEARER_TOKEN"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 1.5,
|
|
19
|
-
"max_token_cost": 250,
|
|
20
|
-
"min_confidence_score": 0.95
|
|
21
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-10",
|
|
3
|
+
"name": "Authentication Header Prefix Upgrade",
|
|
4
|
+
"category": "api_contract_drift",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "API gateway changed auth format from 'Authorization: Token <token>' to 'Bearer <token>'.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "AUTH_HEADER_DRIFT",
|
|
9
|
+
"broken_code": "headers = {'Authorization': f'Token {auth_token}'}",
|
|
10
|
+
"error_log": "HTTP 401 Unauthorized: Invalid authorization scheme. Expected 'Bearer'.",
|
|
11
|
+
"api_payload": "{\"error\": \"Unauthorized\", \"code\": 401}"
|
|
12
|
+
},
|
|
13
|
+
"expected_resolution": {
|
|
14
|
+
"healed_code": "headers = {'Authorization': f'Bearer {auth_token}'}",
|
|
15
|
+
"strategy": "STANDARD_BEARER_TOKEN"
|
|
16
|
+
},
|
|
17
|
+
"benchmark_targets": {
|
|
18
|
+
"max_heal_time_sec": 1.5,
|
|
19
|
+
"max_token_cost": 250,
|
|
20
|
+
"min_confidence_score": 0.95
|
|
21
|
+
}
|
|
22
22
|
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-21_fake_pass_tautology.json
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-21",
|
|
3
|
-
"name": "Swallowed HTTP 500 Error in Empty Catch Block",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Test silently catches runtime exceptions in an empty catch, masking severe backend 500 crash.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "SWALLOWED_EXCEPTION",
|
|
9
|
-
"code_snippet": "try { page.click('#submit-payment') } catch (Exception e) {} // Test passes even on 500",
|
|
10
|
-
"detection_rule": "EMPTY_CATCH_PATTERN"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "page.click('#submit-payment'); assert page.locator('.success-toast').isVisible();",
|
|
14
|
-
"strategy": "REMOVE_SWALLOWED_CATCH_AND_ASSERT"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.0,
|
|
18
|
-
"max_token_cost": 200,
|
|
19
|
-
"min_confidence_score": 0.99
|
|
20
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-21",
|
|
3
|
+
"name": "Swallowed HTTP 500 Error in Empty Catch Block",
|
|
4
|
+
"category": "fake_pass_tautology",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "Test silently catches runtime exceptions in an empty catch, masking severe backend 500 crash.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "SWALLOWED_EXCEPTION",
|
|
9
|
+
"code_snippet": "try { page.click('#submit-payment') } catch (Exception e) {} // Test passes even on 500",
|
|
10
|
+
"detection_rule": "EMPTY_CATCH_PATTERN"
|
|
11
|
+
},
|
|
12
|
+
"expected_resolution": {
|
|
13
|
+
"healed_code": "page.click('#submit-payment'); assert page.locator('.success-toast').isVisible();",
|
|
14
|
+
"strategy": "REMOVE_SWALLOWED_CATCH_AND_ASSERT"
|
|
15
|
+
},
|
|
16
|
+
"benchmark_targets": {
|
|
17
|
+
"max_heal_time_sec": 1.0,
|
|
18
|
+
"max_token_cost": 200,
|
|
19
|
+
"min_confidence_score": 0.99
|
|
20
|
+
}
|
|
21
21
|
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-22_fake_pass_tautology.json
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-22",
|
|
3
|
-
"name": "Zero-Assertion Smoke Test (Tautological Pass)",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Test executes form inputs and clicks submit but contains exactly 0 assertion statements.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "ZERO_ASSERTION_PASS",
|
|
9
|
-
"code_snippet": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn');",
|
|
10
|
-
"detection_rule": "FAKE_PASS_NO_ASSERTION"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn'); expect(page.locator('#dashboard-header')).toBeVisible();",
|
|
14
|
-
"strategy": "INJECT_POST_CONDITION_ASSERTION"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.5,
|
|
18
|
-
"max_token_cost": 240,
|
|
19
|
-
"min_confidence_score": 0.97
|
|
20
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-22",
|
|
3
|
+
"name": "Zero-Assertion Smoke Test (Tautological Pass)",
|
|
4
|
+
"category": "fake_pass_tautology",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "Test executes form inputs and clicks submit but contains exactly 0 assertion statements.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "ZERO_ASSERTION_PASS",
|
|
9
|
+
"code_snippet": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn');",
|
|
10
|
+
"detection_rule": "FAKE_PASS_NO_ASSERTION"
|
|
11
|
+
},
|
|
12
|
+
"expected_resolution": {
|
|
13
|
+
"healed_code": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn'); expect(page.locator('#dashboard-header')).toBeVisible();",
|
|
14
|
+
"strategy": "INJECT_POST_CONDITION_ASSERTION"
|
|
15
|
+
},
|
|
16
|
+
"benchmark_targets": {
|
|
17
|
+
"max_heal_time_sec": 1.5,
|
|
18
|
+
"max_token_cost": 240,
|
|
19
|
+
"min_confidence_score": 0.97
|
|
20
|
+
}
|
|
21
21
|
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-23_fake_pass_tautology.json
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-23",
|
|
3
|
-
"name": "Tautological assert true == true",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Test asserts trivial identity 'assert true == true' instead of verifying business state.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "TAUTOLOGY_ASSERTION",
|
|
9
|
-
"code_snippet": "def response = api.createInvoice(); assert true == true;",
|
|
10
|
-
"detection_rule": "TAUTOLOGY_PATTERN"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "def response = api.createInvoice(); assert response.status == 201 && response.id != null;",
|
|
14
|
-
"strategy": "MUTATION_TESTING_KILLS_TAUTOLOGY"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.0,
|
|
18
|
-
"max_token_cost": 220,
|
|
19
|
-
"min_confidence_score": 0.98
|
|
20
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-23",
|
|
3
|
+
"name": "Tautological assert true == true",
|
|
4
|
+
"category": "fake_pass_tautology",
|
|
5
|
+
"difficulty": "EASY",
|
|
6
|
+
"description": "Test asserts trivial identity 'assert true == true' instead of verifying business state.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "TAUTOLOGY_ASSERTION",
|
|
9
|
+
"code_snippet": "def response = api.createInvoice(); assert true == true;",
|
|
10
|
+
"detection_rule": "TAUTOLOGY_PATTERN"
|
|
11
|
+
},
|
|
12
|
+
"expected_resolution": {
|
|
13
|
+
"healed_code": "def response = api.createInvoice(); assert response.status == 201 && response.id != null;",
|
|
14
|
+
"strategy": "MUTATION_TESTING_KILLS_TAUTOLOGY"
|
|
15
|
+
},
|
|
16
|
+
"benchmark_targets": {
|
|
17
|
+
"max_heal_time_sec": 1.0,
|
|
18
|
+
"max_token_cost": 220,
|
|
19
|
+
"min_confidence_score": 0.98
|
|
20
|
+
}
|
|
21
21
|
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-24_fake_pass_tautology.json
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-24",
|
|
3
|
-
"name": "Unawaited Async Assertion (Silent Green Trap)",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Missing 'await' on expect(locator).toBeVisible(); Promise resolves ignored after test exit.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "UNAWAITED_ASYNC_ASSERTION",
|
|
9
|
-
"code_snippet": "page.click('#save'); expect(page.locator('.alert')).toBeVisible(); // Missing await!",
|
|
10
|
-
"detection_rule": "UNAWAITED_ASYNC_EXPECT"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "page.click('#save'); await expect(page.locator('.alert')).toBeVisible();",
|
|
14
|
-
"strategy": "AST_ASYNC_AWAIT_ENFORCEMENT"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.5,
|
|
18
|
-
"max_token_cost": 250,
|
|
19
|
-
"min_confidence_score": 0.96
|
|
20
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"id": "QA-CHALLENGE-24",
|
|
3
|
+
"name": "Unawaited Async Assertion (Silent Green Trap)",
|
|
4
|
+
"category": "fake_pass_tautology",
|
|
5
|
+
"difficulty": "MEDIUM",
|
|
6
|
+
"description": "Missing 'await' on expect(locator).toBeVisible(); Promise resolves ignored after test exit.",
|
|
7
|
+
"failure_vector": {
|
|
8
|
+
"type": "UNAWAITED_ASYNC_ASSERTION",
|
|
9
|
+
"code_snippet": "page.click('#save'); expect(page.locator('.alert')).toBeVisible(); // Missing await!",
|
|
10
|
+
"detection_rule": "UNAWAITED_ASYNC_EXPECT"
|
|
11
|
+
},
|
|
12
|
+
"expected_resolution": {
|
|
13
|
+
"healed_code": "page.click('#save'); await expect(page.locator('.alert')).toBeVisible();",
|
|
14
|
+
"strategy": "AST_ASYNC_AWAIT_ENFORCEMENT"
|
|
15
|
+
},
|
|
16
|
+
"benchmark_targets": {
|
|
17
|
+
"max_heal_time_sec": 1.5,
|
|
18
|
+
"max_token_cost": 250,
|
|
19
|
+
"min_confidence_score": 0.96
|
|
20
|
+
}
|
|
21
21
|
}
|