synapse-hub 2.4.3 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/agents/failure-analyst.md +13 -0
- package/.agents/agents/orchestrator.md +5 -2
- package/.agents/agents/pom-crafter.md +13 -0
- package/.agents/agents/quality-gatekeeper.md +21 -2
- package/.agents/agents/test-runner.md +11 -0
- package/.agents/rules/context-engineering.md +22 -102
- package/.agents/rules/mobile_appium_rules.md +19 -115
- package/.agents/rules/performance_testing_rules.md +18 -62
- package/.agents/rules/skill-precedence.md +24 -24
- package/.agents/rules/systematic-debugging.md +10 -27
- package/.agents/skills/lean-code-ladder/SKILL.md +12 -1
- package/.agents/skills/lean-impact-scoreboard/SKILL.md +48 -0
- package/.agents/skills/lean-impact-scoreboard/examples/sample_benchmark_gain.md +20 -0
- package/.agents/skills/lean-impact-scoreboard/examples/sample_scoreboard.md +15 -0
- package/.agents/skills/repo-quality-audit/SKILL.md +68 -0
- package/.agents/skills/repo-quality-audit/examples/sample_audit_walkthrough.md +15 -0
- package/.agents/skills/repo-quality-audit/examples/sample_repo_audit.md +33 -0
- package/.agents/skills/senior-code-review/SKILL.md +90 -0
- package/.agents/skills/senior-code-review/examples/sample_review_walkthrough.md +29 -0
- package/.agents/skills/technical-debt-harvester/SKILL.md +70 -0
- package/.agents/skills/technical-debt-harvester/examples/sample_debt_ledger.md +25 -0
- package/.agents/skills/technical-debt-harvester/examples/sample_ledger_output.md +20 -0
- package/.agents/skills/token-and-context-optimizer/references/context_engineering_guide.md +49 -0
- package/.cursor/rules/synapse-qa.mdc +4 -3
- package/.cursorrules +1 -0
- package/.windsurfrules +1 -0
- package/AGENTS.md +57 -180
- package/CLAUDE.md +5 -0
- package/NOTICE +40 -4
- package/README.en.md +33 -13
- package/README.md +36 -16
- package/bin/agent-qa.js +3 -2
- package/docs/ARCHITECTURE.md +39 -11
- package/docs/CLI_REFERENCE.md +15 -1
- package/docs/adopted-ecosystem-tools.md +9 -0
- package/docs/research/agent_skill_training_analysis.md +1357 -0
- package/package.json +1 -3
- package/skills_manifest.json +39 -3
- package/synapse.config.json +3 -2
- package/templates/ai_bridges/CLAUDE.md +5 -0
- package/templates/ai_bridges/copilot-instructions.md +2 -0
- package/templates/ai_bridges/cursor_synapse_qa.mdc +4 -3
- package/templates/ai_bridges/windsurfrules +1 -0
- package/templates/welcome.html.tmpl +34 -25
- package/tools/.infer_baseline.json +7 -1079
- package/tools/TOOLS_RUNBOOK.md +182 -13
- package/tools/__init__.py +1 -1
- package/tools/agent_canary.py +4 -4
- package/tools/analyze_project.py +2 -2
- package/tools/api_fuzzer.py +4 -4
- package/tools/auto_heal.py +14 -2
- package/tools/auto_heal_reflexion.py +172 -0
- package/tools/browser_helpers.js +57 -2
- package/tools/bump_version.py +30 -2
- package/tools/cache_resolver.py +6 -6
- package/tools/check_env.py +13 -7
- package/tools/cli.py +265 -197
- package/tools/count_tokens.py +7 -7
- package/tools/dashboard_generator.py +39 -23
- package/tools/db_sandbox.py +5 -5
- package/tools/debt_harvester.py +236 -0
- package/tools/distill_log.py +2 -2
- package/tools/dspy_qa_signatures.py +357 -0
- package/tools/flaky_detector.py +5 -5
- package/tools/generate_tc_from_story.py +2 -2
- package/tools/generate_test_reports.py +4 -4
- package/tools/git_pre_commit_guard.py +6 -6
- package/tools/infer_linter.py +12 -7
- package/tools/interview_story.py +117 -0
- package/tools/lint_ai_docs.py +3 -3
- package/tools/matrix_sharder.py +4 -4
- package/tools/mcp_excel_server.py +1 -1
- package/tools/mcp_server.py +3 -3
- package/tools/micro_mutator.py +6 -6
- package/tools/omni_transpiler.py +2 -2
- package/tools/output_formatter.py +37 -2
- package/tools/pinpoint_failure.py +4 -1
- package/tools/repo_map.py +2 -2
- package/tools/reversible_sandbox.py +157 -0
- package/tools/safety_firewall.py +30 -2
- package/tools/senior_code_review.py +341 -0
- package/tools/setup_project.py +9 -9
- package/tools/skill_collision_guard.py +231 -0
- package/tools/skill_vault.py +343 -0
- package/tools/skillspector.py +2 -2
- package/tools/synapse_qa_bench.py +190 -0
- package/tools/sync_agent_rules.py +14 -3
- package/tools/test_impact.py +4 -4
- package/tools/textgrad_mutation_optimizer.py +234 -0
- package/tools/tms_sync_bridge.py +2 -2
- package/tools/update_sentinel.py +17 -17
- package/tools/verify_all.py +47 -16
- package/tools/visual_som_helper.py +8 -8
- package/tools/watch_mode.py +7 -8
- package/tools/wcag_audit.py +5 -1
- package/tools/welcome_page.py +12 -12
- package/benchmarks/qa_eval/README.md +0 -71
- package/benchmarks/qa_eval/build_dataset.py +0 -677
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-06_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-07_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-08_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-09_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-10_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-21_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-22_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-23_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-24_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-25_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-01_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-02_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-03_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-04_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-05_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-16_stale_and_mutation.json +0 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-17_stale_and_mutation.json +0 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-18_stale_and_mutation.json +0 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-19_stale_and_mutation.json +0 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-20_stale_and_mutation.json +0 -23
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-11_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-12_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-13_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-14_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-15_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/eval_runner.py +0 -418
- package/examples/cypress-ecommerce-suite/README.md +0 -45
- package/examples/cypress-ecommerce-suite/cypress/e2e/checkout.cy.js +0 -17
- package/examples/cypress-ecommerce-suite/cypress/pages/BasePage.js +0 -29
- package/examples/cypress-ecommerce-suite/cypress/pages/CartPage.js +0 -51
- package/examples/cypress-ecommerce-suite/cypress.config.js +0 -13
- package/examples/cypress-ecommerce-suite/package.json +0 -13
- package/examples/katalon-suite/Keywords/pages/BasePage.groovy +0 -19
- package/examples/katalon-suite/Keywords/pages/DashboardPage.groovy +0 -29
- package/examples/katalon-suite/Keywords/pages/LoginPage.groovy +0 -45
- package/examples/katalon-suite/Profiles/default.glbl +0 -12
- package/examples/katalon-suite/README.md +0 -18
- package/examples/katalon-suite/Scripts/Auth/TC_LoginSuccess/Script1234567890.groovy +0 -12
- package/examples/katalon-suite/Scripts/Dashboard/TC_ViewSummary/Script1234567891.groovy +0 -15
- package/examples/katalon-suite/katalon_suite.prj +0 -21
- package/examples/playwright-crm-suite/README.md +0 -40
- package/examples/playwright-crm-suite/package.json +0 -14
- package/examples/playwright-crm-suite/pages/BasePage.ts +0 -37
- package/examples/playwright-crm-suite/pages/CustomerPage.ts +0 -57
- package/examples/playwright-crm-suite/pages/LoginPage.ts +0 -32
- package/examples/playwright-crm-suite/playwright.config.ts +0 -27
- package/examples/playwright-crm-suite/tests/auth.spec.ts +0 -22
- package/examples/playwright-crm-suite/tests/customers.spec.ts +0 -33
- package/examples/plugins/sample-custom-plugin/README.md +0 -14
- package/examples/plugins/sample-custom-plugin/plugin.json +0 -16
- package/examples/plugins/sample-custom-plugin/skills/custom-security-headers-audit/SKILL.md +0 -29
- package/examples/plugins/sample-custom-plugin/skills/custom-security-headers-audit/examples/audit_fixture.json +0 -8
- package/examples/plugins/sample-custom-plugin/tools/custom_security_headers.py +0 -125
- package/examples/practices/README.md +0 -22
- package/examples/practices/requirements/crm/requirements_contracts.md +0 -292
- package/examples/practices/requirements/crm/requirements_customers.md +0 -462
- package/examples/practices/requirements/crm/requirements_login.md +0 -202
- package/examples/practices/requirements/crm/requirements_projects.md +0 -396
- package/examples/practices/requirements/crm/requirements_tasks.md +0 -343
- package/examples/practices/requirements/ecommerce/requirements_category_brand.txt +0 -402
- package/examples/practices/requirements/jira/KAN-5/KAN-1/2026-04-03_04-28-27.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-1/KAN-1_requirement.md +0 -52
- package/examples/practices/requirements/jira/KAN-5/KAN-4/2026-04-03_04-28-19.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-4/2026-04-03_04-28-57.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-4/KAN-4_requirement.md +0 -119
- package/examples/practices/requirements/jira/KAN-5/KAN-5_overview.md +0 -9
- package/examples/practices/requirements/jira/KAN-5/KAN-6/2026-04-03_05-58-08.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-6/2026-04-03_05-58-23.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-6/KAN-6_requirement.md +0 -32
- package/examples/practices/testcases/api/book_management_api_test_cases.md +0 -144
- package/examples/practices/testcases/api/user_management_api_test_cases.md +0 -139
- package/examples/practices/testcases/crm/test_cases_crm_login.md +0 -148
- package/examples/practices/testcases/crm/test_cases_customers.md +0 -156
- package/examples/pytest-api-suite/README.md +0 -45
- package/examples/pytest-api-suite/conftest.py +0 -31
- package/examples/pytest-api-suite/pages/user_api_client.py +0 -36
- package/examples/pytest-api-suite/requirements.txt +0 -2
- package/examples/pytest-api-suite/tests/test_user_api.py +0 -44
- package/examples/selenium-suite/README.md +0 -18
- package/examples/selenium-suite/pages/base_page.py +0 -27
- package/examples/selenium-suite/pages/login_page.py +0 -29
- package/examples/selenium-suite/pages/order_page.py +0 -25
- package/examples/selenium-suite/pytest.ini +0 -6
- package/examples/selenium-suite/requirements.txt +0 -2
- package/examples/selenium-suite/tests/test_login.py +0 -10
- package/examples/selenium-suite/tests/test_order.py +0 -10
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-06_api_contract_drift.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-06",
|
|
3
|
-
"name": "Numeric ID Serialized as String (Type Mutation)",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Backend response changed 'invoice_id: 10423' (int) to 'invoice_id: \"10423\"' (string).",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "TYPE_MUTATION",
|
|
9
|
-
"broken_assertion": "assert response.data.invoice_id == 10423",
|
|
10
|
-
"error_log": "AssertionError: Expected integer 10423 but received string '10423'",
|
|
11
|
-
"api_payload": "{\"status\": \"SUCCESS\", \"invoice_id\": \"10423\", \"amount\": 250.0}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "assert str(response.data.invoice_id) == '10423'",
|
|
15
|
-
"strategy": "TYPE_COERCION_ASSERTION"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 280,
|
|
20
|
-
"min_confidence_score": 0.9
|
|
21
|
-
}
|
|
22
|
-
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-07_api_contract_drift.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-07",
|
|
3
|
-
"name": "Payload Response Field Rename",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "User profile endpoint renamed 'user_email' to 'email_address'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "FIELD_RENAME",
|
|
9
|
-
"broken_assertion": "assert response.json()['user_email'] == 'qa@example.com'",
|
|
10
|
-
"error_log": "KeyError: 'user_email' not found in response dictionary",
|
|
11
|
-
"api_payload": "{\"id\": 42, \"email_address\": \"qa@example.com\", \"role\": \"QA\"}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "assert (response.json().get('email_address') or response.json().get('user_email')) == 'qa@example.com'",
|
|
15
|
-
"strategy": "RESILIENT_FIELD_LOOKUP"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 1.5,
|
|
19
|
-
"max_token_cost": 260,
|
|
20
|
-
"min_confidence_score": 0.95
|
|
21
|
-
}
|
|
22
|
-
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-08_api_contract_drift.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-08",
|
|
3
|
-
"name": "Nested Response Object Flattened",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Backend flattened nested profile: 'data.profile.tier' changed to 'data.tier'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "STRUCTURE_FLATTEN",
|
|
9
|
-
"broken_assertion": "assert response.json()['profile']['tier'] == 'GOLD'",
|
|
10
|
-
"error_log": "TypeError: 'NoneType' or KeyError: 'profile'",
|
|
11
|
-
"api_payload": "{\"tier\": \"GOLD\", \"points\": 1200}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "tier = response.json().get('tier') or response.json().get('profile', {}).get('tier'); assert tier == 'GOLD'",
|
|
15
|
-
"strategy": "SCHEMA_PATH_FALLBACK"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 320,
|
|
20
|
-
"min_confidence_score": 0.9
|
|
21
|
-
}
|
|
22
|
-
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-09_api_contract_drift.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-09",
|
|
3
|
-
"name": "HTTP Creation Status Code Shift (200 OK -> 201 Created)",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "POST endpoint updated to RFC-compliant 201 Created instead of legacy 200 OK.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "STATUS_CODE_SHIFT",
|
|
9
|
-
"broken_assertion": "assert response.status_code == 200",
|
|
10
|
-
"error_log": "AssertionError: Expected status code 200 but received 201",
|
|
11
|
-
"api_payload": "{\"status\": 201, \"created\": true}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_assertion": "assert response.status_code in (200, 201)",
|
|
15
|
-
"strategy": "SUCCESS_STATUS_RANGE"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 1.0,
|
|
19
|
-
"max_token_cost": 220,
|
|
20
|
-
"min_confidence_score": 0.98
|
|
21
|
-
}
|
|
22
|
-
}
|
package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-10_api_contract_drift.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-10",
|
|
3
|
-
"name": "Authentication Header Prefix Upgrade",
|
|
4
|
-
"category": "api_contract_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "API gateway changed auth format from 'Authorization: Token <token>' to 'Bearer <token>'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "AUTH_HEADER_DRIFT",
|
|
9
|
-
"broken_code": "headers = {'Authorization': f'Token {auth_token}'}",
|
|
10
|
-
"error_log": "HTTP 401 Unauthorized: Invalid authorization scheme. Expected 'Bearer'.",
|
|
11
|
-
"api_payload": "{\"error\": \"Unauthorized\", \"code\": 401}"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "headers = {'Authorization': f'Bearer {auth_token}'}",
|
|
15
|
-
"strategy": "STANDARD_BEARER_TOKEN"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 1.5,
|
|
19
|
-
"max_token_cost": 250,
|
|
20
|
-
"min_confidence_score": 0.95
|
|
21
|
-
}
|
|
22
|
-
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-21_fake_pass_tautology.json
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-21",
|
|
3
|
-
"name": "Swallowed HTTP 500 Error in Empty Catch Block",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Test silently catches runtime exceptions in an empty catch, masking severe backend 500 crash.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "SWALLOWED_EXCEPTION",
|
|
9
|
-
"code_snippet": "try { page.click('#submit-payment') } catch (Exception e) {} // Test passes even on 500",
|
|
10
|
-
"detection_rule": "EMPTY_CATCH_PATTERN"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "page.click('#submit-payment'); assert page.locator('.success-toast').isVisible();",
|
|
14
|
-
"strategy": "REMOVE_SWALLOWED_CATCH_AND_ASSERT"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.0,
|
|
18
|
-
"max_token_cost": 200,
|
|
19
|
-
"min_confidence_score": 0.99
|
|
20
|
-
}
|
|
21
|
-
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-22_fake_pass_tautology.json
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-22",
|
|
3
|
-
"name": "Zero-Assertion Smoke Test (Tautological Pass)",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Test executes form inputs and clicks submit but contains exactly 0 assertion statements.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "ZERO_ASSERTION_PASS",
|
|
9
|
-
"code_snippet": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn');",
|
|
10
|
-
"detection_rule": "FAKE_PASS_NO_ASSERTION"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "page.fill('#username', 'admin'); page.fill('#password', 'secret'); page.click('#login-btn'); expect(page.locator('#dashboard-header')).toBeVisible();",
|
|
14
|
-
"strategy": "INJECT_POST_CONDITION_ASSERTION"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.5,
|
|
18
|
-
"max_token_cost": 240,
|
|
19
|
-
"min_confidence_score": 0.97
|
|
20
|
-
}
|
|
21
|
-
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-23_fake_pass_tautology.json
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-23",
|
|
3
|
-
"name": "Tautological assert true == true",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Test asserts trivial identity 'assert true == true' instead of verifying business state.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "TAUTOLOGY_ASSERTION",
|
|
9
|
-
"code_snippet": "def response = api.createInvoice(); assert true == true;",
|
|
10
|
-
"detection_rule": "TAUTOLOGY_PATTERN"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "def response = api.createInvoice(); assert response.status == 201 && response.id != null;",
|
|
14
|
-
"strategy": "MUTATION_TESTING_KILLS_TAUTOLOGY"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.0,
|
|
18
|
-
"max_token_cost": 220,
|
|
19
|
-
"min_confidence_score": 0.98
|
|
20
|
-
}
|
|
21
|
-
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-24_fake_pass_tautology.json
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-24",
|
|
3
|
-
"name": "Unawaited Async Assertion (Silent Green Trap)",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Missing 'await' on expect(locator).toBeVisible(); Promise resolves ignored after test exit.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "UNAWAITED_ASYNC_ASSERTION",
|
|
9
|
-
"code_snippet": "page.click('#save'); expect(page.locator('.alert')).toBeVisible(); // Missing await!",
|
|
10
|
-
"detection_rule": "UNAWAITED_ASYNC_EXPECT"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "page.click('#save'); await expect(page.locator('.alert')).toBeVisible();",
|
|
14
|
-
"strategy": "AST_ASYNC_AWAIT_ENFORCEMENT"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.5,
|
|
18
|
-
"max_token_cost": 250,
|
|
19
|
-
"min_confidence_score": 0.96
|
|
20
|
-
}
|
|
21
|
-
}
|
package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-25_fake_pass_tautology.json
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-25",
|
|
3
|
-
"name": "Unbounded Relational Assertion (status >= 200)",
|
|
4
|
-
"category": "fake_pass_tautology",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Assertion checks 'status >= 200' without ceiling, so HTTP 500 error causes false pass.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "UNBOUNDED_STATUS_ASSERTION",
|
|
9
|
-
"code_snippet": "def resp = client.send(); assert resp.statusCode >= 200; // Passes on 500!",
|
|
10
|
-
"detection_rule": "MUTATION_BOUNDARY_SHIFT"
|
|
11
|
-
},
|
|
12
|
-
"expected_resolution": {
|
|
13
|
-
"healed_code": "def resp = client.send(); assert resp.statusCode >= 200 && resp.statusCode < 300;",
|
|
14
|
-
"strategy": "BOUNDED_INTERVAL_ASSERTION"
|
|
15
|
-
},
|
|
16
|
-
"benchmark_targets": {
|
|
17
|
-
"max_heal_time_sec": 1.2,
|
|
18
|
-
"max_token_cost": 230,
|
|
19
|
-
"min_confidence_score": 0.98
|
|
20
|
-
}
|
|
21
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-01",
|
|
3
|
-
"name": "Dynamic CSS-in-JS Hash Locator Shift",
|
|
4
|
-
"category": "locator_drift",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Button class changed from '.btn-submit' to dynamic hashed class '.btn-submit_a8f9x2'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "LOCATOR_DRIFT",
|
|
9
|
-
"broken_locator": "button.btn-submit",
|
|
10
|
-
"error_log": "ElementNotFoundException: Cannot locate element with selector 'button.btn-submit' within 5000ms",
|
|
11
|
-
"dom_snapshot": "<button class='btn-submit_a8f9x2' data-testid='submit-order-btn' role='button' aria-label='Submit Order'>Submit Order</button>"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_locator": "[data-testid='submit-order-btn']",
|
|
15
|
-
"fallback_locator": "button:has-text('Submit Order')",
|
|
16
|
-
"strategy": "AOM_ROLE_OR_TESTID"
|
|
17
|
-
},
|
|
18
|
-
"benchmark_targets": {
|
|
19
|
-
"max_heal_time_sec": 2.0,
|
|
20
|
-
"max_token_cost": 350,
|
|
21
|
-
"min_confidence_score": 0.85
|
|
22
|
-
}
|
|
23
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-02",
|
|
3
|
-
"name": "Nested Shadow DOM Component Piercing",
|
|
4
|
-
"category": "locator_drift",
|
|
5
|
-
"difficulty": "HARD",
|
|
6
|
-
"description": "Input field moved inside Web Component with '#shadow-root (open)'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "SHADOW_DOM_ISOLATION",
|
|
9
|
-
"broken_locator": "#customer-email",
|
|
10
|
-
"error_log": "NoSuchElementException: Element '#customer-email' not found in document root",
|
|
11
|
-
"dom_snapshot": "<user-profile-card><template shadowrootmode='open'><input id='customer-email' type='email' data-testid='user-email'/></template></user-profile-card>"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_locator": "user-profile-card >> internal:control=enter-shadow >> input#customer-email",
|
|
15
|
-
"fallback_locator": "user-profile-card >> input[type='email']",
|
|
16
|
-
"strategy": "SHADOW_DOM_PIERCING"
|
|
17
|
-
},
|
|
18
|
-
"benchmark_targets": {
|
|
19
|
-
"max_heal_time_sec": 3.0,
|
|
20
|
-
"max_token_cost": 400,
|
|
21
|
-
"min_confidence_score": 0.88
|
|
22
|
-
}
|
|
23
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-03",
|
|
3
|
-
"name": "Semantic Text Label Evolution",
|
|
4
|
-
"category": "locator_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "CTA button label text changed from 'Submit Order' to 'Complete Purchase'.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "LABEL_EVOLUTION",
|
|
9
|
-
"broken_locator": "//button[text()='Submit Order']",
|
|
10
|
-
"error_log": "TimeoutException: Timed out waiting for element with XPath //button[text()='Submit Order']",
|
|
11
|
-
"dom_snapshot": "<button type='submit' class='p-button-success' id='order-cta'>Complete Purchase</button>"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_locator": "#order-cta",
|
|
15
|
-
"fallback_locator": "button[type='submit']",
|
|
16
|
-
"strategy": "ID_AND_SEMANTIC_INTENT"
|
|
17
|
-
},
|
|
18
|
-
"benchmark_targets": {
|
|
19
|
-
"max_heal_time_sec": 1.5,
|
|
20
|
-
"max_token_cost": 300,
|
|
21
|
-
"min_confidence_score": 0.9
|
|
22
|
-
}
|
|
23
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-04",
|
|
3
|
-
"name": "Legacy ID Removed in Favor of data-testid",
|
|
4
|
-
"category": "locator_drift",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Frontend framework removed legacy 'id=customer_name' during React refactor.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "ID_REMOVAL",
|
|
9
|
-
"broken_locator": "#customer_name",
|
|
10
|
-
"error_log": "ElementNotFoundException: Cannot locate element '#customer_name'",
|
|
11
|
-
"dom_snapshot": "<input type='text' name='customerName' data-testid='customer-name-field' class='form-control'/>"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_locator": "[data-testid='customer-name-field']",
|
|
15
|
-
"fallback_locator": "input[name='customerName']",
|
|
16
|
-
"strategy": "DATA_TESTID_UPGRADE"
|
|
17
|
-
},
|
|
18
|
-
"benchmark_targets": {
|
|
19
|
-
"max_heal_time_sec": 1.5,
|
|
20
|
-
"max_token_cost": 250,
|
|
21
|
-
"min_confidence_score": 0.92
|
|
22
|
-
}
|
|
23
|
-
}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-05",
|
|
3
|
-
"name": "HTML Tag Evolution: Anchor to ARIA Button",
|
|
4
|
-
"category": "locator_drift",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Navigation link <a href='#'> converted into accessible <button role='button'>.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "TAG_MUTATION",
|
|
9
|
-
"broken_locator": "a.nav-export",
|
|
10
|
-
"error_log": "ElementNotInteractableException: Element <a class='nav-export'> is no longer attached or interactive",
|
|
11
|
-
"dom_snapshot": "<button role='button' class='btn-export' aria-label='Export Report' data-action='export'>Export Report</button>"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_locator": "button[aria-label='Export Report']",
|
|
15
|
-
"fallback_locator": "button:has-text('Export Report')",
|
|
16
|
-
"strategy": "AOM_ROLE_MAPPING"
|
|
17
|
-
},
|
|
18
|
-
"benchmark_targets": {
|
|
19
|
-
"max_heal_time_sec": 2.0,
|
|
20
|
-
"max_token_cost": 300,
|
|
21
|
-
"min_confidence_score": 0.88
|
|
22
|
-
}
|
|
23
|
-
}
|
package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-16_stale_and_mutation.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-16",
|
|
3
|
-
"name": "WebSocket Table Auto-Refresh Stale Reference",
|
|
4
|
-
"category": "stale_and_mutation",
|
|
5
|
-
"difficulty": "HARD",
|
|
6
|
-
"description": "WebSocket pushes live quotes every 1.5s, wiping and recreating <tr> elements.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "STALE_ELEMENT",
|
|
9
|
-
"broken_code": "def row = page.locator('#stocks tr').first(); page.waitForTimeout(1600); row.click();",
|
|
10
|
-
"error_log": "StaleElementReferenceException: Element is no longer attached to DOM",
|
|
11
|
-
"environment_condition": "WebSocket DOM Mutation"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "page.locator('#stocks tr').first().click(); // Re-queries locator at moment of dispatch",
|
|
15
|
-
"strategy": "DYNAMIC_LOCATOR_REQUERY"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.5,
|
|
19
|
-
"max_token_cost": 340,
|
|
20
|
-
"min_confidence_score": 0.9
|
|
21
|
-
}
|
|
22
|
-
}
|
package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-17_stale_and_mutation.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-17",
|
|
3
|
-
"name": "Tab Switch Component Unmount Detach",
|
|
4
|
-
"category": "stale_and_mutation",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Switching active tabs unmounts previous panel component entirely from virtual DOM.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "COMPONENT_UNMOUNT",
|
|
9
|
-
"broken_code": "page.click('#tab-billing'); page.fill('#billing-tax-id', 'TAX-999');",
|
|
10
|
-
"error_log": "ElementNotInteractableException: Target element '#billing-tax-id' is hidden or unmounted",
|
|
11
|
-
"environment_condition": "Tab Component View Switching"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "page.click('#tab-billing'); page.waitForSelector('#billing-tab-content[aria-hidden=\"false\"]'); page.fill('#billing-tax-id', 'TAX-999');",
|
|
15
|
-
"strategy": "TAB_MOUNT_SYNCHRONIZATION"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 300,
|
|
20
|
-
"min_confidence_score": 0.91
|
|
21
|
-
}
|
|
22
|
-
}
|
package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-18_stale_and_mutation.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-18",
|
|
3
|
-
"name": "Pagination Replaces tbody (Stale Row Index)",
|
|
4
|
-
"category": "stale_and_mutation",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Navigating to page 2 updates table rows; old row references become detached.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "PAGINATION_DETACH",
|
|
9
|
-
"broken_code": "page.click('.p-paginator-next'); page.locator('tbody tr').nth(0).click();",
|
|
10
|
-
"error_log": "StaleElementReferenceException: Stale element handle during click dispatch",
|
|
11
|
-
"environment_condition": "Data Table Pagination"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "page.click('.p-paginator-next'); page.waitForResponse(r => r.url().includes('page=2')); page.locator('tbody tr').first().click();",
|
|
15
|
-
"strategy": "PAGINATION_NETWORK_SYNC"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 310,
|
|
20
|
-
"min_confidence_score": 0.92
|
|
21
|
-
}
|
|
22
|
-
}
|
package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-19_stale_and_mutation.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-19",
|
|
3
|
-
"name": "Reopened Modal Generates Fresh DOM Node",
|
|
4
|
-
"category": "stale_and_mutation",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Closing and reopening dialog instantiates new DOM element with same CSS selector.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "FRESH_INSTANCE_STALE",
|
|
9
|
-
"broken_code": "def modal = page.locator('#edit-dialog'); page.click('#close-btn'); page.click('#open-btn'); modal.fill('input', 'new');",
|
|
10
|
-
"error_log": "ElementNotVisibleException: Cached element reference points to detached/hidden dialog instance",
|
|
11
|
-
"environment_condition": "Modal Lifecycle Instantiation"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "page.click('#close-btn'); page.click('#open-btn'); page.locator('#edit-dialog:visible input').fill('new');",
|
|
15
|
-
"strategy": "VISIBLE_SELECTOR_REBIND"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 290,
|
|
20
|
-
"min_confidence_score": 0.94
|
|
21
|
-
}
|
|
22
|
-
}
|
package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-20_stale_and_mutation.json
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-20",
|
|
3
|
-
"name": "Locale Swap Invalidates Hardcoded Text Selectors",
|
|
4
|
-
"category": "stale_and_mutation",
|
|
5
|
-
"difficulty": "EASY",
|
|
6
|
-
"description": "Application language changed to French ('Enregistrer'); text selector 'Save' fails.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "I18N_TEXT_MUTATION",
|
|
9
|
-
"broken_locator": "button:has-text('Save')",
|
|
10
|
-
"error_log": "TimeoutException: No element found matching text 'Save'",
|
|
11
|
-
"dom_snapshot": "<button type='submit' data-action='save' class='btn-primary' aria-label='save-record'>Enregistrer</button>"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_locator": "button[data-action='save']",
|
|
15
|
-
"fallback_locator": "button[aria-label='save-record']",
|
|
16
|
-
"strategy": "LOCALE_AGNOSTIC_DATA_ATTRIBUTES"
|
|
17
|
-
},
|
|
18
|
-
"benchmark_targets": {
|
|
19
|
-
"max_heal_time_sec": 1.5,
|
|
20
|
-
"max_token_cost": 270,
|
|
21
|
-
"min_confidence_score": 0.95
|
|
22
|
-
}
|
|
23
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-11",
|
|
3
|
-
"name": "Network Latency Spike (3,500ms Throttle) Breaking Static Sleep",
|
|
4
|
-
"category": "timing_and_race",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Microservice latency under load takes 3.5s; test had Thread.sleep(2000).",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "STATIC_SLEEP_TIMEOUT",
|
|
9
|
-
"broken_code": "Thread.sleep(2000); assert WebUI.verifyElementPresent('#result')",
|
|
10
|
-
"error_log": "ElementNotFoundException: Element '#result' not present after 2000ms delay",
|
|
11
|
-
"environment_condition": "CDP Network Throttling 3500ms delay"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "page.waitForSelector('#result', timeout: 10000); page.waitForLoadState('networkidle')",
|
|
15
|
-
"strategy": "DYNAMIC_SMART_WAIT"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.5,
|
|
19
|
-
"max_token_cost": 320,
|
|
20
|
-
"min_confidence_score": 0.92
|
|
21
|
-
}
|
|
22
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-12",
|
|
3
|
-
"name": "PrimeNG Modal Backdrop Fading Intercept",
|
|
4
|
-
"category": "timing_and_race",
|
|
5
|
-
"difficulty": "HARD",
|
|
6
|
-
"description": "Modal backdrop takes 450ms CSS transition to fade out; click immediately triggers intercept.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "CLICK_INTERCEPTED",
|
|
9
|
-
"broken_code": "page.click('#confirm-btn')",
|
|
10
|
-
"error_log": "ElementClickInterceptedException: <div class='p-component-overlay p-component-overlay-enter'> intercepts pointer-events",
|
|
11
|
-
"environment_condition": "PrimeNG Modal Fade Transition"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "page.waitForSelector('.p-component-overlay', state: 'detached'); BasePage.clickJS('#confirm-btn')",
|
|
15
|
-
"strategy": "OVERLAY_SAFE_DISPATCH"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 3.0,
|
|
19
|
-
"max_token_cost": 380,
|
|
20
|
-
"min_confidence_score": 0.9
|
|
21
|
-
}
|
|
22
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-13",
|
|
3
|
-
"name": "Debounced Search Input Autocomplete Race",
|
|
4
|
-
"category": "timing_and_race",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Search input debounces 400ms before sending query; test clicked dropdown item before query fired.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "DEBOUNCE_RACE",
|
|
9
|
-
"broken_code": "page.fill('#search-input', 'Invoice'); page.click('.dropdown-item:first-child')",
|
|
10
|
-
"error_log": "TimeoutException: Timed out waiting for '.dropdown-item' to appear in DOM",
|
|
11
|
-
"environment_condition": "Debounce Timer 400ms"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "page.fill('#search-input', 'Invoice'); page.waitForResponse(r => r.url().includes('/search') && r.status() == 200); page.click('.dropdown-item:first-child')",
|
|
15
|
-
"strategy": "CDP_RESPONSE_BARRIER"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.5,
|
|
19
|
-
"max_token_cost": 350,
|
|
20
|
-
"min_confidence_score": 0.88
|
|
21
|
-
}
|
|
22
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-14",
|
|
3
|
-
"name": "Async Chart SVG Animation Race Condition",
|
|
4
|
-
"category": "timing_and_race",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Financial chart takes 1200ms to calculate series and render SVG paths.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "ASYNC_RENDER_RACE",
|
|
9
|
-
"broken_code": "page.click('.chart-bar:last-child')",
|
|
10
|
-
"error_log": "ElementNotFoundException: Selector '.chart-bar:last-child' not found",
|
|
11
|
-
"environment_condition": "Highcharts Rendering Animation"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "page.waitForSelector('.highcharts-series path', state: 'visible'); page.click('.chart-bar:last-child')",
|
|
15
|
-
"strategy": "SVG_VISIBILITY_SMART_WAIT"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 310,
|
|
20
|
-
"min_confidence_score": 0.9
|
|
21
|
-
}
|
|
22
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"id": "QA-CHALLENGE-15",
|
|
3
|
-
"name": "Infinite Scroll Lazy Loaded Chunk Detach",
|
|
4
|
-
"category": "timing_and_race",
|
|
5
|
-
"difficulty": "MEDIUM",
|
|
6
|
-
"description": "Table rows 50-100 require viewport scroll to trigger network chunk download.",
|
|
7
|
-
"failure_vector": {
|
|
8
|
-
"type": "LAZY_LOAD_RACE",
|
|
9
|
-
"broken_code": "page.locator('tr[data-id=\"75\"]').click()",
|
|
10
|
-
"error_log": "TimeoutException: Timed out waiting for 'tr[data-id=\"75\"]' to be visible",
|
|
11
|
-
"environment_condition": "Virtual Scrolling Table"
|
|
12
|
-
},
|
|
13
|
-
"expected_resolution": {
|
|
14
|
-
"healed_code": "page.locator('tr[data-id=\"75\"]').scrollIntoViewIfNeeded(); page.locator('tr[data-id=\"75\"]').click()",
|
|
15
|
-
"strategy": "VIRTUAL_SCROLL_INTO_VIEW"
|
|
16
|
-
},
|
|
17
|
-
"benchmark_targets": {
|
|
18
|
-
"max_heal_time_sec": 2.0,
|
|
19
|
-
"max_token_cost": 300,
|
|
20
|
-
"min_confidence_score": 0.92
|
|
21
|
-
}
|
|
22
|
-
}
|