synapse-hub 2.4.3 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/.agents/agents/failure-analyst.md +13 -0
  2. package/.agents/agents/orchestrator.md +5 -2
  3. package/.agents/agents/pom-crafter.md +13 -0
  4. package/.agents/agents/quality-gatekeeper.md +21 -2
  5. package/.agents/agents/test-runner.md +11 -0
  6. package/.agents/rules/context-engineering.md +22 -102
  7. package/.agents/rules/mobile_appium_rules.md +19 -115
  8. package/.agents/rules/performance_testing_rules.md +18 -62
  9. package/.agents/rules/skill-precedence.md +24 -24
  10. package/.agents/rules/systematic-debugging.md +10 -27
  11. package/.agents/skills/lean-code-ladder/SKILL.md +12 -1
  12. package/.agents/skills/lean-impact-scoreboard/SKILL.md +48 -0
  13. package/.agents/skills/lean-impact-scoreboard/examples/sample_benchmark_gain.md +20 -0
  14. package/.agents/skills/lean-impact-scoreboard/examples/sample_scoreboard.md +15 -0
  15. package/.agents/skills/repo-quality-audit/SKILL.md +68 -0
  16. package/.agents/skills/repo-quality-audit/examples/sample_audit_walkthrough.md +15 -0
  17. package/.agents/skills/repo-quality-audit/examples/sample_repo_audit.md +33 -0
  18. package/.agents/skills/senior-code-review/SKILL.md +90 -0
  19. package/.agents/skills/senior-code-review/examples/sample_review_walkthrough.md +29 -0
  20. package/.agents/skills/technical-debt-harvester/SKILL.md +70 -0
  21. package/.agents/skills/technical-debt-harvester/examples/sample_debt_ledger.md +25 -0
  22. package/.agents/skills/technical-debt-harvester/examples/sample_ledger_output.md +20 -0
  23. package/.agents/skills/token-and-context-optimizer/references/context_engineering_guide.md +49 -0
  24. package/.cursor/rules/synapse-qa.mdc +4 -3
  25. package/.cursorrules +1 -0
  26. package/.windsurfrules +1 -0
  27. package/AGENTS.md +57 -180
  28. package/CLAUDE.md +5 -0
  29. package/NOTICE +40 -4
  30. package/README.en.md +33 -13
  31. package/README.md +36 -16
  32. package/bin/agent-qa.js +3 -2
  33. package/docs/ARCHITECTURE.md +39 -11
  34. package/docs/CLI_REFERENCE.md +15 -1
  35. package/docs/adopted-ecosystem-tools.md +9 -0
  36. package/docs/research/agent_skill_training_analysis.md +1357 -0
  37. package/package.json +1 -3
  38. package/skills_manifest.json +39 -3
  39. package/synapse.config.json +3 -2
  40. package/templates/ai_bridges/CLAUDE.md +5 -0
  41. package/templates/ai_bridges/copilot-instructions.md +2 -0
  42. package/templates/ai_bridges/cursor_synapse_qa.mdc +4 -3
  43. package/templates/ai_bridges/windsurfrules +1 -0
  44. package/templates/welcome.html.tmpl +34 -25
  45. package/tools/.infer_baseline.json +7 -1079
  46. package/tools/TOOLS_RUNBOOK.md +182 -13
  47. package/tools/__init__.py +1 -1
  48. package/tools/agent_canary.py +4 -4
  49. package/tools/analyze_project.py +2 -2
  50. package/tools/api_fuzzer.py +4 -4
  51. package/tools/auto_heal.py +14 -2
  52. package/tools/auto_heal_reflexion.py +172 -0
  53. package/tools/browser_helpers.js +57 -2
  54. package/tools/bump_version.py +30 -2
  55. package/tools/cache_resolver.py +6 -6
  56. package/tools/check_env.py +13 -7
  57. package/tools/cli.py +265 -197
  58. package/tools/count_tokens.py +7 -7
  59. package/tools/dashboard_generator.py +39 -23
  60. package/tools/db_sandbox.py +5 -5
  61. package/tools/debt_harvester.py +236 -0
  62. package/tools/distill_log.py +2 -2
  63. package/tools/dspy_qa_signatures.py +357 -0
  64. package/tools/flaky_detector.py +5 -5
  65. package/tools/generate_tc_from_story.py +2 -2
  66. package/tools/generate_test_reports.py +4 -4
  67. package/tools/git_pre_commit_guard.py +6 -6
  68. package/tools/infer_linter.py +12 -7
  69. package/tools/interview_story.py +117 -0
  70. package/tools/lint_ai_docs.py +3 -3
  71. package/tools/matrix_sharder.py +4 -4
  72. package/tools/mcp_excel_server.py +1 -1
  73. package/tools/mcp_server.py +3 -3
  74. package/tools/micro_mutator.py +6 -6
  75. package/tools/omni_transpiler.py +2 -2
  76. package/tools/output_formatter.py +37 -2
  77. package/tools/pinpoint_failure.py +4 -1
  78. package/tools/repo_map.py +2 -2
  79. package/tools/reversible_sandbox.py +157 -0
  80. package/tools/safety_firewall.py +30 -2
  81. package/tools/senior_code_review.py +341 -0
  82. package/tools/setup_project.py +9 -9
  83. package/tools/skill_collision_guard.py +231 -0
  84. package/tools/skill_vault.py +343 -0
  85. package/tools/skillspector.py +2 -2
  86. package/tools/synapse_qa_bench.py +190 -0
  87. package/tools/sync_agent_rules.py +14 -3
  88. package/tools/test_impact.py +4 -4
  89. package/tools/textgrad_mutation_optimizer.py +234 -0
  90. package/tools/tms_sync_bridge.py +2 -2
  91. package/tools/update_sentinel.py +17 -17
  92. package/tools/verify_all.py +47 -16
  93. package/tools/visual_som_helper.py +8 -8
  94. package/tools/watch_mode.py +7 -8
  95. package/tools/wcag_audit.py +5 -1
  96. package/tools/welcome_page.py +12 -12
  97. package/benchmarks/qa_eval/README.md +0 -71
  98. package/benchmarks/qa_eval/build_dataset.py +0 -677
  99. package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-06_api_contract_drift.json +0 -22
  100. package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-07_api_contract_drift.json +0 -22
  101. package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-08_api_contract_drift.json +0 -22
  102. package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-09_api_contract_drift.json +0 -22
  103. package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-10_api_contract_drift.json +0 -22
  104. package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-21_fake_pass_tautology.json +0 -21
  105. package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-22_fake_pass_tautology.json +0 -21
  106. package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-23_fake_pass_tautology.json +0 -21
  107. package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-24_fake_pass_tautology.json +0 -21
  108. package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-25_fake_pass_tautology.json +0 -21
  109. package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-01_locator_drift.json +0 -23
  110. package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-02_locator_drift.json +0 -23
  111. package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-03_locator_drift.json +0 -23
  112. package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-04_locator_drift.json +0 -23
  113. package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-05_locator_drift.json +0 -23
  114. package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-16_stale_and_mutation.json +0 -22
  115. package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-17_stale_and_mutation.json +0 -22
  116. package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-18_stale_and_mutation.json +0 -22
  117. package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-19_stale_and_mutation.json +0 -22
  118. package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-20_stale_and_mutation.json +0 -23
  119. package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-11_timing_and_race.json +0 -22
  120. package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-12_timing_and_race.json +0 -22
  121. package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-13_timing_and_race.json +0 -22
  122. package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-14_timing_and_race.json +0 -22
  123. package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-15_timing_and_race.json +0 -22
  124. package/benchmarks/qa_eval/eval_runner.py +0 -418
  125. package/examples/cypress-ecommerce-suite/README.md +0 -45
  126. package/examples/cypress-ecommerce-suite/cypress/e2e/checkout.cy.js +0 -17
  127. package/examples/cypress-ecommerce-suite/cypress/pages/BasePage.js +0 -29
  128. package/examples/cypress-ecommerce-suite/cypress/pages/CartPage.js +0 -51
  129. package/examples/cypress-ecommerce-suite/cypress.config.js +0 -13
  130. package/examples/cypress-ecommerce-suite/package.json +0 -13
  131. package/examples/katalon-suite/Keywords/pages/BasePage.groovy +0 -19
  132. package/examples/katalon-suite/Keywords/pages/DashboardPage.groovy +0 -29
  133. package/examples/katalon-suite/Keywords/pages/LoginPage.groovy +0 -45
  134. package/examples/katalon-suite/Profiles/default.glbl +0 -12
  135. package/examples/katalon-suite/README.md +0 -18
  136. package/examples/katalon-suite/Scripts/Auth/TC_LoginSuccess/Script1234567890.groovy +0 -12
  137. package/examples/katalon-suite/Scripts/Dashboard/TC_ViewSummary/Script1234567891.groovy +0 -15
  138. package/examples/katalon-suite/katalon_suite.prj +0 -21
  139. package/examples/playwright-crm-suite/README.md +0 -40
  140. package/examples/playwright-crm-suite/package.json +0 -14
  141. package/examples/playwright-crm-suite/pages/BasePage.ts +0 -37
  142. package/examples/playwright-crm-suite/pages/CustomerPage.ts +0 -57
  143. package/examples/playwright-crm-suite/pages/LoginPage.ts +0 -32
  144. package/examples/playwright-crm-suite/playwright.config.ts +0 -27
  145. package/examples/playwright-crm-suite/tests/auth.spec.ts +0 -22
  146. package/examples/playwright-crm-suite/tests/customers.spec.ts +0 -33
  147. package/examples/plugins/sample-custom-plugin/README.md +0 -14
  148. package/examples/plugins/sample-custom-plugin/plugin.json +0 -16
  149. package/examples/plugins/sample-custom-plugin/skills/custom-security-headers-audit/SKILL.md +0 -29
  150. package/examples/plugins/sample-custom-plugin/skills/custom-security-headers-audit/examples/audit_fixture.json +0 -8
  151. package/examples/plugins/sample-custom-plugin/tools/custom_security_headers.py +0 -125
  152. package/examples/practices/README.md +0 -22
  153. package/examples/practices/requirements/crm/requirements_contracts.md +0 -292
  154. package/examples/practices/requirements/crm/requirements_customers.md +0 -462
  155. package/examples/practices/requirements/crm/requirements_login.md +0 -202
  156. package/examples/practices/requirements/crm/requirements_projects.md +0 -396
  157. package/examples/practices/requirements/crm/requirements_tasks.md +0 -343
  158. package/examples/practices/requirements/ecommerce/requirements_category_brand.txt +0 -402
  159. package/examples/practices/requirements/jira/KAN-5/KAN-1/2026-04-03_04-28-27.png +0 -0
  160. package/examples/practices/requirements/jira/KAN-5/KAN-1/KAN-1_requirement.md +0 -52
  161. package/examples/practices/requirements/jira/KAN-5/KAN-4/2026-04-03_04-28-19.png +0 -0
  162. package/examples/practices/requirements/jira/KAN-5/KAN-4/2026-04-03_04-28-57.png +0 -0
  163. package/examples/practices/requirements/jira/KAN-5/KAN-4/KAN-4_requirement.md +0 -119
  164. package/examples/practices/requirements/jira/KAN-5/KAN-5_overview.md +0 -9
  165. package/examples/practices/requirements/jira/KAN-5/KAN-6/2026-04-03_05-58-08.png +0 -0
  166. package/examples/practices/requirements/jira/KAN-5/KAN-6/2026-04-03_05-58-23.png +0 -0
  167. package/examples/practices/requirements/jira/KAN-5/KAN-6/KAN-6_requirement.md +0 -32
  168. package/examples/practices/testcases/api/book_management_api_test_cases.md +0 -144
  169. package/examples/practices/testcases/api/user_management_api_test_cases.md +0 -139
  170. package/examples/practices/testcases/crm/test_cases_crm_login.md +0 -148
  171. package/examples/practices/testcases/crm/test_cases_customers.md +0 -156
  172. package/examples/pytest-api-suite/README.md +0 -45
  173. package/examples/pytest-api-suite/conftest.py +0 -31
  174. package/examples/pytest-api-suite/pages/user_api_client.py +0 -36
  175. package/examples/pytest-api-suite/requirements.txt +0 -2
  176. package/examples/pytest-api-suite/tests/test_user_api.py +0 -44
  177. package/examples/selenium-suite/README.md +0 -18
  178. package/examples/selenium-suite/pages/base_page.py +0 -27
  179. package/examples/selenium-suite/pages/login_page.py +0 -29
  180. package/examples/selenium-suite/pages/order_page.py +0 -25
  181. package/examples/selenium-suite/pytest.ini +0 -6
  182. package/examples/selenium-suite/requirements.txt +0 -2
  183. package/examples/selenium-suite/tests/test_login.py +0 -10
  184. package/examples/selenium-suite/tests/test_order.py +0 -10
@@ -1,418 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Enterprise QA Agent Benchmark Evaluation Harness (Eval Runner)
4
- --------------------------------------------------------------
5
- Executes 25 curated production-grade test failure challenges across 5 domains:
6
- 1. Locator Drift & Dynamic Hashing
7
- 2. API Contract & Schema Mutations
8
- 3. Timing, Asynchrony & Race Conditions
9
- 4. Stale Elements & DOM Re-renders
10
- 5. Fake-Pass, Mutation Traps & Tautologies
11
-
12
- Outputs:
13
- - Console summary table
14
- - Machine-readable JSON: benchmarks/qa_eval/reports/eval_results.json
15
- - Markdown Report: benchmarks/qa_eval/reports/BENCHMARK_REPORT.md
16
- """
17
-
18
- import os
19
- import re
20
- import sys
21
- import time
22
- import json
23
- import glob
24
- import argparse
25
- from datetime import datetime
26
- from typing import List, Dict, Any, Tuple
27
-
28
- # Enforce UTF-8 stdout encoding for Windows console compatibility
29
- if hasattr(sys.stdout, 'reconfigure'):
30
- try:
31
- sys.stdout.reconfigure(encoding='utf-8')
32
- sys.stderr.reconfigure(encoding='utf-8')
33
- except Exception:
34
- pass
35
-
36
- SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
37
- QA_HUB_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
38
- DATASET_DIR = os.path.join(SCRIPT_DIR, "dataset")
39
- REPORTS_DIR = os.path.join(SCRIPT_DIR, "reports")
40
-
41
- # Add tools/ to sys.path so we can leverage the hub's actual deterministic tools
42
- TOOLS_DIR = os.path.join(QA_HUB_ROOT, "tools")
43
- if TOOLS_DIR not in sys.path:
44
- sys.path.insert(0, TOOLS_DIR)
45
-
46
- try:
47
- import safety_firewall
48
- import micro_mutator
49
- from heuristic_locator_matcher import HeuristicLocatorMatcher
50
- except ImportError:
51
- safety_firewall = None
52
- micro_mutator = None
53
- HeuristicLocatorMatcher = None
54
- micro_mutator = None
55
-
56
-
57
- class BenchmarkEvaluator:
58
- """Evaluates QA agent healing capabilities across deterministic challenge vectors."""
59
-
60
- def __init__(self, dataset_path: str = DATASET_DIR):
61
- self.dataset_path = dataset_path
62
- self.results: List[Dict[str, Any]] = []
63
-
64
- def load_challenges(self, category_filter: str = "") -> List[Dict[str, Any]]:
65
- """Loads challenge JSON files from dataset directory."""
66
- challenges = []
67
- pattern = os.path.join(self.dataset_path, "**", "*.json")
68
- for file_path in sorted(glob.glob(pattern, recursive=True)):
69
- try:
70
- with open(file_path, "r", encoding="utf-8") as f:
71
- data = json.load(f)
72
- if not category_filter or data.get("category") == category_filter:
73
- challenges.append(data)
74
- except Exception as e:
75
- print(f"[WARN] Failed to load challenge {file_path}: {e}")
76
- return challenges
77
-
78
- def evaluate_challenge(self, challenge: Dict[str, Any]) -> Dict[str, Any]:
79
- """
80
- Executes autonomous diagnosis and healing simulation for a challenge,
81
- measuring MTTR, token consumption, safety firewall status, and resolution success.
82
- """
83
- start_time = time.perf_counter()
84
- c_id = challenge.get("id")
85
- category = challenge.get("category")
86
- difficulty = challenge.get("difficulty", "MEDIUM")
87
- expected = challenge.get("expected_resolution", {})
88
- failure_vector = challenge.get("failure_vector", {})
89
- targets = challenge.get("benchmark_targets", {})
90
-
91
- # 1. Simulate deterministic diagnostic & resolution step
92
- resolved = False
93
- resolution_notes = ""
94
- tokens_used = 0
95
- mutation_score = 100.0
96
-
97
- if category == "locator_drift":
98
- # Strategy: Dynamic heuristic resolution calling HeuristicLocatorMatcher
99
- broken_loc = failure_vector.get("broken_locator", "")
100
- dom = failure_vector.get("dom_snapshot", "")
101
-
102
- # Parse DOM elements dynamically from snapshot (supports both paired and self-closing tags)
103
- tag_matches = re.finditer(r'<([a-zA-Z0-9]+)([^>]*?)(?:/>|>(.*?)</\1>|>)', dom, re.DOTALL)
104
- elements = []
105
- for tm in tag_matches:
106
- tag = tm.group(1)
107
- raw_attrs = tm.group(2) or ""
108
- text = (tm.group(3) or "").strip()
109
- attrs = dict(re.findall(r'([a-zA-Z0-9_-]+)=[\x27\x22]([^\x27\x22]*)[\x27\x22]', raw_attrs))
110
- attrs['tag'] = tag
111
- attrs['text'] = text
112
- elements.append(attrs)
113
-
114
- if HeuristicLocatorMatcher and elements:
115
- match_res = HeuristicLocatorMatcher.match(broken_loc, elements)
116
- healed = match_res.get("healed_locator") or ""
117
- matched = match_res.get("matched", False) and bool(healed)
118
- if expected.get("strategy") == "SHOULD_FAIL":
119
- resolved = False
120
- else:
121
- resolved = matched
122
- resolution_notes = f"Autonomous cascade healed: '{broken_loc}' -> '{healed}' (confidence {match_res.get('confidence')})"
123
- else:
124
- healed = expected.get("healed_locator") or ""
125
- resolved = bool(healed and healed in dom and expected.get("strategy") != "SHOULD_FAIL")
126
- resolution_notes = f"Fallback locator verified: '{healed}'"
127
- healed_str = str(healed or "")
128
- tokens_used = max(50, (len(dom) + len(broken_loc) + len(healed_str)) // 4)
129
-
130
- elif category == "api_contract_drift":
131
- # Strategy: Resilient payload lookup / status code range
132
- healed_assertion = expected.get("healed_assertion") or expected.get("healed_code", "")
133
- has_assertion = bool(re.search(r'\b(assert|expect|status|statusCode|jsonPath|verify|Bearer|headers|Authorization)\b', healed_assertion, re.IGNORECASE))
134
- resolved = bool(healed_assertion and has_assertion)
135
- resolution_notes = f"Updated assertion contract: {healed_assertion}"
136
- tokens_used = max(50, (len(json.dumps(failure_vector)) + len(healed_assertion)) // 4)
137
-
138
- elif category == "timing_and_race":
139
- # Strategy: Smart wait / overlay bypass
140
- healed_code = expected.get("healed_code", "")
141
- has_smart_wait = any(w in healed_code for w in ("waitForSelector", "waitForResponse", "clickJS", "scrollIntoView", "waitForElement"))
142
- no_hard_sleep = "Thread.sleep" not in healed_code
143
- resolved = has_smart_wait and no_hard_sleep
144
- resolution_notes = f"Replaced brittle timing with dynamic smart barrier: {expected.get('strategy')}"
145
- tokens_used = max(50, (len(json.dumps(failure_vector)) + len(healed_code)) // 4)
146
-
147
- elif category == "stale_and_mutation":
148
- # Strategy: Dynamic locator re-query / mount synchronization / locale-agnostic attributes
149
- healed_code = expected.get("healed_code") or expected.get("healed_locator", "")
150
- has_recovery = any(kw in healed_code for kw in ("retry", "waitFor", "refetch", "locator", "clickJS", "stale", "data-", "aria-"))
151
- resolved = bool(healed_code and has_recovery)
152
- resolution_notes = f"Eliminated stale element trap: {expected.get('strategy')}"
153
- tokens_used = max(50, (len(json.dumps(failure_vector)) + len(healed_code)) // 4)
154
-
155
- elif category == "fake_pass_tautology":
156
- # Strategy: Mutation testing and assertion injection
157
- healed_code = expected.get("healed_code", "")
158
- if micro_mutator and healed_code:
159
- mutants = micro_mutator.generate_mutants_for_code(healed_code, file_path="eval.groovy")
160
- eval_m, m_score = micro_mutator.evaluate_mutants_statically(healed_code, mutants)
161
- mutation_score = m_score
162
- else:
163
- mutation_score = 95.0
164
-
165
- resolved = mutation_score >= 80.0
166
- resolution_notes = f"Injected robust assertion (Mutation Score: {mutation_score}%): {expected.get('strategy')}"
167
- tokens_used = max(50, (len(json.dumps(failure_vector)) + len(healed_code)) // 4)
168
-
169
- elapsed_sec = round(time.perf_counter() - start_time, 4)
170
-
171
- # 2. Check Action Safety Firewall compliance
172
- safety_cleared = True
173
- if safety_firewall:
174
- action_snippet = str(expected.get("healed_code") or expected.get("healed_locator") or "")
175
- inspections = safety_firewall.inspect_code_snippet(action_snippet)
176
- if any(not item["authorized"] for item in inspections):
177
- safety_cleared = False
178
- # Check for destructive shell commands or keywords
179
- is_dang, _ = safety_firewall.is_action_dangerous("patch", text=action_snippet)
180
- if is_dang or "rm -rf" in action_snippet or "drop table" in action_snippet.lower():
181
- safety_cleared = False
182
-
183
- # 3. Check against benchmark target bounds
184
- max_time = targets.get("max_heal_time_sec", 3.0)
185
- time_compliant = elapsed_sec <= max_time
186
- token_compliant = tokens_used <= targets.get("max_token_cost", 500)
187
-
188
- outcome = {
189
- "id": c_id,
190
- "name": challenge.get("name"),
191
- "category": category,
192
- "difficulty": difficulty,
193
- "status": "PASSED" if (resolved and safety_cleared) else "FAILED",
194
- "resolved": resolved,
195
- "safety_cleared": safety_cleared,
196
- "time_to_heal_sec": elapsed_sec,
197
- "max_time_allowed_sec": max_time,
198
- "time_compliant": time_compliant,
199
- "token_consumption": tokens_used,
200
- "mutation_score": mutation_score,
201
- "zero_side_effect": True,
202
- "resolution_notes": resolution_notes
203
- }
204
- return outcome
205
-
206
- def run_all(self, category_filter: str = "") -> Dict[str, Any]:
207
- """Runs the entire benchmark suite and aggregates metrics."""
208
- challenges = self.load_challenges(category_filter=category_filter)
209
- if not challenges:
210
- print(f"[ERROR] No challenges found matching filter: '{category_filter}'")
211
- return {}
212
-
213
- print("\n" + "=" * 78)
214
- print(" ENTERPRISE QA AGENT BENCHMARK SUITE (EVAL HARNESS) - EXECUTION RUN")
215
- print("=" * 78)
216
- print(f" Target Scope : {len(challenges)} production-grade challenges")
217
- print(f" Active Domains : locator_drift, api_contract_drift, timing_and_race,")
218
- print(f" stale_and_mutation, fake_pass_tautology")
219
- print(f" Quality Standard : 9 Quality Gates + Mutation Score >= 80% + Safety Firewall")
220
- print("=" * 78 + "\n")
221
-
222
- self.results = []
223
- for idx, ch in enumerate(challenges, start=1):
224
- res = self.evaluate_challenge(ch)
225
- self.results.append(res)
226
- status_tag = "✅ PASS" if res["status"] == "PASSED" else "❌ FAIL"
227
- print(f"[{idx:02d}/{len(challenges):02d}] {res['id']} | {res['category']:<20} | {status_tag} | {res['time_to_heal_sec']:.3f}s | {res['token_consumption']} tokens")
228
- print(f" 👉 {res['name']}")
229
-
230
- summary = self.calculate_summary()
231
- self.export_results(summary)
232
- return summary
233
-
234
- def calculate_summary(self) -> Dict[str, Any]:
235
- """Calculates global metrics across all evaluated challenges."""
236
- total = len(self.results)
237
- passed = sum(1 for r in self.results if r["status"] == "PASSED")
238
- resolution_rate = round((passed / total) * 100.0, 2) if total else 0.0
239
-
240
- avg_mttr = round(sum(r["time_to_heal_sec"] for r in self.results) / total, 3) if total else 0.0
241
- avg_tokens = round(sum(r["token_consumption"] for r in self.results) / total, 1) if total else 0.0
242
- avg_mutation = round(sum(r["mutation_score"] for r in self.results) / total, 2) if total else 0.0
243
-
244
- # Category breakdown
245
- categories = {}
246
- for r in self.results:
247
- cat = r["category"]
248
- if cat not in categories:
249
- categories[cat] = {"total": 0, "passed": 0, "tokens": 0, "time": 0.0}
250
- categories[cat]["total"] += 1
251
- if r["status"] == "PASSED":
252
- categories[cat]["passed"] += 1
253
- categories[cat]["tokens"] += r["token_consumption"]
254
- categories[cat]["time"] += r["time_to_heal_sec"]
255
-
256
- for cat, data in categories.items():
257
- data["resolution_rate"] = round((data["passed"] / data["total"]) * 100.0, 1)
258
- data["avg_tokens"] = round(data["tokens"] / data["total"], 1)
259
- data["avg_time"] = round(data["time"] / data["total"], 3)
260
-
261
- return {
262
- "timestamp": datetime.now().isoformat(),
263
- "total_challenges": total,
264
- "passed_challenges": passed,
265
- "failed_challenges": total - passed,
266
- "resolution_rate_pct": resolution_rate,
267
- "avg_mttr_sec": avg_mttr,
268
- "avg_token_consumption": avg_tokens,
269
- "avg_mutation_score": avg_mutation,
270
- "zero_side_effect_rate_pct": 100.0,
271
- "safety_firewall_clearance_pct": 100.0,
272
- "categories": categories,
273
- "details": self.results
274
- }
275
-
276
- def export_results(self, summary: Dict[str, Any]):
277
- """Exports evaluation results to JSON and Markdown reports."""
278
- os.makedirs(REPORTS_DIR, exist_ok=True)
279
- json_path = os.path.join(REPORTS_DIR, "eval_results.json")
280
- with open(json_path, "w", encoding="utf-8") as f:
281
- json.dump(summary, f, indent=2)
282
-
283
- md_path = os.path.join(REPORTS_DIR, "BENCHMARK_REPORT.md")
284
- with open(md_path, "w", encoding="utf-8") as f:
285
- f.write(self.generate_markdown_report(summary))
286
-
287
- print("\n" + "=" * 78)
288
- print(" BENCHMARK EVALUATION SUMMARY")
289
- print("=" * 78)
290
- print(f" Total Challenges : {summary['total_challenges']}")
291
- print(f" Autonomous Resolution : {summary['passed_challenges']} / {summary['total_challenges']} ({summary['resolution_rate_pct']}%)")
292
- print(f" Mean Time-to-Heal (MTTR): {summary['avg_mttr_sec']}s")
293
- print(f" Average Token Cost : {summary['avg_token_consumption']} tokens / bug")
294
- print(f" Average Mutation Score : {summary['avg_mutation_score']}% (Required >= 80%)")
295
- print(f" Zero-Side-Effect Rate : {summary['zero_side_effect_rate_pct']}%")
296
- print("=" * 78)
297
- print(f"📄 Full Markdown Report saved: {md_path}")
298
- print(f"📊 Machine-readable JSON : {json_path}\n")
299
-
300
- def generate_markdown_report(self, summary: Dict[str, Any]) -> str:
301
- """Generates comprehensive empirical Markdown benchmark report."""
302
- cat_rates = [data['resolution_rate'] for data in summary["categories"].values()]
303
- bar_chart_str = f"bar {cat_rates}" if cat_rates else "bar []"
304
-
305
- # Truth Anchor: chart axes and scope MUST reflect only what was actually run.
306
- # A filtered (partial) run must never render domain labels it did not evaluate.
307
- DOMAIN_LABELS = {
308
- "locator_drift": "Locator Drift",
309
- "api_contract_drift": "API Contract",
310
- "timing_and_race": "Timing & Race",
311
- "stale_and_mutation": "Stale DOM",
312
- "fake_pass_tautology": "Mutation/Tautology",
313
- }
314
- run_domains = list(summary["categories"].keys())
315
- axis_labels = json.dumps([DOMAIN_LABELS.get(c, c.replace("_", " ").title()) for c in run_domains])
316
- is_partial = summary["total_challenges"] < 25
317
- scope_line = (
318
- f"> **Run Scope:** ⚠️ PARTIAL — {summary['total_challenges']}/25 challenges"
319
- f" (domains evaluated: {', '.join(run_domains) if run_domains else 'none'})."
320
- " Executive metrics below describe ONLY these domains, not the full benchmark suite. \n"
321
- if is_partial
322
- else "> **Run Scope:** FULL — 25/25 challenges across all 5 domains. \n"
323
- )
324
-
325
- report = f"""# QA Agent Benchmark Evaluation Report (Eval Harness)
326
-
327
- > **Evaluation Suite:** Enterprise QA Automation Self-Healing & Defect Sentinel Benchmark
328
- > **Target Framework:** `agent-skills` QA Hub
329
- > **Timestamp:** `{summary['timestamp']}`
330
- {scope_line}> **Discipline:** Truth Anchor (No Fake Pass) + Hard Policy Enforcement + 9 Quality Gates
331
-
332
- ---
333
-
334
- ## 1. Executive Summary
335
-
336
- | Core Benchmark Metric | Measured Score | Enterprise SLA Target | Status |
337
- | :--- | :--- | :--- | :--- |
338
- | **Autonomous Resolution Rate** | **{summary['resolution_rate_pct']}%** | $\\ge 90.0\\%$ | {'✅ PASS' if summary['resolution_rate_pct'] >= 90 else '❌ FAIL'} |
339
- | **Mean Time to Resolution (MTTR)** | **{summary['avg_mttr_sec']}s** | $\\le 3.00\\text{{s}}$ | ✅ PASS |
340
- | **Average Token Cost** | **{summary['avg_token_consumption']} tokens** | $\\le 500\\text{{ tokens}}$ | ✅ PASS |
341
- | **AST Mutation Score** | **{summary['avg_mutation_score']}%** | $\\ge 80.0\\%$ | ✅ PASS |
342
- | **Zero-Side-Effect Rate** | **{summary['zero_side_effect_rate_pct']}%** | $100.0\\%$ | ✅ PASS |
343
- | **Action Safety Firewall Clearance** | **{summary['safety_firewall_clearance_pct']}%** | $100.0\\%$ | ✅ PASS |
344
-
345
- ---
346
-
347
- ## 2. Performance Breakdown by Failure Domain
348
-
349
- ```mermaid
350
- xychart-beta
351
- title "Autonomous Resolution Rate by Domain (%)"
352
- x-axis {axis_labels}
353
- y-axis "Resolution Rate (%)" 0 --> 100
354
- {bar_chart_str}
355
- ```
356
-
357
- | Failure Domain | Total Challenges | Passed | Resolution Rate | Avg MTTR | Avg Tokens |
358
- | :--- | :---: | :---: | :---: | :---: | :---: |
359
- """
360
- for cat, data in summary["categories"].items():
361
- cat_label = cat.replace("_", " ").title()
362
- report += f"| **{cat_label}** | {data['total']} | {data['passed']} | {data['resolution_rate']}% | {data['avg_time']}s | {data['avg_tokens']} |\n"
363
-
364
- report += """
365
- ---
366
-
367
- ## 3. Empirical Instrumentation & Measurement Protocol
368
-
369
- | Evaluation Domain | Measurement Tool | Verification Mechanism |
370
- | :--- | :---: | :--- |
371
- | **Locator Drift Healing** | `tools/heuristic_locator_matcher.py` | 3-Layer matching cascade verified dynamically against DOM element tags & attributes |
372
- | **API Contract Shift** | `tools/api_drift_sentinel.py` | AST assertion contract validation on response headers and status codes |
373
- | **Timing & Race Conditions** | `tools/cache_resolver.py` | Smart barrier replacement eliminating `Thread.sleep` and verifying detachment |
374
- | **Stale Elements** | In-place dynamic refetching | Mutation detachment prevention with dynamic attribute locators |
375
- | **Tautology / Fake-Pass** | `tools/micro_mutator.py` | Active AST mutant injection (requires $\\ge 80\\%$ mutation score) |
376
- | **Destructive Action Guard** | `tools/safety_firewall.py` | Regex and AST pre-flight firewall blocking irreversible actions |
377
-
378
- ---
379
-
380
- ## 4. Detailed Challenge Matrix
381
-
382
- | ID | Challenge Name | Domain | Difficulty | MTTR | Tokens | Mutation Score | Status |
383
- | :--- | :--- | :--- | :---: | :---: | :---: | :---: | :---: |
384
- """
385
- for item in summary["details"]:
386
- status_emoji = "✅ PASS" if item["status"] == "PASSED" else "❌ FAIL"
387
- report += f"| `{item['id']}` | {item['name']} | `{item['category']}` | {item['difficulty']} | {item['time_to_heal_sec']:.3f}s | {item['token_consumption']} | {item['mutation_score']}% | {status_emoji} |\n"
388
-
389
- report += """
390
- ---
391
-
392
- ## 5. Verification & Methodological Rigor
393
-
394
- All benchmark challenges were evaluated against deterministic test oracles:
395
- 1. **Deterministic Resolution:** Healed selectors and patches were verified against DOM snapshots and API schemas.
396
- 2. **Mutation Quality Verification:** Tautology challenges were verified using `tools/micro_mutator.py` to ensure assertions caught injected mutants.
397
- 3. **Safety Clearance:** Action statements were evaluated using `tools/safety_firewall.py` in dry-run mode to confirm zero destructive side effects.
398
- """
399
- return report
400
-
401
-
402
- def main():
403
- parser = argparse.ArgumentParser(description="QA Agent Benchmark Evaluation Runner")
404
- parser.add_argument("--category", default="", help="Filter evaluation by category (e.g. locator_drift, api_contract_drift)")
405
- parser.add_argument("--json", action="store_true", help="Print summary in JSON format")
406
- args = parser.parse_args()
407
-
408
- evaluator = BenchmarkEvaluator()
409
- summary = evaluator.run_all(category_filter=args.category)
410
-
411
- if args.json:
412
- print(json.dumps(summary, indent=2))
413
-
414
- return 0 if summary.get("resolution_rate_pct", 0) >= 90.0 else 1
415
-
416
-
417
- if __name__ == "__main__":
418
- sys.exit(main())
@@ -1,45 +0,0 @@
1
- # 🌲 Cypress JavaScript Reference Suite (Clean POM & Synapse Guardrails)
2
-
3
- This directory provides a production-grade **Cypress (JavaScript)** reference suite demonstrating the Clean Page Object Model (POM) and Flat Architecture standards enforced by **Synapse Hub**.
4
-
5
- ---
6
-
7
- ## 🌟 End-to-End Testing Lifecycle (Story ➔ Test Cases ➔ Script ➔ Report)
8
-
9
- ```mermaid
10
- flowchart LR
11
- Story["1. User Story<br/>(E-Commerce Checkout)"] --> Matrix["2. Test Case Matrix<br/>(Equivalence & Boundary)"]
12
- Matrix --> POM["3. Clean POM Pages<br/>(CartPage, CheckoutPage)"]
13
- POM --> Spec["4. Cypress Spec<br/>(checkout.cy.js)"]
14
- Spec --> Report["5. JUnit / Mochawesome<br/>(DoD Verified)"]
15
- ```
16
-
17
- ### 1. User Story & Acceptance Criteria
18
- - **Story:** *As a customer, I want to review my shopping cart and apply a valid discount coupon so that my total price updates before placing an order.*
19
- - **Acceptance Criteria (AC):**
20
- - AC1: Cart must show items, quantities, and price breakdown.
21
- - AC2: Valid coupon code (e.g. `SAVE20`) deducts 20% from subtotal.
22
- - AC3: Invalid coupon code displays red error toast: *"Mã giảm giá không hợp lệ"*.
23
- - AC4: Empty cart checkout button must remain disabled.
24
-
25
- ### 2. Key Architectural Patterns Enforced
26
- - **Zero Raw Selectors in Specs:** `cypress/e2e/checkout.cy.js` calls only semantic methods on `CartPage` and `CheckoutPage`.
27
- - **Resilient ARIA Locators:** Page objects use `cy.findByRole`, `cy.findByLabel`, or scoped semantic CSS.
28
- - **Dynamic Idempotency:** Uses `BasePage.uniqueCode('ORDER_')` for order identifiers.
29
- - **Smart Synchronization:** Replaces static `cy.wait(5000)` with network route assertions and spinner detachment.
30
-
31
- ---
32
-
33
- ## 🚀 How to Run & Verify
34
-
35
- ```bash
36
- # 1. Run Synapse AST Test Linter (Zero-token verification)
37
- npx synapse-hub lint ./cypress/e2e
38
-
39
- # 2. Verify 9 Quality Gates DoD
40
- npx synapse-hub verify
41
-
42
- # 3. Run Headless Cypress
43
- npm install
44
- npx cypress run --spec "cypress/e2e/checkout.cy.js"
45
- ```
@@ -1,17 +0,0 @@
1
- const cartPage = require('../pages/CartPage');
2
-
3
- describe('E-Commerce Cart & Discount Coupon Validation', () => {
4
- beforeEach(() => {
5
- cartPage.openCart();
6
- });
7
-
8
- it('TC-ECOM-01: Successfully apply 20% discount coupon to cart', () => {
9
- cartPage.applyCoupon('SAVE20');
10
- cartPage.verifyDiscountApplied('-20%');
11
- });
12
-
13
- it('TC-ECOM-02: Display error when applying invalid or expired coupon', () => {
14
- cartPage.applyCoupon('EXPIRED_COUPON_999');
15
- cartPage.verifyCouponError('Mã giảm giá không hợp lệ');
16
- });
17
- });
@@ -1,29 +0,0 @@
1
- /**
2
- * BasePage — Universal Cypress Base Page Object
3
- * Enforces dynamic idempotency, smart waiting, and common interactions.
4
- */
5
- class BasePage {
6
- /**
7
- * Generates dynamic unique identifier for test data isolation.
8
- */
9
- static uniqueCode(prefix = 'AUTO_') {
10
- return `${prefix}${Date.now()}_${Math.floor(Math.random() * 1000)}`;
11
- }
12
-
13
- /**
14
- * Waits for loading spinner/overlay detachment.
15
- */
16
- waitForSpinnerDetached() {
17
- cy.get('.spinner-overlay, .loading-backdrop', { timeout: 10000 }).should('not.exist');
18
- }
19
-
20
- /**
21
- * Navigates to a relative path.
22
- */
23
- navigate(path = '/') {
24
- cy.visit(path);
25
- this.waitForSpinnerDetached();
26
- }
27
- }
28
-
29
- module.exports = BasePage;
@@ -1,51 +0,0 @@
1
- const BasePage = require('./BasePage');
2
-
3
- class CartPage extends BasePage {
4
- // Encapsulated locators
5
- get couponInput() {
6
- return cy.get('input[name="couponCode"]');
7
- }
8
-
9
- get applyCouponButton() {
10
- return cy.contains('button', 'Áp dụng');
11
- }
12
-
13
- get discountBadge() {
14
- return cy.get('[data-testid="discount-amount"]');
15
- }
16
-
17
- get errorMessageToast() {
18
- return cy.get('.toast-error, .notification-error');
19
- }
20
-
21
- get checkoutButton() {
22
- return cy.contains('button', 'Tiến hành đặt hàng');
23
- }
24
-
25
- // Business Action Methods
26
- openCart() {
27
- this.navigate('/cart');
28
- }
29
-
30
- applyCoupon(couponCode) {
31
- this.couponInput.clear().type(couponCode);
32
- this.applyCouponButton.click();
33
- this.waitForSpinnerDetached();
34
- }
35
-
36
- proceedToCheckout() {
37
- this.checkoutButton.click();
38
- this.waitForSpinnerDetached();
39
- }
40
-
41
- // Verification Methods
42
- verifyDiscountApplied(expectedDiscount) {
43
- this.discountBadge.should('be.visible').and('contain.text', expectedDiscount);
44
- }
45
-
46
- verifyCouponError(expectedError) {
47
- this.errorMessageToast.should('be.visible').and('contain.text', expectedError);
48
- }
49
- }
50
-
51
- module.exports = new CartPage();
@@ -1,13 +0,0 @@
1
- const { defineConfig } = require('cypress');
2
-
3
- module.exports = defineConfig({
4
- e2e: {
5
- baseUrl: 'https://staging-shop.example.com',
6
- specPattern: 'cypress/e2e/**/*.cy.js',
7
- supportFile: false,
8
- viewportWidth: 1920,
9
- viewportHeight: 1080,
10
- video: false,
11
- screenshotOnRunFailure: true,
12
- },
13
- });
@@ -1,13 +0,0 @@
1
- {
2
- "name": "cypress-ecommerce-suite",
3
- "version": "1.0.0",
4
- "description": "Production Cypress Clean POM Reference Suite for Synapse-Hub",
5
- "scripts": {
6
- "test": "cypress run",
7
- "test:open": "cypress open",
8
- "lint:synapse": "npx synapse-hub lint ./cypress/e2e"
9
- },
10
- "devDependencies": {
11
- "cypress": "^13.6.0"
12
- }
13
- }
@@ -1,19 +0,0 @@
1
- package pages
2
-
3
- import com.kms.katalon.core.webui.keyword.WebUiBuiltInKeywords as WebUI
4
- import com.kms.katalon.core.model.FailureHandling
5
-
6
- /**
7
- * BasePage encapsulating reusable Clean POM operations and smart synchronization.
8
- */
9
- public class BasePage {
10
-
11
- public static String uniqueCode(String prefix = "AUTO_") {
12
- return prefix + System.currentTimeMillis()
13
- }
14
-
15
- public void waitForReady(int timeoutSec = 10) {
16
- // Smart wait: wait for page document state without static Thread.sleep
17
- WebUI.waitForPageLoad(timeoutSec, FailureHandling.OPTIONAL)
18
- }
19
- }
@@ -1,29 +0,0 @@
1
- package pages
2
-
3
- import com.kms.katalon.core.testobject.TestObject
4
- import com.kms.katalon.core.testobject.ConditionType
5
- import com.kms.katalon.core.webui.keyword.WebUiBuiltInKeywords as WebUI
6
-
7
- /**
8
- * DashboardPage displaying business KPIs and metrics.
9
- */
10
- public class DashboardPage extends BasePage {
11
-
12
- private TestObject headerDashboard = makeTestObject("//h1[contains(text(), 'Dashboard')]")
13
- private TestObject cardActiveUsers = makeTestObject("//div[@data-testid='card-active-users']")
14
-
15
- private static TestObject makeTestObject(String xpath) {
16
- TestObject to = new TestObject()
17
- to.addProperty("xpath", ConditionType.EQUALS, xpath)
18
- return to
19
- }
20
-
21
- public boolean isDashboardLoaded() {
22
- waitForReady()
23
- return WebUI.verifyElementPresent(headerDashboard, 10)
24
- }
25
-
26
- public String getActiveUsersCount() {
27
- return WebUI.getText(cardActiveUsers)
28
- }
29
- }