synapse-hub 2.4.3 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/agents/orchestrator.md +1 -0
- package/.agents/agents/pom-crafter.md +1 -0
- package/.agents/agents/quality-gatekeeper.md +6 -0
- package/.agents/rules/skill-precedence.md +38 -18
- package/.agents/rules/systematic-debugging.md +2 -0
- package/.agents/skills/lean-code-ladder/SKILL.md +12 -1
- package/.agents/skills/lean-impact-scoreboard/SKILL.md +48 -0
- package/.agents/skills/lean-impact-scoreboard/examples/sample_benchmark_gain.md +20 -0
- package/.agents/skills/lean-impact-scoreboard/examples/sample_scoreboard.md +15 -0
- package/.agents/skills/repo-quality-audit/SKILL.md +68 -0
- package/.agents/skills/repo-quality-audit/examples/sample_audit_walkthrough.md +15 -0
- package/.agents/skills/repo-quality-audit/examples/sample_repo_audit.md +33 -0
- package/.agents/skills/senior-code-review/SKILL.md +90 -0
- package/.agents/skills/senior-code-review/examples/sample_review_walkthrough.md +29 -0
- package/.agents/skills/technical-debt-harvester/SKILL.md +70 -0
- package/.agents/skills/technical-debt-harvester/examples/sample_debt_ledger.md +25 -0
- package/.agents/skills/technical-debt-harvester/examples/sample_ledger_output.md +20 -0
- package/.cursor/rules/synapse-qa.mdc +4 -3
- package/.cursorrules +1 -0
- package/.windsurfrules +1 -0
- package/AGENTS.md +51 -179
- package/CLAUDE.md +5 -0
- package/README.en.md +12 -8
- package/README.md +12 -8
- package/bin/agent-qa.js +3 -2
- package/docs/ARCHITECTURE.md +3 -3
- package/docs/CLI_REFERENCE.md +15 -1
- package/package.json +1 -3
- package/skills_manifest.json +38 -2
- package/synapse.config.json +1 -1
- package/templates/ai_bridges/CLAUDE.md +5 -0
- package/templates/ai_bridges/copilot-instructions.md +2 -0
- package/templates/ai_bridges/cursor_synapse_qa.mdc +4 -3
- package/templates/ai_bridges/windsurfrules +1 -0
- package/tools/.infer_baseline.json +7 -1079
- package/tools/TOOLS_RUNBOOK.md +53 -13
- package/tools/__init__.py +1 -1
- package/tools/agent_canary.py +4 -4
- package/tools/analyze_project.py +2 -2
- package/tools/api_fuzzer.py +4 -4
- package/tools/auto_heal.py +4 -2
- package/tools/browser_helpers.js +2 -2
- package/tools/bump_version.py +30 -2
- package/tools/cache_resolver.py +6 -6
- package/tools/check_env.py +13 -7
- package/tools/cli.py +265 -197
- package/tools/count_tokens.py +7 -7
- package/tools/dashboard_generator.py +14 -14
- package/tools/db_sandbox.py +5 -5
- package/tools/debt_harvester.py +236 -0
- package/tools/distill_log.py +2 -2
- package/tools/flaky_detector.py +5 -5
- package/tools/generate_tc_from_story.py +2 -2
- package/tools/generate_test_reports.py +4 -4
- package/tools/git_pre_commit_guard.py +6 -6
- package/tools/infer_linter.py +12 -7
- package/tools/lint_ai_docs.py +2 -2
- package/tools/matrix_sharder.py +4 -4
- package/tools/mcp_excel_server.py +1 -1
- package/tools/mcp_server.py +3 -3
- package/tools/micro_mutator.py +6 -6
- package/tools/omni_transpiler.py +2 -2
- package/tools/output_formatter.py +37 -2
- package/tools/pinpoint_failure.py +4 -1
- package/tools/repo_map.py +2 -2
- package/tools/safety_firewall.py +2 -2
- package/tools/senior_code_review.py +341 -0
- package/tools/setup_project.py +9 -9
- package/tools/skillspector.py +2 -2
- package/tools/sync_agent_rules.py +14 -3
- package/tools/test_impact.py +4 -4
- package/tools/tms_sync_bridge.py +2 -2
- package/tools/update_sentinel.py +17 -17
- package/tools/verify_all.py +12 -12
- package/tools/visual_som_helper.py +8 -8
- package/tools/watch_mode.py +6 -7
- package/tools/wcag_audit.py +5 -1
- package/tools/welcome_page.py +11 -11
- package/benchmarks/qa_eval/README.md +0 -71
- package/benchmarks/qa_eval/build_dataset.py +0 -677
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-06_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-07_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-08_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-09_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/api_contract_drift/qa-challenge-10_api_contract_drift.json +0 -22
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-21_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-22_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-23_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-24_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/fake_pass_tautology/qa-challenge-25_fake_pass_tautology.json +0 -21
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-01_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-02_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-03_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-04_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/locator_drift/qa-challenge-05_locator_drift.json +0 -23
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-16_stale_and_mutation.json +0 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-17_stale_and_mutation.json +0 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-18_stale_and_mutation.json +0 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-19_stale_and_mutation.json +0 -22
- package/benchmarks/qa_eval/dataset/stale_and_mutation/qa-challenge-20_stale_and_mutation.json +0 -23
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-11_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-12_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-13_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-14_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/dataset/timing_and_race/qa-challenge-15_timing_and_race.json +0 -22
- package/benchmarks/qa_eval/eval_runner.py +0 -418
- package/examples/cypress-ecommerce-suite/README.md +0 -45
- package/examples/cypress-ecommerce-suite/cypress/e2e/checkout.cy.js +0 -17
- package/examples/cypress-ecommerce-suite/cypress/pages/BasePage.js +0 -29
- package/examples/cypress-ecommerce-suite/cypress/pages/CartPage.js +0 -51
- package/examples/cypress-ecommerce-suite/cypress.config.js +0 -13
- package/examples/cypress-ecommerce-suite/package.json +0 -13
- package/examples/katalon-suite/Keywords/pages/BasePage.groovy +0 -19
- package/examples/katalon-suite/Keywords/pages/DashboardPage.groovy +0 -29
- package/examples/katalon-suite/Keywords/pages/LoginPage.groovy +0 -45
- package/examples/katalon-suite/Profiles/default.glbl +0 -12
- package/examples/katalon-suite/README.md +0 -18
- package/examples/katalon-suite/Scripts/Auth/TC_LoginSuccess/Script1234567890.groovy +0 -12
- package/examples/katalon-suite/Scripts/Dashboard/TC_ViewSummary/Script1234567891.groovy +0 -15
- package/examples/katalon-suite/katalon_suite.prj +0 -21
- package/examples/playwright-crm-suite/README.md +0 -40
- package/examples/playwright-crm-suite/package.json +0 -14
- package/examples/playwright-crm-suite/pages/BasePage.ts +0 -37
- package/examples/playwright-crm-suite/pages/CustomerPage.ts +0 -57
- package/examples/playwright-crm-suite/pages/LoginPage.ts +0 -32
- package/examples/playwright-crm-suite/playwright.config.ts +0 -27
- package/examples/playwright-crm-suite/tests/auth.spec.ts +0 -22
- package/examples/playwright-crm-suite/tests/customers.spec.ts +0 -33
- package/examples/plugins/sample-custom-plugin/README.md +0 -14
- package/examples/plugins/sample-custom-plugin/plugin.json +0 -16
- package/examples/plugins/sample-custom-plugin/skills/custom-security-headers-audit/SKILL.md +0 -29
- package/examples/plugins/sample-custom-plugin/skills/custom-security-headers-audit/examples/audit_fixture.json +0 -8
- package/examples/plugins/sample-custom-plugin/tools/custom_security_headers.py +0 -125
- package/examples/practices/README.md +0 -22
- package/examples/practices/requirements/crm/requirements_contracts.md +0 -292
- package/examples/practices/requirements/crm/requirements_customers.md +0 -462
- package/examples/practices/requirements/crm/requirements_login.md +0 -202
- package/examples/practices/requirements/crm/requirements_projects.md +0 -396
- package/examples/practices/requirements/crm/requirements_tasks.md +0 -343
- package/examples/practices/requirements/ecommerce/requirements_category_brand.txt +0 -402
- package/examples/practices/requirements/jira/KAN-5/KAN-1/2026-04-03_04-28-27.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-1/KAN-1_requirement.md +0 -52
- package/examples/practices/requirements/jira/KAN-5/KAN-4/2026-04-03_04-28-19.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-4/2026-04-03_04-28-57.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-4/KAN-4_requirement.md +0 -119
- package/examples/practices/requirements/jira/KAN-5/KAN-5_overview.md +0 -9
- package/examples/practices/requirements/jira/KAN-5/KAN-6/2026-04-03_05-58-08.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-6/2026-04-03_05-58-23.png +0 -0
- package/examples/practices/requirements/jira/KAN-5/KAN-6/KAN-6_requirement.md +0 -32
- package/examples/practices/testcases/api/book_management_api_test_cases.md +0 -144
- package/examples/practices/testcases/api/user_management_api_test_cases.md +0 -139
- package/examples/practices/testcases/crm/test_cases_crm_login.md +0 -148
- package/examples/practices/testcases/crm/test_cases_customers.md +0 -156
- package/examples/pytest-api-suite/README.md +0 -45
- package/examples/pytest-api-suite/conftest.py +0 -31
- package/examples/pytest-api-suite/pages/user_api_client.py +0 -36
- package/examples/pytest-api-suite/requirements.txt +0 -2
- package/examples/pytest-api-suite/tests/test_user_api.py +0 -44
- package/examples/selenium-suite/README.md +0 -18
- package/examples/selenium-suite/pages/base_page.py +0 -27
- package/examples/selenium-suite/pages/login_page.py +0 -29
- package/examples/selenium-suite/pages/order_page.py +0 -25
- package/examples/selenium-suite/pytest.ini +0 -6
- package/examples/selenium-suite/requirements.txt +0 -2
- package/examples/selenium-suite/tests/test_login.py +0 -10
- package/examples/selenium-suite/tests/test_order.py +0 -10
|
@@ -1,418 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""
|
|
3
|
-
Enterprise QA Agent Benchmark Evaluation Harness (Eval Runner)
|
|
4
|
-
--------------------------------------------------------------
|
|
5
|
-
Executes 25 curated production-grade test failure challenges across 5 domains:
|
|
6
|
-
1. Locator Drift & Dynamic Hashing
|
|
7
|
-
2. API Contract & Schema Mutations
|
|
8
|
-
3. Timing, Asynchrony & Race Conditions
|
|
9
|
-
4. Stale Elements & DOM Re-renders
|
|
10
|
-
5. Fake-Pass, Mutation Traps & Tautologies
|
|
11
|
-
|
|
12
|
-
Outputs:
|
|
13
|
-
- Console summary table
|
|
14
|
-
- Machine-readable JSON: benchmarks/qa_eval/reports/eval_results.json
|
|
15
|
-
- Markdown Report: benchmarks/qa_eval/reports/BENCHMARK_REPORT.md
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
import os
|
|
19
|
-
import re
|
|
20
|
-
import sys
|
|
21
|
-
import time
|
|
22
|
-
import json
|
|
23
|
-
import glob
|
|
24
|
-
import argparse
|
|
25
|
-
from datetime import datetime
|
|
26
|
-
from typing import List, Dict, Any, Tuple
|
|
27
|
-
|
|
28
|
-
# Enforce UTF-8 stdout encoding for Windows console compatibility
|
|
29
|
-
if hasattr(sys.stdout, 'reconfigure'):
|
|
30
|
-
try:
|
|
31
|
-
sys.stdout.reconfigure(encoding='utf-8')
|
|
32
|
-
sys.stderr.reconfigure(encoding='utf-8')
|
|
33
|
-
except Exception:
|
|
34
|
-
pass
|
|
35
|
-
|
|
36
|
-
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
37
|
-
QA_HUB_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", ".."))
|
|
38
|
-
DATASET_DIR = os.path.join(SCRIPT_DIR, "dataset")
|
|
39
|
-
REPORTS_DIR = os.path.join(SCRIPT_DIR, "reports")
|
|
40
|
-
|
|
41
|
-
# Add tools/ to sys.path so we can leverage the hub's actual deterministic tools
|
|
42
|
-
TOOLS_DIR = os.path.join(QA_HUB_ROOT, "tools")
|
|
43
|
-
if TOOLS_DIR not in sys.path:
|
|
44
|
-
sys.path.insert(0, TOOLS_DIR)
|
|
45
|
-
|
|
46
|
-
try:
|
|
47
|
-
import safety_firewall
|
|
48
|
-
import micro_mutator
|
|
49
|
-
from heuristic_locator_matcher import HeuristicLocatorMatcher
|
|
50
|
-
except ImportError:
|
|
51
|
-
safety_firewall = None
|
|
52
|
-
micro_mutator = None
|
|
53
|
-
HeuristicLocatorMatcher = None
|
|
54
|
-
micro_mutator = None
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
class BenchmarkEvaluator:
|
|
58
|
-
"""Evaluates QA agent healing capabilities across deterministic challenge vectors."""
|
|
59
|
-
|
|
60
|
-
def __init__(self, dataset_path: str = DATASET_DIR):
|
|
61
|
-
self.dataset_path = dataset_path
|
|
62
|
-
self.results: List[Dict[str, Any]] = []
|
|
63
|
-
|
|
64
|
-
def load_challenges(self, category_filter: str = "") -> List[Dict[str, Any]]:
|
|
65
|
-
"""Loads challenge JSON files from dataset directory."""
|
|
66
|
-
challenges = []
|
|
67
|
-
pattern = os.path.join(self.dataset_path, "**", "*.json")
|
|
68
|
-
for file_path in sorted(glob.glob(pattern, recursive=True)):
|
|
69
|
-
try:
|
|
70
|
-
with open(file_path, "r", encoding="utf-8") as f:
|
|
71
|
-
data = json.load(f)
|
|
72
|
-
if not category_filter or data.get("category") == category_filter:
|
|
73
|
-
challenges.append(data)
|
|
74
|
-
except Exception as e:
|
|
75
|
-
print(f"[WARN] Failed to load challenge {file_path}: {e}")
|
|
76
|
-
return challenges
|
|
77
|
-
|
|
78
|
-
def evaluate_challenge(self, challenge: Dict[str, Any]) -> Dict[str, Any]:
|
|
79
|
-
"""
|
|
80
|
-
Executes autonomous diagnosis and healing simulation for a challenge,
|
|
81
|
-
measuring MTTR, token consumption, safety firewall status, and resolution success.
|
|
82
|
-
"""
|
|
83
|
-
start_time = time.perf_counter()
|
|
84
|
-
c_id = challenge.get("id")
|
|
85
|
-
category = challenge.get("category")
|
|
86
|
-
difficulty = challenge.get("difficulty", "MEDIUM")
|
|
87
|
-
expected = challenge.get("expected_resolution", {})
|
|
88
|
-
failure_vector = challenge.get("failure_vector", {})
|
|
89
|
-
targets = challenge.get("benchmark_targets", {})
|
|
90
|
-
|
|
91
|
-
# 1. Simulate deterministic diagnostic & resolution step
|
|
92
|
-
resolved = False
|
|
93
|
-
resolution_notes = ""
|
|
94
|
-
tokens_used = 0
|
|
95
|
-
mutation_score = 100.0
|
|
96
|
-
|
|
97
|
-
if category == "locator_drift":
|
|
98
|
-
# Strategy: Dynamic heuristic resolution calling HeuristicLocatorMatcher
|
|
99
|
-
broken_loc = failure_vector.get("broken_locator", "")
|
|
100
|
-
dom = failure_vector.get("dom_snapshot", "")
|
|
101
|
-
|
|
102
|
-
# Parse DOM elements dynamically from snapshot (supports both paired and self-closing tags)
|
|
103
|
-
tag_matches = re.finditer(r'<([a-zA-Z0-9]+)([^>]*?)(?:/>|>(.*?)</\1>|>)', dom, re.DOTALL)
|
|
104
|
-
elements = []
|
|
105
|
-
for tm in tag_matches:
|
|
106
|
-
tag = tm.group(1)
|
|
107
|
-
raw_attrs = tm.group(2) or ""
|
|
108
|
-
text = (tm.group(3) or "").strip()
|
|
109
|
-
attrs = dict(re.findall(r'([a-zA-Z0-9_-]+)=[\x27\x22]([^\x27\x22]*)[\x27\x22]', raw_attrs))
|
|
110
|
-
attrs['tag'] = tag
|
|
111
|
-
attrs['text'] = text
|
|
112
|
-
elements.append(attrs)
|
|
113
|
-
|
|
114
|
-
if HeuristicLocatorMatcher and elements:
|
|
115
|
-
match_res = HeuristicLocatorMatcher.match(broken_loc, elements)
|
|
116
|
-
healed = match_res.get("healed_locator") or ""
|
|
117
|
-
matched = match_res.get("matched", False) and bool(healed)
|
|
118
|
-
if expected.get("strategy") == "SHOULD_FAIL":
|
|
119
|
-
resolved = False
|
|
120
|
-
else:
|
|
121
|
-
resolved = matched
|
|
122
|
-
resolution_notes = f"Autonomous cascade healed: '{broken_loc}' -> '{healed}' (confidence {match_res.get('confidence')})"
|
|
123
|
-
else:
|
|
124
|
-
healed = expected.get("healed_locator") or ""
|
|
125
|
-
resolved = bool(healed and healed in dom and expected.get("strategy") != "SHOULD_FAIL")
|
|
126
|
-
resolution_notes = f"Fallback locator verified: '{healed}'"
|
|
127
|
-
healed_str = str(healed or "")
|
|
128
|
-
tokens_used = max(50, (len(dom) + len(broken_loc) + len(healed_str)) // 4)
|
|
129
|
-
|
|
130
|
-
elif category == "api_contract_drift":
|
|
131
|
-
# Strategy: Resilient payload lookup / status code range
|
|
132
|
-
healed_assertion = expected.get("healed_assertion") or expected.get("healed_code", "")
|
|
133
|
-
has_assertion = bool(re.search(r'\b(assert|expect|status|statusCode|jsonPath|verify|Bearer|headers|Authorization)\b', healed_assertion, re.IGNORECASE))
|
|
134
|
-
resolved = bool(healed_assertion and has_assertion)
|
|
135
|
-
resolution_notes = f"Updated assertion contract: {healed_assertion}"
|
|
136
|
-
tokens_used = max(50, (len(json.dumps(failure_vector)) + len(healed_assertion)) // 4)
|
|
137
|
-
|
|
138
|
-
elif category == "timing_and_race":
|
|
139
|
-
# Strategy: Smart wait / overlay bypass
|
|
140
|
-
healed_code = expected.get("healed_code", "")
|
|
141
|
-
has_smart_wait = any(w in healed_code for w in ("waitForSelector", "waitForResponse", "clickJS", "scrollIntoView", "waitForElement"))
|
|
142
|
-
no_hard_sleep = "Thread.sleep" not in healed_code
|
|
143
|
-
resolved = has_smart_wait and no_hard_sleep
|
|
144
|
-
resolution_notes = f"Replaced brittle timing with dynamic smart barrier: {expected.get('strategy')}"
|
|
145
|
-
tokens_used = max(50, (len(json.dumps(failure_vector)) + len(healed_code)) // 4)
|
|
146
|
-
|
|
147
|
-
elif category == "stale_and_mutation":
|
|
148
|
-
# Strategy: Dynamic locator re-query / mount synchronization / locale-agnostic attributes
|
|
149
|
-
healed_code = expected.get("healed_code") or expected.get("healed_locator", "")
|
|
150
|
-
has_recovery = any(kw in healed_code for kw in ("retry", "waitFor", "refetch", "locator", "clickJS", "stale", "data-", "aria-"))
|
|
151
|
-
resolved = bool(healed_code and has_recovery)
|
|
152
|
-
resolution_notes = f"Eliminated stale element trap: {expected.get('strategy')}"
|
|
153
|
-
tokens_used = max(50, (len(json.dumps(failure_vector)) + len(healed_code)) // 4)
|
|
154
|
-
|
|
155
|
-
elif category == "fake_pass_tautology":
|
|
156
|
-
# Strategy: Mutation testing and assertion injection
|
|
157
|
-
healed_code = expected.get("healed_code", "")
|
|
158
|
-
if micro_mutator and healed_code:
|
|
159
|
-
mutants = micro_mutator.generate_mutants_for_code(healed_code, file_path="eval.groovy")
|
|
160
|
-
eval_m, m_score = micro_mutator.evaluate_mutants_statically(healed_code, mutants)
|
|
161
|
-
mutation_score = m_score
|
|
162
|
-
else:
|
|
163
|
-
mutation_score = 95.0
|
|
164
|
-
|
|
165
|
-
resolved = mutation_score >= 80.0
|
|
166
|
-
resolution_notes = f"Injected robust assertion (Mutation Score: {mutation_score}%): {expected.get('strategy')}"
|
|
167
|
-
tokens_used = max(50, (len(json.dumps(failure_vector)) + len(healed_code)) // 4)
|
|
168
|
-
|
|
169
|
-
elapsed_sec = round(time.perf_counter() - start_time, 4)
|
|
170
|
-
|
|
171
|
-
# 2. Check Action Safety Firewall compliance
|
|
172
|
-
safety_cleared = True
|
|
173
|
-
if safety_firewall:
|
|
174
|
-
action_snippet = str(expected.get("healed_code") or expected.get("healed_locator") or "")
|
|
175
|
-
inspections = safety_firewall.inspect_code_snippet(action_snippet)
|
|
176
|
-
if any(not item["authorized"] for item in inspections):
|
|
177
|
-
safety_cleared = False
|
|
178
|
-
# Check for destructive shell commands or keywords
|
|
179
|
-
is_dang, _ = safety_firewall.is_action_dangerous("patch", text=action_snippet)
|
|
180
|
-
if is_dang or "rm -rf" in action_snippet or "drop table" in action_snippet.lower():
|
|
181
|
-
safety_cleared = False
|
|
182
|
-
|
|
183
|
-
# 3. Check against benchmark target bounds
|
|
184
|
-
max_time = targets.get("max_heal_time_sec", 3.0)
|
|
185
|
-
time_compliant = elapsed_sec <= max_time
|
|
186
|
-
token_compliant = tokens_used <= targets.get("max_token_cost", 500)
|
|
187
|
-
|
|
188
|
-
outcome = {
|
|
189
|
-
"id": c_id,
|
|
190
|
-
"name": challenge.get("name"),
|
|
191
|
-
"category": category,
|
|
192
|
-
"difficulty": difficulty,
|
|
193
|
-
"status": "PASSED" if (resolved and safety_cleared) else "FAILED",
|
|
194
|
-
"resolved": resolved,
|
|
195
|
-
"safety_cleared": safety_cleared,
|
|
196
|
-
"time_to_heal_sec": elapsed_sec,
|
|
197
|
-
"max_time_allowed_sec": max_time,
|
|
198
|
-
"time_compliant": time_compliant,
|
|
199
|
-
"token_consumption": tokens_used,
|
|
200
|
-
"mutation_score": mutation_score,
|
|
201
|
-
"zero_side_effect": True,
|
|
202
|
-
"resolution_notes": resolution_notes
|
|
203
|
-
}
|
|
204
|
-
return outcome
|
|
205
|
-
|
|
206
|
-
def run_all(self, category_filter: str = "") -> Dict[str, Any]:
|
|
207
|
-
"""Runs the entire benchmark suite and aggregates metrics."""
|
|
208
|
-
challenges = self.load_challenges(category_filter=category_filter)
|
|
209
|
-
if not challenges:
|
|
210
|
-
print(f"[ERROR] No challenges found matching filter: '{category_filter}'")
|
|
211
|
-
return {}
|
|
212
|
-
|
|
213
|
-
print("\n" + "=" * 78)
|
|
214
|
-
print(" ENTERPRISE QA AGENT BENCHMARK SUITE (EVAL HARNESS) - EXECUTION RUN")
|
|
215
|
-
print("=" * 78)
|
|
216
|
-
print(f" Target Scope : {len(challenges)} production-grade challenges")
|
|
217
|
-
print(f" Active Domains : locator_drift, api_contract_drift, timing_and_race,")
|
|
218
|
-
print(f" stale_and_mutation, fake_pass_tautology")
|
|
219
|
-
print(f" Quality Standard : 9 Quality Gates + Mutation Score >= 80% + Safety Firewall")
|
|
220
|
-
print("=" * 78 + "\n")
|
|
221
|
-
|
|
222
|
-
self.results = []
|
|
223
|
-
for idx, ch in enumerate(challenges, start=1):
|
|
224
|
-
res = self.evaluate_challenge(ch)
|
|
225
|
-
self.results.append(res)
|
|
226
|
-
status_tag = "✅ PASS" if res["status"] == "PASSED" else "❌ FAIL"
|
|
227
|
-
print(f"[{idx:02d}/{len(challenges):02d}] {res['id']} | {res['category']:<20} | {status_tag} | {res['time_to_heal_sec']:.3f}s | {res['token_consumption']} tokens")
|
|
228
|
-
print(f" 👉 {res['name']}")
|
|
229
|
-
|
|
230
|
-
summary = self.calculate_summary()
|
|
231
|
-
self.export_results(summary)
|
|
232
|
-
return summary
|
|
233
|
-
|
|
234
|
-
def calculate_summary(self) -> Dict[str, Any]:
|
|
235
|
-
"""Calculates global metrics across all evaluated challenges."""
|
|
236
|
-
total = len(self.results)
|
|
237
|
-
passed = sum(1 for r in self.results if r["status"] == "PASSED")
|
|
238
|
-
resolution_rate = round((passed / total) * 100.0, 2) if total else 0.0
|
|
239
|
-
|
|
240
|
-
avg_mttr = round(sum(r["time_to_heal_sec"] for r in self.results) / total, 3) if total else 0.0
|
|
241
|
-
avg_tokens = round(sum(r["token_consumption"] for r in self.results) / total, 1) if total else 0.0
|
|
242
|
-
avg_mutation = round(sum(r["mutation_score"] for r in self.results) / total, 2) if total else 0.0
|
|
243
|
-
|
|
244
|
-
# Category breakdown
|
|
245
|
-
categories = {}
|
|
246
|
-
for r in self.results:
|
|
247
|
-
cat = r["category"]
|
|
248
|
-
if cat not in categories:
|
|
249
|
-
categories[cat] = {"total": 0, "passed": 0, "tokens": 0, "time": 0.0}
|
|
250
|
-
categories[cat]["total"] += 1
|
|
251
|
-
if r["status"] == "PASSED":
|
|
252
|
-
categories[cat]["passed"] += 1
|
|
253
|
-
categories[cat]["tokens"] += r["token_consumption"]
|
|
254
|
-
categories[cat]["time"] += r["time_to_heal_sec"]
|
|
255
|
-
|
|
256
|
-
for cat, data in categories.items():
|
|
257
|
-
data["resolution_rate"] = round((data["passed"] / data["total"]) * 100.0, 1)
|
|
258
|
-
data["avg_tokens"] = round(data["tokens"] / data["total"], 1)
|
|
259
|
-
data["avg_time"] = round(data["time"] / data["total"], 3)
|
|
260
|
-
|
|
261
|
-
return {
|
|
262
|
-
"timestamp": datetime.now().isoformat(),
|
|
263
|
-
"total_challenges": total,
|
|
264
|
-
"passed_challenges": passed,
|
|
265
|
-
"failed_challenges": total - passed,
|
|
266
|
-
"resolution_rate_pct": resolution_rate,
|
|
267
|
-
"avg_mttr_sec": avg_mttr,
|
|
268
|
-
"avg_token_consumption": avg_tokens,
|
|
269
|
-
"avg_mutation_score": avg_mutation,
|
|
270
|
-
"zero_side_effect_rate_pct": 100.0,
|
|
271
|
-
"safety_firewall_clearance_pct": 100.0,
|
|
272
|
-
"categories": categories,
|
|
273
|
-
"details": self.results
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
def export_results(self, summary: Dict[str, Any]):
|
|
277
|
-
"""Exports evaluation results to JSON and Markdown reports."""
|
|
278
|
-
os.makedirs(REPORTS_DIR, exist_ok=True)
|
|
279
|
-
json_path = os.path.join(REPORTS_DIR, "eval_results.json")
|
|
280
|
-
with open(json_path, "w", encoding="utf-8") as f:
|
|
281
|
-
json.dump(summary, f, indent=2)
|
|
282
|
-
|
|
283
|
-
md_path = os.path.join(REPORTS_DIR, "BENCHMARK_REPORT.md")
|
|
284
|
-
with open(md_path, "w", encoding="utf-8") as f:
|
|
285
|
-
f.write(self.generate_markdown_report(summary))
|
|
286
|
-
|
|
287
|
-
print("\n" + "=" * 78)
|
|
288
|
-
print(" BENCHMARK EVALUATION SUMMARY")
|
|
289
|
-
print("=" * 78)
|
|
290
|
-
print(f" Total Challenges : {summary['total_challenges']}")
|
|
291
|
-
print(f" Autonomous Resolution : {summary['passed_challenges']} / {summary['total_challenges']} ({summary['resolution_rate_pct']}%)")
|
|
292
|
-
print(f" Mean Time-to-Heal (MTTR): {summary['avg_mttr_sec']}s")
|
|
293
|
-
print(f" Average Token Cost : {summary['avg_token_consumption']} tokens / bug")
|
|
294
|
-
print(f" Average Mutation Score : {summary['avg_mutation_score']}% (Required >= 80%)")
|
|
295
|
-
print(f" Zero-Side-Effect Rate : {summary['zero_side_effect_rate_pct']}%")
|
|
296
|
-
print("=" * 78)
|
|
297
|
-
print(f"📄 Full Markdown Report saved: {md_path}")
|
|
298
|
-
print(f"📊 Machine-readable JSON : {json_path}\n")
|
|
299
|
-
|
|
300
|
-
def generate_markdown_report(self, summary: Dict[str, Any]) -> str:
|
|
301
|
-
"""Generates comprehensive empirical Markdown benchmark report."""
|
|
302
|
-
cat_rates = [data['resolution_rate'] for data in summary["categories"].values()]
|
|
303
|
-
bar_chart_str = f"bar {cat_rates}" if cat_rates else "bar []"
|
|
304
|
-
|
|
305
|
-
# Truth Anchor: chart axes and scope MUST reflect only what was actually run.
|
|
306
|
-
# A filtered (partial) run must never render domain labels it did not evaluate.
|
|
307
|
-
DOMAIN_LABELS = {
|
|
308
|
-
"locator_drift": "Locator Drift",
|
|
309
|
-
"api_contract_drift": "API Contract",
|
|
310
|
-
"timing_and_race": "Timing & Race",
|
|
311
|
-
"stale_and_mutation": "Stale DOM",
|
|
312
|
-
"fake_pass_tautology": "Mutation/Tautology",
|
|
313
|
-
}
|
|
314
|
-
run_domains = list(summary["categories"].keys())
|
|
315
|
-
axis_labels = json.dumps([DOMAIN_LABELS.get(c, c.replace("_", " ").title()) for c in run_domains])
|
|
316
|
-
is_partial = summary["total_challenges"] < 25
|
|
317
|
-
scope_line = (
|
|
318
|
-
f"> **Run Scope:** ⚠️ PARTIAL — {summary['total_challenges']}/25 challenges"
|
|
319
|
-
f" (domains evaluated: {', '.join(run_domains) if run_domains else 'none'})."
|
|
320
|
-
" Executive metrics below describe ONLY these domains, not the full benchmark suite. \n"
|
|
321
|
-
if is_partial
|
|
322
|
-
else "> **Run Scope:** FULL — 25/25 challenges across all 5 domains. \n"
|
|
323
|
-
)
|
|
324
|
-
|
|
325
|
-
report = f"""# QA Agent Benchmark Evaluation Report (Eval Harness)
|
|
326
|
-
|
|
327
|
-
> **Evaluation Suite:** Enterprise QA Automation Self-Healing & Defect Sentinel Benchmark
|
|
328
|
-
> **Target Framework:** `agent-skills` QA Hub
|
|
329
|
-
> **Timestamp:** `{summary['timestamp']}`
|
|
330
|
-
{scope_line}> **Discipline:** Truth Anchor (No Fake Pass) + Hard Policy Enforcement + 9 Quality Gates
|
|
331
|
-
|
|
332
|
-
---
|
|
333
|
-
|
|
334
|
-
## 1. Executive Summary
|
|
335
|
-
|
|
336
|
-
| Core Benchmark Metric | Measured Score | Enterprise SLA Target | Status |
|
|
337
|
-
| :--- | :--- | :--- | :--- |
|
|
338
|
-
| **Autonomous Resolution Rate** | **{summary['resolution_rate_pct']}%** | $\\ge 90.0\\%$ | {'✅ PASS' if summary['resolution_rate_pct'] >= 90 else '❌ FAIL'} |
|
|
339
|
-
| **Mean Time to Resolution (MTTR)** | **{summary['avg_mttr_sec']}s** | $\\le 3.00\\text{{s}}$ | ✅ PASS |
|
|
340
|
-
| **Average Token Cost** | **{summary['avg_token_consumption']} tokens** | $\\le 500\\text{{ tokens}}$ | ✅ PASS |
|
|
341
|
-
| **AST Mutation Score** | **{summary['avg_mutation_score']}%** | $\\ge 80.0\\%$ | ✅ PASS |
|
|
342
|
-
| **Zero-Side-Effect Rate** | **{summary['zero_side_effect_rate_pct']}%** | $100.0\\%$ | ✅ PASS |
|
|
343
|
-
| **Action Safety Firewall Clearance** | **{summary['safety_firewall_clearance_pct']}%** | $100.0\\%$ | ✅ PASS |
|
|
344
|
-
|
|
345
|
-
---
|
|
346
|
-
|
|
347
|
-
## 2. Performance Breakdown by Failure Domain
|
|
348
|
-
|
|
349
|
-
```mermaid
|
|
350
|
-
xychart-beta
|
|
351
|
-
title "Autonomous Resolution Rate by Domain (%)"
|
|
352
|
-
x-axis {axis_labels}
|
|
353
|
-
y-axis "Resolution Rate (%)" 0 --> 100
|
|
354
|
-
{bar_chart_str}
|
|
355
|
-
```
|
|
356
|
-
|
|
357
|
-
| Failure Domain | Total Challenges | Passed | Resolution Rate | Avg MTTR | Avg Tokens |
|
|
358
|
-
| :--- | :---: | :---: | :---: | :---: | :---: |
|
|
359
|
-
"""
|
|
360
|
-
for cat, data in summary["categories"].items():
|
|
361
|
-
cat_label = cat.replace("_", " ").title()
|
|
362
|
-
report += f"| **{cat_label}** | {data['total']} | {data['passed']} | {data['resolution_rate']}% | {data['avg_time']}s | {data['avg_tokens']} |\n"
|
|
363
|
-
|
|
364
|
-
report += """
|
|
365
|
-
---
|
|
366
|
-
|
|
367
|
-
## 3. Empirical Instrumentation & Measurement Protocol
|
|
368
|
-
|
|
369
|
-
| Evaluation Domain | Measurement Tool | Verification Mechanism |
|
|
370
|
-
| :--- | :---: | :--- |
|
|
371
|
-
| **Locator Drift Healing** | `tools/heuristic_locator_matcher.py` | 3-Layer matching cascade verified dynamically against DOM element tags & attributes |
|
|
372
|
-
| **API Contract Shift** | `tools/api_drift_sentinel.py` | AST assertion contract validation on response headers and status codes |
|
|
373
|
-
| **Timing & Race Conditions** | `tools/cache_resolver.py` | Smart barrier replacement eliminating `Thread.sleep` and verifying detachment |
|
|
374
|
-
| **Stale Elements** | In-place dynamic refetching | Mutation detachment prevention with dynamic attribute locators |
|
|
375
|
-
| **Tautology / Fake-Pass** | `tools/micro_mutator.py` | Active AST mutant injection (requires $\\ge 80\\%$ mutation score) |
|
|
376
|
-
| **Destructive Action Guard** | `tools/safety_firewall.py` | Regex and AST pre-flight firewall blocking irreversible actions |
|
|
377
|
-
|
|
378
|
-
---
|
|
379
|
-
|
|
380
|
-
## 4. Detailed Challenge Matrix
|
|
381
|
-
|
|
382
|
-
| ID | Challenge Name | Domain | Difficulty | MTTR | Tokens | Mutation Score | Status |
|
|
383
|
-
| :--- | :--- | :--- | :---: | :---: | :---: | :---: | :---: |
|
|
384
|
-
"""
|
|
385
|
-
for item in summary["details"]:
|
|
386
|
-
status_emoji = "✅ PASS" if item["status"] == "PASSED" else "❌ FAIL"
|
|
387
|
-
report += f"| `{item['id']}` | {item['name']} | `{item['category']}` | {item['difficulty']} | {item['time_to_heal_sec']:.3f}s | {item['token_consumption']} | {item['mutation_score']}% | {status_emoji} |\n"
|
|
388
|
-
|
|
389
|
-
report += """
|
|
390
|
-
---
|
|
391
|
-
|
|
392
|
-
## 5. Verification & Methodological Rigor
|
|
393
|
-
|
|
394
|
-
All benchmark challenges were evaluated against deterministic test oracles:
|
|
395
|
-
1. **Deterministic Resolution:** Healed selectors and patches were verified against DOM snapshots and API schemas.
|
|
396
|
-
2. **Mutation Quality Verification:** Tautology challenges were verified using `tools/micro_mutator.py` to ensure assertions caught injected mutants.
|
|
397
|
-
3. **Safety Clearance:** Action statements were evaluated using `tools/safety_firewall.py` in dry-run mode to confirm zero destructive side effects.
|
|
398
|
-
"""
|
|
399
|
-
return report
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
def main():
|
|
403
|
-
parser = argparse.ArgumentParser(description="QA Agent Benchmark Evaluation Runner")
|
|
404
|
-
parser.add_argument("--category", default="", help="Filter evaluation by category (e.g. locator_drift, api_contract_drift)")
|
|
405
|
-
parser.add_argument("--json", action="store_true", help="Print summary in JSON format")
|
|
406
|
-
args = parser.parse_args()
|
|
407
|
-
|
|
408
|
-
evaluator = BenchmarkEvaluator()
|
|
409
|
-
summary = evaluator.run_all(category_filter=args.category)
|
|
410
|
-
|
|
411
|
-
if args.json:
|
|
412
|
-
print(json.dumps(summary, indent=2))
|
|
413
|
-
|
|
414
|
-
return 0 if summary.get("resolution_rate_pct", 0) >= 90.0 else 1
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
if __name__ == "__main__":
|
|
418
|
-
sys.exit(main())
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
# 🌲 Cypress JavaScript Reference Suite (Clean POM & Synapse Guardrails)
|
|
2
|
-
|
|
3
|
-
This directory provides a production-grade **Cypress (JavaScript)** reference suite demonstrating the Clean Page Object Model (POM) and Flat Architecture standards enforced by **Synapse Hub**.
|
|
4
|
-
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
## 🌟 End-to-End Testing Lifecycle (Story ➔ Test Cases ➔ Script ➔ Report)
|
|
8
|
-
|
|
9
|
-
```mermaid
|
|
10
|
-
flowchart LR
|
|
11
|
-
Story["1. User Story<br/>(E-Commerce Checkout)"] --> Matrix["2. Test Case Matrix<br/>(Equivalence & Boundary)"]
|
|
12
|
-
Matrix --> POM["3. Clean POM Pages<br/>(CartPage, CheckoutPage)"]
|
|
13
|
-
POM --> Spec["4. Cypress Spec<br/>(checkout.cy.js)"]
|
|
14
|
-
Spec --> Report["5. JUnit / Mochawesome<br/>(DoD Verified)"]
|
|
15
|
-
```
|
|
16
|
-
|
|
17
|
-
### 1. User Story & Acceptance Criteria
|
|
18
|
-
- **Story:** *As a customer, I want to review my shopping cart and apply a valid discount coupon so that my total price updates before placing an order.*
|
|
19
|
-
- **Acceptance Criteria (AC):**
|
|
20
|
-
- AC1: Cart must show items, quantities, and price breakdown.
|
|
21
|
-
- AC2: Valid coupon code (e.g. `SAVE20`) deducts 20% from subtotal.
|
|
22
|
-
- AC3: Invalid coupon code displays red error toast: *"Mã giảm giá không hợp lệ"*.
|
|
23
|
-
- AC4: Empty cart checkout button must remain disabled.
|
|
24
|
-
|
|
25
|
-
### 2. Key Architectural Patterns Enforced
|
|
26
|
-
- **Zero Raw Selectors in Specs:** `cypress/e2e/checkout.cy.js` calls only semantic methods on `CartPage` and `CheckoutPage`.
|
|
27
|
-
- **Resilient ARIA Locators:** Page objects use `cy.findByRole`, `cy.findByLabel`, or scoped semantic CSS.
|
|
28
|
-
- **Dynamic Idempotency:** Uses `BasePage.uniqueCode('ORDER_')` for order identifiers.
|
|
29
|
-
- **Smart Synchronization:** Replaces static `cy.wait(5000)` with network route assertions and spinner detachment.
|
|
30
|
-
|
|
31
|
-
---
|
|
32
|
-
|
|
33
|
-
## 🚀 How to Run & Verify
|
|
34
|
-
|
|
35
|
-
```bash
|
|
36
|
-
# 1. Run Synapse AST Test Linter (Zero-token verification)
|
|
37
|
-
npx synapse-hub lint ./cypress/e2e
|
|
38
|
-
|
|
39
|
-
# 2. Verify 9 Quality Gates DoD
|
|
40
|
-
npx synapse-hub verify
|
|
41
|
-
|
|
42
|
-
# 3. Run Headless Cypress
|
|
43
|
-
npm install
|
|
44
|
-
npx cypress run --spec "cypress/e2e/checkout.cy.js"
|
|
45
|
-
```
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
const cartPage = require('../pages/CartPage');
|
|
2
|
-
|
|
3
|
-
describe('E-Commerce Cart & Discount Coupon Validation', () => {
|
|
4
|
-
beforeEach(() => {
|
|
5
|
-
cartPage.openCart();
|
|
6
|
-
});
|
|
7
|
-
|
|
8
|
-
it('TC-ECOM-01: Successfully apply 20% discount coupon to cart', () => {
|
|
9
|
-
cartPage.applyCoupon('SAVE20');
|
|
10
|
-
cartPage.verifyDiscountApplied('-20%');
|
|
11
|
-
});
|
|
12
|
-
|
|
13
|
-
it('TC-ECOM-02: Display error when applying invalid or expired coupon', () => {
|
|
14
|
-
cartPage.applyCoupon('EXPIRED_COUPON_999');
|
|
15
|
-
cartPage.verifyCouponError('Mã giảm giá không hợp lệ');
|
|
16
|
-
});
|
|
17
|
-
});
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* BasePage — Universal Cypress Base Page Object
|
|
3
|
-
* Enforces dynamic idempotency, smart waiting, and common interactions.
|
|
4
|
-
*/
|
|
5
|
-
class BasePage {
|
|
6
|
-
/**
|
|
7
|
-
* Generates dynamic unique identifier for test data isolation.
|
|
8
|
-
*/
|
|
9
|
-
static uniqueCode(prefix = 'AUTO_') {
|
|
10
|
-
return `${prefix}${Date.now()}_${Math.floor(Math.random() * 1000)}`;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* Waits for loading spinner/overlay detachment.
|
|
15
|
-
*/
|
|
16
|
-
waitForSpinnerDetached() {
|
|
17
|
-
cy.get('.spinner-overlay, .loading-backdrop', { timeout: 10000 }).should('not.exist');
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Navigates to a relative path.
|
|
22
|
-
*/
|
|
23
|
-
navigate(path = '/') {
|
|
24
|
-
cy.visit(path);
|
|
25
|
-
this.waitForSpinnerDetached();
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
module.exports = BasePage;
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
const BasePage = require('./BasePage');
|
|
2
|
-
|
|
3
|
-
class CartPage extends BasePage {
|
|
4
|
-
// Encapsulated locators
|
|
5
|
-
get couponInput() {
|
|
6
|
-
return cy.get('input[name="couponCode"]');
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
get applyCouponButton() {
|
|
10
|
-
return cy.contains('button', 'Áp dụng');
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
get discountBadge() {
|
|
14
|
-
return cy.get('[data-testid="discount-amount"]');
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
get errorMessageToast() {
|
|
18
|
-
return cy.get('.toast-error, .notification-error');
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
get checkoutButton() {
|
|
22
|
-
return cy.contains('button', 'Tiến hành đặt hàng');
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
// Business Action Methods
|
|
26
|
-
openCart() {
|
|
27
|
-
this.navigate('/cart');
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
applyCoupon(couponCode) {
|
|
31
|
-
this.couponInput.clear().type(couponCode);
|
|
32
|
-
this.applyCouponButton.click();
|
|
33
|
-
this.waitForSpinnerDetached();
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
proceedToCheckout() {
|
|
37
|
-
this.checkoutButton.click();
|
|
38
|
-
this.waitForSpinnerDetached();
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
// Verification Methods
|
|
42
|
-
verifyDiscountApplied(expectedDiscount) {
|
|
43
|
-
this.discountBadge.should('be.visible').and('contain.text', expectedDiscount);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
verifyCouponError(expectedError) {
|
|
47
|
-
this.errorMessageToast.should('be.visible').and('contain.text', expectedError);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
module.exports = new CartPage();
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
const { defineConfig } = require('cypress');
|
|
2
|
-
|
|
3
|
-
module.exports = defineConfig({
|
|
4
|
-
e2e: {
|
|
5
|
-
baseUrl: 'https://staging-shop.example.com',
|
|
6
|
-
specPattern: 'cypress/e2e/**/*.cy.js',
|
|
7
|
-
supportFile: false,
|
|
8
|
-
viewportWidth: 1920,
|
|
9
|
-
viewportHeight: 1080,
|
|
10
|
-
video: false,
|
|
11
|
-
screenshotOnRunFailure: true,
|
|
12
|
-
},
|
|
13
|
-
});
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "cypress-ecommerce-suite",
|
|
3
|
-
"version": "1.0.0",
|
|
4
|
-
"description": "Production Cypress Clean POM Reference Suite for Synapse-Hub",
|
|
5
|
-
"scripts": {
|
|
6
|
-
"test": "cypress run",
|
|
7
|
-
"test:open": "cypress open",
|
|
8
|
-
"lint:synapse": "npx synapse-hub lint ./cypress/e2e"
|
|
9
|
-
},
|
|
10
|
-
"devDependencies": {
|
|
11
|
-
"cypress": "^13.6.0"
|
|
12
|
-
}
|
|
13
|
-
}
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
package pages
|
|
2
|
-
|
|
3
|
-
import com.kms.katalon.core.webui.keyword.WebUiBuiltInKeywords as WebUI
|
|
4
|
-
import com.kms.katalon.core.model.FailureHandling
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* BasePage encapsulating reusable Clean POM operations and smart synchronization.
|
|
8
|
-
*/
|
|
9
|
-
public class BasePage {
|
|
10
|
-
|
|
11
|
-
public static String uniqueCode(String prefix = "AUTO_") {
|
|
12
|
-
return prefix + System.currentTimeMillis()
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
public void waitForReady(int timeoutSec = 10) {
|
|
16
|
-
// Smart wait: wait for page document state without static Thread.sleep
|
|
17
|
-
WebUI.waitForPageLoad(timeoutSec, FailureHandling.OPTIONAL)
|
|
18
|
-
}
|
|
19
|
-
}
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
package pages
|
|
2
|
-
|
|
3
|
-
import com.kms.katalon.core.testobject.TestObject
|
|
4
|
-
import com.kms.katalon.core.testobject.ConditionType
|
|
5
|
-
import com.kms.katalon.core.webui.keyword.WebUiBuiltInKeywords as WebUI
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* DashboardPage displaying business KPIs and metrics.
|
|
9
|
-
*/
|
|
10
|
-
public class DashboardPage extends BasePage {
|
|
11
|
-
|
|
12
|
-
private TestObject headerDashboard = makeTestObject("//h1[contains(text(), 'Dashboard')]")
|
|
13
|
-
private TestObject cardActiveUsers = makeTestObject("//div[@data-testid='card-active-users']")
|
|
14
|
-
|
|
15
|
-
private static TestObject makeTestObject(String xpath) {
|
|
16
|
-
TestObject to = new TestObject()
|
|
17
|
-
to.addProperty("xpath", ConditionType.EQUALS, xpath)
|
|
18
|
-
return to
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
public boolean isDashboardLoaded() {
|
|
22
|
-
waitForReady()
|
|
23
|
-
return WebUI.verifyElementPresent(headerDashboard, 10)
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
public String getActiveUsersCount() {
|
|
27
|
-
return WebUI.getText(cardActiveUsers)
|
|
28
|
-
}
|
|
29
|
-
}
|