synapse-hub 0.0.0-stage → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/agents/failure-analyst.md +47 -0
- package/.agents/agents/live-explorer.md +41 -0
- package/.agents/agents/orchestrator.md +43 -0
- package/.agents/agents/pom-crafter.md +81 -0
- package/.agents/agents/quality-gatekeeper.md +47 -0
- package/.agents/agents/test-runner.md +43 -0
- package/.agents/rules/context-engineering.md +88 -0
- package/.agents/rules/git-push-protocol.md +101 -0
- package/.agents/rules/graphify.md +14 -0
- package/.agents/rules/lessons-learned.md +23 -0
- package/.agents/rules/mandatory-playwright-live.md +29 -0
- package/.agents/rules/skill-precedence.md +42 -0
- package/.agents/rules/systematic-debugging.md +99 -0
- package/.agents/skills/agent-orchestration-multi-agent-optimize/SKILL.md +239 -0
- package/.agents/skills/antigravity-test-sentinel/SKILL.md +64 -0
- package/.agents/skills/api-network-mocking-interception/SKILL.md +142 -0
- package/.agents/skills/api-spec-to-e2e-matrix/SKILL.md +70 -0
- package/.agents/skills/architecture-decision-records/SKILL.md +441 -0
- package/.agents/skills/auth-session-and-cookie-manager/SKILL.md +72 -0
- package/.agents/skills/backend-load-and-stress-patterns/SKILL.md +214 -0
- package/.agents/skills/bats-testing-patterns/SKILL.md +100 -0
- package/.agents/skills/bats-testing-patterns/resources/implementation-playbook.md +614 -0
- package/.agents/skills/clean-pom-crafter/SKILL.md +75 -0
- package/.agents/skills/clean-pom-crafter/examples/standard_test_case.groovy +28 -0
- package/.agents/skills/clean-pom-crafter/references/pom_patterns.groovy +138 -0
- package/.agents/skills/code-refactoring-refactor-clean/SKILL.md +64 -0
- package/.agents/skills/code-refactoring-refactor-clean/resources/implementation-playbook.md +879 -0
- package/.agents/skills/db-state-verifier-and-seeder/SKILL.md +123 -0
- package/.agents/skills/e2e-testing-patterns/SKILL.md +103 -0
- package/.agents/skills/e2e-testing-patterns/resources/implementation-playbook.md +522 -0
- package/.agents/skills/failure-autopsy-and-healing/SKILL.md +54 -0
- package/.agents/skills/failure-autopsy-and-healing/references/healing_matrix.md +64 -0
- package/.agents/skills/failure-autopsy-and-healing/references/qa_investigation_protocol.md +56 -0
- package/.agents/skills/failure-autopsy-and-healing/scripts/triage_failures.sh +98 -0
- package/.agents/skills/file-and-report-validator/SKILL.md +90 -0
- package/.agents/skills/flaky-test-sentinel-and-quarantine/SKILL.md +114 -0
- package/.agents/skills/github-actions-templates/SKILL.md +334 -0
- package/.agents/skills/graphify-impact-analyzer/SKILL.md +78 -0
- package/.agents/skills/katalon-cli-runner/SKILL.md +76 -0
- package/.agents/skills/katalon-cli-runner/references/groovy-patterns.md +209 -0
- package/.agents/skills/katalon-cli-runner/references/katalon-api.md +155 -0
- package/.agents/skills/katalon-cli-runner/references/runner_commands.md +67 -0
- package/.agents/skills/lam-script-generator/SKILL.md +83 -0
- package/.agents/skills/playwright-live-interaction/SKILL.md +86 -0
- package/.agents/skills/playwright-live-interaction/references/advanced_superpowers.md +160 -0
- package/.agents/skills/playwright-live-interaction/references/compact_accessibility_snapshots.md +75 -0
- package/.agents/skills/playwright-live-interaction/references/recipes.md +388 -0
- package/.agents/skills/playwright-live-interaction/references/swarm_orchestration.md +134 -0
- package/.agents/skills/security-dast-sentinel/SKILL.md +176 -0
- package/.agents/skills/self-healing-locator/SKILL.md +73 -0
- package/.agents/skills/synthetic-test-data-generator/SKILL.md +135 -0
- package/.agents/skills/token-and-context-optimizer/SKILL.md +70 -0
- package/.agents/skills/visual-regression-sentinel/SKILL.md +62 -0
- package/.agents/skills/wcag-audit-patterns/SKILL.md +77 -0
- package/.agents/skills/wcag-audit-patterns/resources/implementation-playbook.md +541 -0
- package/.agents/skills/web-vitals-and-performance-profiler/SKILL.md +65 -0
- package/.agents/templates/completion_report.md +57 -0
- package/.agents/workflows/graphify.md +10 -0
- package/.agents/workflows/test-impact.md +15 -0
- package/.agents/workflows/verify.md +34 -0
- package/AGENTS.md +115 -0
- package/README.md +394 -2
- package/bin/agent-qa.js +90 -0
- package/package.json +59 -4
- package/skills_manifest.json +262 -0
- package/templates/PROJECT_AI_GUIDE_TEMPLATE.md +97 -0
- package/templates/PROJECT_STATE_TEMPLATE.md +20 -0
- package/tools/TOOLS_RUNBOOK.md +298 -0
- package/tools/agent_canary.py +501 -0
- package/tools/analyze_project.py +242 -0
- package/tools/api_drift_sentinel.py +208 -0
- package/tools/auto_heal.py +189 -0
- package/tools/browser_helpers.js +593 -0
- package/tools/bug_reporter.py +224 -0
- package/tools/cache_resolver.py +846 -0
- package/tools/check_env.py +288 -0
- package/tools/ci_healing_hook.py +128 -0
- package/tools/cli.py +402 -0
- package/tools/config_decoupler.py +253 -0
- package/tools/db_sandbox.py +341 -0
- package/tools/distill_log.py +152 -0
- package/tools/e2e_flow_prober.ps1 +65 -0
- package/tools/flaky_detector.py +222 -0
- package/tools/fuzz_payloads.py +197 -0
- package/tools/generate_allure_report.ps1 +30 -0
- package/tools/generate_api_matrix.py +193 -0
- package/tools/generate_tc_from_story.py +182 -0
- package/tools/generate_test_reports.ps1 +135 -0
- package/tools/git_pre_commit_guard.py +203 -0
- package/tools/heal_pr_automator.py +195 -0
- package/tools/heuristic_locator_matcher.py +202 -0
- package/tools/infer_linter.py +238 -0
- package/tools/lint_ai_docs.py +254 -0
- package/tools/lookup_tc.py +165 -0
- package/tools/lookup_xpath.py +167 -0
- package/tools/matrix_sharder.py +146 -0
- package/tools/mcp_excel_server.ps1 +251 -0
- package/tools/mcp_server.py +490 -0
- package/tools/micro_mutator.py +309 -0
- package/tools/network_mock_shield.py +170 -0
- package/tools/omni_emitter.py +500 -0
- package/tools/omni_parser.py +435 -0
- package/tools/omni_transpiler.py +168 -0
- package/tools/pinpoint_failure.py +88 -0
- package/tools/project_state.py +169 -0
- package/tools/repo_map.json +12 -0
- package/tools/repo_map.py +277 -0
- package/tools/repo_map.txt +7 -0
- package/tools/safety_firewall.py +245 -0
- package/tools/setup_project.py +265 -0
- package/tools/smart_visual_masker.py +134 -0
- package/tools/test_impact.py +321 -0
- package/tools/tms_sync_bridge.py +342 -0
- package/tools/trace_data_flow.py +240 -0
- package/tools/traffic_session_converter.py +256 -0
- package/tools/universal_ir.py +480 -0
- package/tools/validate_groovy_scripts.py +173 -0
- package/tools/verify_all.py +496 -0
- package/tools/verify_all.sh +4 -0
- package/tools/visual_diff_canvas.py +142 -0
- package/tools/visual_som_helper.py +418 -0
- package/tools/wcag_audit.py +214 -0
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Enterprise Vision-Grounded Fallback & Set-of-Marks (SoM) Engine
|
|
4
|
+
--------------------------------------------------------------
|
|
5
|
+
Provides multimodal coordinate grounding when standard DOM locators fail
|
|
6
|
+
(HTML5 Canvas, WebGL financial charts, deeply nested Shadow DOM, cross-origin iframes).
|
|
7
|
+
|
|
8
|
+
Features:
|
|
9
|
+
1. Set-of-Marks (SoM) Overlay Generation:
|
|
10
|
+
- Detects interactive visual regions (or accepts DOM bounding boxes).
|
|
11
|
+
- Renders high-contrast colored bounding boxes with numeric IDs (1, 2, 3...) on screenshots.
|
|
12
|
+
2. Coordinate Resolution & Physical Mouse Dispatch:
|
|
13
|
+
- Maps Mark IDs and visual targets to exact center coordinates (x, y).
|
|
14
|
+
- Generates direct Playwright physical mouse actions: await page.mouse.click(x, y).
|
|
15
|
+
3. Visual Perceptual Caching & Token Economy:
|
|
16
|
+
- Generates difference hash (dHash) fingerprints for visual components.
|
|
17
|
+
- Caches coordinates in SQLite (memory_cache.db). If visual similarity > 98%,
|
|
18
|
+
instantly retrieves cached coordinates without expensive Vision LLM calls (Zero-Token).
|
|
19
|
+
4. Compact Vision Prompt Formatter:
|
|
20
|
+
- Formats token-efficient briefs for Multimodal Vision Models (Gemini / Claude / GPT-4o).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import os
|
|
24
|
+
import sys
|
|
25
|
+
import json
|
|
26
|
+
import sqlite3
|
|
27
|
+
import argparse
|
|
28
|
+
from typing import Dict, Any, List, Optional, Tuple, Union
|
|
29
|
+
from PIL import Image, ImageDraw, ImageFont
|
|
30
|
+
|
|
31
|
+
# Enforce UTF-8 stdout encoding for Windows console compatibility
|
|
32
|
+
if hasattr(sys.stdout, 'reconfigure'):
|
|
33
|
+
try:
|
|
34
|
+
sys.stdout.reconfigure(encoding='utf-8')
|
|
35
|
+
sys.stderr.reconfigure(encoding='utf-8')
|
|
36
|
+
except Exception:
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
40
|
+
QA_HUB_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
|
|
41
|
+
DEFAULT_CACHE_DB = os.path.join(SCRIPT_DIR, "memory_cache.db")
|
|
42
|
+
|
|
43
|
+
MARK_COLORS = [
|
|
44
|
+
"#FF0055", # Vibrant Red/Pink
|
|
45
|
+
"#00CCFF", # Cyan
|
|
46
|
+
"#00FF66", # Neon Green
|
|
47
|
+
"#FFCC00", # Bright Yellow
|
|
48
|
+
"#9933FF", # Purple
|
|
49
|
+
"#FF6600", # Orange
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def compute_visual_hash(image_source: Union[Image.Image, str], bbox: Optional[List[int]] = None, hash_size: int = 8) -> str:
|
|
54
|
+
"""
|
|
55
|
+
Computes a 64-bit difference hash (dHash) for an image or cropped component.
|
|
56
|
+
Fast, deterministic, and 100% offline.
|
|
57
|
+
"""
|
|
58
|
+
if isinstance(image_source, str):
|
|
59
|
+
if not os.path.exists(image_source):
|
|
60
|
+
raise FileNotFoundError(f"Image not found: {image_source}")
|
|
61
|
+
img = Image.open(image_source)
|
|
62
|
+
else:
|
|
63
|
+
img = image_source
|
|
64
|
+
|
|
65
|
+
if bbox and len(bbox) == 4:
|
|
66
|
+
img = img.crop(bbox)
|
|
67
|
+
|
|
68
|
+
# Resize to (hash_size + 1, hash_size) in grayscale
|
|
69
|
+
resized = img.convert("L").resize((hash_size + 1, hash_size), Image.Resampling.LANCZOS)
|
|
70
|
+
pixels = list(resized.tobytes())
|
|
71
|
+
|
|
72
|
+
diff = []
|
|
73
|
+
for row in range(hash_size):
|
|
74
|
+
row_start = row * (hash_size + 1)
|
|
75
|
+
for col in range(hash_size):
|
|
76
|
+
left = pixels[row_start + col]
|
|
77
|
+
right = pixels[row_start + col + 1]
|
|
78
|
+
diff.append("1" if left > right else "0")
|
|
79
|
+
|
|
80
|
+
return "".join(diff)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def calculate_visual_similarity(hash1: str, hash2: str) -> float:
|
|
84
|
+
"""Calculates normalized Hamming similarity between two binary hashes (0.0 to 1.0)."""
|
|
85
|
+
if not hash1 or not hash2 or len(hash1) != len(hash2):
|
|
86
|
+
return 0.0
|
|
87
|
+
hamming_dist = sum(c1 != c2 for c1, c2 in zip(hash1, hash2))
|
|
88
|
+
return round(1.0 - (hamming_dist / len(hash1)), 4)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class VisualSoMHelper:
|
|
92
|
+
"""Manages visual grounding, bounding box marking, visual caching, and coordinate dispatch."""
|
|
93
|
+
|
|
94
|
+
def __init__(self, viewport_width: int = 1920, viewport_height: int = 1080, db_path: Optional[str] = None):
|
|
95
|
+
self.viewport_width = viewport_width
|
|
96
|
+
self.viewport_height = viewport_height
|
|
97
|
+
self.db_path = db_path or DEFAULT_CACHE_DB
|
|
98
|
+
self.init_cache_table()
|
|
99
|
+
|
|
100
|
+
def init_cache_table(self):
|
|
101
|
+
"""Initializes the visual_som_cache SQLite table if not already present."""
|
|
102
|
+
try:
|
|
103
|
+
conn = sqlite3.connect(self.db_path, timeout=10.0)
|
|
104
|
+
try:
|
|
105
|
+
conn.execute("""
|
|
106
|
+
CREATE TABLE IF NOT EXISTS visual_som_cache (
|
|
107
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
108
|
+
component_name TEXT NOT NULL,
|
|
109
|
+
visual_hash TEXT NOT NULL,
|
|
110
|
+
center_x INTEGER NOT NULL,
|
|
111
|
+
center_y INTEGER NOT NULL,
|
|
112
|
+
rel_x REAL DEFAULT 0.0,
|
|
113
|
+
rel_y REAL DEFAULT 0.0,
|
|
114
|
+
hit_count INTEGER DEFAULT 0,
|
|
115
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
116
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
117
|
+
UNIQUE(component_name, visual_hash)
|
|
118
|
+
);
|
|
119
|
+
""")
|
|
120
|
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_som_component ON visual_som_cache (component_name);")
|
|
121
|
+
conn.commit()
|
|
122
|
+
finally:
|
|
123
|
+
conn.close()
|
|
124
|
+
except Exception:
|
|
125
|
+
pass
|
|
126
|
+
|
|
127
|
+
def lookup_cached_coordinates(
|
|
128
|
+
self,
|
|
129
|
+
component_name: str,
|
|
130
|
+
image_source: Union[Image.Image, str],
|
|
131
|
+
bbox: Optional[List[int]] = None,
|
|
132
|
+
threshold: float = 0.98,
|
|
133
|
+
db_path: Optional[str] = None
|
|
134
|
+
) -> Optional[Dict[str, Any]]:
|
|
135
|
+
"""
|
|
136
|
+
Looks up cached coordinates for a component if visual similarity exceeds threshold (default 98%).
|
|
137
|
+
Returns resolved coordinate data if cache hit, None if cache miss.
|
|
138
|
+
"""
|
|
139
|
+
current_hash = compute_visual_hash(image_source, bbox=bbox)
|
|
140
|
+
db = db_path or self.db_path
|
|
141
|
+
|
|
142
|
+
try:
|
|
143
|
+
conn = sqlite3.connect(db, timeout=10.0)
|
|
144
|
+
try:
|
|
145
|
+
conn.row_factory = sqlite3.Row
|
|
146
|
+
cursor = conn.cursor()
|
|
147
|
+
cursor.execute(
|
|
148
|
+
"SELECT id, component_name, visual_hash, center_x, center_y, hit_count FROM visual_som_cache WHERE component_name = ?",
|
|
149
|
+
(component_name,)
|
|
150
|
+
)
|
|
151
|
+
rows = cursor.fetchall()
|
|
152
|
+
|
|
153
|
+
best_match = None
|
|
154
|
+
best_sim = -1.0
|
|
155
|
+
|
|
156
|
+
for row in rows:
|
|
157
|
+
cached_hash = row["visual_hash"]
|
|
158
|
+
sim = calculate_visual_similarity(current_hash, cached_hash)
|
|
159
|
+
if sim >= threshold and sim > best_sim:
|
|
160
|
+
best_sim = sim
|
|
161
|
+
best_match = row
|
|
162
|
+
|
|
163
|
+
if best_match:
|
|
164
|
+
match_id = best_match["id"]
|
|
165
|
+
cursor.execute(
|
|
166
|
+
"UPDATE visual_som_cache SET hit_count = hit_count + 1, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
167
|
+
(match_id,)
|
|
168
|
+
)
|
|
169
|
+
conn.commit()
|
|
170
|
+
|
|
171
|
+
cx = best_match["center_x"]
|
|
172
|
+
cy = best_match["center_y"]
|
|
173
|
+
return {
|
|
174
|
+
"component_name": component_name,
|
|
175
|
+
"center_x": cx,
|
|
176
|
+
"center_y": cy,
|
|
177
|
+
"visual_similarity": best_sim,
|
|
178
|
+
"visual_hash": current_hash,
|
|
179
|
+
"cached": True,
|
|
180
|
+
"hit_count": best_match["hit_count"] + 1,
|
|
181
|
+
"playwright_action": f"await page.mouse.click({cx}, {cy});"
|
|
182
|
+
}
|
|
183
|
+
finally:
|
|
184
|
+
conn.close()
|
|
185
|
+
except Exception:
|
|
186
|
+
pass
|
|
187
|
+
|
|
188
|
+
return None
|
|
189
|
+
|
|
190
|
+
def save_cached_coordinates(
|
|
191
|
+
self,
|
|
192
|
+
component_name: str,
|
|
193
|
+
image_source: Union[Image.Image, str],
|
|
194
|
+
center_x: int,
|
|
195
|
+
center_y: int,
|
|
196
|
+
bbox: Optional[List[int]] = None,
|
|
197
|
+
db_path: Optional[str] = None
|
|
198
|
+
) -> str:
|
|
199
|
+
"""
|
|
200
|
+
Saves resolved coordinates and visual hash into SQLite cache.
|
|
201
|
+
"""
|
|
202
|
+
visual_hash = compute_visual_hash(image_source, bbox=bbox)
|
|
203
|
+
db = db_path or self.db_path
|
|
204
|
+
|
|
205
|
+
try:
|
|
206
|
+
conn = sqlite3.connect(db, timeout=10.0)
|
|
207
|
+
try:
|
|
208
|
+
conn.execute("""
|
|
209
|
+
INSERT INTO visual_som_cache (component_name, visual_hash, center_x, center_y, hit_count)
|
|
210
|
+
VALUES (?, ?, ?, ?, 1)
|
|
211
|
+
ON CONFLICT(component_name, visual_hash) DO UPDATE SET
|
|
212
|
+
center_x = excluded.center_x,
|
|
213
|
+
center_y = excluded.center_y,
|
|
214
|
+
hit_count = visual_som_cache.hit_count + 1,
|
|
215
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
216
|
+
""", (component_name, visual_hash, center_x, center_y))
|
|
217
|
+
conn.commit()
|
|
218
|
+
finally:
|
|
219
|
+
conn.close()
|
|
220
|
+
except Exception:
|
|
221
|
+
pass
|
|
222
|
+
|
|
223
|
+
return visual_hash
|
|
224
|
+
|
|
225
|
+
def generate_som_overlay(
|
|
226
|
+
self,
|
|
227
|
+
image_path: str,
|
|
228
|
+
elements: Optional[List[Dict[str, Any]]] = None,
|
|
229
|
+
output_path: Optional[str] = None
|
|
230
|
+
) -> Tuple[str, Dict[int, Dict[str, Any]]]:
|
|
231
|
+
"""
|
|
232
|
+
Draws high-contrast numbered badges (Set-of-Marks) over each element bounding box.
|
|
233
|
+
Returns (marked_image_path, marks_map).
|
|
234
|
+
"""
|
|
235
|
+
if not elements:
|
|
236
|
+
elements = [
|
|
237
|
+
{"x": 100, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom In", "tag": "canvas_control"},
|
|
238
|
+
{"x": 240, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom Out", "tag": "canvas_control"},
|
|
239
|
+
{"x": 500, "y": 300, "width": 200, "height": 50, "label": "Execute Trade Order", "tag": "shadow_button"},
|
|
240
|
+
{"x": 720, "y": 300, "width": 150, "height": 50, "label": "Cancel Order", "tag": "shadow_button"}
|
|
241
|
+
]
|
|
242
|
+
|
|
243
|
+
if not os.path.exists(image_path):
|
|
244
|
+
img = Image.new("RGB", (self.viewport_width, self.viewport_height), color="#1E1E2E")
|
|
245
|
+
draw = ImageDraw.Draw(img)
|
|
246
|
+
draw.text((50, 50), "Mock Financial Dashboard (HTML5 Canvas)", fill="#FFFFFF")
|
|
247
|
+
img.save(image_path)
|
|
248
|
+
|
|
249
|
+
img = Image.open(image_path).convert("RGBA")
|
|
250
|
+
overlay = Image.new("RGBA", img.size, (255, 255, 255, 0))
|
|
251
|
+
draw = ImageDraw.Draw(overlay)
|
|
252
|
+
|
|
253
|
+
marks_map = {}
|
|
254
|
+
for idx, el in enumerate(elements, start=1):
|
|
255
|
+
x = el.get("x", 0)
|
|
256
|
+
y = el.get("y", 0)
|
|
257
|
+
w = el.get("width", 80)
|
|
258
|
+
h = el.get("height", 35)
|
|
259
|
+
|
|
260
|
+
center_x = round(x + w / 2)
|
|
261
|
+
center_y = round(y + h / 2)
|
|
262
|
+
color = MARK_COLORS[(idx - 1) % len(MARK_COLORS)]
|
|
263
|
+
|
|
264
|
+
# Draw bounding box
|
|
265
|
+
draw.rectangle([x, y, x + w, y + h], outline=color, width=3)
|
|
266
|
+
|
|
267
|
+
# Draw badge label (e.g. [1])
|
|
268
|
+
badge_w, badge_h = 24, 20
|
|
269
|
+
badge_x = max(0, x)
|
|
270
|
+
badge_y = max(0, y - badge_h)
|
|
271
|
+
draw.rectangle([badge_x, badge_y, badge_x + badge_w, badge_y + badge_h], fill=color)
|
|
272
|
+
draw.text((badge_x + 6, badge_y + 3), str(idx), fill="#000000")
|
|
273
|
+
|
|
274
|
+
marks_map[idx] = {
|
|
275
|
+
"mark_id": idx,
|
|
276
|
+
"label": el.get("label") or el.get("text") or el.get("role") or f"Element_{idx}",
|
|
277
|
+
"tag": el.get("tag", "element"),
|
|
278
|
+
"bounds": [x, y, x + w, y + h],
|
|
279
|
+
"center_x": center_x,
|
|
280
|
+
"center_y": center_y,
|
|
281
|
+
"playwright_action": f"await page.mouse.click({center_x}, {center_y});"
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
combined = Image.alpha_composite(img, overlay).convert("RGB")
|
|
285
|
+
out_file = output_path or image_path.replace(".png", "_som_marked.png")
|
|
286
|
+
combined.save(out_file)
|
|
287
|
+
|
|
288
|
+
return out_file, marks_map
|
|
289
|
+
|
|
290
|
+
def resolve_mark_coordinates(
|
|
291
|
+
self,
|
|
292
|
+
marks_map: Dict[int, Dict[str, Any]],
|
|
293
|
+
target_mark_id: Optional[int] = None,
|
|
294
|
+
target_label: Optional[str] = None
|
|
295
|
+
) -> Optional[Dict[str, Any]]:
|
|
296
|
+
"""
|
|
297
|
+
Resolves the exact (x, y) coordinates for a given Mark ID or fuzzy label.
|
|
298
|
+
"""
|
|
299
|
+
if target_mark_id and target_mark_id in marks_map:
|
|
300
|
+
return marks_map[target_mark_id]
|
|
301
|
+
|
|
302
|
+
if target_label:
|
|
303
|
+
target_clean = target_label.strip().lower()
|
|
304
|
+
for mark in marks_map.values():
|
|
305
|
+
if target_clean in mark.get("label", "").lower():
|
|
306
|
+
return mark
|
|
307
|
+
|
|
308
|
+
return None
|
|
309
|
+
|
|
310
|
+
def format_vision_prompt(self, marks_map: Dict[int, Dict[str, Any]], goal: str) -> str:
|
|
311
|
+
"""
|
|
312
|
+
Formats a compact Vision brief for multimodal models (Gemini / Claude).
|
|
313
|
+
"""
|
|
314
|
+
lines = [
|
|
315
|
+
f"GOAL: {goal}",
|
|
316
|
+
f"SET-OF-MARKS (SoM) CANDIDATE REGIONS DETECTED: {len(marks_map)}",
|
|
317
|
+
"",
|
|
318
|
+
"Inspect the attached marked screenshot and identify the target Mark ID:",
|
|
319
|
+
]
|
|
320
|
+
for m_id, data in marks_map.items():
|
|
321
|
+
lines.append(f" [{m_id}] '{data['label']}' -> Center ({data['center_x']}, {data['center_y']})")
|
|
322
|
+
lines.append("")
|
|
323
|
+
lines.append("RESPOND WITH FORMAT: CLICK_MARK: <ID>")
|
|
324
|
+
return "\n".join(lines)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def main():
|
|
328
|
+
parser = argparse.ArgumentParser(description="Multimodal Vision Grounding & Set-of-Marks Engine")
|
|
329
|
+
parser.add_argument("--image", default="screenshot.png", help="Path to input screenshot image")
|
|
330
|
+
parser.add_argument("--output", default="", help="Path to output marked image")
|
|
331
|
+
parser.add_argument("--elements-json", default="", help="JSON string or file of candidate bounding boxes")
|
|
332
|
+
parser.add_argument("--mark-id", type=int, default=None, help="Target Mark ID to resolve")
|
|
333
|
+
parser.add_argument("--label", default="", help="Target semantic label to resolve")
|
|
334
|
+
parser.add_argument("--component", default="", help="Target component name for visual cache lookup/save")
|
|
335
|
+
parser.add_argument("--cache-lookup", action="store_true", help="Attempt zero-token visual cache resolution first")
|
|
336
|
+
parser.add_argument("--cache-save", action="store_true", help="Save resolved coordinates to visual cache")
|
|
337
|
+
parser.add_argument("--threshold", type=float, default=0.98, help="Visual similarity threshold for cache hit (default: 0.98)")
|
|
338
|
+
parser.add_argument("--json", action="store_true", help="Output results in JSON format")
|
|
339
|
+
args = parser.parse_args()
|
|
340
|
+
|
|
341
|
+
helper = VisualSoMHelper()
|
|
342
|
+
|
|
343
|
+
# Step 1: Zero-Token Visual Cache Lookup
|
|
344
|
+
if (args.cache_lookup or args.component) and args.component and os.path.exists(args.image):
|
|
345
|
+
cached_result = helper.lookup_cached_coordinates(
|
|
346
|
+
component_name=args.component,
|
|
347
|
+
image_source=args.image,
|
|
348
|
+
threshold=args.threshold
|
|
349
|
+
)
|
|
350
|
+
if cached_result:
|
|
351
|
+
if args.json:
|
|
352
|
+
print(json.dumps({"status": "CACHE_HIT", "data": cached_result}, indent=2))
|
|
353
|
+
else:
|
|
354
|
+
print(f"\n⚡ [CACHE HIT - ZERO VISION TOKEN] Component '{args.component}' matched ({cached_result['visual_similarity']*100:.1f}% Similarity)")
|
|
355
|
+
print(f" Coordinates : ({cached_result['center_x']}, {cached_result['center_y']})")
|
|
356
|
+
print(f" Hit Count : {cached_result['hit_count']}")
|
|
357
|
+
print(f" Action : {cached_result['playwright_action']}\n")
|
|
358
|
+
return 0
|
|
359
|
+
|
|
360
|
+
# Load elements
|
|
361
|
+
elements = []
|
|
362
|
+
if args.elements_json:
|
|
363
|
+
if os.path.exists(args.elements_json):
|
|
364
|
+
with open(args.elements_json, "r", encoding="utf-8") as f:
|
|
365
|
+
elements = json.load(f)
|
|
366
|
+
else:
|
|
367
|
+
try:
|
|
368
|
+
elements = json.loads(args.elements_json)
|
|
369
|
+
except Exception:
|
|
370
|
+
pass
|
|
371
|
+
|
|
372
|
+
if not elements:
|
|
373
|
+
elements = [
|
|
374
|
+
{"x": 100, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom In", "tag": "canvas_control"},
|
|
375
|
+
{"x": 240, "y": 150, "width": 120, "height": 40, "label": "TradingView Zoom Out", "tag": "canvas_control"},
|
|
376
|
+
{"x": 500, "y": 300, "width": 200, "height": 50, "label": "Execute Trade Order", "tag": "shadow_button"},
|
|
377
|
+
{"x": 720, "y": 300, "width": 150, "height": 50, "label": "Cancel Order", "tag": "shadow_button"}
|
|
378
|
+
]
|
|
379
|
+
|
|
380
|
+
marked_img, marks_map = helper.generate_som_overlay(args.image, elements, output_path=args.output)
|
|
381
|
+
|
|
382
|
+
resolved = None
|
|
383
|
+
target_query = args.label or args.component
|
|
384
|
+
if args.mark_id is not None or target_query:
|
|
385
|
+
resolved = helper.resolve_mark_coordinates(marks_map, target_mark_id=args.mark_id, target_label=target_query)
|
|
386
|
+
|
|
387
|
+
# Cache Save if requested or if component is specified and resolved
|
|
388
|
+
if resolved and (args.cache_save or args.component):
|
|
389
|
+
comp_name = args.component or resolved["label"]
|
|
390
|
+
helper.save_cached_coordinates(
|
|
391
|
+
component_name=comp_name,
|
|
392
|
+
image_source=args.image,
|
|
393
|
+
center_x=resolved["center_x"],
|
|
394
|
+
center_y=resolved["center_y"]
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
if args.json:
|
|
398
|
+
result = {
|
|
399
|
+
"marked_image": marked_img,
|
|
400
|
+
"total_marks": len(marks_map),
|
|
401
|
+
"marks": marks_map,
|
|
402
|
+
"resolved_target": resolved
|
|
403
|
+
}
|
|
404
|
+
print(json.dumps(result, indent=2))
|
|
405
|
+
else:
|
|
406
|
+
print(f"\n🎨 [SET-OF-MARKS GENERATED] Marked image: {marked_img}")
|
|
407
|
+
print(f" Candidate Visual Regions: {len(marks_map)}")
|
|
408
|
+
for mid, data in marks_map.items():
|
|
409
|
+
print(f" [{mid}] {data['label']} at ({data['center_x']}, {data['center_y']})")
|
|
410
|
+
if resolved:
|
|
411
|
+
print(f"\n🎯 [COORDINATE GROUNDED] Target '{resolved['label']}' -> Mark #{resolved['mark_id']}")
|
|
412
|
+
print(f" Playwright Mouse Action: {resolved['playwright_action']}\n")
|
|
413
|
+
|
|
414
|
+
return 0
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
if __name__ == "__main__":
|
|
418
|
+
sys.exit(main())
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
tools/wcag_audit.py - Automated Accessibility (WCAG 2.2) Linter & Auditor
|
|
4
|
+
Operationalizes the .agents/skills/wcag-audit-patterns skill for Web & Enterprise Applications.
|
|
5
|
+
|
|
6
|
+
Checks WCAG 2.2 Level A and AA criteria:
|
|
7
|
+
- WCAG 1.1.1: Images missing alt attributes
|
|
8
|
+
- WCAG 1.3.1: Form controls missing labels or aria-label/aria-labelledby
|
|
9
|
+
- WCAG 1.3.1: Table headers structure
|
|
10
|
+
- WCAG 2.4.4: Links with empty text / accessible name
|
|
11
|
+
- WCAG 4.1.2: Buttons with empty text / accessible name
|
|
12
|
+
- WCAG 1.3.1: Heading hierarchy gaps (e.g., h1 to h3)
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import os
|
|
17
|
+
import sys
|
|
18
|
+
import urllib.request
|
|
19
|
+
from html.parser import HTMLParser
|
|
20
|
+
|
|
21
|
+
# Enforce UTF-8 stdout encoding for Windows console compatibility
|
|
22
|
+
if hasattr(sys.stdout, 'reconfigure'):
|
|
23
|
+
try:
|
|
24
|
+
sys.stdout.reconfigure(encoding='utf-8')
|
|
25
|
+
sys.stderr.reconfigure(encoding='utf-8')
|
|
26
|
+
except Exception:
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class WCAGAuditor(HTMLParser):
|
|
31
|
+
def __init__(self):
|
|
32
|
+
super().__init__()
|
|
33
|
+
self.violations = []
|
|
34
|
+
self.tag_stack = []
|
|
35
|
+
self.current_heading_level = 0
|
|
36
|
+
self.in_button = False
|
|
37
|
+
self.button_text = ""
|
|
38
|
+
self.button_attrs = {}
|
|
39
|
+
self.in_a = False
|
|
40
|
+
self.a_text = ""
|
|
41
|
+
self.a_attrs = {}
|
|
42
|
+
self.has_main = False
|
|
43
|
+
self.elements_scanned = 0
|
|
44
|
+
|
|
45
|
+
def handle_starttag(self, tag, attrs):
|
|
46
|
+
self.elements_scanned += 1
|
|
47
|
+
attr_dict = dict(attrs)
|
|
48
|
+
|
|
49
|
+
# WCAG 1.1.1 Non-text Content (Images)
|
|
50
|
+
if tag == "img":
|
|
51
|
+
if "alt" not in attr_dict:
|
|
52
|
+
self.violations.append({
|
|
53
|
+
"criterion": "1.1.1 Non-text Content (Level A)",
|
|
54
|
+
"severity": "CRITICAL",
|
|
55
|
+
"element": f"<img src='{attr_dict.get('src', '')}'>",
|
|
56
|
+
"message": "Missing 'alt' attribute on image."
|
|
57
|
+
})
|
|
58
|
+
|
|
59
|
+
# WCAG 1.3.1 Info and Relationships (Form Controls)
|
|
60
|
+
if tag in ["input", "select", "textarea"]:
|
|
61
|
+
input_type = attr_dict.get("type", "text")
|
|
62
|
+
if input_type not in ["hidden", "submit", "button", "reset"]:
|
|
63
|
+
has_label = "aria-label" in attr_dict or "aria-labelledby" in attr_dict or "id" in attr_dict
|
|
64
|
+
if not has_label:
|
|
65
|
+
self.violations.append({
|
|
66
|
+
"criterion": "1.3.1 Info and Relationships / 4.1.2 (Level A)",
|
|
67
|
+
"severity": "HIGH",
|
|
68
|
+
"element": f"<{tag} type='{input_type}' name='{attr_dict.get('name', '')}'>",
|
|
69
|
+
"message": "Form control missing accessible name (no aria-label, aria-labelledby, or matching <label> id)."
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
# Heading hierarchy
|
|
73
|
+
if tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
|
|
74
|
+
level = int(tag[1])
|
|
75
|
+
if self.current_heading_level > 0 and level > self.current_heading_level + 1:
|
|
76
|
+
self.violations.append({
|
|
77
|
+
"criterion": "1.3.1 Info and Relationships (Level A)",
|
|
78
|
+
"severity": "MEDIUM",
|
|
79
|
+
"element": f"<{tag}>",
|
|
80
|
+
"message": f"Heading level skipped from h{self.current_heading_level} to h{level}."
|
|
81
|
+
})
|
|
82
|
+
self.current_heading_level = level
|
|
83
|
+
|
|
84
|
+
# Track Buttons
|
|
85
|
+
if tag == "button":
|
|
86
|
+
self.in_button = True
|
|
87
|
+
self.button_text = ""
|
|
88
|
+
self.button_attrs = attr_dict
|
|
89
|
+
|
|
90
|
+
# Track Anchors
|
|
91
|
+
if tag == "a":
|
|
92
|
+
self.in_a = True
|
|
93
|
+
self.a_text = ""
|
|
94
|
+
self.a_attrs = attr_dict
|
|
95
|
+
|
|
96
|
+
def handle_data(self, data):
|
|
97
|
+
if self.in_button:
|
|
98
|
+
self.button_text += data.strip()
|
|
99
|
+
if self.in_a:
|
|
100
|
+
self.a_text += data.strip()
|
|
101
|
+
|
|
102
|
+
def handle_endtag(self, tag):
|
|
103
|
+
if tag == "button" and self.in_button:
|
|
104
|
+
has_name = bool(self.button_text) or "aria-label" in self.button_attrs or "title" in self.button_attrs
|
|
105
|
+
if not has_name:
|
|
106
|
+
self.violations.append({
|
|
107
|
+
"criterion": "4.1.2 Name, Role, Value (Level A)",
|
|
108
|
+
"severity": "CRITICAL",
|
|
109
|
+
"element": "<button> (empty)",
|
|
110
|
+
"message": "Button has no accessible name (empty text, no aria-label, no title)."
|
|
111
|
+
})
|
|
112
|
+
self.in_button = False
|
|
113
|
+
|
|
114
|
+
if tag == "a" and self.in_a:
|
|
115
|
+
has_name = bool(self.a_text) or "aria-label" in self.a_attrs or "title" in self.a_attrs
|
|
116
|
+
href = self.a_attrs.get("href", "")
|
|
117
|
+
if href and not has_name:
|
|
118
|
+
self.violations.append({
|
|
119
|
+
"criterion": "2.4.4 Link Purpose (In Context) (Level A)",
|
|
120
|
+
"severity": "HIGH",
|
|
121
|
+
"element": f"<a href='{href}'> (empty)",
|
|
122
|
+
"message": "Link has no accessible text or aria-label."
|
|
123
|
+
})
|
|
124
|
+
self.in_a = False
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def run_audit(target, is_url=False):
|
|
128
|
+
auditor = WCAGAuditor()
|
|
129
|
+
if is_url:
|
|
130
|
+
req = urllib.request.Request(target, headers={'User-Agent': 'Mozilla/5.0 (WCAG-Audit-Bot)'})
|
|
131
|
+
with urllib.request.urlopen(req) as resp:
|
|
132
|
+
html = resp.read().decode('utf-8', errors='ignore')
|
|
133
|
+
else:
|
|
134
|
+
with open(target, 'r', encoding='utf-8', errors='ignore') as f:
|
|
135
|
+
html = f.read()
|
|
136
|
+
|
|
137
|
+
auditor.feed(html)
|
|
138
|
+
return auditor
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def format_markdown_report(target, auditor):
|
|
142
|
+
violations = auditor.violations
|
|
143
|
+
stats = {
|
|
144
|
+
"elements_scanned": auditor.elements_scanned,
|
|
145
|
+
"violations_found": len(violations),
|
|
146
|
+
"critical": sum(1 for v in violations if v["severity"] == "CRITICAL"),
|
|
147
|
+
"high": sum(1 for v in violations if v["severity"] == "HIGH"),
|
|
148
|
+
"medium": sum(1 for v in violations if v["severity"] == "MEDIUM")
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
# Score calculation
|
|
152
|
+
penalty = (stats["critical"] * 25) + (stats["high"] * 10) + (stats["medium"] * 5)
|
|
153
|
+
score = max(0, 100 - penalty)
|
|
154
|
+
grade = "PASS" if score >= 80 else "NEEDS REMEDIATION"
|
|
155
|
+
|
|
156
|
+
out = [
|
|
157
|
+
f"# WCAG 2.2 Accessibility Audit Report",
|
|
158
|
+
f"**Target:** `{target}`",
|
|
159
|
+
f"**Elements Scanned:** {stats['elements_scanned']}",
|
|
160
|
+
f"**Violations Found:** {stats['violations_found']}",
|
|
161
|
+
f"**Accessibility Score:** {score}/100 - **[{grade}]**",
|
|
162
|
+
"",
|
|
163
|
+
"---",
|
|
164
|
+
"",
|
|
165
|
+
"## Violations Detail",
|
|
166
|
+
"",
|
|
167
|
+
"| Severity | Criterion | Element Snippet | Issue Description |",
|
|
168
|
+
"|---|---|---|---|"
|
|
169
|
+
]
|
|
170
|
+
|
|
171
|
+
if not violations:
|
|
172
|
+
out.append("| PASS | - | - | Zero WCAG 2.2 violations detected! Element tree is fully compliant. |")
|
|
173
|
+
else:
|
|
174
|
+
for v in violations:
|
|
175
|
+
out.append(f"| **{v['severity']}** | {v['criterion']} | `{v['element']}` | {v['message']} |")
|
|
176
|
+
|
|
177
|
+
out.append("")
|
|
178
|
+
out.append("---")
|
|
179
|
+
out.append("### Recommended Remediation Actions:")
|
|
180
|
+
out.append("1. Add descriptive `alt=\"...\"` attributes to all images.")
|
|
181
|
+
out.append("2. Add `aria-label=\"...\"` or explicit `<label for=\"...\">` bindings to form controls and icon buttons.")
|
|
182
|
+
out.append("3. Maintain strict heading hierarchy (h1 -> h2 -> h3).")
|
|
183
|
+
return "\n".join(out)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def main():
|
|
187
|
+
parser = argparse.ArgumentParser(description="Automated WCAG 2.2 Accessibility Auditor for Web & Enterprise Applications.")
|
|
188
|
+
parser.add_argument("target", help="HTML file path or URL to audit.")
|
|
189
|
+
parser.add_argument("--url", action="store_true", help="Treat target as a live URL.")
|
|
190
|
+
parser.add_argument("--output", "-o", help="Path to save markdown report.")
|
|
191
|
+
args = parser.parse_args()
|
|
192
|
+
|
|
193
|
+
if not args.url and not os.path.exists(args.target):
|
|
194
|
+
print(f"Error: File '{args.target}' does not exist.")
|
|
195
|
+
sys.exit(1)
|
|
196
|
+
|
|
197
|
+
print(f"[AUDIT] Running WCAG 2.2 Accessibility Audit on: {args.target}...")
|
|
198
|
+
auditor = run_audit(args.target, is_url=args.url)
|
|
199
|
+
report = format_markdown_report(args.target, auditor)
|
|
200
|
+
|
|
201
|
+
if args.output:
|
|
202
|
+
with open(args.output, "w", encoding="utf-8") as f:
|
|
203
|
+
f.write(report)
|
|
204
|
+
print(f"[OK] Report saved to: {args.output}")
|
|
205
|
+
else:
|
|
206
|
+
print(report)
|
|
207
|
+
|
|
208
|
+
if len(auditor.violations) > 0:
|
|
209
|
+
sys.exit(2)
|
|
210
|
+
sys.exit(0)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
if __name__ == "__main__":
|
|
214
|
+
main()
|