nat-engine 1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (299) hide show
  1. mannf/__init__.py +33 -0
  2. mannf/__main__.py +10 -0
  3. mannf/_version.py +8 -0
  4. mannf/agents/__init__.py +7 -0
  5. mannf/agents/analyzer_agent.py +9 -0
  6. mannf/agents/base.py +9 -0
  7. mannf/agents/bdi_agent.py +9 -0
  8. mannf/agents/belief_state.py +9 -0
  9. mannf/agents/coordinator_agent.py +9 -0
  10. mannf/agents/executor_agent.py +9 -0
  11. mannf/agents/monitor_agent.py +9 -0
  12. mannf/agents/oracle_agent.py +9 -0
  13. mannf/agents/planner_agent.py +9 -0
  14. mannf/agents/test_agent.py +9 -0
  15. mannf/anomaly/__init__.py +7 -0
  16. mannf/anomaly/enhanced_detector.py +9 -0
  17. mannf/cli.py +9 -0
  18. mannf/core/__init__.py +26 -0
  19. mannf/core/agents/__init__.py +52 -0
  20. mannf/core/agents/accessibility_scanner_agent.py +245 -0
  21. mannf/core/agents/analyzer_agent.py +224 -0
  22. mannf/core/agents/autonomous_loop_agent.py +1086 -0
  23. mannf/core/agents/autonomous_loop_models.py +62 -0
  24. mannf/core/agents/autonomous_run_differ.py +427 -0
  25. mannf/core/agents/base.py +128 -0
  26. mannf/core/agents/bdi_agent.py +330 -0
  27. mannf/core/agents/belief_state.py +202 -0
  28. mannf/core/agents/browser_coordinator_agent.py +224 -0
  29. mannf/core/agents/browser_executor_agent.py +410 -0
  30. mannf/core/agents/coordinator_agent.py +262 -0
  31. mannf/core/agents/executor_agent.py +222 -0
  32. mannf/core/agents/monitor_agent.py +188 -0
  33. mannf/core/agents/oracle_agent.py +150 -0
  34. mannf/core/agents/performance_testing_agent.py +279 -0
  35. mannf/core/agents/planner_agent.py +128 -0
  36. mannf/core/agents/test_agent.py +249 -0
  37. mannf/core/agents/visual_regression_agent.py +311 -0
  38. mannf/core/agents/web_crawler_agent.py +510 -0
  39. mannf/core/agents/worker_pool.py +366 -0
  40. mannf/core/anomaly/__init__.py +14 -0
  41. mannf/core/anomaly/enhanced_detector.py +541 -0
  42. mannf/core/browser/__init__.py +63 -0
  43. mannf/core/browser/accessibility_scanner.py +424 -0
  44. mannf/core/browser/discovery_model.py +178 -0
  45. mannf/core/browser/dom_snapshot.py +349 -0
  46. mannf/core/browser/ingestor_bridge.py +371 -0
  47. mannf/core/browser/performance_metrics.py +217 -0
  48. mannf/core/browser/reflection_analyzer.py +442 -0
  49. mannf/core/browser/scenario_generator.py +1100 -0
  50. mannf/core/browser/security_scenario_generator.py +695 -0
  51. mannf/core/browser/visual_comparer.py +159 -0
  52. mannf/core/diagnostics/__init__.py +28 -0
  53. mannf/core/diagnostics/failure_clusterer.py +211 -0
  54. mannf/core/diagnostics/flake_detector.py +233 -0
  55. mannf/core/diagnostics/root_cause_analyzer.py +273 -0
  56. mannf/core/distributed/__init__.py +16 -0
  57. mannf/core/distributed/endpoint.py +139 -0
  58. mannf/core/distributed/system_under_test.py +207 -0
  59. mannf/core/functional_orchestrator.py +428 -0
  60. mannf/core/messaging/__init__.py +11 -0
  61. mannf/core/messaging/bus.py +113 -0
  62. mannf/core/messaging/messages.py +89 -0
  63. mannf/core/nat_orchestrator.py +342 -0
  64. mannf/core/neural/__init__.py +183 -0
  65. mannf/core/orchestrator.py +272 -0
  66. mannf/core/prioritization/__init__.py +17 -0
  67. mannf/core/prioritization/adaptive_controller.py +509 -0
  68. mannf/core/prioritization/belief_prioritizer.py +231 -0
  69. mannf/core/prioritization/risk_scorer.py +430 -0
  70. mannf/core/reporting/__init__.py +12 -0
  71. mannf/core/reporting/unified_report.py +664 -0
  72. mannf/core/testing/__init__.py +17 -0
  73. mannf/core/testing/adaptive_controller.py +149 -0
  74. mannf/core/testing/models.py +179 -0
  75. mannf/core/validation/__init__.py +10 -0
  76. mannf/core/validation/self_validation_runner.py +180 -0
  77. mannf/dashboard/__init__.py +7 -0
  78. mannf/dashboard/app.py +9 -0
  79. mannf/dashboard/models.py +9 -0
  80. mannf/dashboard/static/index.html +2538 -0
  81. mannf/dashboard/telemetry.py +9 -0
  82. mannf/distributed/__init__.py +7 -0
  83. mannf/distributed/endpoint.py +9 -0
  84. mannf/distributed/system_under_test.py +9 -0
  85. mannf/healing/__init__.py +7 -0
  86. mannf/healing/graphql_schema_diff.py +9 -0
  87. mannf/healing/healer.py +9 -0
  88. mannf/healing/models.py +9 -0
  89. mannf/healing/schema_diff.py +9 -0
  90. mannf/integrations/__init__.py +7 -0
  91. mannf/integrations/auth.py +9 -0
  92. mannf/integrations/graphql_parser.py +9 -0
  93. mannf/integrations/graphql_sut.py +9 -0
  94. mannf/integrations/http_sut.py +9 -0
  95. mannf/integrations/openapi_parser.py +9 -0
  96. mannf/integrations/postman_parser.py +9 -0
  97. mannf/llm/__init__.py +7 -0
  98. mannf/llm/anthropic_provider.py +9 -0
  99. mannf/llm/base.py +9 -0
  100. mannf/llm/config.py +9 -0
  101. mannf/llm/factory.py +9 -0
  102. mannf/llm/openai_provider.py +9 -0
  103. mannf/llm/prompts.py +9 -0
  104. mannf/messaging/__init__.py +7 -0
  105. mannf/messaging/bus.py +9 -0
  106. mannf/messaging/messages.py +9 -0
  107. mannf/nat_orchestrator.py +9 -0
  108. mannf/neural/__init__.py +7 -0
  109. mannf/orchestrator.py +9 -0
  110. mannf/prioritization/__init__.py +7 -0
  111. mannf/prioritization/adaptive_controller.py +9 -0
  112. mannf/prioritization/belief_prioritizer.py +9 -0
  113. mannf/prioritization/risk_scorer.py +9 -0
  114. mannf/product/__init__.py +29 -0
  115. mannf/product/admin/__init__.py +3 -0
  116. mannf/product/admin/routes.py +514 -0
  117. mannf/product/auth/__init__.py +5 -0
  118. mannf/product/auth/saml.py +212 -0
  119. mannf/product/billing/__init__.py +5 -0
  120. mannf/product/billing/audit.py +160 -0
  121. mannf/product/billing/feature_gates.py +180 -0
  122. mannf/product/billing/metering.py +179 -0
  123. mannf/product/billing/notifications.py +181 -0
  124. mannf/product/billing/plans.py +133 -0
  125. mannf/product/billing/rate_limits.py +35 -0
  126. mannf/product/billing/stripe_billing.py +906 -0
  127. mannf/product/billing/tenant_auth.py +233 -0
  128. mannf/product/billing/tenant_manager.py +873 -0
  129. mannf/product/cli.py +3900 -0
  130. mannf/product/cli_admin.py +408 -0
  131. mannf/product/dashboard/__init__.py +61 -0
  132. mannf/product/dashboard/app.py +3567 -0
  133. mannf/product/dashboard/models.py +460 -0
  134. mannf/product/dashboard/static/index.html +6347 -0
  135. mannf/product/dashboard/static/manifest.json +25 -0
  136. mannf/product/dashboard/static/pwa-icon-192.png +0 -0
  137. mannf/product/dashboard/static/pwa-icon-512.png +0 -0
  138. mannf/product/dashboard/static/sw.js +64 -0
  139. mannf/product/dashboard/telemetry.py +547 -0
  140. mannf/product/database.py +145 -0
  141. mannf/product/demo.py +844 -0
  142. mannf/product/doctor.py +509 -0
  143. mannf/product/exporters/__init__.py +65 -0
  144. mannf/product/exporters/azuredevops_exporter.py +257 -0
  145. mannf/product/exporters/base.py +307 -0
  146. mannf/product/exporters/bugzilla_exporter.py +200 -0
  147. mannf/product/exporters/dedup.py +275 -0
  148. mannf/product/exporters/finding_adapter.py +216 -0
  149. mannf/product/exporters/github_exporter.py +197 -0
  150. mannf/product/exporters/gitlab_exporter.py +215 -0
  151. mannf/product/exporters/jira_exporter.py +180 -0
  152. mannf/product/exporters/linear_exporter.py +195 -0
  153. mannf/product/exporters/loader.py +233 -0
  154. mannf/product/exporters/pagerduty_exporter.py +363 -0
  155. mannf/product/exporters/sentry_exporter.py +322 -0
  156. mannf/product/exporters/servicenow_exporter.py +240 -0
  157. mannf/product/exporters/shortcut_exporter.py +231 -0
  158. mannf/product/exporters/webhook_exporter.py +383 -0
  159. mannf/product/formatters/__init__.py +18 -0
  160. mannf/product/formatters/allure_formatter.py +161 -0
  161. mannf/product/formatters/ctrf_formatter.py +149 -0
  162. mannf/product/healing/__init__.py +30 -0
  163. mannf/product/healing/graphql_schema_diff.py +152 -0
  164. mannf/product/healing/healer.py +141 -0
  165. mannf/product/healing/models.py +175 -0
  166. mannf/product/healing/schema_diff.py +251 -0
  167. mannf/product/ingestors/__init__.py +77 -0
  168. mannf/product/ingestors/base.py +256 -0
  169. mannf/product/ingestors/bgstm_ingestor.py +764 -0
  170. mannf/product/ingestors/curl_ingestor.py +1019 -0
  171. mannf/product/ingestors/cypress_ingestor.py +487 -0
  172. mannf/product/ingestors/gherkin_ingestor.py +967 -0
  173. mannf/product/ingestors/graphql_ingestor.py +845 -0
  174. mannf/product/ingestors/grpc_ingestor.py +591 -0
  175. mannf/product/ingestors/har_ingestor.py +976 -0
  176. mannf/product/ingestors/loader.py +284 -0
  177. mannf/product/ingestors/models.py +146 -0
  178. mannf/product/ingestors/openapi_ingestor.py +606 -0
  179. mannf/product/ingestors/playwright_ingestor.py +449 -0
  180. mannf/product/ingestors/postman_ingestor.py +631 -0
  181. mannf/product/ingestors/traffic_ingestor.py +679 -0
  182. mannf/product/ingestors/websocket_ingestor.py +526 -0
  183. mannf/product/integrations/__init__.py +21 -0
  184. mannf/product/integrations/auth.py +190 -0
  185. mannf/product/integrations/graphql_parser.py +436 -0
  186. mannf/product/integrations/graphql_sut.py +247 -0
  187. mannf/product/integrations/grpc_sut.py +469 -0
  188. mannf/product/integrations/http_sut.py +237 -0
  189. mannf/product/integrations/kafka_adapter.py +342 -0
  190. mannf/product/integrations/openapi_parser.py +513 -0
  191. mannf/product/integrations/postman_parser.py +467 -0
  192. mannf/product/integrations/webhook_receiver.py +344 -0
  193. mannf/product/integrations/websocket_sut.py +434 -0
  194. mannf/product/llm/__init__.py +25 -0
  195. mannf/product/llm/anthropic_provider.py +94 -0
  196. mannf/product/llm/base.py +267 -0
  197. mannf/product/llm/config.py +48 -0
  198. mannf/product/llm/factory.py +42 -0
  199. mannf/product/llm/openai_provider.py +93 -0
  200. mannf/product/llm/prompts.py +403 -0
  201. mannf/product/llm/root_cause_service.py +311 -0
  202. mannf/product/llm/test_plan_models.py +78 -0
  203. mannf/product/metrics.py +149 -0
  204. mannf/product/middleware/__init__.py +3 -0
  205. mannf/product/middleware/audit_middleware.py +112 -0
  206. mannf/product/middleware/tenant_isolation.py +114 -0
  207. mannf/product/models.py +347 -0
  208. mannf/product/notifications/__init__.py +24 -0
  209. mannf/product/notifications/dispatcher.py +411 -0
  210. mannf/product/onboarding.py +190 -0
  211. mannf/product/orchestration/__init__.py +39 -0
  212. mannf/product/orchestration/ingest_scan_orchestrator.py +339 -0
  213. mannf/product/orchestration/pipeline.py +401 -0
  214. mannf/product/orchestrator.py +987 -0
  215. mannf/product/orchestrator_models.py +269 -0
  216. mannf/product/regression/__init__.py +36 -0
  217. mannf/product/regression/differ.py +172 -0
  218. mannf/product/regression/masking.py +100 -0
  219. mannf/product/regression/models.py +232 -0
  220. mannf/product/regression/recorder.py +124 -0
  221. mannf/product/regression/replayer.py +168 -0
  222. mannf/product/reports/__init__.py +10 -0
  223. mannf/product/reports/pdf.py +132 -0
  224. mannf/product/scheduling/__init__.py +57 -0
  225. mannf/product/scheduling/cron_utils.py +251 -0
  226. mannf/product/scheduling/engine.py +473 -0
  227. mannf/product/scheduling/models.py +86 -0
  228. mannf/product/scheduling/queue.py +894 -0
  229. mannf/product/scheduling/store.py +235 -0
  230. mannf/product/security/__init__.py +21 -0
  231. mannf/product/security/belief_guided.py +143 -0
  232. mannf/product/security/checks/__init__.py +55 -0
  233. mannf/product/security/checks/base.py +69 -0
  234. mannf/product/security/checks/bfla.py +77 -0
  235. mannf/product/security/checks/bola.py +77 -0
  236. mannf/product/security/checks/bopla.py +80 -0
  237. mannf/product/security/checks/broken_auth.py +86 -0
  238. mannf/product/security/checks/graphql_security.py +299 -0
  239. mannf/product/security/checks/inventory.py +70 -0
  240. mannf/product/security/checks/misconfig.py +158 -0
  241. mannf/product/security/checks/resource_consumption.py +70 -0
  242. mannf/product/security/checks/sensitive_flows.py +80 -0
  243. mannf/product/security/checks/ssrf.py +101 -0
  244. mannf/product/security/checks/unsafe_consumption.py +120 -0
  245. mannf/product/security/models.py +92 -0
  246. mannf/product/security/plugin_loader.py +182 -0
  247. mannf/product/security/reporter.py +92 -0
  248. mannf/product/security/scanner.py +183 -0
  249. mannf/product/server.py +6220 -0
  250. mannf/product/setup_wizard.py +873 -0
  251. mannf/product/status.py +404 -0
  252. mannf/product/storage/__init__.py +10 -0
  253. mannf/product/storage/artifact_store.py +343 -0
  254. mannf/product/telemetry.py +300 -0
  255. mannf/product/uninstall.py +169 -0
  256. mannf/product/upgrade.py +139 -0
  257. mannf/product/weights/__init__.py +13 -0
  258. mannf/product/weights/blob_store.py +299 -0
  259. mannf/product/weights/factory.py +42 -0
  260. mannf/product/weights/registry.py +159 -0
  261. mannf/product/weights/store.py +210 -0
  262. mannf/regression/__init__.py +7 -0
  263. mannf/regression/differ.py +9 -0
  264. mannf/regression/masking.py +9 -0
  265. mannf/regression/models.py +9 -0
  266. mannf/regression/recorder.py +9 -0
  267. mannf/regression/replayer.py +9 -0
  268. mannf/security/__init__.py +7 -0
  269. mannf/security/belief_guided.py +9 -0
  270. mannf/security/checks/__init__.py +7 -0
  271. mannf/security/checks/base.py +9 -0
  272. mannf/security/checks/bfla.py +9 -0
  273. mannf/security/checks/bola.py +9 -0
  274. mannf/security/checks/bopla.py +9 -0
  275. mannf/security/checks/broken_auth.py +9 -0
  276. mannf/security/checks/graphql_security.py +9 -0
  277. mannf/security/checks/inventory.py +9 -0
  278. mannf/security/checks/misconfig.py +9 -0
  279. mannf/security/checks/resource_consumption.py +9 -0
  280. mannf/security/checks/sensitive_flows.py +9 -0
  281. mannf/security/checks/ssrf.py +9 -0
  282. mannf/security/checks/unsafe_consumption.py +9 -0
  283. mannf/security/models.py +9 -0
  284. mannf/security/reporter.py +9 -0
  285. mannf/security/scanner.py +9 -0
  286. mannf/server.py +9 -0
  287. mannf/testing/__init__.py +7 -0
  288. mannf/testing/adaptive_controller.py +9 -0
  289. mannf/testing/models.py +9 -0
  290. mannf/weights/__init__.py +7 -0
  291. mannf/weights/registry.py +9 -0
  292. mannf/weights/store.py +9 -0
  293. nat_engine-1.dist-info/METADATA +555 -0
  294. nat_engine-1.dist-info/RECORD +299 -0
  295. nat_engine-1.dist-info/WHEEL +5 -0
  296. nat_engine-1.dist-info/entry_points.txt +4 -0
  297. nat_engine-1.dist-info/licenses/LICENSE +651 -0
  298. nat_engine-1.dist-info/licenses/NOTICE +178 -0
  299. nat_engine-1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,664 @@
1
+ # Copyright (C) 2026 Brad Guider
2
+ # This file is part of NAT (Neural Agent Testing Framework).
3
+ # Licensed under the AGPL-3.0. See LICENSE for details.
4
+ # Commercial licensing available — see COMMERCIAL_LICENSE.md.
5
+
6
+ """UnifiedReportGenerator – aggregates all testing dimensions into a single report.
7
+
8
+ Consumes the raw dict returned by
9
+ :meth:`~mannf.core.functional_orchestrator.FunctionalTestOrchestrator.report`
10
+ and produces:
11
+
12
+ * :meth:`generate_json` — a fully JSON-serializable dict with summary
13
+ statistics across all four testing dimensions.
14
+ * :meth:`generate_html` — a self-contained HTML dashboard (no external
15
+ dependencies) with embedded CSS.
16
+ * :meth:`save_report` — writes HTML and/or JSON files to a given directory.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import os
23
+ from datetime import datetime, timezone
24
+ from typing import Any
25
+
26
+
27
+ class UnifiedReportGenerator:
28
+ """Aggregates functional, visual, accessibility, and performance results.
29
+
30
+ All public methods are stateless and accept the raw report dict directly.
31
+ """
32
+
33
+ # ------------------------------------------------------------------
34
+ # JSON report
35
+ # ------------------------------------------------------------------
36
+
37
+ def generate_json(self, report_dict: dict[str, Any]) -> dict[str, Any]:
38
+ """Build a structured, JSON-serializable summary of *report_dict*.
39
+
40
+ Parameters
41
+ ----------
42
+ report_dict:
43
+ The dict returned by
44
+ :meth:`~mannf.core.functional_orchestrator.FunctionalTestOrchestrator.report`.
45
+
46
+ Returns
47
+ -------
48
+ dict
49
+ Unified summary with keys ``generated_at``, ``elapsed_seconds``,
50
+ ``agent_count``, ``overall_verdict``, ``functional``, and
51
+ optionally ``visual_regression``, ``accessibility``,
52
+ ``performance``.
53
+ """
54
+ overall_verdict = self._compute_overall_verdict(report_dict)
55
+
56
+ result: dict[str, Any] = {
57
+ "generated_at": datetime.now(timezone.utc).isoformat(),
58
+ "elapsed_seconds": report_dict.get("elapsed_seconds", 0.0),
59
+ "agent_count": report_dict.get("agent_count", 0),
60
+ "overall_verdict": overall_verdict,
61
+ "functional": self._functional_summary(report_dict),
62
+ }
63
+
64
+ if "visual_regression" in report_dict:
65
+ result["visual_regression"] = self._visual_summary(report_dict["visual_regression"])
66
+
67
+ if "accessibility" in report_dict:
68
+ result["accessibility"] = self._accessibility_summary(report_dict["accessibility"])
69
+
70
+ if "performance" in report_dict:
71
+ result["performance"] = self._performance_summary(report_dict["performance"])
72
+
73
+ if "root_cause_analysis" in report_dict:
74
+ result["root_cause_analysis"] = report_dict["root_cause_analysis"]
75
+
76
+ return result
77
+
78
+ # ------------------------------------------------------------------
79
+ # HTML report
80
+ # ------------------------------------------------------------------
81
+
82
+ def generate_html(self, report_dict: dict[str, Any]) -> str:
83
+ """Build a self-contained HTML dashboard from *report_dict*.
84
+
85
+ Parameters
86
+ ----------
87
+ report_dict:
88
+ The dict returned by
89
+ :meth:`~mannf.core.functional_orchestrator.FunctionalTestOrchestrator.report`.
90
+
91
+ Returns
92
+ -------
93
+ str
94
+ A complete HTML document with embedded CSS and no external
95
+ dependencies.
96
+ """
97
+ json_data = self.generate_json(report_dict)
98
+ verdict = json_data["overall_verdict"]
99
+ generated_at = json_data["generated_at"]
100
+ elapsed = json_data["elapsed_seconds"]
101
+ agent_count = json_data["agent_count"]
102
+
103
+ verdict_class = "badge-pass" if verdict == "pass" else "badge-fail"
104
+ verdict_label = "PASS" if verdict == "pass" else "FAIL"
105
+
106
+ sections = []
107
+ sections.append(self._html_functional_section(json_data["functional"], report_dict))
108
+
109
+ if "visual_regression" in json_data:
110
+ sections.append(self._html_visual_section(json_data["visual_regression"], report_dict.get("visual_regression", {})))
111
+
112
+ if "accessibility" in json_data:
113
+ sections.append(self._html_accessibility_section(json_data["accessibility"], report_dict.get("accessibility", {})))
114
+
115
+ if "performance" in json_data:
116
+ sections.append(self._html_performance_section(json_data["performance"], report_dict.get("performance", {})))
117
+
118
+ if "root_cause_analysis" in json_data:
119
+ sections.append(self._html_root_cause_section(json_data["root_cause_analysis"]))
120
+
121
+ sections_html = "\n".join(sections)
122
+
123
+ return f"""<!DOCTYPE html>
124
+ <html lang="en">
125
+ <head>
126
+ <meta charset="UTF-8">
127
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
128
+ <title>NAT Unified Test Report</title>
129
+ <style>
130
+ {self._css()}
131
+ </style>
132
+ </head>
133
+ <body>
134
+ <div class="container">
135
+ <header>
136
+ <div class="header-content">
137
+ <h1>NAT Unified Test Report</h1>
138
+ <span class="badge {verdict_class}">{verdict_label}</span>
139
+ </div>
140
+ <div class="header-meta">
141
+ <span>Run timestamp: {generated_at}</span>
142
+ <span>Elapsed: {elapsed}s</span>
143
+ <span>Agents: {agent_count}</span>
144
+ </div>
145
+ </header>
146
+ {sections_html}
147
+ <footer>
148
+ <p>Generated by NAT Framework &mdash; Neural Agent Testing &mdash; {generated_at}</p>
149
+ </footer>
150
+ </div>
151
+ </body>
152
+ </html>"""
153
+
154
+ # ------------------------------------------------------------------
155
+ # Save report
156
+ # ------------------------------------------------------------------
157
+
158
+ def save_report(
159
+ self,
160
+ report_dict: dict[str, Any],
161
+ output_dir: str,
162
+ formats: list[str] | None = None,
163
+ ) -> list[str]:
164
+ """Write report files to *output_dir*.
165
+
166
+ Parameters
167
+ ----------
168
+ report_dict:
169
+ The dict returned by
170
+ :meth:`~mannf.core.functional_orchestrator.FunctionalTestOrchestrator.report`.
171
+ output_dir:
172
+ Directory to write files into. Created if it does not exist.
173
+ formats:
174
+ List of ``"html"`` and/or ``"json"``. Defaults to
175
+ ``["html", "json"]``.
176
+
177
+ Returns
178
+ -------
179
+ list[str]
180
+ Absolute paths of all files written.
181
+ """
182
+ if formats is None:
183
+ formats = ["html", "json"]
184
+
185
+ os.makedirs(output_dir, exist_ok=True)
186
+
187
+ written: list[str] = []
188
+
189
+ if "html" in formats:
190
+ html_path = os.path.join(output_dir, "nat_report.html")
191
+ with open(html_path, "w", encoding="utf-8") as fh:
192
+ fh.write(self.generate_html(report_dict))
193
+ written.append(html_path)
194
+
195
+ if "json" in formats:
196
+ json_path = os.path.join(output_dir, "nat_report.json")
197
+ with open(json_path, "w", encoding="utf-8") as fh:
198
+ json.dump(self.generate_json(report_dict), fh, indent=2)
199
+ written.append(json_path)
200
+
201
+ return written
202
+
203
+ # ------------------------------------------------------------------
204
+ # Overall verdict
205
+ # ------------------------------------------------------------------
206
+
207
+ def _compute_overall_verdict(self, report_dict: dict[str, Any]) -> str:
208
+ """Return ``"pass"`` only when all enabled dimensions pass."""
209
+ # Functional: any failures → fail
210
+ if report_dict.get("failed", 0) > 0:
211
+ return "fail"
212
+
213
+ # Visual regression: any diffs detected → fail
214
+ vr = report_dict.get("visual_regression")
215
+ if vr is not None and vr.get("failed", 0) > 0:
216
+ return "fail"
217
+
218
+ # Accessibility: any failed scans → fail
219
+ acc = report_dict.get("accessibility")
220
+ if acc is not None and acc.get("failed", 0) > 0:
221
+ return "fail"
222
+
223
+ # Performance: any failed evaluations → fail
224
+ perf = report_dict.get("performance")
225
+ if perf is not None and perf.get("failed", 0) > 0:
226
+ return "fail"
227
+
228
+ return "pass"
229
+
230
+ # ------------------------------------------------------------------
231
+ # Summary builders
232
+ # ------------------------------------------------------------------
233
+
234
+ @staticmethod
235
+ def _functional_summary(report_dict: dict[str, Any]) -> dict[str, Any]:
236
+ return {
237
+ "total": report_dict.get("total", 0),
238
+ "passed": report_dict.get("passed", 0),
239
+ "failed": report_dict.get("failed", 0),
240
+ }
241
+
242
+ @staticmethod
243
+ def _visual_summary(vr: dict[str, Any]) -> dict[str, Any]:
244
+ return {
245
+ "total_comparisons": vr.get("total_comparisons", 0),
246
+ "matched": vr.get("passed", 0),
247
+ "diffs_detected": vr.get("failed", 0),
248
+ "baselines_saved": vr.get("baselines_created", 0),
249
+ }
250
+
251
+ @staticmethod
252
+ def _accessibility_summary(acc: dict[str, Any]) -> dict[str, Any]:
253
+ results = acc.get("results", [])
254
+ total_violations = sum(r.get("violation_count", 0) for r in results)
255
+ # Aggregate violations by impact level
256
+ violation_counts: dict[str, int] = {
257
+ "critical": 0, "serious": 0, "moderate": 0, "minor": 0
258
+ }
259
+ for r in results:
260
+ for v in r.get("violations", []):
261
+ impact = v.get("impact", "moderate")
262
+ if impact in violation_counts:
263
+ violation_counts[impact] += 1
264
+
265
+ return {
266
+ "total_pages_scanned": acc.get("total_scans", 0),
267
+ "total_violations": total_violations,
268
+ "avg_compliance_score": acc.get("avg_compliance_score", 100.0),
269
+ "violation_counts_by_impact": violation_counts,
270
+ }
271
+
272
+ @staticmethod
273
+ def _performance_summary(perf: dict[str, Any]) -> dict[str, Any]:
274
+ results = perf.get("results", [])
275
+
276
+ # Per Core Web Vital aggregation
277
+ cwv_metrics: dict[str, list[float]] = {}
278
+ for r in results:
279
+ for m in r.get("metrics", []):
280
+ name = m["name"]
281
+ cwv_metrics.setdefault(name, []).append(m["value"])
282
+
283
+ cwv_stats: dict[str, dict[str, float]] = {}
284
+ for name, values in cwv_metrics.items():
285
+ cwv_stats[name] = {
286
+ "avg": round(sum(values) / len(values), 2),
287
+ "min": round(min(values), 2),
288
+ "max": round(max(values), 2),
289
+ }
290
+
291
+ # Overall grade: based on avg performance score
292
+ avg_score = perf.get("avg_performance_score", 100.0)
293
+ if avg_score >= 90:
294
+ grade = "good"
295
+ elif avg_score >= 50:
296
+ grade = "needs-improvement"
297
+ else:
298
+ grade = "poor"
299
+
300
+ return {
301
+ "total_pages_measured": perf.get("total_evaluations", 0),
302
+ "avg_performance_score": avg_score,
303
+ "overall_grade": grade,
304
+ "cwv_stats": cwv_stats,
305
+ "threshold_violations": len(perf.get("threshold_violations", [])),
306
+ }
307
+
308
+ # ------------------------------------------------------------------
309
+ # HTML section builders
310
+ # ------------------------------------------------------------------
311
+
312
+ @staticmethod
313
+ def _badge(passed: bool) -> str:
314
+ if passed:
315
+ return '<span class="badge badge-pass">PASS</span>'
316
+ return '<span class="badge badge-fail">FAIL</span>'
317
+
318
+ def _html_functional_section(
319
+ self,
320
+ summary: dict[str, Any],
321
+ report_dict: dict[str, Any],
322
+ ) -> str:
323
+ total = summary["total"]
324
+ passed = summary["passed"]
325
+ failed = summary["failed"]
326
+ verdict = self._badge(failed == 0)
327
+
328
+ tasks = report_dict.get("tasks", [])
329
+ rows = ""
330
+ for t in tasks:
331
+ task_passed = t.get("passed", not t.get("error"))
332
+ row_class = "row-pass" if task_passed else "row-fail"
333
+ status_badge = self._badge(task_passed)
334
+ url = t.get("url", "")
335
+ error = t.get("error", "") or ""
336
+ rows += (
337
+ f'<tr class="{row_class}">'
338
+ f"<td>{url}</td>"
339
+ f"<td>{status_badge}</td>"
340
+ f"<td>{error}</td>"
341
+ f"</tr>\n"
342
+ )
343
+
344
+ table = ""
345
+ if rows:
346
+ table = f"""
347
+ <table>
348
+ <thead><tr><th>URL</th><th>Status</th><th>Error</th></tr></thead>
349
+ <tbody>{rows}</tbody>
350
+ </table>"""
351
+
352
+ return f"""
353
+ <section class="report-section" id="functional-testing">
354
+ <h2>Functional Testing {verdict}</h2>
355
+ <div class="stats-row">
356
+ <div class="stat-card"><div class="stat-value">{total}</div><div class="stat-label">Total</div></div>
357
+ <div class="stat-card stat-pass"><div class="stat-value">{passed}</div><div class="stat-label">Passed</div></div>
358
+ <div class="stat-card stat-fail"><div class="stat-value">{failed}</div><div class="stat-label">Failed</div></div>
359
+ </div>
360
+ {table}
361
+ </section>"""
362
+
363
+ def _html_visual_section(
364
+ self,
365
+ summary: dict[str, Any],
366
+ vr_raw: dict[str, Any],
367
+ ) -> str:
368
+ total = summary["total_comparisons"]
369
+ matched = summary["matched"]
370
+ diffs = summary["diffs_detected"]
371
+ baselines = summary["baselines_saved"]
372
+ verdict = self._badge(diffs == 0)
373
+
374
+ rows = ""
375
+ for r in vr_raw.get("results", []):
376
+ matched_val = r.get("matched", False)
377
+ diff_pct = r.get("diff_percentage", 0.0)
378
+ row_class = "row-pass" if matched_val else "row-fail"
379
+ rows += (
380
+ f'<tr class="{row_class}">'
381
+ f'<td>{r.get("url", "")}</td>'
382
+ f"<td>{self._badge(matched_val)}</td>"
383
+ f"<td>{diff_pct:.2f}%</td>"
384
+ f"</tr>\n"
385
+ )
386
+
387
+ table = ""
388
+ if rows:
389
+ table = f"""
390
+ <table>
391
+ <thead><tr><th>URL</th><th>Status</th><th>Diff %</th></tr></thead>
392
+ <tbody>{rows}</tbody>
393
+ </table>"""
394
+
395
+ return f"""
396
+ <section class="report-section" id="visual-regression">
397
+ <h2>Visual Regression {verdict}</h2>
398
+ <div class="stats-row">
399
+ <div class="stat-card"><div class="stat-value">{total}</div><div class="stat-label">Total</div></div>
400
+ <div class="stat-card stat-pass"><div class="stat-value">{matched}</div><div class="stat-label">Matched</div></div>
401
+ <div class="stat-card stat-fail"><div class="stat-value">{diffs}</div><div class="stat-label">Diffs</div></div>
402
+ <div class="stat-card"><div class="stat-value">{baselines}</div><div class="stat-label">Baselines Saved</div></div>
403
+ </div>
404
+ {table}
405
+ </section>"""
406
+
407
+ def _html_accessibility_section(
408
+ self,
409
+ summary: dict[str, Any],
410
+ acc_raw: dict[str, Any],
411
+ ) -> str:
412
+ pages = summary["total_pages_scanned"]
413
+ avg_score = summary["avg_compliance_score"]
414
+ total_violations = summary["total_violations"]
415
+ v_counts = summary["violation_counts_by_impact"]
416
+ verdict = self._badge(total_violations == 0)
417
+
418
+ score_class = "stat-pass" if avg_score >= 80 else "stat-fail"
419
+
420
+ rows = ""
421
+ for r in acc_raw.get("results", []):
422
+ score = r.get("compliance_score", 100.0)
423
+ passed = r.get("passed", True)
424
+ row_class = "row-pass" if passed else "row-fail"
425
+ rows += (
426
+ f'<tr class="{row_class}">'
427
+ f'<td>{r.get("url", "")}</td>'
428
+ f"<td>{self._badge(passed)}</td>"
429
+ f"<td>{score:.1f}%</td>"
430
+ f'<td>{r.get("violation_count", 0)}</td>'
431
+ f"</tr>\n"
432
+ )
433
+
434
+ table = ""
435
+ if rows:
436
+ table = f"""
437
+ <table>
438
+ <thead><tr><th>URL</th><th>Status</th><th>Compliance Score</th><th>Violations</th></tr></thead>
439
+ <tbody>{rows}</tbody>
440
+ </table>"""
441
+
442
+ return f"""
443
+ <section class="report-section" id="accessibility">
444
+ <h2>Accessibility {verdict}</h2>
445
+ <div class="stats-row">
446
+ <div class="stat-card"><div class="stat-value">{pages}</div><div class="stat-label">Pages Scanned</div></div>
447
+ <div class="stat-card {score_class}"><div class="stat-value">{avg_score:.1f}%</div><div class="stat-label">Avg Compliance</div></div>
448
+ <div class="stat-card stat-fail"><div class="stat-value">{total_violations}</div><div class="stat-label">Total Violations</div></div>
449
+ <div class="stat-card"><div class="stat-value">{v_counts.get('critical', 0)}</div><div class="stat-label">Critical</div></div>
450
+ <div class="stat-card"><div class="stat-value">{v_counts.get('serious', 0)}</div><div class="stat-label">Serious</div></div>
451
+ <div class="stat-card"><div class="stat-value">{v_counts.get('moderate', 0)}</div><div class="stat-label">Moderate</div></div>
452
+ <div class="stat-card"><div class="stat-value">{v_counts.get('minor', 0)}</div><div class="stat-label">Minor</div></div>
453
+ </div>
454
+ {table}
455
+ </section>"""
456
+
457
+ def _html_performance_section(
458
+ self,
459
+ summary: dict[str, Any],
460
+ perf_raw: dict[str, Any],
461
+ ) -> str:
462
+ pages = summary["total_pages_measured"]
463
+ avg_score = summary["avg_performance_score"]
464
+ grade = summary["overall_grade"]
465
+ cwv_stats = summary["cwv_stats"]
466
+ violations = summary["threshold_violations"]
467
+ verdict = self._badge(violations == 0)
468
+
469
+ grade_class = {
470
+ "good": "stat-pass",
471
+ "needs-improvement": "stat-warn",
472
+ "poor": "stat-fail",
473
+ }.get(grade, "stat-warn")
474
+
475
+ # CWV cards
476
+ cwv_order = ["LCP", "FID", "FCP", "CLS", "TTFB", "TTI", "TBT", "page_load_time"]
477
+ cwv_cards = ""
478
+ for metric_name in cwv_order:
479
+ if metric_name not in cwv_stats:
480
+ continue
481
+ stats = cwv_stats[metric_name]
482
+ cwv_cards += (
483
+ f'<div class="cwv-card">'
484
+ f"<div class=\"cwv-name\">{metric_name}</div>"
485
+ f"<div class=\"cwv-avg\">avg: {stats['avg']}</div>"
486
+ f"<div class=\"cwv-range\">min: {stats['min']} / max: {stats['max']}</div>"
487
+ f"</div>\n"
488
+ )
489
+
490
+ rows = ""
491
+ for r in perf_raw.get("results", []):
492
+ passed = r.get("passed", True)
493
+ row_class = "row-pass" if passed else "row-fail"
494
+ rows += (
495
+ f'<tr class="{row_class}">'
496
+ f'<td>{r.get("url", "")}</td>'
497
+ f"<td>{self._badge(passed)}</td>"
498
+ f'<td>{r.get("performance_score", 0):.1f}</td>'
499
+ f'<td>{r.get("page_load_time_ms", 0):.0f} ms</td>'
500
+ f'<td>{r.get("good_count", 0)} good / {r.get("poor_count", 0)} poor</td>'
501
+ f"</tr>\n"
502
+ )
503
+
504
+ table = ""
505
+ if rows:
506
+ table = f"""
507
+ <table>
508
+ <thead><tr><th>URL</th><th>Status</th><th>Score</th><th>Load Time</th><th>Metrics</th></tr></thead>
509
+ <tbody>{rows}</tbody>
510
+ </table>"""
511
+
512
+ return f"""
513
+ <section class="report-section" id="performance">
514
+ <h2>Performance {verdict}</h2>
515
+ <div class="stats-row">
516
+ <div class="stat-card"><div class="stat-value">{pages}</div><div class="stat-label">Pages Measured</div></div>
517
+ <div class="stat-card {grade_class}"><div class="stat-value">{avg_score:.1f}</div><div class="stat-label">Avg Score</div></div>
518
+ <div class="stat-card {grade_class}"><div class="stat-value">{grade}</div><div class="stat-label">Grade</div></div>
519
+ <div class="stat-card stat-fail"><div class="stat-value">{violations}</div><div class="stat-label">Threshold Violations</div></div>
520
+ </div>
521
+ <div class="cwv-grid">
522
+ {cwv_cards}
523
+ </div>
524
+ {table}
525
+ </section>"""
526
+
527
+ @staticmethod
528
+ def _html_root_cause_section(suggestions: list[dict[str, Any]]) -> str:
529
+ """Render the Root Cause Analysis section for the HTML report."""
530
+ if not suggestions:
531
+ return """
532
+ <section class="report-section" id="root-cause-analysis">
533
+ <h2>🧠 Root Cause Analysis</h2>
534
+ <p style="color:#666;font-size:.9rem;">No failures were analyzed.</p>
535
+ </section>"""
536
+
537
+ analyzed = [s for s in suggestions if not s.get("quota_limited")]
538
+ limited = [s for s in suggestions if s.get("quota_limited")]
539
+
540
+ cards_html = ""
541
+ for s in analyzed:
542
+ confidence_pct = int(s.get("confidence", 0.0) * 100)
543
+ severity = s.get("severity", "")
544
+ sev_class = {
545
+ "critical": "sev-critical",
546
+ "high": "sev-high",
547
+ "medium": "sev-medium",
548
+ "low": "sev-low",
549
+ }.get(severity, "")
550
+ files = s.get("files_to_check", [])
551
+ files_html = (
552
+ "<ul>" + "".join(f"<li><code>{f}</code></li>" for f in files) + "</ul>"
553
+ if files
554
+ else "<em>None identified</em>"
555
+ )
556
+ cards_html += f"""
557
+ <div class="rca-card">
558
+ <div class="rca-header">
559
+ <span class="rca-url">{s.get("url", "")}</span>
560
+ {f'<span class="rca-sev {sev_class}">{severity.upper()}</span>' if severity else ""}
561
+ <span class="rca-confidence">Confidence: {confidence_pct}%</span>
562
+ </div>
563
+ <div class="rca-body">
564
+ <div class="rca-field"><strong>Likely Cause:</strong> {s.get("likely_cause", "")}</div>
565
+ <div class="rca-field"><strong>Suggested Fix:</strong> {s.get("suggested_fix", "")}</div>
566
+ <div class="rca-field"><strong>Files to Check:</strong> {files_html}</div>
567
+ </div>
568
+ </div>"""
569
+
570
+ limited_html = ""
571
+ if limited:
572
+ limited_html = f"""
573
+ <div class="rca-quota-notice">
574
+ 🔒 {len(limited)} finding(s) not analyzed — upgrade your plan to analyze all findings.
575
+ </div>"""
576
+
577
+ return f"""
578
+ <section class="report-section" id="root-cause-analysis">
579
+ <h2>🧠 Root Cause Analysis</h2>
580
+ <div class="stats-row">
581
+ <div class="stat-card"><div class="stat-value">{len(analyzed)}</div><div class="stat-label">Analyzed</div></div>
582
+ <div class="stat-card"><div class="stat-value">{len(limited)}</div><div class="stat-label">Quota Limited</div></div>
583
+ </div>
584
+ {cards_html}
585
+ {limited_html}
586
+ </section>"""
587
+
588
+ # ------------------------------------------------------------------
589
+ # Embedded CSS
590
+ # ------------------------------------------------------------------
591
+
592
+ @staticmethod
593
+ def _css() -> str:
594
+ return """
595
+ * { box-sizing: border-box; margin: 0; padding: 0; }
596
+ body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
597
+ background: #f0f2f5; color: #1a1a2e; line-height: 1.5; }
598
+ .container { max-width: 1100px; margin: 0 auto; padding: 24px; }
599
+
600
+ header { background: #fff; border-radius: 8px; padding: 24px;
601
+ box-shadow: 0 2px 8px rgba(0,0,0,.08); margin-bottom: 24px; }
602
+ .header-content { display: flex; align-items: center; gap: 16px; margin-bottom: 8px; }
603
+ .header-content h1 { font-size: 1.6rem; color: #1a1a2e; }
604
+ .header-meta { display: flex; gap: 24px; font-size: .85rem; color: #666; }
605
+
606
+ .badge { display: inline-block; padding: 4px 12px; border-radius: 12px;
607
+ font-weight: 700; font-size: .85rem; text-transform: uppercase; }
608
+ .badge-pass { background: #d4edda; color: #155724; }
609
+ .badge-fail { background: #f8d7da; color: #721c24; }
610
+
611
+ .report-section { background: #fff; border-radius: 8px; padding: 24px;
612
+ box-shadow: 0 2px 8px rgba(0,0,0,.08); margin-bottom: 24px; }
613
+ .report-section h2 { font-size: 1.2rem; margin-bottom: 16px; display: flex;
614
+ align-items: center; gap: 10px; }
615
+
616
+ .stats-row { display: flex; flex-wrap: wrap; gap: 12px; margin-bottom: 16px; }
617
+ .stat-card { background: #f8f9fa; border-radius: 6px; padding: 12px 16px;
618
+ min-width: 90px; text-align: center; border: 1px solid #e9ecef; }
619
+ .stat-value { font-size: 1.4rem; font-weight: 700; }
620
+ .stat-label { font-size: .75rem; color: #666; margin-top: 2px; }
621
+ .stat-pass { background: #d4edda; border-color: #c3e6cb; }
622
+ .stat-pass .stat-value { color: #155724; }
623
+ .stat-fail { background: #f8d7da; border-color: #f5c6cb; }
624
+ .stat-fail .stat-value { color: #721c24; }
625
+ .stat-warn { background: #fff3cd; border-color: #ffc107; }
626
+ .stat-warn .stat-value { color: #856404; }
627
+
628
+ table { width: 100%; border-collapse: collapse; font-size: .9rem; margin-top: 8px; }
629
+ thead { background: #f8f9fa; }
630
+ th { padding: 10px 12px; text-align: left; font-weight: 600;
631
+ border-bottom: 2px solid #dee2e6; }
632
+ td { padding: 8px 12px; border-bottom: 1px solid #dee2e6; }
633
+ .row-pass td { background: #f9fff9; }
634
+ .row-fail td { background: #fff9f9; }
635
+
636
+ .cwv-grid { display: flex; flex-wrap: wrap; gap: 12px; margin-bottom: 16px; }
637
+ .cwv-card { background: #f8f9fa; border-radius: 6px; padding: 12px;
638
+ min-width: 130px; border: 1px solid #e9ecef; }
639
+ .cwv-name { font-weight: 700; font-size: .95rem; margin-bottom: 4px; }
640
+ .cwv-avg { font-size: .85rem; color: #333; }
641
+ .cwv-range { font-size: .75rem; color: #888; margin-top: 2px; }
642
+
643
+ /* Root Cause Analysis */
644
+ .rca-card { border: 1px solid #e9ecef; border-radius: 6px; margin-bottom: 12px;
645
+ overflow: hidden; }
646
+ .rca-header { background: #f8f9fa; padding: 10px 14px; display: flex;
647
+ align-items: center; gap: 10px; flex-wrap: wrap; }
648
+ .rca-url { font-weight: 600; font-size: .9rem; flex: 1; }
649
+ .rca-confidence { font-size: .8rem; color: #666; margin-left: auto; }
650
+ .rca-body { padding: 12px 14px; }
651
+ .rca-field { margin-bottom: 8px; font-size: .9rem; }
652
+ .rca-field ul { margin: 4px 0 0 16px; }
653
+ .rca-field code { background: #f8f9fa; padding: 1px 5px; border-radius: 3px;
654
+ font-size: .85em; }
655
+ .rca-quota-notice { background: #fff3cd; border: 1px solid #ffc107; border-radius: 6px;
656
+ padding: 10px 14px; font-size: .85rem; color: #856404; margin-top: 8px; }
657
+ .rca-sev { padding: 2px 8px; border-radius: 10px; font-size: .75rem; font-weight: 700; }
658
+ .sev-critical { background: #f8d7da; color: #721c24; }
659
+ .sev-high { background: #fde8d2; color: #8a3a00; }
660
+ .sev-medium { background: #fff3cd; color: #856404; }
661
+ .sev-low { background: #d4edda; color: #155724; }
662
+
663
+ footer { text-align: center; color: #aaa; font-size: .8rem; padding: 16px 0; }
664
+ """
@@ -0,0 +1,17 @@
1
+ # Copyright (C) 2026 Brad Guider
2
+ # This file is part of NAT (Neural Agent Testing Framework).
3
+ # Licensed under the AGPL-3.0. See LICENSE for details.
4
+ # Commercial licensing available — see COMMERCIAL_LICENSE.md.
5
+
6
+ """Testing sub-package for the Multi-Agent Neural Network Framework."""
7
+
8
+ from mannf.core.testing.models import TestCase, TestResult, TestSuite, TestStatus
9
+ from mannf.core.testing.adaptive_controller import AdaptiveController
10
+
11
+ __all__ = [
12
+ "TestCase",
13
+ "TestResult",
14
+ "TestSuite",
15
+ "TestStatus",
16
+ "AdaptiveController",
17
+ ]