ai-testing-swarm 0.1.14__py3-none-any.whl → 0.1.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_testing_swarm/__init__.py +1 -1
- ai_testing_swarm/cli.py +20 -1
- ai_testing_swarm/core/risk.py +139 -0
- ai_testing_swarm/orchestrator.py +45 -5
- ai_testing_swarm/reporting/dashboard.py +188 -0
- ai_testing_swarm/reporting/report_writer.py +50 -3
- {ai_testing_swarm-0.1.14.dist-info → ai_testing_swarm-0.1.15.dist-info}/METADATA +3 -1
- {ai_testing_swarm-0.1.14.dist-info → ai_testing_swarm-0.1.15.dist-info}/RECORD +11 -9
- {ai_testing_swarm-0.1.14.dist-info → ai_testing_swarm-0.1.15.dist-info}/WHEEL +0 -0
- {ai_testing_swarm-0.1.14.dist-info → ai_testing_swarm-0.1.15.dist-info}/entry_points.txt +0 -0
- {ai_testing_swarm-0.1.14.dist-info → ai_testing_swarm-0.1.15.dist-info}/top_level.txt +0 -0
ai_testing_swarm/__init__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.1.
|
|
1
|
+
__version__ = "0.1.15"
|
ai_testing_swarm/cli.py
CHANGED
|
@@ -107,6 +107,20 @@ def main():
|
|
|
107
107
|
help="Report format to write (default: json)",
|
|
108
108
|
)
|
|
109
109
|
|
|
110
|
+
# Batch1: risk gate thresholds (backward compatible defaults)
|
|
111
|
+
parser.add_argument(
|
|
112
|
+
"--gate-warn",
|
|
113
|
+
type=int,
|
|
114
|
+
default=30,
|
|
115
|
+
help="Gate WARN threshold for endpoint risk score (default: 30)",
|
|
116
|
+
)
|
|
117
|
+
parser.add_argument(
|
|
118
|
+
"--gate-block",
|
|
119
|
+
type=int,
|
|
120
|
+
default=80,
|
|
121
|
+
help="Gate BLOCK threshold for endpoint risk score (default: 80)",
|
|
122
|
+
)
|
|
123
|
+
|
|
110
124
|
args = parser.parse_args()
|
|
111
125
|
|
|
112
126
|
# ------------------------------------------------------------
|
|
@@ -127,7 +141,12 @@ def main():
|
|
|
127
141
|
import os
|
|
128
142
|
os.environ["AI_SWARM_PUBLIC_ONLY"] = "1"
|
|
129
143
|
|
|
130
|
-
decision, results = SwarmOrchestrator().run(
|
|
144
|
+
decision, results = SwarmOrchestrator().run(
|
|
145
|
+
request,
|
|
146
|
+
report_format=args.report_format,
|
|
147
|
+
gate_warn=args.gate_warn,
|
|
148
|
+
gate_block=args.gate_block,
|
|
149
|
+
)
|
|
131
150
|
|
|
132
151
|
# ------------------------------------------------------------
|
|
133
152
|
# Console output
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Risk scoring for AI Testing Swarm.
|
|
2
|
+
|
|
3
|
+
Batch1 additions:
|
|
4
|
+
- Compute a numeric risk_score per test result (0..100)
|
|
5
|
+
- Aggregate endpoint risk and drive gate thresholds (PASS/WARN/BLOCK)
|
|
6
|
+
|
|
7
|
+
Design goals:
|
|
8
|
+
- Backward compatible: consumers can ignore risk_score fields.
|
|
9
|
+
- Deterministic and explainable.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class RiskThresholds:
|
|
19
|
+
"""Thresholds for the endpoint gate.
|
|
20
|
+
|
|
21
|
+
Gate is computed from endpoint_risk_score (currently max test risk).
|
|
22
|
+
|
|
23
|
+
- PASS: score < warn
|
|
24
|
+
- WARN: warn <= score < block
|
|
25
|
+
- BLOCK: score >= block
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
warn: int = 30
|
|
29
|
+
block: int = 80
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# Keep aligned with orchestrator/release gate semantics
|
|
33
|
+
EXPECTED_FAILURES: set[str] = {
|
|
34
|
+
"success",
|
|
35
|
+
"missing_param",
|
|
36
|
+
"invalid_param",
|
|
37
|
+
"security",
|
|
38
|
+
"method_not_allowed",
|
|
39
|
+
"not_found",
|
|
40
|
+
"content_negotiation",
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
RISKY_FAILURES: set[str] = {
|
|
44
|
+
"unknown",
|
|
45
|
+
"missing_param_accepted",
|
|
46
|
+
"null_param_accepted",
|
|
47
|
+
"invalid_param_accepted",
|
|
48
|
+
"headers_accepted",
|
|
49
|
+
"method_risk",
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
BLOCKING_FAILURES: set[str] = {
|
|
53
|
+
"auth_issue",
|
|
54
|
+
"infra",
|
|
55
|
+
"security_risk",
|
|
56
|
+
"server_error",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def clamp_int(x: int, lo: int, hi: int) -> int:
|
|
61
|
+
return lo if x < lo else hi if x > hi else x
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def compute_test_risk_score(result: dict, *, sla_ms: int | None = None) -> int:
|
|
65
|
+
"""Compute a risk score for a single test result.
|
|
66
|
+
|
|
67
|
+
Inputs expected (best-effort):
|
|
68
|
+
- result['failure_type']
|
|
69
|
+
- result['status']
|
|
70
|
+
- result['response']['status_code']
|
|
71
|
+
- result['response']['elapsed_ms']
|
|
72
|
+
- result['response']['openapi_validation'] (list)
|
|
73
|
+
|
|
74
|
+
Returns: int in range 0..100.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
ft = str(result.get("failure_type") or "unknown")
|
|
78
|
+
status = str(result.get("status") or "")
|
|
79
|
+
resp = result.get("response") or {}
|
|
80
|
+
sc = resp.get("status_code")
|
|
81
|
+
|
|
82
|
+
# Base score from semantic classification.
|
|
83
|
+
if ft in EXPECTED_FAILURES:
|
|
84
|
+
base = 0
|
|
85
|
+
elif ft in RISKY_FAILURES:
|
|
86
|
+
base = 35
|
|
87
|
+
elif ft in BLOCKING_FAILURES:
|
|
88
|
+
base = 90
|
|
89
|
+
else:
|
|
90
|
+
# Unknown failure types are treated as high risk but not always a hard blocker.
|
|
91
|
+
base = 60
|
|
92
|
+
|
|
93
|
+
# Status-code adjustments (defense in depth)
|
|
94
|
+
if isinstance(sc, int):
|
|
95
|
+
if 500 <= sc:
|
|
96
|
+
base = max(base, 90)
|
|
97
|
+
elif sc in (401, 403):
|
|
98
|
+
base = max(base, 80)
|
|
99
|
+
elif 400 <= sc < 500:
|
|
100
|
+
# Client errors for negative tests are expected; keep base.
|
|
101
|
+
base = max(base, base)
|
|
102
|
+
|
|
103
|
+
# Explicit FAILED status implies something unexpected.
|
|
104
|
+
if status.upper() == "FAILED":
|
|
105
|
+
base = max(base, 70)
|
|
106
|
+
|
|
107
|
+
# OpenAPI validation issues are a small additive risk.
|
|
108
|
+
issues = resp.get("openapi_validation") or []
|
|
109
|
+
if isinstance(issues, list) and issues:
|
|
110
|
+
base += 10
|
|
111
|
+
|
|
112
|
+
# SLA breach is a small additive risk.
|
|
113
|
+
if sla_ms is not None:
|
|
114
|
+
elapsed = resp.get("elapsed_ms")
|
|
115
|
+
if isinstance(elapsed, int) and elapsed > sla_ms:
|
|
116
|
+
base += 5
|
|
117
|
+
|
|
118
|
+
return clamp_int(int(base), 0, 100)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def compute_endpoint_risk_score(results: list[dict]) -> int:
|
|
122
|
+
"""Aggregate endpoint risk score.
|
|
123
|
+
|
|
124
|
+
Current policy: endpoint risk is the max risk_score across tests.
|
|
125
|
+
(This makes gating stable and easy to interpret.)
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
scores = [r.get("risk_score") for r in (results or []) if isinstance(r.get("risk_score"), int)]
|
|
129
|
+
if not scores:
|
|
130
|
+
return 0
|
|
131
|
+
return int(max(scores))
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def gate_from_score(score: int, thresholds: RiskThresholds) -> str:
|
|
135
|
+
if score >= thresholds.block:
|
|
136
|
+
return "BLOCK"
|
|
137
|
+
if score >= thresholds.warn:
|
|
138
|
+
return "WARN"
|
|
139
|
+
return "PASS"
|
ai_testing_swarm/orchestrator.py
CHANGED
|
@@ -5,6 +5,12 @@ from ai_testing_swarm.agents.learning_agent import LearningAgent
|
|
|
5
5
|
from ai_testing_swarm.agents.release_gate_agent import ReleaseGateAgent
|
|
6
6
|
from ai_testing_swarm.reporting.report_writer import write_report
|
|
7
7
|
from ai_testing_swarm.core.safety import enforce_public_only
|
|
8
|
+
from ai_testing_swarm.core.risk import (
|
|
9
|
+
RiskThresholds,
|
|
10
|
+
compute_test_risk_score,
|
|
11
|
+
compute_endpoint_risk_score,
|
|
12
|
+
gate_from_score,
|
|
13
|
+
)
|
|
8
14
|
|
|
9
15
|
import logging
|
|
10
16
|
|
|
@@ -40,8 +46,20 @@ class SwarmOrchestrator:
|
|
|
40
46
|
self.learner = LearningAgent()
|
|
41
47
|
self.release_gate = ReleaseGateAgent()
|
|
42
48
|
|
|
43
|
-
def run(
|
|
44
|
-
|
|
49
|
+
def run(
|
|
50
|
+
self,
|
|
51
|
+
request: dict,
|
|
52
|
+
*,
|
|
53
|
+
report_format: str = "json",
|
|
54
|
+
gate_warn: int = 30,
|
|
55
|
+
gate_block: int = 80,
|
|
56
|
+
):
|
|
57
|
+
"""Runs the full AI testing swarm and returns (decision, results).
|
|
58
|
+
|
|
59
|
+
gate_warn/gate_block:
|
|
60
|
+
Thresholds for PASS/WARN/BLOCK gate based on endpoint risk.
|
|
61
|
+
(Kept optional for backward compatibility.)
|
|
62
|
+
"""
|
|
45
63
|
|
|
46
64
|
# Safety hook (currently no-op; kept for backward compatibility)
|
|
47
65
|
enforce_public_only(request["url"])
|
|
@@ -78,6 +96,19 @@ class SwarmOrchestrator:
|
|
|
78
96
|
else "FAILED"
|
|
79
97
|
),
|
|
80
98
|
})
|
|
99
|
+
|
|
100
|
+
# Batch1: numeric risk score per test (0..100)
|
|
101
|
+
try:
|
|
102
|
+
from ai_testing_swarm.core.config import AI_SWARM_SLA_MS
|
|
103
|
+
|
|
104
|
+
execution_result["risk_score"] = compute_test_risk_score(
|
|
105
|
+
execution_result,
|
|
106
|
+
sla_ms=AI_SWARM_SLA_MS,
|
|
107
|
+
)
|
|
108
|
+
except Exception:
|
|
109
|
+
# Keep backwards compatibility: don't fail the run if scoring breaks.
|
|
110
|
+
execution_result["risk_score"] = 0
|
|
111
|
+
|
|
81
112
|
# Optional learning step
|
|
82
113
|
try:
|
|
83
114
|
self.learner.learn(test_name, classification)
|
|
@@ -97,14 +128,23 @@ class SwarmOrchestrator:
|
|
|
97
128
|
results.append(results_by_name[t["name"]])
|
|
98
129
|
|
|
99
130
|
# --------------------------------------------------------
|
|
100
|
-
# 3️⃣ RELEASE DECISION
|
|
131
|
+
# 3️⃣ RELEASE DECISION + RISK GATE
|
|
101
132
|
# --------------------------------------------------------
|
|
102
133
|
decision = self.release_gate.decide(results)
|
|
103
134
|
|
|
135
|
+
thresholds = RiskThresholds(warn=int(gate_warn), block=int(gate_block))
|
|
136
|
+
endpoint_risk_score = compute_endpoint_risk_score(results)
|
|
137
|
+
gate_status = gate_from_score(endpoint_risk_score, thresholds)
|
|
138
|
+
|
|
104
139
|
# --------------------------------------------------------
|
|
105
|
-
# 4️⃣ WRITE
|
|
140
|
+
# 4️⃣ WRITE REPORT
|
|
106
141
|
# --------------------------------------------------------
|
|
107
|
-
meta = {
|
|
142
|
+
meta = {
|
|
143
|
+
"decision": decision,
|
|
144
|
+
"gate_status": gate_status,
|
|
145
|
+
"gate_thresholds": {"warn": thresholds.warn, "block": thresholds.block},
|
|
146
|
+
"endpoint_risk_score": endpoint_risk_score,
|
|
147
|
+
}
|
|
108
148
|
|
|
109
149
|
# Optional AI summary for humans (best-effort)
|
|
110
150
|
try:
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _html_escape(s: object) -> str:
|
|
9
|
+
return (
|
|
10
|
+
str(s)
|
|
11
|
+
.replace("&", "&")
|
|
12
|
+
.replace("<", "<")
|
|
13
|
+
.replace(">", ">")
|
|
14
|
+
.replace('"', """)
|
|
15
|
+
.replace("'", "'")
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class EndpointRow:
|
|
21
|
+
endpoint_dir: str
|
|
22
|
+
endpoint: str
|
|
23
|
+
run_time: str
|
|
24
|
+
gate_status: str
|
|
25
|
+
endpoint_risk_score: int
|
|
26
|
+
decision: str
|
|
27
|
+
report_relpath: str
|
|
28
|
+
top_risks: list[dict]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _latest_json_report(endpoint_dir: Path) -> Path | None:
|
|
32
|
+
if not endpoint_dir.exists() or not endpoint_dir.is_dir():
|
|
33
|
+
return None
|
|
34
|
+
candidates = sorted(endpoint_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
|
|
35
|
+
return candidates[0] if candidates else None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _load_report(p: Path) -> dict | None:
|
|
39
|
+
try:
|
|
40
|
+
return json.loads(p.read_text(encoding="utf-8"))
|
|
41
|
+
except Exception:
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def write_dashboard_index(reports_dir: Path) -> str:
|
|
46
|
+
"""Create/overwrite a simple static index.html under reports_dir."""
|
|
47
|
+
|
|
48
|
+
reports_dir = Path(reports_dir)
|
|
49
|
+
reports_dir.mkdir(parents=True, exist_ok=True)
|
|
50
|
+
|
|
51
|
+
rows: list[EndpointRow] = []
|
|
52
|
+
|
|
53
|
+
for child in sorted([p for p in reports_dir.iterdir() if p.is_dir()]):
|
|
54
|
+
latest = _latest_json_report(child)
|
|
55
|
+
if not latest:
|
|
56
|
+
continue
|
|
57
|
+
rpt = _load_report(latest)
|
|
58
|
+
if not rpt:
|
|
59
|
+
continue
|
|
60
|
+
|
|
61
|
+
meta = rpt.get("meta") or {}
|
|
62
|
+
summary = rpt.get("summary") or {}
|
|
63
|
+
|
|
64
|
+
endpoint_risk = meta.get("endpoint_risk_score")
|
|
65
|
+
if endpoint_risk is None:
|
|
66
|
+
endpoint_risk = summary.get("endpoint_risk_score", 0)
|
|
67
|
+
|
|
68
|
+
row = EndpointRow(
|
|
69
|
+
endpoint_dir=child.name,
|
|
70
|
+
endpoint=str(rpt.get("endpoint") or child.name),
|
|
71
|
+
run_time=str(rpt.get("run_time") or ""),
|
|
72
|
+
gate_status=str(meta.get("gate_status") or ""),
|
|
73
|
+
endpoint_risk_score=int(endpoint_risk or 0),
|
|
74
|
+
decision=str(meta.get("decision") or ""),
|
|
75
|
+
report_relpath=str(child.name + "/" + latest.name),
|
|
76
|
+
top_risks=list(summary.get("top_risks") or []),
|
|
77
|
+
)
|
|
78
|
+
rows.append(row)
|
|
79
|
+
|
|
80
|
+
# Sort by risk (desc) then recent
|
|
81
|
+
rows.sort(key=lambda r: (r.endpoint_risk_score, r.run_time), reverse=True)
|
|
82
|
+
|
|
83
|
+
# Global top risks across endpoints
|
|
84
|
+
global_risks = []
|
|
85
|
+
for r in rows:
|
|
86
|
+
for item in (r.top_risks or [])[:3]:
|
|
87
|
+
global_risks.append(
|
|
88
|
+
{
|
|
89
|
+
"endpoint": r.endpoint,
|
|
90
|
+
"name": item.get("name"),
|
|
91
|
+
"risk_score": item.get("risk_score"),
|
|
92
|
+
"status": item.get("status"),
|
|
93
|
+
"failure_type": item.get("failure_type"),
|
|
94
|
+
"report": r.report_relpath,
|
|
95
|
+
}
|
|
96
|
+
)
|
|
97
|
+
global_risks.sort(key=lambda x: int(x.get("risk_score") or 0), reverse=True)
|
|
98
|
+
global_risks = global_risks[:15]
|
|
99
|
+
|
|
100
|
+
def badge(gate: str) -> str:
|
|
101
|
+
gate = (gate or "").upper()
|
|
102
|
+
cls = {"PASS": "pass", "WARN": "warn", "BLOCK": "block"}.get(gate, "")
|
|
103
|
+
return f"<span class='gate {cls}'>{_html_escape(gate)}</span>"
|
|
104
|
+
|
|
105
|
+
endpoint_rows_html = "".join(
|
|
106
|
+
"<tr>"
|
|
107
|
+
f"<td>{badge(r.gate_status)}</td>"
|
|
108
|
+
f"<td><code>{_html_escape(r.endpoint_risk_score)}</code></td>"
|
|
109
|
+
f"<td><a href='{_html_escape(r.report_relpath)}'>{_html_escape(r.endpoint)}</a></td>"
|
|
110
|
+
f"<td><code>{_html_escape(r.run_time)}</code></td>"
|
|
111
|
+
f"<td><code>{_html_escape(r.decision)}</code></td>"
|
|
112
|
+
"</tr>"
|
|
113
|
+
for r in rows
|
|
114
|
+
) or "<tr><td colspan='5'>(no JSON reports found)</td></tr>"
|
|
115
|
+
|
|
116
|
+
top_risks_html = "".join(
|
|
117
|
+
"<tr>"
|
|
118
|
+
f"<td><code>{_html_escape(x.get('risk_score'))}</code></td>"
|
|
119
|
+
f"<td>{_html_escape(x.get('status'))}</td>"
|
|
120
|
+
f"<td><code>{_html_escape(x.get('failure_type'))}</code></td>"
|
|
121
|
+
f"<td>{_html_escape(x.get('name'))}</td>"
|
|
122
|
+
f"<td><a href='{_html_escape(x.get('report'))}'>{_html_escape(x.get('endpoint'))}</a></td>"
|
|
123
|
+
"</tr>"
|
|
124
|
+
for x in global_risks
|
|
125
|
+
) or "<tr><td colspan='5'>(none)</td></tr>"
|
|
126
|
+
|
|
127
|
+
css = """
|
|
128
|
+
body{font-family:ui-sans-serif,system-ui,Segoe UI,Roboto,Arial; margin:20px;}
|
|
129
|
+
table{border-collapse:collapse; width:100%;}
|
|
130
|
+
th,td{border:1px solid #ddd; padding:8px; vertical-align:top}
|
|
131
|
+
th{background:#f6f7f9; text-align:left}
|
|
132
|
+
code{background:#f1f2f4; padding:1px 5px; border-radius:6px;}
|
|
133
|
+
.gate{display:inline-block; padding:2px 10px; border-radius:999px; border:1px solid #aaa; font-size:12px;}
|
|
134
|
+
.gate.pass{background:#e6ffed; border-color:#36b37e;}
|
|
135
|
+
.gate.warn{background:#fff7e6; border-color:#ffab00;}
|
|
136
|
+
.gate.block{background:#ffebe6; border-color:#ff5630;}
|
|
137
|
+
.muted{color:#555}
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
html = f"""<!doctype html>
|
|
141
|
+
<html>
|
|
142
|
+
<head>
|
|
143
|
+
<meta charset='utf-8'/>
|
|
144
|
+
<title>AI Testing Swarm — Dashboard</title>
|
|
145
|
+
<style>{css}</style>
|
|
146
|
+
</head>
|
|
147
|
+
<body>
|
|
148
|
+
<h1>AI Testing Swarm — Dashboard</h1>
|
|
149
|
+
<p class='muted'>Static index generated from latest JSON report per endpoint.</p>
|
|
150
|
+
|
|
151
|
+
<h2>Endpoints</h2>
|
|
152
|
+
<table>
|
|
153
|
+
<thead>
|
|
154
|
+
<tr>
|
|
155
|
+
<th>Gate</th>
|
|
156
|
+
<th>Risk</th>
|
|
157
|
+
<th>Endpoint</th>
|
|
158
|
+
<th>Run time</th>
|
|
159
|
+
<th>Decision</th>
|
|
160
|
+
</tr>
|
|
161
|
+
</thead>
|
|
162
|
+
<tbody>
|
|
163
|
+
{endpoint_rows_html}
|
|
164
|
+
</tbody>
|
|
165
|
+
</table>
|
|
166
|
+
|
|
167
|
+
<h2>Top risks (across endpoints)</h2>
|
|
168
|
+
<table>
|
|
169
|
+
<thead>
|
|
170
|
+
<tr>
|
|
171
|
+
<th>Risk</th>
|
|
172
|
+
<th>Status</th>
|
|
173
|
+
<th>Failure type</th>
|
|
174
|
+
<th>Test</th>
|
|
175
|
+
<th>Endpoint</th>
|
|
176
|
+
</tr>
|
|
177
|
+
</thead>
|
|
178
|
+
<tbody>
|
|
179
|
+
{top_risks_html}
|
|
180
|
+
</tbody>
|
|
181
|
+
</table>
|
|
182
|
+
|
|
183
|
+
</body>
|
|
184
|
+
</html>"""
|
|
185
|
+
|
|
186
|
+
out = reports_dir / "index.html"
|
|
187
|
+
out.write_text(html, encoding="utf-8")
|
|
188
|
+
return str(out)
|
|
@@ -58,6 +58,7 @@ def extract_endpoint_name(method: str, url: str) -> str:
|
|
|
58
58
|
# 📝 REPORT WRITER
|
|
59
59
|
# ============================================================
|
|
60
60
|
from ai_testing_swarm.core.config import AI_SWARM_SLA_MS
|
|
61
|
+
from ai_testing_swarm.reporting.dashboard import write_dashboard_index
|
|
61
62
|
|
|
62
63
|
SENSITIVE_HEADER_KEYS = {
|
|
63
64
|
"authorization",
|
|
@@ -103,6 +104,13 @@ def _render_markdown(report: dict) -> str:
|
|
|
103
104
|
lines.append(f"**Endpoint:** `{_markdown_escape(report.get('endpoint'))}` ")
|
|
104
105
|
lines.append(f"**Run time:** `{_markdown_escape(report.get('run_time'))}` ")
|
|
105
106
|
lines.append(f"**Total tests:** `{report.get('total_tests')}`")
|
|
107
|
+
|
|
108
|
+
meta = report.get("meta") or {}
|
|
109
|
+
if meta.get("gate_status"):
|
|
110
|
+
lines.append(f"**Gate:** `{_markdown_escape(meta.get('gate_status'))}` ")
|
|
111
|
+
if meta.get("endpoint_risk_score") is not None:
|
|
112
|
+
lines.append(f"**Endpoint risk:** `{_markdown_escape(meta.get('endpoint_risk_score'))}`")
|
|
113
|
+
|
|
106
114
|
lines.append("")
|
|
107
115
|
|
|
108
116
|
summary = report.get("summary") or {}
|
|
@@ -136,7 +144,8 @@ def _render_markdown(report: dict) -> str:
|
|
|
136
144
|
sc = resp.get("status_code")
|
|
137
145
|
lines.append(
|
|
138
146
|
f"- **{_markdown_escape(r.get('status'))}** `{_markdown_escape(r.get('name'))}` "
|
|
139
|
-
f"(
|
|
147
|
+
f"(risk={_markdown_escape(r.get('risk_score'))}, status={sc}, "
|
|
148
|
+
f"failure_type={_markdown_escape(r.get('failure_type'))})"
|
|
140
149
|
)
|
|
141
150
|
|
|
142
151
|
return "\n".join(lines) + "\n"
|
|
@@ -158,6 +167,7 @@ def _render_html(report: dict) -> str:
|
|
|
158
167
|
run_time = _html_escape(report.get("run_time"))
|
|
159
168
|
total_tests = report.get("total_tests")
|
|
160
169
|
summary = report.get("summary") or {}
|
|
170
|
+
meta = report.get("meta") or {}
|
|
161
171
|
|
|
162
172
|
results = report.get("results") or []
|
|
163
173
|
risky = [r for r in results if str(r.get("status")) == "RISK"]
|
|
@@ -184,6 +194,7 @@ def _render_html(report: dict) -> str:
|
|
|
184
194
|
failure_type = _html_escape(r.get("failure_type"))
|
|
185
195
|
sc = _html_escape(resp.get("status_code"))
|
|
186
196
|
elapsed = _html_escape(resp.get("elapsed_ms"))
|
|
197
|
+
risk_score = _html_escape(r.get("risk_score"))
|
|
187
198
|
badge = {
|
|
188
199
|
"PASSED": "badge passed",
|
|
189
200
|
"FAILED": "badge failed",
|
|
@@ -193,13 +204,13 @@ def _render_html(report: dict) -> str:
|
|
|
193
204
|
rows.append(
|
|
194
205
|
"<details class='case'>"
|
|
195
206
|
f"<summary><span class='{badge}'>{status}</span> "
|
|
196
|
-
f"<b>{name}</b> — status={sc}, elapsed_ms={elapsed}, failure_type={failure_type}"
|
|
207
|
+
f"<b>{name}</b> — risk={risk_score}, status={sc}, elapsed_ms={elapsed}, failure_type={failure_type}"
|
|
197
208
|
f"{(' — openapi_issues=' + str(len(issues))) if issues else ''}"
|
|
198
209
|
"</summary>"
|
|
199
210
|
"<div class='grid'>"
|
|
200
211
|
f"<div><h4>Request</h4>{_pre(r.get('request'))}</div>"
|
|
201
212
|
f"<div><h4>Response</h4>{_pre(resp)}</div>"
|
|
202
|
-
f"<div><h4>Reasoning</h4>{_pre({k: r.get(k) for k in ('reason','confidence','failure_type','status')})}</div>"
|
|
213
|
+
f"<div><h4>Reasoning</h4>{_pre({k: r.get(k) for k in ('reason','confidence','failure_type','status','risk_score')})}</div>"
|
|
203
214
|
"</div>"
|
|
204
215
|
"</details>"
|
|
205
216
|
)
|
|
@@ -236,6 +247,8 @@ def _render_html(report: dict) -> str:
|
|
|
236
247
|
<div><b>Endpoint:</b> <code>{endpoint}</code></div>
|
|
237
248
|
<div><b>Run time:</b> <code>{run_time}</code></div>
|
|
238
249
|
<div><b>Total tests:</b> <code>{total_tests}</code></div>
|
|
250
|
+
<div><b>Gate:</b> <code>{_html_escape(meta.get('gate_status'))}</code></div>
|
|
251
|
+
<div><b>Endpoint risk:</b> <code>{_html_escape(meta.get('endpoint_risk_score'))}</code></div>
|
|
239
252
|
</div>
|
|
240
253
|
|
|
241
254
|
<h2>Summary</h2>
|
|
@@ -258,6 +271,7 @@ def write_report(
|
|
|
258
271
|
*,
|
|
259
272
|
meta: dict | None = None,
|
|
260
273
|
report_format: str = "json",
|
|
274
|
+
update_index: bool = True,
|
|
261
275
|
) -> str:
|
|
262
276
|
"""Write a swarm report.
|
|
263
277
|
|
|
@@ -284,6 +298,9 @@ def write_report(
|
|
|
284
298
|
"counts_by_failure_type": {},
|
|
285
299
|
"counts_by_status_code": {},
|
|
286
300
|
"slow_tests": [],
|
|
301
|
+
# Batch1: risk aggregation (optional fields)
|
|
302
|
+
"endpoint_risk_score": 0,
|
|
303
|
+
"top_risks": [],
|
|
287
304
|
}
|
|
288
305
|
|
|
289
306
|
for r in safe_results:
|
|
@@ -298,6 +315,28 @@ def write_report(
|
|
|
298
315
|
if isinstance(elapsed, int) and elapsed > AI_SWARM_SLA_MS:
|
|
299
316
|
summary["slow_tests"].append({"name": r.get("name"), "elapsed_ms": elapsed})
|
|
300
317
|
|
|
318
|
+
# Batch1: endpoint risk summary (best-effort)
|
|
319
|
+
try:
|
|
320
|
+
scores = [r.get("risk_score") for r in safe_results if isinstance(r.get("risk_score"), int)]
|
|
321
|
+
summary["endpoint_risk_score"] = int(max(scores)) if scores else 0
|
|
322
|
+
top = sorted(
|
|
323
|
+
(
|
|
324
|
+
{
|
|
325
|
+
"name": r.get("name"),
|
|
326
|
+
"status": r.get("status"),
|
|
327
|
+
"failure_type": r.get("failure_type"),
|
|
328
|
+
"risk_score": r.get("risk_score"),
|
|
329
|
+
}
|
|
330
|
+
for r in safe_results
|
|
331
|
+
if isinstance(r.get("risk_score"), int)
|
|
332
|
+
),
|
|
333
|
+
key=lambda x: int(x.get("risk_score") or 0),
|
|
334
|
+
reverse=True,
|
|
335
|
+
)
|
|
336
|
+
summary["top_risks"] = top[:10]
|
|
337
|
+
except Exception:
|
|
338
|
+
pass
|
|
339
|
+
|
|
301
340
|
report = {
|
|
302
341
|
"endpoint": f"{method} {url}",
|
|
303
342
|
"run_time": timestamp,
|
|
@@ -324,4 +363,12 @@ def write_report(
|
|
|
324
363
|
with open(report_path, "w", encoding="utf-8") as f:
|
|
325
364
|
f.write(_render_html(report))
|
|
326
365
|
|
|
366
|
+
# Batch1: update a simple static dashboard index.html
|
|
367
|
+
if update_index:
|
|
368
|
+
try:
|
|
369
|
+
write_dashboard_index(REPORTS_DIR)
|
|
370
|
+
except Exception:
|
|
371
|
+
# Dashboard is best-effort; never fail the run
|
|
372
|
+
pass
|
|
373
|
+
|
|
327
374
|
return str(report_path)
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ai-testing-swarm
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.15
|
|
4
4
|
Summary: AI-powered testing swarm
|
|
5
5
|
Author-email: Arif Shah <ashah7775@gmail.com>
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.9
|
|
8
8
|
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: requests>=2.28
|
|
10
|
+
Requires-Dist: PyYAML>=6.0
|
|
9
11
|
Provides-Extra: openapi
|
|
10
12
|
Requires-Dist: jsonschema>=4.0; extra == "openapi"
|
|
11
13
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
ai_testing_swarm/__init__.py,sha256=
|
|
2
|
-
ai_testing_swarm/cli.py,sha256=
|
|
3
|
-
ai_testing_swarm/orchestrator.py,sha256=
|
|
1
|
+
ai_testing_swarm/__init__.py,sha256=qb0TalpSt1CbprnFyeLUKqgrqNtmnk9IoQQ7umAoXVY,23
|
|
2
|
+
ai_testing_swarm/cli.py,sha256=IeCU0E1Ju8hfkHXT4ySI1KVSvKAlYK429Fa5atcAjdo,5626
|
|
3
|
+
ai_testing_swarm/orchestrator.py,sha256=8uaUuyGy2lm3QzTNVl_8AeYtbyzrrxuykGQchRn1EEo,6295
|
|
4
4
|
ai_testing_swarm/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
ai_testing_swarm/agents/execution_agent.py,sha256=lkipsVOWe7agMt3kY9gDxjHfWujLtMh9DJCsAKhYzD8,8050
|
|
6
6
|
ai_testing_swarm/agents/learning_agent.py,sha256=lHqikc8A8s2-Yln_qCm8hCxB9KTqVrTcNvFp20_QQU0,777
|
|
@@ -16,11 +16,13 @@ ai_testing_swarm/core/curl_parser.py,sha256=0dPEhRGqn-4u00t-bp7kW9Ii7GSiO0wteB4k
|
|
|
16
16
|
ai_testing_swarm/core/openai_client.py,sha256=gxbrZZZUh_jBcXfxtsBei2t72_xEQrKZ0Z10HjS62Ss,5668
|
|
17
17
|
ai_testing_swarm/core/openapi_loader.py,sha256=lZ1Y7lyyEL4Y90pc-naZQbCyNGndr4GmmUAvR71bpos,3639
|
|
18
18
|
ai_testing_swarm/core/openapi_validator.py,sha256=8oZNSCBSf6O6CyHZj9MB8Ojaqw54DzJqSm6pbF57HuE,5153
|
|
19
|
+
ai_testing_swarm/core/risk.py,sha256=9FtBWwq3_a1xZd5r-F78g9G6WaxA76xJBMe4LR90A_4,3646
|
|
19
20
|
ai_testing_swarm/core/safety.py,sha256=MvOMr7pKFAn0z37pBcE8X8dOqPK-juxhypCZQ4YcriI,825
|
|
20
21
|
ai_testing_swarm/reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
-
ai_testing_swarm/reporting/
|
|
22
|
-
ai_testing_swarm
|
|
23
|
-
ai_testing_swarm-0.1.
|
|
24
|
-
ai_testing_swarm-0.1.
|
|
25
|
-
ai_testing_swarm-0.1.
|
|
26
|
-
ai_testing_swarm-0.1.
|
|
22
|
+
ai_testing_swarm/reporting/dashboard.py,sha256=W53SRD6ttT7C3VCcsB9nkXOGWaoRwzOp-cZaj1NcRx8,5682
|
|
23
|
+
ai_testing_swarm/reporting/report_writer.py,sha256=wLndx2VPPldGxbCuMyCaWd3VesVRLBPGQ8n9jyyTctI,12554
|
|
24
|
+
ai_testing_swarm-0.1.15.dist-info/METADATA,sha256=hAwNBIjwMxfpDBXHuomFLBHNM0csFl9ELwaXgAZtuyk,5562
|
|
25
|
+
ai_testing_swarm-0.1.15.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
26
|
+
ai_testing_swarm-0.1.15.dist-info/entry_points.txt,sha256=vbW-IBcVcls5I-NA3xFUZxH4Ktevt7lA4w9P4Me0yXo,54
|
|
27
|
+
ai_testing_swarm-0.1.15.dist-info/top_level.txt,sha256=OSqbej3vG04SKqgEcgzDTMn8QzpVsxwOzpSG7quhWJw,17
|
|
28
|
+
ai_testing_swarm-0.1.15.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|