spanforge 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spanforge/__init__.py +695 -0
- spanforge/_batch_exporter.py +322 -0
- spanforge/_cli.py +3081 -0
- spanforge/_hooks.py +340 -0
- spanforge/_server.py +953 -0
- spanforge/_span.py +1015 -0
- spanforge/_store.py +287 -0
- spanforge/_stream.py +654 -0
- spanforge/_trace.py +334 -0
- spanforge/_tracer.py +253 -0
- spanforge/actor.py +141 -0
- spanforge/alerts.py +464 -0
- spanforge/auto.py +181 -0
- spanforge/baseline.py +336 -0
- spanforge/config.py +460 -0
- spanforge/consent.py +227 -0
- spanforge/consumer.py +379 -0
- spanforge/core/__init__.py +5 -0
- spanforge/core/compliance_mapping.py +1060 -0
- spanforge/cost.py +597 -0
- spanforge/debug.py +514 -0
- spanforge/drift.py +488 -0
- spanforge/egress.py +63 -0
- spanforge/eval.py +575 -0
- spanforge/event.py +1052 -0
- spanforge/exceptions.py +246 -0
- spanforge/explain.py +181 -0
- spanforge/export/__init__.py +50 -0
- spanforge/export/append_only.py +342 -0
- spanforge/export/cloud.py +349 -0
- spanforge/export/datadog.py +495 -0
- spanforge/export/grafana.py +331 -0
- spanforge/export/jsonl.py +198 -0
- spanforge/export/otel_bridge.py +291 -0
- spanforge/export/otlp.py +817 -0
- spanforge/export/otlp_bridge.py +231 -0
- spanforge/export/redis_backend.py +282 -0
- spanforge/export/webhook.py +302 -0
- spanforge/exporters/__init__.py +29 -0
- spanforge/exporters/console.py +271 -0
- spanforge/exporters/jsonl.py +144 -0
- spanforge/hitl.py +297 -0
- spanforge/inspect.py +429 -0
- spanforge/integrations/__init__.py +39 -0
- spanforge/integrations/_pricing.py +277 -0
- spanforge/integrations/anthropic.py +388 -0
- spanforge/integrations/bedrock.py +306 -0
- spanforge/integrations/crewai.py +251 -0
- spanforge/integrations/gemini.py +349 -0
- spanforge/integrations/groq.py +444 -0
- spanforge/integrations/langchain.py +349 -0
- spanforge/integrations/llamaindex.py +370 -0
- spanforge/integrations/ollama.py +286 -0
- spanforge/integrations/openai.py +370 -0
- spanforge/integrations/together.py +485 -0
- spanforge/metrics.py +393 -0
- spanforge/metrics_export.py +342 -0
- spanforge/migrate.py +278 -0
- spanforge/model_registry.py +282 -0
- spanforge/models.py +407 -0
- spanforge/namespaces/__init__.py +215 -0
- spanforge/namespaces/audit.py +253 -0
- spanforge/namespaces/cache.py +209 -0
- spanforge/namespaces/chain.py +74 -0
- spanforge/namespaces/confidence.py +69 -0
- spanforge/namespaces/consent.py +85 -0
- spanforge/namespaces/cost.py +175 -0
- spanforge/namespaces/decision.py +135 -0
- spanforge/namespaces/diff.py +146 -0
- spanforge/namespaces/drift.py +79 -0
- spanforge/namespaces/eval_.py +232 -0
- spanforge/namespaces/fence.py +180 -0
- spanforge/namespaces/guard.py +104 -0
- spanforge/namespaces/hitl.py +92 -0
- spanforge/namespaces/latency.py +69 -0
- spanforge/namespaces/prompt.py +185 -0
- spanforge/namespaces/redact.py +172 -0
- spanforge/namespaces/template.py +197 -0
- spanforge/namespaces/tool_call.py +76 -0
- spanforge/namespaces/trace.py +1006 -0
- spanforge/normalizer.py +183 -0
- spanforge/presidio_backend.py +149 -0
- spanforge/processor.py +258 -0
- spanforge/prompt_registry.py +415 -0
- spanforge/py.typed +0 -0
- spanforge/redact.py +780 -0
- spanforge/sampling.py +500 -0
- spanforge/schemas/v1.0/schema.json +170 -0
- spanforge/schemas/v2.0/schema.json +536 -0
- spanforge/signing.py +1152 -0
- spanforge/stream.py +559 -0
- spanforge/testing.py +376 -0
- spanforge/trace.py +199 -0
- spanforge/types.py +696 -0
- spanforge/ulid.py +304 -0
- spanforge/validate.py +383 -0
- spanforge-2.0.0.dist-info/METADATA +1777 -0
- spanforge-2.0.0.dist-info/RECORD +101 -0
- spanforge-2.0.0.dist-info/WHEEL +4 -0
- spanforge-2.0.0.dist-info/entry_points.txt +5 -0
- spanforge-2.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1060 @@
|
|
|
1
|
+
"""spanforge.core.compliance_mapping — Compliance evidence engine.
|
|
2
|
+
|
|
3
|
+
Maps spanforge telemetry events to regulatory framework clauses and produces
|
|
4
|
+
signed attestation packages suitable for audit submission.
|
|
5
|
+
|
|
6
|
+
Supported frameworks
|
|
7
|
+
--------------------
|
|
8
|
+
* soc2 — SOC 2 Type II (CC series)
|
|
9
|
+
* hipaa — HIPAA Security Rule
|
|
10
|
+
* gdpr — GDPR (EU) 2016/679
|
|
11
|
+
* nist_ai_rmf — NIST AI Risk Management Framework 1.0
|
|
12
|
+
* eu_ai_act — EU AI Act (Annex IV documentation requirements)
|
|
13
|
+
* iso_42001 — ISO/IEC 42001:2023 AI Management System
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import enum
|
|
19
|
+
import hashlib
|
|
20
|
+
import hmac as _hmac
|
|
21
|
+
import json
|
|
22
|
+
import logging
|
|
23
|
+
import os
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from datetime import datetime, timedelta, timezone
|
|
26
|
+
from typing import Any
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"ClauseStatus",
|
|
30
|
+
"ComplianceAttestation",
|
|
31
|
+
"ComplianceEvidencePackage",
|
|
32
|
+
"ComplianceFramework",
|
|
33
|
+
"ComplianceMappingEngine",
|
|
34
|
+
"EvidenceRecord",
|
|
35
|
+
"GapReport",
|
|
36
|
+
"verify_attestation_signature",
|
|
37
|
+
"verify_pdf_attestation",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
_log = logging.getLogger("spanforge.core.compliance_mapping")
|
|
41
|
+
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
# Framework enum
|
|
44
|
+
# ---------------------------------------------------------------------------
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ComplianceFramework(enum.Enum):
|
|
48
|
+
"""Supported regulatory frameworks."""
|
|
49
|
+
|
|
50
|
+
SOC2 = "SOC 2 Type II"
|
|
51
|
+
HIPAA = "HIPAA"
|
|
52
|
+
GDPR = "GDPR"
|
|
53
|
+
NIST_AI_RMF = "NIST AI RMF"
|
|
54
|
+
EU_AI_ACT = "EU AI Act"
|
|
55
|
+
ISO_42001 = "ISO/IEC 42001"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# Maps enum value strings and slug strings to _FRAMEWORK_CLAUSES keys
|
|
59
|
+
_FRAMEWORK_KEY_MAP: dict[str, str] = {
|
|
60
|
+
# enum values
|
|
61
|
+
"soc 2 type ii": "soc2",
|
|
62
|
+
"hipaa": "hipaa",
|
|
63
|
+
"gdpr": "gdpr",
|
|
64
|
+
"nist ai rmf": "nist_ai_rmf",
|
|
65
|
+
"eu ai act": "eu_ai_act",
|
|
66
|
+
"iso/iec 42001": "iso_42001",
|
|
67
|
+
# slugs (already match keys, but listed for completeness)
|
|
68
|
+
"soc2": "soc2",
|
|
69
|
+
"nist_ai_rmf": "nist_ai_rmf",
|
|
70
|
+
"eu_ai_act": "eu_ai_act",
|
|
71
|
+
"iso_42001": "iso_42001",
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# ---------------------------------------------------------------------------
|
|
76
|
+
# Framework → clause definitions
|
|
77
|
+
# ---------------------------------------------------------------------------
|
|
78
|
+
|
|
79
|
+
# Each clause maps to a list of event-type prefixes that provide evidence.
|
|
80
|
+
_FRAMEWORK_CLAUSES: dict[str, dict[str, dict[str, Any]]] = {
|
|
81
|
+
"soc2": {
|
|
82
|
+
"CC6.1": {
|
|
83
|
+
"title": "Logical and Physical Access Controls — access management",
|
|
84
|
+
"event_prefixes": ["llm.audit.", "llm.trace.", "model_registry."],
|
|
85
|
+
"description": "Events demonstrating actor-based access controls, audit trails, and model lifecycle tracking.",
|
|
86
|
+
"min_event_count": 5,
|
|
87
|
+
"time_window_hours": None,
|
|
88
|
+
},
|
|
89
|
+
"CC6.6": {
|
|
90
|
+
"title": "PII / Sensitive Data Protection",
|
|
91
|
+
"event_prefixes": ["llm.redact."],
|
|
92
|
+
"description": "Evidence of PII detection and redaction before model transmission.",
|
|
93
|
+
"min_event_count": 5,
|
|
94
|
+
"time_window_hours": None,
|
|
95
|
+
},
|
|
96
|
+
"CC7.2": {
|
|
97
|
+
"title": "Anomaly and Threat Detection",
|
|
98
|
+
"event_prefixes": ["llm.drift.", "llm.guard."],
|
|
99
|
+
"description": "Drift detection and guard-rail events demonstrating anomaly monitoring.",
|
|
100
|
+
"min_event_count": 5,
|
|
101
|
+
"time_window_hours": None,
|
|
102
|
+
},
|
|
103
|
+
"CC8.1": {
|
|
104
|
+
"title": "Change Management — schema validation",
|
|
105
|
+
"event_prefixes": ["llm.trace.", "llm.eval."],
|
|
106
|
+
"description": "Schema-validated telemetry providing a tamper-evident event chain.",
|
|
107
|
+
"min_event_count": 5,
|
|
108
|
+
"time_window_hours": None,
|
|
109
|
+
},
|
|
110
|
+
"CC9.2": {
|
|
111
|
+
"title": "Risk Mitigation — cost and budget controls",
|
|
112
|
+
"event_prefixes": ["llm.cost."],
|
|
113
|
+
"description": "Cost budget and spend telemetry supporting financial risk controls.",
|
|
114
|
+
"min_event_count": 5,
|
|
115
|
+
"time_window_hours": None,
|
|
116
|
+
},
|
|
117
|
+
},
|
|
118
|
+
"hipaa": {
|
|
119
|
+
"164.312(b)": {
|
|
120
|
+
"title": "Audit Controls",
|
|
121
|
+
"event_prefixes": ["llm.audit.", "llm.trace."],
|
|
122
|
+
"description": "HMAC-signed event chain providing tamper-evident audit record of PHI activity.",
|
|
123
|
+
"min_event_count": 10,
|
|
124
|
+
"time_window_hours": None,
|
|
125
|
+
},
|
|
126
|
+
"164.312(a)(1)": {
|
|
127
|
+
"title": "Access Control",
|
|
128
|
+
"event_prefixes": ["llm.trace.", "llm.audit."],
|
|
129
|
+
"description": "Actor-tagged events demonstrating user/system access to PHI workloads.",
|
|
130
|
+
"min_event_count": 5,
|
|
131
|
+
"time_window_hours": None,
|
|
132
|
+
},
|
|
133
|
+
"164.312(e)(2)(ii)": {
|
|
134
|
+
"title": "Encryption and Decryption",
|
|
135
|
+
"event_prefixes": ["llm.redact."],
|
|
136
|
+
"description": "PII redaction events demonstrating PHI de-identification before model use.",
|
|
137
|
+
"min_event_count": 5,
|
|
138
|
+
"time_window_hours": None,
|
|
139
|
+
},
|
|
140
|
+
"164.530(j)": {
|
|
141
|
+
"title": "Documentation and Retention",
|
|
142
|
+
"event_prefixes": ["llm.trace.", "llm.audit.", "llm.cost."],
|
|
143
|
+
"description": "Complete event log supporting required 6-year audit retention.",
|
|
144
|
+
"min_event_count": 10,
|
|
145
|
+
"time_window_hours": 168,
|
|
146
|
+
},
|
|
147
|
+
},
|
|
148
|
+
"gdpr": {
|
|
149
|
+
"Art.30": {
|
|
150
|
+
"title": "Records of Processing Activities",
|
|
151
|
+
"event_prefixes": ["llm.trace.", "llm.cost.", "llm.audit."],
|
|
152
|
+
"description": "Structured event log mapping to Article 30 processing record requirements.",
|
|
153
|
+
"min_event_count": 10,
|
|
154
|
+
"time_window_hours": None,
|
|
155
|
+
},
|
|
156
|
+
"Art.35": {
|
|
157
|
+
"title": "Data Protection Impact Assessment",
|
|
158
|
+
"event_prefixes": ["llm.redact.", "llm.guard.", "llm.drift."],
|
|
159
|
+
"description": "Redaction, guard-rail, and drift events supporting DPIA risk evidence.",
|
|
160
|
+
"min_event_count": 5,
|
|
161
|
+
"time_window_hours": None,
|
|
162
|
+
},
|
|
163
|
+
"Art.22": {
|
|
164
|
+
"title": "Automated Individual Decision-Making — consent and oversight",
|
|
165
|
+
"event_prefixes": ["consent.", "hitl."],
|
|
166
|
+
"description": "Consent boundary and human-in-the-loop events demonstrating safeguards for automated decisions affecting individuals.",
|
|
167
|
+
"min_event_count": 5,
|
|
168
|
+
"time_window_hours": None,
|
|
169
|
+
},
|
|
170
|
+
"Art.25": {
|
|
171
|
+
"title": "Data Protection by Design and by Default",
|
|
172
|
+
"event_prefixes": ["llm.redact.", "consent."],
|
|
173
|
+
"description": "PII stripping and consent enforcement at instrumentation level demonstrates privacy-by-design.",
|
|
174
|
+
"min_event_count": 5,
|
|
175
|
+
"time_window_hours": None,
|
|
176
|
+
},
|
|
177
|
+
},
|
|
178
|
+
"nist_ai_rmf": {
|
|
179
|
+
"MAP.1.1": {
|
|
180
|
+
"title": "AI System Documentation",
|
|
181
|
+
"event_prefixes": ["llm.trace.", "llm.eval.", "model_registry.", "explanation."],
|
|
182
|
+
"description": "Trace, evaluation, model registry, and explainability events documenting AI system behaviour.",
|
|
183
|
+
"min_event_count": 5,
|
|
184
|
+
"time_window_hours": None,
|
|
185
|
+
},
|
|
186
|
+
"MEASURE.2.6": {
|
|
187
|
+
"title": "AI Risk Monitoring",
|
|
188
|
+
"event_prefixes": ["llm.drift.", "llm.guard.", "llm.eval."],
|
|
189
|
+
"description": "Drift and guard events demonstrating continuous risk monitoring.",
|
|
190
|
+
"min_event_count": 5,
|
|
191
|
+
"time_window_hours": None,
|
|
192
|
+
},
|
|
193
|
+
"MANAGE.3.2": {
|
|
194
|
+
"title": "Incident Response",
|
|
195
|
+
"event_prefixes": ["llm.guard.", "llm.audit."],
|
|
196
|
+
"description": "Guard and audit events providing evidence of incident detection and response.",
|
|
197
|
+
"min_event_count": 5,
|
|
198
|
+
"time_window_hours": None,
|
|
199
|
+
},
|
|
200
|
+
"GOVERN.1.7": {
|
|
201
|
+
"title": "AI Policies and Processes",
|
|
202
|
+
"event_prefixes": ["llm.audit.", "llm.trace."],
|
|
203
|
+
"description": "Audit chain demonstrating policy enforcement in AI pipelines.",
|
|
204
|
+
"min_event_count": 5,
|
|
205
|
+
"time_window_hours": None,
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
"eu_ai_act": {
|
|
209
|
+
"AnnexIV.1": {
|
|
210
|
+
"title": "General Description of the AI System",
|
|
211
|
+
"event_prefixes": ["llm.trace.", "llm.eval."],
|
|
212
|
+
"description": "Trace and evaluation telemetry documenting system purpose and behaviour.",
|
|
213
|
+
"min_event_count": 5,
|
|
214
|
+
"time_window_hours": None,
|
|
215
|
+
},
|
|
216
|
+
"Art.13": {
|
|
217
|
+
"title": "Transparency — explainability of AI decisions",
|
|
218
|
+
"event_prefixes": ["explanation."],
|
|
219
|
+
"description": "Explainability records demonstrating that high-risk AI decisions are accompanied by human-readable rationale.",
|
|
220
|
+
"min_event_count": 5,
|
|
221
|
+
"time_window_hours": None,
|
|
222
|
+
},
|
|
223
|
+
"Art.14": {
|
|
224
|
+
"title": "Human Oversight — HITL review and escalation",
|
|
225
|
+
"event_prefixes": ["hitl.", "consent."],
|
|
226
|
+
"description": "Human-in-the-loop review, escalation, and consent events demonstrating mandatory human oversight of high-risk AI.",
|
|
227
|
+
"min_event_count": 5,
|
|
228
|
+
"time_window_hours": None,
|
|
229
|
+
},
|
|
230
|
+
"AnnexIV.5": {
|
|
231
|
+
"title": "Human Oversight Measures",
|
|
232
|
+
"event_prefixes": ["llm.guard.", "llm.audit.", "hitl."],
|
|
233
|
+
"description": "Guard, audit, and human-in-the-loop events demonstrating human oversight mechanisms.",
|
|
234
|
+
"min_event_count": 5,
|
|
235
|
+
"time_window_hours": None,
|
|
236
|
+
},
|
|
237
|
+
"AnnexIV.6": {
|
|
238
|
+
"title": "Robustness, Accuracy and Cybersecurity",
|
|
239
|
+
"event_prefixes": ["llm.eval.", "llm.drift."],
|
|
240
|
+
"description": "Evaluation and drift telemetry supporting robustness and accuracy evidence.",
|
|
241
|
+
"min_event_count": 5,
|
|
242
|
+
"time_window_hours": None,
|
|
243
|
+
},
|
|
244
|
+
},
|
|
245
|
+
"iso_42001": {
|
|
246
|
+
"6.1": {
|
|
247
|
+
"title": "Actions to Address AI Risks and Opportunities",
|
|
248
|
+
"event_prefixes": ["llm.drift.", "llm.guard.", "llm.eval."],
|
|
249
|
+
"description": "Drift, guard, and evaluation events supporting risk treatment records.",
|
|
250
|
+
"min_event_count": 5,
|
|
251
|
+
"time_window_hours": None,
|
|
252
|
+
},
|
|
253
|
+
"9.1": {
|
|
254
|
+
"title": "Monitoring, Measurement, Analysis and Evaluation",
|
|
255
|
+
"event_prefixes": ["llm.trace.", "llm.eval.", "llm.cost."],
|
|
256
|
+
"description": "Continuous telemetry supporting measurement and evaluation requirements.",
|
|
257
|
+
"min_event_count": 5,
|
|
258
|
+
"time_window_hours": None,
|
|
259
|
+
},
|
|
260
|
+
"10.1": {
|
|
261
|
+
"title": "Nonconformity and Corrective Action",
|
|
262
|
+
"event_prefixes": ["llm.audit.", "llm.guard."],
|
|
263
|
+
"description": "Audit and guard events documenting corrective actions.",
|
|
264
|
+
"min_event_count": 5,
|
|
265
|
+
"time_window_hours": None,
|
|
266
|
+
},
|
|
267
|
+
},
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
# Minimum event count to consider a clause "passed" (not just partial)
|
|
271
|
+
_MIN_PASS_THRESHOLD = 5
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
# ---------------------------------------------------------------------------
|
|
275
|
+
# Data classes
|
|
276
|
+
# ---------------------------------------------------------------------------
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class ClauseStatus(enum.Enum):
|
|
280
|
+
"""Pass/fail/coverage status for a single compliance clause."""
|
|
281
|
+
|
|
282
|
+
PASS = "pass"
|
|
283
|
+
FAIL = "fail"
|
|
284
|
+
PARTIAL = "partial"
|
|
285
|
+
NOT_APPLICABLE = "not_applicable"
|
|
286
|
+
UNKNOWN = "unknown"
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
@dataclass
|
|
290
|
+
class EvidenceRecord:
|
|
291
|
+
"""Evidence collected for one framework clause."""
|
|
292
|
+
|
|
293
|
+
clause_id: str
|
|
294
|
+
status: ClauseStatus
|
|
295
|
+
evidence_count: int
|
|
296
|
+
audit_ids: list[str]
|
|
297
|
+
summary: str
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
@dataclass
|
|
301
|
+
class ComplianceAttestation:
|
|
302
|
+
"""HMAC-signed attestation package for a model + framework + period."""
|
|
303
|
+
|
|
304
|
+
model_id: str
|
|
305
|
+
framework: str
|
|
306
|
+
period_from: str
|
|
307
|
+
period_to: str
|
|
308
|
+
generated_at: str
|
|
309
|
+
generated_by: str
|
|
310
|
+
clauses: list[EvidenceRecord]
|
|
311
|
+
overall_status: ClauseStatus
|
|
312
|
+
hmac_sig: str
|
|
313
|
+
model_owner: str | None = None
|
|
314
|
+
model_risk_tier: str | None = None
|
|
315
|
+
model_status: str | None = None
|
|
316
|
+
model_warnings: list[str] = field(default_factory=list)
|
|
317
|
+
explanation_coverage_pct: float | None = None
|
|
318
|
+
|
|
319
|
+
def to_json(self) -> str:
|
|
320
|
+
doc: dict[str, Any] = {
|
|
321
|
+
"model_id": self.model_id,
|
|
322
|
+
"framework": self.framework,
|
|
323
|
+
"period_from": self.period_from,
|
|
324
|
+
"period_to": self.period_to,
|
|
325
|
+
"generated_at": self.generated_at,
|
|
326
|
+
"generated_by": self.generated_by,
|
|
327
|
+
"overall_status": self.overall_status.value,
|
|
328
|
+
"hmac_sig": self.hmac_sig,
|
|
329
|
+
"clauses": [
|
|
330
|
+
{
|
|
331
|
+
"clause_id": r.clause_id,
|
|
332
|
+
"status": r.status.value,
|
|
333
|
+
"evidence_count": r.evidence_count,
|
|
334
|
+
"audit_ids": r.audit_ids[:20], # cap for readability
|
|
335
|
+
"summary": r.summary,
|
|
336
|
+
}
|
|
337
|
+
for r in self.clauses
|
|
338
|
+
],
|
|
339
|
+
}
|
|
340
|
+
if self.model_owner is not None:
|
|
341
|
+
doc["model_owner"] = self.model_owner
|
|
342
|
+
if self.model_risk_tier is not None:
|
|
343
|
+
doc["model_risk_tier"] = self.model_risk_tier
|
|
344
|
+
if self.model_status is not None:
|
|
345
|
+
doc["model_status"] = self.model_status
|
|
346
|
+
if self.model_warnings:
|
|
347
|
+
doc["model_warnings"] = self.model_warnings
|
|
348
|
+
if self.explanation_coverage_pct is not None:
|
|
349
|
+
doc["explanation_coverage_pct"] = self.explanation_coverage_pct
|
|
350
|
+
return json.dumps(doc, indent=2)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
@dataclass
|
|
354
|
+
class GapReport:
|
|
355
|
+
"""Summary of compliance gaps found during the analysis."""
|
|
356
|
+
|
|
357
|
+
model_id: str
|
|
358
|
+
framework: str
|
|
359
|
+
period_from: str
|
|
360
|
+
period_to: str
|
|
361
|
+
generated_at: str
|
|
362
|
+
gap_clause_ids: list[str]
|
|
363
|
+
partial_clause_ids: list[str]
|
|
364
|
+
|
|
365
|
+
@property
|
|
366
|
+
def has_gaps(self) -> bool:
|
|
367
|
+
return bool(self.gap_clause_ids)
|
|
368
|
+
|
|
369
|
+
@property
|
|
370
|
+
def total_issues(self) -> int:
|
|
371
|
+
return len(self.gap_clause_ids) + len(self.partial_clause_ids)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
@dataclass
|
|
375
|
+
class ComplianceEvidencePackage:
|
|
376
|
+
"""Full deliverable: attestation + human-readable report + gap analysis + audit exports."""
|
|
377
|
+
|
|
378
|
+
attestation: ComplianceAttestation
|
|
379
|
+
report_text: str
|
|
380
|
+
gap_report: GapReport
|
|
381
|
+
audit_exports: dict[str, list[dict[str, Any]]] = field(default_factory=dict)
|
|
382
|
+
|
|
383
|
+
def to_json(self) -> str:
|
|
384
|
+
"""Return a signed JSON attestation with HMAC covering canonical bytes.
|
|
385
|
+
|
|
386
|
+
The output includes: framework, model_id, period, generated_at,
|
|
387
|
+
clauses with status/evidence_count/summary, gap_clause_ids,
|
|
388
|
+
overall_status, and hmac_sig.
|
|
389
|
+
"""
|
|
390
|
+
att = self.attestation
|
|
391
|
+
doc: dict[str, Any] = {
|
|
392
|
+
"framework": att.framework,
|
|
393
|
+
"model_id": att.model_id,
|
|
394
|
+
"period_from": att.period_from,
|
|
395
|
+
"period_to": att.period_to,
|
|
396
|
+
"generated_at": att.generated_at,
|
|
397
|
+
"generated_by": att.generated_by,
|
|
398
|
+
"overall_status": att.overall_status.value,
|
|
399
|
+
"clauses": [
|
|
400
|
+
{
|
|
401
|
+
"clause_id": r.clause_id,
|
|
402
|
+
"status": r.status.value,
|
|
403
|
+
"evidence_count": r.evidence_count,
|
|
404
|
+
"summary": r.summary,
|
|
405
|
+
}
|
|
406
|
+
for r in att.clauses
|
|
407
|
+
],
|
|
408
|
+
"gap_clause_ids": self.gap_report.gap_clause_ids,
|
|
409
|
+
"hmac_sig": att.hmac_sig,
|
|
410
|
+
}
|
|
411
|
+
if att.model_owner is not None:
|
|
412
|
+
doc["model_owner"] = att.model_owner
|
|
413
|
+
if att.model_risk_tier is not None:
|
|
414
|
+
doc["model_risk_tier"] = att.model_risk_tier
|
|
415
|
+
if att.model_status is not None:
|
|
416
|
+
doc["model_status"] = att.model_status
|
|
417
|
+
if att.model_warnings:
|
|
418
|
+
doc["model_warnings"] = att.model_warnings
|
|
419
|
+
if att.explanation_coverage_pct is not None:
|
|
420
|
+
doc["explanation_coverage_pct"] = att.explanation_coverage_pct
|
|
421
|
+
return json.dumps(doc, sort_keys=True, separators=(",", ":"))
|
|
422
|
+
|
|
423
|
+
def to_pdf(self, path: str | Any) -> Any:
|
|
424
|
+
"""Generate a signed PDF attestation report.
|
|
425
|
+
|
|
426
|
+
Requires ``reportlab``: ``pip install 'spanforge[compliance]'``.
|
|
427
|
+
|
|
428
|
+
Args:
|
|
429
|
+
path: File path for the output PDF.
|
|
430
|
+
|
|
431
|
+
Returns:
|
|
432
|
+
:class:`pathlib.Path` to the written PDF file.
|
|
433
|
+
|
|
434
|
+
Raises:
|
|
435
|
+
ImportError: If ``reportlab`` is not installed.
|
|
436
|
+
"""
|
|
437
|
+
from pathlib import Path as _Path # noqa: PLC0415
|
|
438
|
+
|
|
439
|
+
try:
|
|
440
|
+
from reportlab.lib.pagesizes import A4 # noqa: PLC0415
|
|
441
|
+
from reportlab.lib.units import mm # noqa: PLC0415
|
|
442
|
+
from reportlab.pdfgen import canvas # noqa: PLC0415
|
|
443
|
+
except ImportError:
|
|
444
|
+
raise ImportError(
|
|
445
|
+
"PDF attestation export requires reportlab. "
|
|
446
|
+
"Install it with: pip install 'spanforge[compliance]'"
|
|
447
|
+
) from None
|
|
448
|
+
|
|
449
|
+
out_path = _Path(path)
|
|
450
|
+
att = self.attestation
|
|
451
|
+
|
|
452
|
+
c = canvas.Canvas(str(out_path), pagesize=A4)
|
|
453
|
+
width, height = A4
|
|
454
|
+
y = height - 40 * mm
|
|
455
|
+
|
|
456
|
+
# Cover page
|
|
457
|
+
c.setFont("Helvetica-Bold", 18)
|
|
458
|
+
c.drawString(30 * mm, y, "SpanForge Compliance Attestation")
|
|
459
|
+
y -= 12 * mm
|
|
460
|
+
c.setFont("Helvetica", 11)
|
|
461
|
+
c.drawString(30 * mm, y, f"Framework: {att.framework.upper()}")
|
|
462
|
+
y -= 7 * mm
|
|
463
|
+
c.drawString(30 * mm, y, f"Model: {att.model_id}")
|
|
464
|
+
y -= 7 * mm
|
|
465
|
+
c.drawString(30 * mm, y, f"Period: {att.period_from} — {att.period_to}")
|
|
466
|
+
y -= 7 * mm
|
|
467
|
+
c.drawString(30 * mm, y, f"Generated: {att.generated_at}")
|
|
468
|
+
y -= 7 * mm
|
|
469
|
+
c.drawString(30 * mm, y, f"Overall Status: {att.overall_status.value.upper()}")
|
|
470
|
+
y -= 14 * mm
|
|
471
|
+
|
|
472
|
+
# Clause table
|
|
473
|
+
c.setFont("Helvetica-Bold", 12)
|
|
474
|
+
c.drawString(30 * mm, y, "Clause Analysis")
|
|
475
|
+
y -= 8 * mm
|
|
476
|
+
c.setFont("Helvetica", 9)
|
|
477
|
+
for rec in att.clauses:
|
|
478
|
+
if y < 30 * mm:
|
|
479
|
+
c.showPage()
|
|
480
|
+
y = height - 30 * mm
|
|
481
|
+
c.setFont("Helvetica", 9)
|
|
482
|
+
icon = {"pass": "PASS", "fail": "FAIL", "partial": "PARTIAL"}.get(rec.status.value, "?")
|
|
483
|
+
c.drawString(30 * mm, y, f"[{icon}] {rec.clause_id}: {rec.summary[:80]}")
|
|
484
|
+
y -= 5 * mm
|
|
485
|
+
|
|
486
|
+
# Gap list
|
|
487
|
+
if self.gap_report.has_gaps:
|
|
488
|
+
y -= 6 * mm
|
|
489
|
+
c.setFont("Helvetica-Bold", 11)
|
|
490
|
+
c.drawString(30 * mm, y, "Gaps Requiring Action")
|
|
491
|
+
y -= 6 * mm
|
|
492
|
+
c.setFont("Helvetica", 9)
|
|
493
|
+
for cid in self.gap_report.gap_clause_ids:
|
|
494
|
+
c.drawString(32 * mm, y, f"- {cid}")
|
|
495
|
+
y -= 5 * mm
|
|
496
|
+
|
|
497
|
+
# HMAC footer
|
|
498
|
+
y -= 10 * mm
|
|
499
|
+
c.setFont("Helvetica", 8)
|
|
500
|
+
c.drawString(30 * mm, y, f"HMAC-SHA256: {att.hmac_sig}")
|
|
501
|
+
|
|
502
|
+
c.save()
|
|
503
|
+
|
|
504
|
+
# Sign the PDF bytes and store in metadata
|
|
505
|
+
pdf_bytes = out_path.read_bytes()
|
|
506
|
+
pdf_hash = hashlib.sha256(pdf_bytes).hexdigest()
|
|
507
|
+
signing_key = os.environ.get("SPANFORGE_SIGNING_KEY", "")
|
|
508
|
+
if not signing_key or signing_key == "spanforge-default":
|
|
509
|
+
raise ValueError(
|
|
510
|
+
"SPANFORGE_SIGNING_KEY is not set or uses the insecure default value. "
|
|
511
|
+
"Set a strong secret before generating PDF attestations for production. "
|
|
512
|
+
"Example: export SPANFORGE_SIGNING_KEY=$(openssl rand -hex 32)"
|
|
513
|
+
)
|
|
514
|
+
pdf_hmac = _hmac.new(
|
|
515
|
+
signing_key.encode(),
|
|
516
|
+
pdf_hash.encode(),
|
|
517
|
+
hashlib.sha256,
|
|
518
|
+
).hexdigest()
|
|
519
|
+
|
|
520
|
+
# Re-open and set metadata
|
|
521
|
+
from reportlab.lib.pagesizes import A4 as _A4 # noqa: PLC0415, F811
|
|
522
|
+
# Store the HMAC as a sidecar JSON (reportlab doesn't support PDF metadata update easily)
|
|
523
|
+
sidecar = out_path.with_suffix(".pdf.sig")
|
|
524
|
+
sidecar.write_text(
|
|
525
|
+
json.dumps({"SpanForgeHMAC": pdf_hmac, "pdf_sha256": pdf_hash}),
|
|
526
|
+
encoding="utf-8",
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
return out_path
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
# ---------------------------------------------------------------------------
|
|
533
|
+
# Engine
|
|
534
|
+
# ---------------------------------------------------------------------------
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
class ComplianceMappingEngine:
|
|
538
|
+
"""Map spanforge telemetry events to framework clauses and generate evidence packages."""
|
|
539
|
+
|
|
540
|
+
def generate_evidence_package(
|
|
541
|
+
self,
|
|
542
|
+
model_id: str,
|
|
543
|
+
framework: str,
|
|
544
|
+
from_date: str,
|
|
545
|
+
to_date: str,
|
|
546
|
+
audit_events: list[dict[str, Any]] | None = None,
|
|
547
|
+
) -> ComplianceEvidencePackage:
|
|
548
|
+
"""Analyse *audit_events* and produce a full compliance evidence package.
|
|
549
|
+
|
|
550
|
+
Parameters
|
|
551
|
+
----------
|
|
552
|
+
model_id:
|
|
553
|
+
The AI model identifier (e.g. ``gpt-4o``).
|
|
554
|
+
framework:
|
|
555
|
+
One of ``soc2``, ``hipaa``, ``gdpr``, ``nist_ai_rmf``,
|
|
556
|
+
``eu_ai_act``, ``iso_42001``.
|
|
557
|
+
from_date / to_date:
|
|
558
|
+
ISO-8601 date strings defining the audit period.
|
|
559
|
+
audit_events:
|
|
560
|
+
Raw event dicts. If *None* or empty the engine will load from the
|
|
561
|
+
active ``TraceStore`` instead.
|
|
562
|
+
"""
|
|
563
|
+
# ------------------------------------------------------------------
|
|
564
|
+
# Resolve framework
|
|
565
|
+
# ------------------------------------------------------------------
|
|
566
|
+
if isinstance(framework, ComplianceFramework):
|
|
567
|
+
raw = framework.value.lower()
|
|
568
|
+
else:
|
|
569
|
+
raw = str(framework).lower().replace("-", "_").replace(" ", "_")
|
|
570
|
+
|
|
571
|
+
framework_key = _FRAMEWORK_KEY_MAP.get(raw, raw)
|
|
572
|
+
if framework_key not in _FRAMEWORK_CLAUSES:
|
|
573
|
+
supported = ", ".join(sorted(_FRAMEWORK_CLAUSES))
|
|
574
|
+
raise ValueError(
|
|
575
|
+
f"Unknown framework {framework!r}. Supported: {supported}"
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
# ------------------------------------------------------------------
|
|
579
|
+
# Load events
|
|
580
|
+
# ------------------------------------------------------------------
|
|
581
|
+
if not audit_events:
|
|
582
|
+
audit_events = self._load_from_store()
|
|
583
|
+
|
|
584
|
+
# ------------------------------------------------------------------
|
|
585
|
+
# Filter to period
|
|
586
|
+
# ------------------------------------------------------------------
|
|
587
|
+
period_events = self._filter_period(audit_events, from_date, to_date)
|
|
588
|
+
|
|
589
|
+
# ------------------------------------------------------------------
|
|
590
|
+
# Map events to clauses
|
|
591
|
+
# ------------------------------------------------------------------
|
|
592
|
+
clauses_def = _FRAMEWORK_CLAUSES[framework_key]
|
|
593
|
+
evidence_records: list[EvidenceRecord] = []
|
|
594
|
+
audit_exports: dict[str, list[dict[str, Any]]] = {}
|
|
595
|
+
|
|
596
|
+
_now_utc = datetime.now(timezone.utc)
|
|
597
|
+
|
|
598
|
+
for clause_id, clause_info in clauses_def.items():
|
|
599
|
+
prefixes: list[str] = clause_info["event_prefixes"]
|
|
600
|
+
# Per-clause minimum, falling back to global default
|
|
601
|
+
clause_min: int = clause_info.get("min_event_count") or _MIN_PASS_THRESHOLD
|
|
602
|
+
# Optional rolling time window (hours) scoped to this clause
|
|
603
|
+
tw_hours: int | None = clause_info.get("time_window_hours")
|
|
604
|
+
|
|
605
|
+
matching = [
|
|
606
|
+
e for e in period_events
|
|
607
|
+
if any(str(e.get("event_type", "")).startswith(p) for p in prefixes)
|
|
608
|
+
]
|
|
609
|
+
|
|
610
|
+
# Apply clause-level time window filter when defined
|
|
611
|
+
if tw_hours is not None:
|
|
612
|
+
_tw_cutoff = _now_utc - timedelta(hours=tw_hours)
|
|
613
|
+
_cutoff_iso = _tw_cutoff.isoformat()
|
|
614
|
+
matching = [
|
|
615
|
+
e for e in matching
|
|
616
|
+
if str(e.get("timestamp", "")) >= _cutoff_iso
|
|
617
|
+
]
|
|
618
|
+
|
|
619
|
+
model_matching = [
|
|
620
|
+
e for e in matching
|
|
621
|
+
if self._event_matches_model(e, model_id)
|
|
622
|
+
]
|
|
623
|
+
# When a model_id is given, restrict to model-specific events only.
|
|
624
|
+
# Fall back to all matching events only when no model_id is specified.
|
|
625
|
+
effective = model_matching if model_id else matching
|
|
626
|
+
|
|
627
|
+
audit_ids = [str(e.get("event_id", "")) for e in effective[:50]]
|
|
628
|
+
count = len(effective)
|
|
629
|
+
audit_exports[clause_id] = [
|
|
630
|
+
{k: v for k, v in e.items() if k != "signature"}
|
|
631
|
+
for e in effective[:100]
|
|
632
|
+
]
|
|
633
|
+
|
|
634
|
+
if count >= clause_min:
|
|
635
|
+
status = ClauseStatus.PASS
|
|
636
|
+
summary = (
|
|
637
|
+
f"{count} events from prefixes {prefixes} satisfy this clause."
|
|
638
|
+
)
|
|
639
|
+
elif count > 0:
|
|
640
|
+
status = ClauseStatus.PARTIAL
|
|
641
|
+
summary = (
|
|
642
|
+
f"Only {count} events found (need ≥{clause_min}). "
|
|
643
|
+
f"Increase instrumentation coverage."
|
|
644
|
+
)
|
|
645
|
+
else:
|
|
646
|
+
status = ClauseStatus.FAIL
|
|
647
|
+
summary = (
|
|
648
|
+
f"No events found matching {prefixes}. "
|
|
649
|
+
f"Add {framework_key.upper()} instrumentation for this clause."
|
|
650
|
+
)
|
|
651
|
+
|
|
652
|
+
evidence_records.append(
|
|
653
|
+
EvidenceRecord(
|
|
654
|
+
clause_id=clause_id,
|
|
655
|
+
status=status,
|
|
656
|
+
evidence_count=count,
|
|
657
|
+
audit_ids=audit_ids,
|
|
658
|
+
summary=summary,
|
|
659
|
+
)
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
# ------------------------------------------------------------------
|
|
663
|
+
# Overall status
|
|
664
|
+
# ------------------------------------------------------------------
|
|
665
|
+
statuses = [r.status for r in evidence_records]
|
|
666
|
+
if all(s == ClauseStatus.PASS for s in statuses):
|
|
667
|
+
overall = ClauseStatus.PASS
|
|
668
|
+
elif any(s == ClauseStatus.FAIL for s in statuses):
|
|
669
|
+
overall = ClauseStatus.FAIL
|
|
670
|
+
else:
|
|
671
|
+
overall = ClauseStatus.PARTIAL
|
|
672
|
+
|
|
673
|
+
# ------------------------------------------------------------------
|
|
674
|
+
# HMAC signature
|
|
675
|
+
# ------------------------------------------------------------------
|
|
676
|
+
generated_at = datetime.now(timezone.utc).isoformat()
|
|
677
|
+
sig_payload = json.dumps(
|
|
678
|
+
{
|
|
679
|
+
"model_id": model_id,
|
|
680
|
+
"framework": framework_key,
|
|
681
|
+
"from": from_date,
|
|
682
|
+
"to": to_date,
|
|
683
|
+
"generated_at": generated_at,
|
|
684
|
+
"clauses": {r.clause_id: r.status.value for r in evidence_records},
|
|
685
|
+
"event_count": len(period_events),
|
|
686
|
+
},
|
|
687
|
+
sort_keys=True,
|
|
688
|
+
)
|
|
689
|
+
signing_key = os.environ.get("SPANFORGE_SIGNING_KEY", "")
|
|
690
|
+
if not signing_key or signing_key == "spanforge-default":
|
|
691
|
+
raise ValueError(
|
|
692
|
+
"SPANFORGE_SIGNING_KEY is not set or uses the insecure default value. "
|
|
693
|
+
"Set a strong secret before generating compliance attestations for production. "
|
|
694
|
+
"Example: export SPANFORGE_SIGNING_KEY=$(openssl rand -hex 32)"
|
|
695
|
+
)
|
|
696
|
+
hmac_sig = _hmac.new(
|
|
697
|
+
signing_key.encode(),
|
|
698
|
+
sig_payload.encode(),
|
|
699
|
+
hashlib.sha256,
|
|
700
|
+
).hexdigest()
|
|
701
|
+
|
|
702
|
+
attestation = ComplianceAttestation(
|
|
703
|
+
model_id=model_id,
|
|
704
|
+
framework=framework_key,
|
|
705
|
+
period_from=from_date,
|
|
706
|
+
period_to=to_date,
|
|
707
|
+
generated_at=generated_at,
|
|
708
|
+
generated_by=f"spanforge.core.compliance_mapping v1",
|
|
709
|
+
clauses=evidence_records,
|
|
710
|
+
overall_status=overall,
|
|
711
|
+
hmac_sig=hmac_sig,
|
|
712
|
+
)
|
|
713
|
+
|
|
714
|
+
# ------------------------------------------------------------------
|
|
715
|
+
# Model registry enrichment (Fix 3)
|
|
716
|
+
# ------------------------------------------------------------------
|
|
717
|
+
self._enrich_from_model_registry(attestation, model_id)
|
|
718
|
+
|
|
719
|
+
# ------------------------------------------------------------------
|
|
720
|
+
# Explanation coverage metric (Fix 4)
|
|
721
|
+
# ------------------------------------------------------------------
|
|
722
|
+
self._compute_explanation_coverage(attestation, period_events, model_id)
|
|
723
|
+
|
|
724
|
+
# ------------------------------------------------------------------
|
|
725
|
+
# Gap report
|
|
726
|
+
# ------------------------------------------------------------------
|
|
727
|
+
gap_ids = [r.clause_id for r in evidence_records if r.status == ClauseStatus.FAIL]
|
|
728
|
+
partial_ids = [r.clause_id for r in evidence_records if r.status == ClauseStatus.PARTIAL]
|
|
729
|
+
gap_report = GapReport(
|
|
730
|
+
model_id=model_id,
|
|
731
|
+
framework=framework_key,
|
|
732
|
+
period_from=from_date,
|
|
733
|
+
period_to=to_date,
|
|
734
|
+
generated_at=generated_at,
|
|
735
|
+
gap_clause_ids=gap_ids,
|
|
736
|
+
partial_clause_ids=partial_ids,
|
|
737
|
+
)
|
|
738
|
+
|
|
739
|
+
# ------------------------------------------------------------------
|
|
740
|
+
# Human-readable report
|
|
741
|
+
# ------------------------------------------------------------------
|
|
742
|
+
report_text = self._build_report(
|
|
743
|
+
attestation, gap_report, period_events, clauses_def
|
|
744
|
+
)
|
|
745
|
+
|
|
746
|
+
return ComplianceEvidencePackage(
|
|
747
|
+
attestation=attestation,
|
|
748
|
+
report_text=report_text,
|
|
749
|
+
gap_report=gap_report,
|
|
750
|
+
audit_exports=audit_exports,
|
|
751
|
+
)
|
|
752
|
+
|
|
753
|
+
# ------------------------------------------------------------------
|
|
754
|
+
# Internal helpers
|
|
755
|
+
# ------------------------------------------------------------------
|
|
756
|
+
|
|
757
|
+
@staticmethod
|
|
758
|
+
def _enrich_from_model_registry(
|
|
759
|
+
attestation: ComplianceAttestation, model_id: str
|
|
760
|
+
) -> None:
|
|
761
|
+
"""Enrich *attestation* with model registry metadata (owner, risk_tier, status)."""
|
|
762
|
+
if not model_id:
|
|
763
|
+
return
|
|
764
|
+
try:
|
|
765
|
+
from spanforge.model_registry import get_model # noqa: PLC0415
|
|
766
|
+
|
|
767
|
+
entry = get_model(model_id)
|
|
768
|
+
if entry is None:
|
|
769
|
+
attestation.model_warnings.append(
|
|
770
|
+
f"Model {model_id!r} is not registered in the model registry. "
|
|
771
|
+
"Register it for full compliance traceability."
|
|
772
|
+
)
|
|
773
|
+
return
|
|
774
|
+
|
|
775
|
+
attestation.model_owner = entry.owner
|
|
776
|
+
attestation.model_risk_tier = entry.risk_tier
|
|
777
|
+
attestation.model_status = entry.status
|
|
778
|
+
|
|
779
|
+
if entry.status == "deprecated":
|
|
780
|
+
attestation.model_warnings.append(
|
|
781
|
+
f"Model {model_id!r} is DEPRECATED in the registry. "
|
|
782
|
+
"Consider migrating to an active model before the next audit period."
|
|
783
|
+
)
|
|
784
|
+
elif entry.status == "retired":
|
|
785
|
+
attestation.model_warnings.append(
|
|
786
|
+
f"Model {model_id!r} is RETIRED in the registry. "
|
|
787
|
+
"Generating a compliance attestation for a retired model is unusual — "
|
|
788
|
+
"verify this is intentional."
|
|
789
|
+
)
|
|
790
|
+
except Exception: # noqa: BLE001
|
|
791
|
+
pass
|
|
792
|
+
|
|
793
|
+
@staticmethod
|
|
794
|
+
def _compute_explanation_coverage(
|
|
795
|
+
attestation: ComplianceAttestation,
|
|
796
|
+
period_events: list[dict[str, Any]],
|
|
797
|
+
model_id: str,
|
|
798
|
+
) -> None:
|
|
799
|
+
"""Compute explanation coverage: % of high-risk decisions with an explanation."""
|
|
800
|
+
# Count decisions (trace spans) for this model in the period
|
|
801
|
+
decision_events = [
|
|
802
|
+
e for e in period_events
|
|
803
|
+
if str(e.get("event_type", "")).startswith(("llm.trace.", "hitl."))
|
|
804
|
+
and (
|
|
805
|
+
not model_id
|
|
806
|
+
or (e.get("payload") or {}).get("model", {}).get("name", "").lower() == model_id.lower()
|
|
807
|
+
or (e.get("payload") or {}).get("model_id", "").lower() == model_id.lower()
|
|
808
|
+
or str((e.get("payload") or {}).get("model", "")).lower() == model_id.lower()
|
|
809
|
+
)
|
|
810
|
+
]
|
|
811
|
+
explanation_events = [
|
|
812
|
+
e for e in period_events
|
|
813
|
+
if str(e.get("event_type", "")).startswith("explanation.")
|
|
814
|
+
]
|
|
815
|
+
|
|
816
|
+
decision_count = len(decision_events)
|
|
817
|
+
explanation_count = len(explanation_events)
|
|
818
|
+
|
|
819
|
+
if decision_count > 0:
|
|
820
|
+
attestation.explanation_coverage_pct = round(
|
|
821
|
+
min(explanation_count / decision_count * 100, 100.0), 1
|
|
822
|
+
)
|
|
823
|
+
else:
|
|
824
|
+
# No decisions → coverage is N/A; store None to omit from output
|
|
825
|
+
attestation.explanation_coverage_pct = None
|
|
826
|
+
|
|
827
|
+
def _load_from_store(self) -> list[dict[str, Any]]:
|
|
828
|
+
"""Load events from the active TraceStore."""
|
|
829
|
+
try:
|
|
830
|
+
from spanforge._store import get_store # noqa: PLC0415
|
|
831
|
+
|
|
832
|
+
store = get_store()
|
|
833
|
+
with store._lock:
|
|
834
|
+
events = [e for evts in store._traces.values() for e in evts]
|
|
835
|
+
return [
|
|
836
|
+
{
|
|
837
|
+
"event_id": getattr(e, "event_id", None),
|
|
838
|
+
"event_type": getattr(e, "event_type", None),
|
|
839
|
+
"timestamp": getattr(e, "timestamp", None),
|
|
840
|
+
"source": getattr(e, "source", None),
|
|
841
|
+
"trace_id": getattr(e, "trace_id", None),
|
|
842
|
+
"span_id": getattr(e, "span_id", None),
|
|
843
|
+
"payload": getattr(e, "payload", {}),
|
|
844
|
+
"tags": getattr(e, "tags", {}),
|
|
845
|
+
"signature": getattr(e, "signature", None),
|
|
846
|
+
}
|
|
847
|
+
for e in events
|
|
848
|
+
]
|
|
849
|
+
except Exception: # noqa: BLE001
|
|
850
|
+
return []
|
|
851
|
+
|
|
852
|
+
@staticmethod
|
|
853
|
+
def _filter_period(
|
|
854
|
+
events: list[dict[str, Any]], from_date: str, to_date: str
|
|
855
|
+
) -> list[dict[str, Any]]:
|
|
856
|
+
"""Filter events to the requested date range (inclusive)."""
|
|
857
|
+
|
|
858
|
+
def _parse(s: str) -> datetime | None:
|
|
859
|
+
"""Parse a date or datetime string to an aware UTC datetime."""
|
|
860
|
+
try:
|
|
861
|
+
dt = datetime.fromisoformat(s.replace("Z", "+00:00"))
|
|
862
|
+
except ValueError:
|
|
863
|
+
return None
|
|
864
|
+
if dt.tzinfo is None:
|
|
865
|
+
dt = dt.replace(tzinfo=timezone.utc)
|
|
866
|
+
return dt
|
|
867
|
+
|
|
868
|
+
from_dt = _parse(from_date)
|
|
869
|
+
to_dt = _parse(to_date)
|
|
870
|
+
if from_dt is None or to_dt is None:
|
|
871
|
+
raise ValueError(
|
|
872
|
+
f"Cannot parse date range: from_date={from_date!r}, to_date={to_date!r}. "
|
|
873
|
+
"Use ISO-8601 format: YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ"
|
|
874
|
+
)
|
|
875
|
+
|
|
876
|
+
filtered = []
|
|
877
|
+
for e in events:
|
|
878
|
+
ts_raw = e.get("timestamp")
|
|
879
|
+
if not ts_raw:
|
|
880
|
+
continue
|
|
881
|
+
ts = _parse(str(ts_raw))
|
|
882
|
+
if ts is not None and from_dt <= ts <= to_dt:
|
|
883
|
+
filtered.append(e)
|
|
884
|
+
|
|
885
|
+
return filtered # empty list is correct when no events fall in the period
|
|
886
|
+
|
|
887
|
+
@staticmethod
|
|
888
|
+
def _event_matches_model(event: dict[str, Any], model_id: str) -> bool:
|
|
889
|
+
"""Return True if event is associated with *model_id*."""
|
|
890
|
+
if not model_id:
|
|
891
|
+
return True
|
|
892
|
+
payload = event.get("payload") or {}
|
|
893
|
+
model_block = payload.get("model") or {}
|
|
894
|
+
model_name = (
|
|
895
|
+
model_block.get("name")
|
|
896
|
+
or payload.get("model_id")
|
|
897
|
+
or payload.get("model")
|
|
898
|
+
or ""
|
|
899
|
+
)
|
|
900
|
+
return model_id.lower() == str(model_name).lower()
|
|
901
|
+
|
|
902
|
+
@staticmethod
|
|
903
|
+
def _build_report(
|
|
904
|
+
att: ComplianceAttestation,
|
|
905
|
+
gap: GapReport,
|
|
906
|
+
events: list[dict[str, Any]],
|
|
907
|
+
clauses_def: dict[str, Any],
|
|
908
|
+
) -> str:
|
|
909
|
+
"""Build a human-readable markdown-style compliance report."""
|
|
910
|
+
lines = [
|
|
911
|
+
f"# spanforge Compliance Report",
|
|
912
|
+
f"",
|
|
913
|
+
f"| Field | Value |",
|
|
914
|
+
f"|---------------|-------|",
|
|
915
|
+
f"| Framework | {att.framework.upper()} |",
|
|
916
|
+
f"| Model | {att.model_id} |",
|
|
917
|
+
f"| Period | {att.period_from} → {att.period_to} |",
|
|
918
|
+
f"| Generated | {att.generated_at} |",
|
|
919
|
+
f"| Overall | **{att.overall_status.value.upper()}** |",
|
|
920
|
+
f"| Events in scope | {len(events)} |",
|
|
921
|
+
f"| HMAC Sig | `{att.hmac_sig[:32]}…` |",
|
|
922
|
+
]
|
|
923
|
+
|
|
924
|
+
# Model registry metadata
|
|
925
|
+
if att.model_owner is not None:
|
|
926
|
+
lines.append(f"| Model Owner | {att.model_owner} |")
|
|
927
|
+
if att.model_risk_tier is not None:
|
|
928
|
+
lines.append(f"| Risk Tier | {att.model_risk_tier} |")
|
|
929
|
+
if att.model_status is not None:
|
|
930
|
+
lines.append(f"| Model Status | {att.model_status} |")
|
|
931
|
+
|
|
932
|
+
# Explanation coverage
|
|
933
|
+
if att.explanation_coverage_pct is not None:
|
|
934
|
+
lines.append(f"| Explanation Coverage | {att.explanation_coverage_pct}% |")
|
|
935
|
+
|
|
936
|
+
lines.append(f"")
|
|
937
|
+
|
|
938
|
+
# Model warnings
|
|
939
|
+
if att.model_warnings:
|
|
940
|
+
lines.append(f"## ⚠️ Model Registry Warnings")
|
|
941
|
+
lines.append(f"")
|
|
942
|
+
for w in att.model_warnings:
|
|
943
|
+
lines.append(f"- {w}")
|
|
944
|
+
lines.append(f"")
|
|
945
|
+
|
|
946
|
+
lines.extend([
|
|
947
|
+
f"## Clause Analysis",
|
|
948
|
+
f"",
|
|
949
|
+
])
|
|
950
|
+
for rec in att.clauses:
|
|
951
|
+
info = clauses_def.get(rec.clause_id, {})
|
|
952
|
+
icon = {"pass": "✅", "fail": "❌", "partial": "⚠️"}.get(rec.status.value, "❓")
|
|
953
|
+
lines.append(f"### {icon} {rec.clause_id} — {info.get('title', rec.clause_id)}")
|
|
954
|
+
lines.append(f"")
|
|
955
|
+
lines.append(f"- **Status**: {rec.status.value.upper()}")
|
|
956
|
+
lines.append(f"- **Evidence events**: {rec.evidence_count}")
|
|
957
|
+
lines.append(f"- **Summary**: {rec.summary}")
|
|
958
|
+
lines.append(f"")
|
|
959
|
+
|
|
960
|
+
if gap.has_gaps:
|
|
961
|
+
lines.append(f"## ❌ Gaps Requiring Action")
|
|
962
|
+
lines.append(f"")
|
|
963
|
+
for cid in gap.gap_clause_ids:
|
|
964
|
+
info = clauses_def.get(cid, {})
|
|
965
|
+
lines.append(f"- **{cid}** — {info.get('title', cid)}: {info.get('description', '')}")
|
|
966
|
+
lines.append(f"")
|
|
967
|
+
|
|
968
|
+
if gap.partial_clause_ids:
|
|
969
|
+
lines.append(f"## ⚠️ Partial Coverage")
|
|
970
|
+
lines.append(f"")
|
|
971
|
+
for cid in gap.partial_clause_ids:
|
|
972
|
+
info = clauses_def.get(cid, {})
|
|
973
|
+
lines.append(f"- **{cid}** — {info.get('title', cid)}")
|
|
974
|
+
lines.append(f"")
|
|
975
|
+
|
|
976
|
+
lines.append(f"---")
|
|
977
|
+
lines.append(f"*Generated by spanforge.core.compliance_mapping. HMAC key: `SPANFORGE_SIGNING_KEY` env var.*")
|
|
978
|
+
return "\n".join(lines)
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
# ---------------------------------------------------------------------------
|
|
982
|
+
# Attestation verification
|
|
983
|
+
# ---------------------------------------------------------------------------
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
def verify_attestation_signature(attestation: ComplianceAttestation) -> bool:
|
|
987
|
+
"""Re-compute the HMAC and compare against the stored signature.
|
|
988
|
+
|
|
989
|
+
Returns ``True`` if the attestation has not been tampered with.
|
|
990
|
+
Note: verification requires the same ``SPANFORGE_SIGNING_KEY`` that was
|
|
991
|
+
used during generation.
|
|
992
|
+
"""
|
|
993
|
+
sig_payload = json.dumps(
|
|
994
|
+
{
|
|
995
|
+
"model_id": attestation.model_id,
|
|
996
|
+
"framework": attestation.framework,
|
|
997
|
+
"from": attestation.period_from,
|
|
998
|
+
"to": attestation.period_to,
|
|
999
|
+
"generated_at": attestation.generated_at,
|
|
1000
|
+
"clauses": {r.clause_id: r.status.value for r in attestation.clauses},
|
|
1001
|
+
"event_count": sum(r.evidence_count for r in attestation.clauses),
|
|
1002
|
+
},
|
|
1003
|
+
sort_keys=True,
|
|
1004
|
+
)
|
|
1005
|
+
signing_key = os.environ.get("SPANFORGE_SIGNING_KEY", "")
|
|
1006
|
+
if not signing_key or signing_key == "spanforge-default":
|
|
1007
|
+
raise ValueError(
|
|
1008
|
+
"SPANFORGE_SIGNING_KEY is not set or uses the insecure default value. "
|
|
1009
|
+
"Attestation verification requires the same key used at signing time. "
|
|
1010
|
+
"Example: export SPANFORGE_SIGNING_KEY=<your-secret>"
|
|
1011
|
+
)
|
|
1012
|
+
expected = _hmac.new(
|
|
1013
|
+
signing_key.encode(),
|
|
1014
|
+
sig_payload.encode(),
|
|
1015
|
+
hashlib.sha256,
|
|
1016
|
+
).hexdigest()
|
|
1017
|
+
return _hmac.compare_digest(attestation.hmac_sig, expected)
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
def verify_pdf_attestation(path: str | Any, org_secret: str | None = None) -> bool:
|
|
1021
|
+
"""Verify that a PDF attestation has not been modified since signing.
|
|
1022
|
+
|
|
1023
|
+
Reads the ``.pdf.sig`` sidecar file to obtain the original HMAC, then
|
|
1024
|
+
re-computes ``HMAC-SHA256(SHA256(pdf_bytes), org_secret)`` and compares.
|
|
1025
|
+
|
|
1026
|
+
Args:
|
|
1027
|
+
path: Path to the PDF file.
|
|
1028
|
+
org_secret: HMAC signing key. If ``None``, reads from
|
|
1029
|
+
``SPANFORGE_SIGNING_KEY`` env var.
|
|
1030
|
+
|
|
1031
|
+
Returns:
|
|
1032
|
+
``True`` if the PDF bytes have not been altered since signing.
|
|
1033
|
+
"""
|
|
1034
|
+
from pathlib import Path as _Path # noqa: PLC0415
|
|
1035
|
+
|
|
1036
|
+
pdf_path = _Path(path)
|
|
1037
|
+
sig_path = pdf_path.with_suffix(".pdf.sig")
|
|
1038
|
+
|
|
1039
|
+
if not sig_path.exists():
|
|
1040
|
+
return False
|
|
1041
|
+
|
|
1042
|
+
sig_data = json.loads(sig_path.read_text(encoding="utf-8"))
|
|
1043
|
+
stored_hmac = sig_data.get("SpanForgeHMAC", "")
|
|
1044
|
+
|
|
1045
|
+
signing_key = org_secret or os.environ.get("SPANFORGE_SIGNING_KEY", "")
|
|
1046
|
+
if not signing_key or signing_key == "spanforge-default":
|
|
1047
|
+
raise ValueError(
|
|
1048
|
+
"SPANFORGE_SIGNING_KEY is not set or uses the insecure default value. "
|
|
1049
|
+
"PDF attestation verification requires the same key used at signing time. "
|
|
1050
|
+
"Example: export SPANFORGE_SIGNING_KEY=<your-secret>"
|
|
1051
|
+
)
|
|
1052
|
+
pdf_bytes = pdf_path.read_bytes()
|
|
1053
|
+
pdf_hash = hashlib.sha256(pdf_bytes).hexdigest()
|
|
1054
|
+
expected_hmac = _hmac.new(
|
|
1055
|
+
signing_key.encode(),
|
|
1056
|
+
pdf_hash.encode(),
|
|
1057
|
+
hashlib.sha256,
|
|
1058
|
+
).hexdigest()
|
|
1059
|
+
|
|
1060
|
+
return _hmac.compare_digest(stored_hmac, expected_hmac)
|