admina-framework 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- admina/__init__.py +34 -0
- admina/cli/__init__.py +14 -0
- admina/cli/commands/__init__.py +14 -0
- admina/cli/main.py +1522 -0
- admina/cli/templates/admina.yaml.j2 +77 -0
- admina/cli/templates/docker-compose.yml.j2 +254 -0
- admina/cli/templates/env.j2 +10 -0
- admina/cli/templates/main.py.j2 +95 -0
- admina/cli/templates/plugin.py.j2 +145 -0
- admina/cli/templates/plugin_pyproject.toml.j2 +15 -0
- admina/cli/templates/plugin_readme.md.j2 +27 -0
- admina/cli/templates/plugin_test.py.j2 +48 -0
- admina/core/__init__.py +14 -0
- admina/core/config.py +497 -0
- admina/core/event_bus.py +112 -0
- admina/core/secrets.py +257 -0
- admina/core/types.py +146 -0
- admina/dashboard/__init__.py +8 -0
- admina/dashboard/static/heimdall.png +0 -0
- admina/dashboard/static/index.html +1045 -0
- admina/dashboard/static/vendor/alpinejs.min.js +5 -0
- admina/domains/__init__.py +14 -0
- admina/domains/agent_security/__init__.py +41 -0
- admina/domains/agent_security/firewall.py +634 -0
- admina/domains/agent_security/loop_breaker.py +176 -0
- admina/domains/ai_infra/__init__.py +79 -0
- admina/domains/ai_infra/llm_engine.py +477 -0
- admina/domains/ai_infra/rag.py +817 -0
- admina/domains/ai_infra/webui.py +292 -0
- admina/domains/compliance/__init__.py +109 -0
- admina/domains/compliance/cross_regulation.py +314 -0
- admina/domains/compliance/eu_ai_act.py +367 -0
- admina/domains/compliance/forensic.py +380 -0
- admina/domains/compliance/gdpr.py +331 -0
- admina/domains/compliance/nis2.py +258 -0
- admina/domains/compliance/oisg.py +658 -0
- admina/domains/compliance/otel.py +101 -0
- admina/domains/data_sovereignty/__init__.py +42 -0
- admina/domains/data_sovereignty/classification.py +102 -0
- admina/domains/data_sovereignty/pii.py +260 -0
- admina/domains/data_sovereignty/residency.py +121 -0
- admina/integrations/__init__.py +14 -0
- admina/integrations/_engines.py +63 -0
- admina/integrations/cheshirecat/__init__.py +13 -0
- admina/integrations/cheshirecat/admina-plugin/admina_governance.py +207 -0
- admina/integrations/crewai/__init__.py +13 -0
- admina/integrations/crewai/callbacks.py +347 -0
- admina/integrations/langchain/__init__.py +13 -0
- admina/integrations/langchain/callbacks.py +341 -0
- admina/integrations/n8n/__init__.py +14 -0
- admina/integrations/openclaw/__init__.py +14 -0
- admina/plugins/__init__.py +49 -0
- admina/plugins/base.py +633 -0
- admina/plugins/builtin/__init__.py +14 -0
- admina/plugins/builtin/adapters/__init__.py +14 -0
- admina/plugins/builtin/adapters/ollama.py +120 -0
- admina/plugins/builtin/adapters/openai.py +138 -0
- admina/plugins/builtin/alerts/__init__.py +14 -0
- admina/plugins/builtin/alerts/log.py +66 -0
- admina/plugins/builtin/alerts/webhook.py +102 -0
- admina/plugins/builtin/auth/__init__.py +14 -0
- admina/plugins/builtin/auth/apikey.py +138 -0
- admina/plugins/builtin/compliance/__init__.py +14 -0
- admina/plugins/builtin/compliance/eu_ai_act.py +202 -0
- admina/plugins/builtin/connectors/__init__.py +14 -0
- admina/plugins/builtin/connectors/chromadb.py +137 -0
- admina/plugins/builtin/connectors/filesystem.py +111 -0
- admina/plugins/builtin/forensic/__init__.py +14 -0
- admina/plugins/builtin/forensic/filesystem.py +163 -0
- admina/plugins/builtin/forensic/minio.py +180 -0
- admina/plugins/builtin/guards/__init__.py +0 -0
- admina/plugins/builtin/guards/guardrailsai_guard.py +172 -0
- admina/plugins/builtin/pii/__init__.py +14 -0
- admina/plugins/builtin/pii/spacy_regex.py +160 -0
- admina/plugins/builtin/transports/__init__.py +14 -0
- admina/plugins/builtin/transports/http_rest.py +97 -0
- admina/plugins/builtin/transports/mcp.py +173 -0
- admina/plugins/registry.py +356 -0
- admina/proxy/__init__.py +15 -0
- admina/proxy/api/__init__.py +17 -0
- admina/proxy/api/dashboard.py +925 -0
- admina/proxy/api/integration.py +153 -0
- admina/proxy/config.py +214 -0
- admina/proxy/engine_bridge.py +306 -0
- admina/proxy/governance.py +232 -0
- admina/proxy/main.py +1484 -0
- admina/proxy/multi_upstream.py +156 -0
- admina/proxy/state.py +97 -0
- admina/py.typed +0 -0
- admina/sdk/__init__.py +34 -0
- admina/sdk/_compat.py +43 -0
- admina/sdk/compliance_kit.py +359 -0
- admina/sdk/governed_agent.py +391 -0
- admina/sdk/governed_data.py +434 -0
- admina/sdk/governed_model.py +241 -0
- admina_framework-0.9.0.dist-info/METADATA +575 -0
- admina_framework-0.9.0.dist-info/RECORD +102 -0
- admina_framework-0.9.0.dist-info/WHEEL +5 -0
- admina_framework-0.9.0.dist-info/entry_points.txt +2 -0
- admina_framework-0.9.0.dist-info/licenses/LICENSE +191 -0
- admina_framework-0.9.0.dist-info/licenses/NOTICE +16 -0
- admina_framework-0.9.0.dist-info/top_level.txt +1 -0
admina/proxy/main.py
ADDED
|
@@ -0,0 +1,1484 @@
|
|
|
1
|
+
# Copyright © 2025–2026 Stefano Noferi & Admina contributors
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
Admina — AI Governance Proxy
|
|
17
|
+
The governance layer for AI agents, LLMs, and autonomous systems.
|
|
18
|
+
https://admina.org
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
import hashlib
|
|
23
|
+
import json
|
|
24
|
+
import logging
|
|
25
|
+
import os
|
|
26
|
+
import re
|
|
27
|
+
import secrets as _secrets
|
|
28
|
+
import time
|
|
29
|
+
from collections.abc import AsyncGenerator
|
|
30
|
+
from contextlib import asynccontextmanager
|
|
31
|
+
from datetime import UTC, datetime
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
import clickhouse_connect
|
|
36
|
+
import httpx
|
|
37
|
+
import redis.asyncio as aioredis
|
|
38
|
+
from fastapi import FastAPI, HTTPException, Request, Response
|
|
39
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
40
|
+
from fastapi.responses import JSONResponse
|
|
41
|
+
from minio import Minio
|
|
42
|
+
from minio.error import S3Error as _S3Error
|
|
43
|
+
|
|
44
|
+
import admina.plugins.builtin.transports.mcp as mcp_transport
|
|
45
|
+
from admina import __version__
|
|
46
|
+
from admina.core.event_bus import GovernanceEvent as BusGovernanceEvent
|
|
47
|
+
from admina.core.event_bus import bus as governance_bus
|
|
48
|
+
from admina.core.types import EventType, GovernanceAction
|
|
49
|
+
from admina.domains.compliance.forensic import ForensicBlackBox
|
|
50
|
+
from admina.domains.compliance.otel import OTELGovernanceExporter
|
|
51
|
+
from admina.proxy.api.dashboard import create_dashboard_endpoints
|
|
52
|
+
from admina.proxy.api.integration import create_integration_endpoints
|
|
53
|
+
from admina.proxy.config import GovernanceEvent, settings
|
|
54
|
+
from admina.proxy.engine_bridge import (
|
|
55
|
+
engine_status,
|
|
56
|
+
get_firewall,
|
|
57
|
+
get_loop_breaker,
|
|
58
|
+
get_pii_scanner,
|
|
59
|
+
)
|
|
60
|
+
from admina.proxy.governance import run_pipeline, safe_serialize
|
|
61
|
+
from admina.proxy.multi_upstream import MultiUpstreamRouter
|
|
62
|
+
from admina.proxy.state import ProxyState
|
|
63
|
+
|
|
64
|
+
# ── Admina config (for OISG score) ──────────────────────────
|
|
65
|
+
try:
|
|
66
|
+
from admina.core.config import load_config as _load_admina_config
|
|
67
|
+
|
|
68
|
+
_admina_config = _load_admina_config()
|
|
69
|
+
except (ImportError, ValueError, OSError): # pragma: no cover
|
|
70
|
+
_admina_config = None
|
|
71
|
+
|
|
72
|
+
# ── SQL identifier validation ────────────────────────────────
|
|
73
|
+
_SAFE_IDENTIFIER = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _validate_identifier(name: str, label: str = "identifier") -> None:
|
|
77
|
+
"""Raise ValueError if *name* is not a safe SQL identifier."""
|
|
78
|
+
if not _SAFE_IDENTIFIER.match(name):
|
|
79
|
+
raise ValueError(f"Unsafe {label}: {name!r}")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# ── Logging ──────────────────────────────────────────────────
|
|
83
|
+
logging.basicConfig(
|
|
84
|
+
level=getattr(logging, settings.LOG_LEVEL),
|
|
85
|
+
format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
|
|
86
|
+
)
|
|
87
|
+
logger = logging.getLogger("admina.proxy")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# ── Background tasks ─────────────────────────────────────────
|
|
91
|
+
# Hold strong references to fire-and-forget tasks. Without this set,
|
|
92
|
+
# Python may garbage-collect the task object before the coroutine
|
|
93
|
+
# completes — see https://docs.python.org/3/library/asyncio-task.html
|
|
94
|
+
# #asyncio.create_task.
|
|
95
|
+
_background_tasks: set[asyncio.Task] = set()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _spawn(coro: Any) -> asyncio.Task:
|
|
99
|
+
"""Schedule *coro* as a background task and keep a strong ref."""
|
|
100
|
+
task = asyncio.create_task(coro)
|
|
101
|
+
_background_tasks.add(task)
|
|
102
|
+
task.add_done_callback(_background_tasks.discard)
|
|
103
|
+
return task
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# ── Startup / Shutdown ───────────────────────────────────────
|
|
107
|
+
@asynccontextmanager
|
|
108
|
+
async def lifespan(app: FastAPI) -> AsyncGenerator[None]:
|
|
109
|
+
logger.info("Admina Proxy starting...")
|
|
110
|
+
|
|
111
|
+
# Build ProxyState
|
|
112
|
+
state = ProxyState(
|
|
113
|
+
firewall=get_firewall(),
|
|
114
|
+
pii_redactor=get_pii_scanner(),
|
|
115
|
+
loop_breaker=get_loop_breaker(
|
|
116
|
+
window_size=settings.LOOP_WINDOW_SIZE,
|
|
117
|
+
similarity_threshold=settings.LOOP_SIMILARITY_THRESHOLD,
|
|
118
|
+
max_consecutive=settings.LOOP_MAX_CONSECUTIVE,
|
|
119
|
+
),
|
|
120
|
+
router=MultiUpstreamRouter(default_upstream=settings.UPSTREAM_MCP_URL),
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
# ── Plugin discovery ──────────────────────────────────────
|
|
124
|
+
state.registry.discover()
|
|
125
|
+
|
|
126
|
+
def _instantiate(category: str) -> list:
|
|
127
|
+
instances = []
|
|
128
|
+
for name, cls in state.registry.list(category).items():
|
|
129
|
+
try:
|
|
130
|
+
instances.append(cls())
|
|
131
|
+
except ImportError as exc:
|
|
132
|
+
logger.warning(
|
|
133
|
+
"Skipping %s plugin %r: optional dependency missing (%s)",
|
|
134
|
+
category,
|
|
135
|
+
name,
|
|
136
|
+
exc,
|
|
137
|
+
)
|
|
138
|
+
return instances
|
|
139
|
+
|
|
140
|
+
state.governance_guards = _instantiate("governance_guard")
|
|
141
|
+
state.alert_channels = _instantiate("alert_channel")
|
|
142
|
+
state.auth_providers = _instantiate("auth_provider")
|
|
143
|
+
if state.governance_guards:
|
|
144
|
+
logger.info(
|
|
145
|
+
"Governance guards loaded: %s",
|
|
146
|
+
[g.name for g in state.governance_guards],
|
|
147
|
+
)
|
|
148
|
+
if state.alert_channels:
|
|
149
|
+
logger.info(
|
|
150
|
+
"Alert channels loaded: %s",
|
|
151
|
+
[c.channel_name for c in state.alert_channels],
|
|
152
|
+
)
|
|
153
|
+
if state.auth_providers:
|
|
154
|
+
logger.info(
|
|
155
|
+
"Auth providers loaded: %s",
|
|
156
|
+
[p.provider_name for p in state.auth_providers],
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
# ── OTEL exporter — subscribe to event bus ────────────────
|
|
160
|
+
otel_endpoint = getattr(settings, "OTEL_ENDPOINT", "http://localhost:4317")
|
|
161
|
+
state.otel_exporter = OTELGovernanceExporter(endpoint=otel_endpoint)
|
|
162
|
+
if state.otel_exporter.enabled:
|
|
163
|
+
|
|
164
|
+
async def _otel_subscriber(event: BusGovernanceEvent) -> None:
|
|
165
|
+
state.otel_exporter.trace_governance_decision(
|
|
166
|
+
domain=event.domain or "unknown",
|
|
167
|
+
action=event.action or "UNKNOWN",
|
|
168
|
+
risk_level=event.risk_level or "LOW",
|
|
169
|
+
latency_us=event.metadata.get("latency_us", 0),
|
|
170
|
+
session_id=event.session_id,
|
|
171
|
+
metadata=event.metadata,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
governance_bus.subscribe(EventType.GOVERNANCE_DECISION, _otel_subscriber)
|
|
175
|
+
logger.info("OTEL exporter subscribed to governance events")
|
|
176
|
+
|
|
177
|
+
# ── Alert channels — subscribe to governance decisions ────
|
|
178
|
+
if state.alert_channels:
|
|
179
|
+
|
|
180
|
+
async def _alert_bus_subscriber(event: BusGovernanceEvent) -> None:
|
|
181
|
+
if event.action in ("BLOCK", "CIRCUIT_BREAK"):
|
|
182
|
+
alert = {
|
|
183
|
+
"level": event.risk_level or "HIGH",
|
|
184
|
+
"domain": event.domain or "unknown",
|
|
185
|
+
"summary": f"{event.action} — session {event.session_id}",
|
|
186
|
+
"details": event.metadata,
|
|
187
|
+
"session_id": event.session_id,
|
|
188
|
+
}
|
|
189
|
+
await _fire_alerts(state.alert_channels, alert)
|
|
190
|
+
|
|
191
|
+
governance_bus.subscribe(EventType.GOVERNANCE_DECISION, _alert_bus_subscriber)
|
|
192
|
+
logger.info("Alert channels subscribed to governance events")
|
|
193
|
+
|
|
194
|
+
# Warn if running without auth in non-dev context
|
|
195
|
+
if not settings.ADMINA_API_KEY and not state.auth_providers:
|
|
196
|
+
logger.warning(
|
|
197
|
+
"ADMINA_API_KEY is not set and no auth providers loaded — "
|
|
198
|
+
"all endpoints are unauthenticated. "
|
|
199
|
+
"Set ADMINA_API_KEY or configure an auth provider for production."
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
# Redis — optional, skip gracefully if URL is empty or malformed
|
|
203
|
+
state.redis = None
|
|
204
|
+
if settings.REDIS_URL and settings.REDIS_URL.startswith(("redis://", "rediss://", "unix://")):
|
|
205
|
+
try:
|
|
206
|
+
state.redis = aioredis.from_url(settings.REDIS_URL, decode_responses=True)
|
|
207
|
+
await state.redis.ping()
|
|
208
|
+
logger.info("Redis connected")
|
|
209
|
+
except (OSError, ValueError, aioredis.RedisError) as e:
|
|
210
|
+
logger.warning("Redis not available: %s — continuing without rate-limit cache", e)
|
|
211
|
+
state.redis = None
|
|
212
|
+
else:
|
|
213
|
+
logger.info("Redis disabled (REDIS_URL is empty or non-redis scheme)")
|
|
214
|
+
|
|
215
|
+
# Forensic backend: filesystem (default) | s3 (boto3 generic) | minio (legacy)
|
|
216
|
+
state.minio = None
|
|
217
|
+
boto3_client = None
|
|
218
|
+
|
|
219
|
+
if settings.FORENSIC_BACKEND == "minio":
|
|
220
|
+
try:
|
|
221
|
+
state.minio = Minio(
|
|
222
|
+
settings.MINIO_ENDPOINT,
|
|
223
|
+
access_key=settings.MINIO_ACCESS_KEY,
|
|
224
|
+
secret_key=settings.MINIO_SECRET_KEY,
|
|
225
|
+
secure=settings.MINIO_SECURE,
|
|
226
|
+
)
|
|
227
|
+
state.minio.list_buckets()
|
|
228
|
+
logger.info("MinIO connected (legacy backend)")
|
|
229
|
+
except (OSError, _S3Error) as e:
|
|
230
|
+
logger.warning("MinIO not available: %s — falling back to filesystem", e)
|
|
231
|
+
state.minio = None
|
|
232
|
+
|
|
233
|
+
elif settings.FORENSIC_BACKEND == "s3":
|
|
234
|
+
try:
|
|
235
|
+
import boto3
|
|
236
|
+
|
|
237
|
+
kwargs = {
|
|
238
|
+
"service_name": "s3",
|
|
239
|
+
"region_name": settings.FORENSIC_S3_REGION,
|
|
240
|
+
}
|
|
241
|
+
if settings.FORENSIC_S3_ENDPOINT:
|
|
242
|
+
kwargs["endpoint_url"] = settings.FORENSIC_S3_ENDPOINT
|
|
243
|
+
if settings.FORENSIC_S3_ACCESS_KEY:
|
|
244
|
+
kwargs["aws_access_key_id"] = settings.FORENSIC_S3_ACCESS_KEY
|
|
245
|
+
kwargs["aws_secret_access_key"] = settings.FORENSIC_S3_SECRET_KEY
|
|
246
|
+
boto3_client = boto3.client(**kwargs)
|
|
247
|
+
boto3_client.list_buckets()
|
|
248
|
+
logger.info(
|
|
249
|
+
"S3 forensic backend connected (endpoint=%s)",
|
|
250
|
+
settings.FORENSIC_S3_ENDPOINT or "default AWS",
|
|
251
|
+
)
|
|
252
|
+
except ImportError:
|
|
253
|
+
logger.warning(
|
|
254
|
+
"FORENSIC_BACKEND=s3 requires boto3 (pip install boto3) — "
|
|
255
|
+
"falling back to filesystem"
|
|
256
|
+
)
|
|
257
|
+
boto3_client = None
|
|
258
|
+
except Exception as e: # noqa: BLE001
|
|
259
|
+
logger.warning("S3 not reachable: %s — falling back to filesystem", e)
|
|
260
|
+
boto3_client = None
|
|
261
|
+
|
|
262
|
+
if state.minio is not None:
|
|
263
|
+
state.forensic_box = ForensicBlackBox(
|
|
264
|
+
minio_client=state.minio, bucket=settings.MINIO_BUCKET
|
|
265
|
+
)
|
|
266
|
+
elif boto3_client is not None:
|
|
267
|
+
state.forensic_box = ForensicBlackBox(
|
|
268
|
+
boto3_client=boto3_client,
|
|
269
|
+
bucket=settings.FORENSIC_S3_BUCKET,
|
|
270
|
+
s3_object_lock=settings.FORENSIC_S3_LOCK,
|
|
271
|
+
s3_lock_days=settings.FORENSIC_S3_LOCK_DAYS,
|
|
272
|
+
s3_auto_create_locked_bucket=settings.FORENSIC_S3_LOCK_AUTO_BUCKET,
|
|
273
|
+
s3_max_retries=settings.FORENSIC_S3_MAX_RETRIES,
|
|
274
|
+
s3_base_delay_s=settings.FORENSIC_S3_BASE_DELAY_S,
|
|
275
|
+
)
|
|
276
|
+
if settings.FORENSIC_S3_LOCK:
|
|
277
|
+
logger.info(
|
|
278
|
+
"Forensic Object Lock ENABLED: every record locked for %d days "
|
|
279
|
+
"in COMPLIANCE mode (WORM)",
|
|
280
|
+
settings.FORENSIC_S3_LOCK_DAYS,
|
|
281
|
+
)
|
|
282
|
+
elif settings.FORENSIC_BACKEND == "filesystem":
|
|
283
|
+
if not settings.FORENSIC_BASE_DIR:
|
|
284
|
+
logger.warning(
|
|
285
|
+
"FORENSIC_BACKEND=filesystem but FORENSIC_BASE_DIR is empty — "
|
|
286
|
+
"downgrading to in-memory backend (records will be lost on restart)"
|
|
287
|
+
)
|
|
288
|
+
state.forensic_box = ForensicBlackBox()
|
|
289
|
+
else:
|
|
290
|
+
state.forensic_box = ForensicBlackBox(filesystem_dir=settings.FORENSIC_BASE_DIR)
|
|
291
|
+
logger.info(
|
|
292
|
+
"Forensic backend: filesystem at %s",
|
|
293
|
+
settings.FORENSIC_BASE_DIR,
|
|
294
|
+
)
|
|
295
|
+
else:
|
|
296
|
+
# Default: in-memory only. Loud warning so the operator
|
|
297
|
+
# knows the proxy is running with no audit persistence.
|
|
298
|
+
state.forensic_box = ForensicBlackBox()
|
|
299
|
+
logger.warning(
|
|
300
|
+
"Forensic backend: IN-MEMORY ONLY — events will be LOST on restart. "
|
|
301
|
+
"Set FORENSIC_BACKEND=filesystem (with FORENSIC_BASE_DIR) or =s3 "
|
|
302
|
+
"for persistence."
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
# ClickHouse — optional, skip if host is empty
|
|
306
|
+
state.clickhouse = None
|
|
307
|
+
if settings.CLICKHOUSE_HOST:
|
|
308
|
+
try:
|
|
309
|
+
state.clickhouse = clickhouse_connect.get_client(
|
|
310
|
+
host=settings.CLICKHOUSE_HOST,
|
|
311
|
+
port=settings.CLICKHOUSE_PORT,
|
|
312
|
+
database=settings.CLICKHOUSE_DB,
|
|
313
|
+
password=settings.CLICKHOUSE_PASSWORD,
|
|
314
|
+
)
|
|
315
|
+
_init_clickhouse_tables(state.clickhouse)
|
|
316
|
+
logger.info("ClickHouse connected")
|
|
317
|
+
except (OSError, clickhouse_connect.driver.exceptions.DatabaseError) as e:
|
|
318
|
+
logger.warning("ClickHouse not available: %s — analytics disabled", e)
|
|
319
|
+
state.clickhouse = None
|
|
320
|
+
else:
|
|
321
|
+
logger.info("ClickHouse disabled (CLICKHOUSE_HOST is empty)")
|
|
322
|
+
|
|
323
|
+
# HTTP Client for upstream MCP
|
|
324
|
+
state.http_client = httpx.AsyncClient(timeout=30.0)
|
|
325
|
+
|
|
326
|
+
# Multi-upstream router (for OpenClaw integration)
|
|
327
|
+
routing_path = os.environ.get("ROUTING_CONFIG_PATH", "")
|
|
328
|
+
if routing_path:
|
|
329
|
+
state.router.load_config(routing_path)
|
|
330
|
+
logger.info("Multi-upstream routing: %d servers configured", len(state.router.routes))
|
|
331
|
+
|
|
332
|
+
# Publish state on app
|
|
333
|
+
app.state.proxy = state
|
|
334
|
+
|
|
335
|
+
logger.info("=" * 60)
|
|
336
|
+
logger.info(" Admina Governance Proxy — READY v%s", __version__)
|
|
337
|
+
_eng = engine_status()
|
|
338
|
+
_eng_label = (
|
|
339
|
+
"%s v%s" % (_eng["engine"].upper(), _eng["rust_version"])
|
|
340
|
+
if _eng["rust_available"]
|
|
341
|
+
else "%s (install admina-core for Rust speed)" % _eng["engine"].upper()
|
|
342
|
+
)
|
|
343
|
+
logger.info(" Engine: %s", _eng_label)
|
|
344
|
+
logger.info(" Upstream MCP: %s", settings.UPSTREAM_MCP_URL)
|
|
345
|
+
logger.info(
|
|
346
|
+
" Auth: %s",
|
|
347
|
+
"ON" if settings.ADMINA_API_KEY else "OFF (set ADMINA_API_KEY for production)",
|
|
348
|
+
)
|
|
349
|
+
logger.info(
|
|
350
|
+
" Rate Limiting: %s",
|
|
351
|
+
"ON (Redis)" if state.redis else "OFF (Redis unavailable)",
|
|
352
|
+
)
|
|
353
|
+
if state.router.is_multi_upstream:
|
|
354
|
+
logger.info(" OpenClaw mode: routing %d MCP servers", len(state.router.routes))
|
|
355
|
+
logger.info(" Firewall: ON | PII Redaction: ON | Loop Breaker: ON")
|
|
356
|
+
logger.info("=" * 60)
|
|
357
|
+
|
|
358
|
+
yield
|
|
359
|
+
|
|
360
|
+
# Shutdown
|
|
361
|
+
if state.redis:
|
|
362
|
+
await state.redis.close()
|
|
363
|
+
if state.http_client:
|
|
364
|
+
await state.http_client.aclose()
|
|
365
|
+
logger.info("Admina Proxy stopped")
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _init_clickhouse_tables(client):
|
|
369
|
+
"""Create governance events table in ClickHouse."""
|
|
370
|
+
_validate_identifier(settings.CLICKHOUSE_DB, "CLICKHOUSE_DB")
|
|
371
|
+
client.command(f"CREATE DATABASE IF NOT EXISTS {settings.CLICKHOUSE_DB}")
|
|
372
|
+
client.command(f"""
|
|
373
|
+
CREATE TABLE IF NOT EXISTS {settings.CLICKHOUSE_DB}.governance_events (
|
|
374
|
+
event_id String,
|
|
375
|
+
timestamp DateTime64(3),
|
|
376
|
+
event_type String,
|
|
377
|
+
agent_id String,
|
|
378
|
+
session_id String,
|
|
379
|
+
method String,
|
|
380
|
+
tool_name String,
|
|
381
|
+
action String,
|
|
382
|
+
risk_level String,
|
|
383
|
+
details String,
|
|
384
|
+
latency_ms Float64,
|
|
385
|
+
request_hash String,
|
|
386
|
+
response_hash String
|
|
387
|
+
) ENGINE = MergeTree()
|
|
388
|
+
ORDER BY (timestamp, event_id)
|
|
389
|
+
TTL toDateTime(timestamp) + INTERVAL 7 YEAR
|
|
390
|
+
""")
|
|
391
|
+
logger.info("ClickHouse tables initialized")
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
# ── FastAPI App ──────────────────────────────────────────────
|
|
395
|
+
app = FastAPI(
|
|
396
|
+
title="Admina Governance Proxy",
|
|
397
|
+
description="AI Governance & Security for Autonomous Agents",
|
|
398
|
+
version=__version__,
|
|
399
|
+
lifespan=lifespan,
|
|
400
|
+
servers=[
|
|
401
|
+
{"url": "http://localhost:3000", "description": "Dashboard (nginx proxy)"},
|
|
402
|
+
{"url": "http://localhost:8080", "description": "Proxy (direct)"},
|
|
403
|
+
],
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
_cors_origins = [o.strip() for o in settings.CORS_ORIGINS.split(",")]
|
|
407
|
+
if "*" in _cors_origins:
|
|
408
|
+
logger.warning(
|
|
409
|
+
"CORS_ORIGINS contains wildcard '*' — all cross-origin requests will be accepted"
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
app.add_middleware(
|
|
413
|
+
CORSMiddleware,
|
|
414
|
+
allow_origins=_cors_origins,
|
|
415
|
+
allow_credentials=False,
|
|
416
|
+
allow_methods=["GET", "POST", "OPTIONS"],
|
|
417
|
+
allow_headers=[
|
|
418
|
+
"Content-Type",
|
|
419
|
+
"X-Session-Id",
|
|
420
|
+
"X-Agent-Id",
|
|
421
|
+
"Authorization",
|
|
422
|
+
"X-API-Key",
|
|
423
|
+
],
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _get_state(request: Request) -> ProxyState:
|
|
428
|
+
"""Retrieve ProxyState from the app."""
|
|
429
|
+
return request.app.state.proxy
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
# ── Dashboard & Integration API Routers ──────────────────────
|
|
433
|
+
# The lambdas close over `app` so they resolve state at call time (after lifespan).
|
|
434
|
+
_dashboard_router = create_dashboard_endpoints(
|
|
435
|
+
get_metrics=lambda: app.state.proxy.metrics,
|
|
436
|
+
get_forensic_box=lambda: app.state.proxy.forensic_box,
|
|
437
|
+
get_compliance=lambda: app.state.proxy.compliance,
|
|
438
|
+
get_clickhouse=lambda: app.state.proxy.clickhouse,
|
|
439
|
+
get_settings=lambda: settings,
|
|
440
|
+
get_redis=lambda: app.state.proxy.redis,
|
|
441
|
+
get_minio=lambda: app.state.proxy.minio,
|
|
442
|
+
get_engine_status=lambda: engine_status(),
|
|
443
|
+
get_http_client=lambda: app.state.proxy.http_client,
|
|
444
|
+
get_firewall=lambda: app.state.proxy.firewall,
|
|
445
|
+
get_pii_redactor=lambda: app.state.proxy.pii_redactor,
|
|
446
|
+
get_loop_breaker=lambda: app.state.proxy.loop_breaker,
|
|
447
|
+
get_otel_exporter=lambda: app.state.proxy.otel_exporter,
|
|
448
|
+
get_governance_guards=lambda: app.state.proxy.governance_guards,
|
|
449
|
+
get_config=lambda: _admina_config,
|
|
450
|
+
)
|
|
451
|
+
app.include_router(_dashboard_router)
|
|
452
|
+
|
|
453
|
+
_integration_router = create_integration_endpoints(
|
|
454
|
+
get_firewall=lambda: app.state.proxy.firewall,
|
|
455
|
+
get_pii_scanner=lambda: app.state.proxy.pii_redactor,
|
|
456
|
+
get_loop_breaker=lambda: app.state.proxy.loop_breaker,
|
|
457
|
+
get_forensic_box=lambda: app.state.proxy.forensic_box,
|
|
458
|
+
)
|
|
459
|
+
app.include_router(_integration_router)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
# ── Bundled dashboard (no-Docker dev mode) ────────────────────
|
|
463
|
+
# When running `admina dev` (default, no Docker), the proxy serves
|
|
464
|
+
# the dashboard SPA on the same port. In Docker mode nginx serves it
|
|
465
|
+
# separately on :3000, so this code path is harmless there too —
|
|
466
|
+
# nginx hits /api/* before /, and / is fine either way.
|
|
467
|
+
_DASHBOARD_DIR = Path(__file__).resolve().parent.parent / "dashboard" / "static"
|
|
468
|
+
_DASHBOARD_CACHE: str | None = None
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _dashboard_index_html() -> str:
|
|
472
|
+
"""Return the dashboard index.html with %%VERSION%% / %%GIT_COMMIT%%
|
|
473
|
+
substituted in-process (no nginx build-time templating)."""
|
|
474
|
+
global _DASHBOARD_CACHE
|
|
475
|
+
if _DASHBOARD_CACHE is not None:
|
|
476
|
+
return _DASHBOARD_CACHE
|
|
477
|
+
index_path = _DASHBOARD_DIR / "index.html"
|
|
478
|
+
html = index_path.read_text(encoding="utf-8")
|
|
479
|
+
html = html.replace("%%VERSION%%", f"v{__version__}")
|
|
480
|
+
html = html.replace("%%GIT_COMMIT%%", "local")
|
|
481
|
+
# Local mode has no nginx, so the basic-auth placeholder must be
|
|
482
|
+
# neutralised to avoid the browser seeing the literal placeholder.
|
|
483
|
+
html = html.replace("__ADMINA_API_KEY__", "")
|
|
484
|
+
_DASHBOARD_CACHE = html
|
|
485
|
+
return html
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
_DASHBOARD_COOKIE = "admina_session"
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
if _DASHBOARD_DIR.is_dir():
|
|
492
|
+
from fastapi.responses import HTMLResponse
|
|
493
|
+
from fastapi.staticfiles import StaticFiles
|
|
494
|
+
|
|
495
|
+
app.mount(
|
|
496
|
+
"/vendor",
|
|
497
|
+
StaticFiles(directory=_DASHBOARD_DIR / "vendor"),
|
|
498
|
+
name="dashboard-vendor",
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
@app.get("/heimdall.png", include_in_schema=False)
|
|
502
|
+
async def _dashboard_logo() -> Response:
|
|
503
|
+
return Response(
|
|
504
|
+
content=(_DASHBOARD_DIR / "heimdall.png").read_bytes(),
|
|
505
|
+
media_type="image/png",
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
@app.get("/", include_in_schema=False)
|
|
509
|
+
async def _dashboard_root() -> HTMLResponse:
|
|
510
|
+
# Issue a session cookie carrying the API key so subsequent
|
|
511
|
+
# /api/* fetches and the WebSocket auto-authenticate without
|
|
512
|
+
# the dashboard JS needing to know the key.
|
|
513
|
+
resp = HTMLResponse(_dashboard_index_html())
|
|
514
|
+
if settings.ADMINA_API_KEY:
|
|
515
|
+
resp.set_cookie(
|
|
516
|
+
_DASHBOARD_COOKIE,
|
|
517
|
+
settings.ADMINA_API_KEY,
|
|
518
|
+
httponly=True,
|
|
519
|
+
samesite="lax",
|
|
520
|
+
# No `secure=True` here — local dev is HTTP. Production
|
|
521
|
+
# behind HTTPS should set secure=True via a reverse proxy.
|
|
522
|
+
max_age=86400,
|
|
523
|
+
)
|
|
524
|
+
return resp
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
# ── Auth Middleware ───────────────────────────────────────────
|
|
528
|
+
# /health and the OpenAPI docs are always public.
|
|
529
|
+
# Dashboard static assets are also public so the SPA can boot.
|
|
530
|
+
_AUTH_EXEMPT = {
|
|
531
|
+
"/",
|
|
532
|
+
"/health",
|
|
533
|
+
"/docs",
|
|
534
|
+
"/openapi.json",
|
|
535
|
+
"/redoc",
|
|
536
|
+
"/metrics",
|
|
537
|
+
"/heimdall.png",
|
|
538
|
+
}
|
|
539
|
+
_AUTH_EXEMPT_PREFIXES = ("/vendor/",)
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
@app.middleware("http")
|
|
543
|
+
async def auth_middleware(request: Request, call_next) -> JSONResponse:
|
|
544
|
+
path = request.url.path
|
|
545
|
+
if path in _AUTH_EXEMPT or path.startswith(_AUTH_EXEMPT_PREFIXES):
|
|
546
|
+
return await call_next(request)
|
|
547
|
+
|
|
548
|
+
state = _get_state(request)
|
|
549
|
+
|
|
550
|
+
# 1. Try plugin auth providers first (if any are loaded)
|
|
551
|
+
if state.auth_providers:
|
|
552
|
+
for provider in state.auth_providers:
|
|
553
|
+
try:
|
|
554
|
+
user = await provider.authenticate(request)
|
|
555
|
+
if user:
|
|
556
|
+
request.state.user = user
|
|
557
|
+
return await call_next(request)
|
|
558
|
+
except (ValueError, RuntimeError, OSError):
|
|
559
|
+
continue # try next provider
|
|
560
|
+
# All providers failed — reject
|
|
561
|
+
return JSONResponse(
|
|
562
|
+
status_code=401,
|
|
563
|
+
content={
|
|
564
|
+
"error": "Unauthorized",
|
|
565
|
+
"detail": "Authentication failed across all providers",
|
|
566
|
+
},
|
|
567
|
+
)
|
|
568
|
+
|
|
569
|
+
# 2. Fallback: static ADMINA_API_KEY check.
|
|
570
|
+
# Accept it from X-API-Key header, Authorization: Bearer, OR the
|
|
571
|
+
# `admina_session` cookie issued by the bundled dashboard at GET /.
|
|
572
|
+
if settings.ADMINA_API_KEY:
|
|
573
|
+
provided = (
|
|
574
|
+
request.headers.get("X-API-Key")
|
|
575
|
+
or request.headers.get("Authorization", "").removeprefix("Bearer ").strip()
|
|
576
|
+
or request.cookies.get(_DASHBOARD_COOKIE, "")
|
|
577
|
+
)
|
|
578
|
+
if not provided or not _secrets.compare_digest(provided, settings.ADMINA_API_KEY):
|
|
579
|
+
return JSONResponse(
|
|
580
|
+
status_code=401,
|
|
581
|
+
content={
|
|
582
|
+
"error": "Unauthorized",
|
|
583
|
+
"detail": "Provide your API key via X-API-Key header or Authorization: Bearer <key>",
|
|
584
|
+
},
|
|
585
|
+
)
|
|
586
|
+
return await call_next(request)
|
|
587
|
+
|
|
588
|
+
# 3. No API key and no auth providers — block unless explicitly allowed
|
|
589
|
+
if settings.ALLOW_UNAUTHENTICATED:
|
|
590
|
+
return await call_next(request)
|
|
591
|
+
|
|
592
|
+
return JSONResponse(
|
|
593
|
+
status_code=401,
|
|
594
|
+
content={
|
|
595
|
+
"error": "Unauthorized",
|
|
596
|
+
"detail": (
|
|
597
|
+
"No authentication configured. "
|
|
598
|
+
"Run 'admina dev' to auto-generate credentials, "
|
|
599
|
+
"or set ADMINA_API_KEY in .env, "
|
|
600
|
+
"or set ALLOW_UNAUTHENTICATED=true for local development."
|
|
601
|
+
),
|
|
602
|
+
},
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
# ── Admin API ─────────────────────────────────────────────────
|
|
607
|
+
@app.get("/health", tags=["admin"], summary="Liveness probe")
|
|
608
|
+
async def health() -> dict[str, Any]:
|
|
609
|
+
return {
|
|
610
|
+
"status": "healthy",
|
|
611
|
+
"service": "admina-proxy",
|
|
612
|
+
"version": __version__,
|
|
613
|
+
"engine": engine_status(),
|
|
614
|
+
"timestamp": datetime.now(UTC).isoformat(),
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
@app.get(
|
|
619
|
+
"/metrics",
|
|
620
|
+
tags=["admin"],
|
|
621
|
+
summary="Prometheus metrics exposition",
|
|
622
|
+
response_class=Response,
|
|
623
|
+
)
|
|
624
|
+
async def prometheus_metrics(request: Request) -> Response:
|
|
625
|
+
"""Plain-text Prometheus exposition for /metrics scraping.
|
|
626
|
+
|
|
627
|
+
Public endpoint (no API key) — Prometheus scraping is normally
|
|
628
|
+
network-restricted at the firewall/pod level, not via auth.
|
|
629
|
+
Producing the format inline avoids a hard dependency on
|
|
630
|
+
prometheus_client (kept lightweight for the OSS distribution).
|
|
631
|
+
"""
|
|
632
|
+
state = _get_state(request)
|
|
633
|
+
m = state.metrics
|
|
634
|
+
fw_stats = state.firewall.get_stats() if state.firewall else {}
|
|
635
|
+
lb_stats = state.loop_breaker.get_stats() if state.loop_breaker else {}
|
|
636
|
+
pii_stats = state.pii_redactor.get_stats() if state.pii_redactor else {}
|
|
637
|
+
fbox_stats = state.forensic_box.get_stats() if state.forensic_box else {}
|
|
638
|
+
eng = engine_status()
|
|
639
|
+
|
|
640
|
+
lines: list[str] = []
|
|
641
|
+
|
|
642
|
+
def _metric(name: str, value, help_text: str, mtype: str = "counter", labels: str = "") -> None:
|
|
643
|
+
lines.append(f"# HELP admina_{name} {help_text}")
|
|
644
|
+
lines.append(f"# TYPE admina_{name} {mtype}")
|
|
645
|
+
suffix = f"{{{labels}}}" if labels else ""
|
|
646
|
+
lines.append(f"admina_{name}{suffix} {value}")
|
|
647
|
+
|
|
648
|
+
_metric("requests_total", m.get("requests_total", 0), "Total governance requests processed")
|
|
649
|
+
_metric(
|
|
650
|
+
"requests_blocked_total", m.get("requests_blocked", 0), "Total governance requests blocked"
|
|
651
|
+
)
|
|
652
|
+
_metric(
|
|
653
|
+
"requests_allowed_total",
|
|
654
|
+
m.get("requests_allowed", 0),
|
|
655
|
+
"Total governance requests allowed through",
|
|
656
|
+
)
|
|
657
|
+
_metric(
|
|
658
|
+
"requests_redacted_total",
|
|
659
|
+
m.get("requests_redacted", 0),
|
|
660
|
+
"Total requests in which PII was redacted",
|
|
661
|
+
)
|
|
662
|
+
_metric(
|
|
663
|
+
"avg_latency_ms",
|
|
664
|
+
round(m.get("avg_latency_ms", 0.0), 2),
|
|
665
|
+
"Rolling average pipeline latency in milliseconds",
|
|
666
|
+
"gauge",
|
|
667
|
+
)
|
|
668
|
+
|
|
669
|
+
# Firewall pattern hits, broken down per category
|
|
670
|
+
for cat, count in (fw_stats.get("detections_by_type") or {}).items():
|
|
671
|
+
safe = "".join(c if c.isalnum() or c == "_" else "_" for c in cat)
|
|
672
|
+
_metric(
|
|
673
|
+
"firewall_detections_total",
|
|
674
|
+
count,
|
|
675
|
+
"Firewall pattern detections per category",
|
|
676
|
+
labels=f'category="{safe}"',
|
|
677
|
+
)
|
|
678
|
+
_metric(
|
|
679
|
+
"firewall_total_checked", fw_stats.get("total_checked", 0), "Firewall total inputs scanned"
|
|
680
|
+
)
|
|
681
|
+
_metric(
|
|
682
|
+
"firewall_total_blocked", fw_stats.get("total_blocked", 0), "Firewall total inputs blocked"
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
_metric(
|
|
686
|
+
"loop_breaker_total_blocked", lb_stats.get("total_blocked", 0), "Loop breaker activations"
|
|
687
|
+
)
|
|
688
|
+
_metric(
|
|
689
|
+
"loop_breaker_active_sessions",
|
|
690
|
+
lb_stats.get("active_sessions", 0),
|
|
691
|
+
"Loop breaker sessions currently tracked",
|
|
692
|
+
"gauge",
|
|
693
|
+
)
|
|
694
|
+
|
|
695
|
+
_metric("pii_total_redacted", pii_stats.get("total_redacted", 0), "Total PII entities redacted")
|
|
696
|
+
|
|
697
|
+
if fbox_stats:
|
|
698
|
+
_metric(
|
|
699
|
+
"forensic_record_count",
|
|
700
|
+
fbox_stats.get("record_count", 0),
|
|
701
|
+
"Forensic chain length (records appended)",
|
|
702
|
+
"gauge",
|
|
703
|
+
)
|
|
704
|
+
|
|
705
|
+
# Engine info as a labelled gauge with constant value 1
|
|
706
|
+
engine_name = eng.get("engine", "unknown")
|
|
707
|
+
rust_avail = "yes" if eng.get("rust_available") else "no"
|
|
708
|
+
lines.append("# HELP admina_engine_info Static info about the running engine")
|
|
709
|
+
lines.append("# TYPE admina_engine_info gauge")
|
|
710
|
+
lines.append(
|
|
711
|
+
f'admina_engine_info{{engine="{engine_name}",rust_available="{rust_avail}",'
|
|
712
|
+
f'version="{__version__}"}} 1'
|
|
713
|
+
)
|
|
714
|
+
|
|
715
|
+
body = "\n".join(lines) + "\n"
|
|
716
|
+
return Response(content=body, media_type="text/plain; version=0.0.4; charset=utf-8")
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
@app.get("/api/stats", tags=["admin"], summary="Proxy and engine statistics")
|
|
720
|
+
async def get_stats(request: Request) -> dict[str, Any]:
|
|
721
|
+
state = _get_state(request)
|
|
722
|
+
return {
|
|
723
|
+
"proxy": state.metrics,
|
|
724
|
+
"engine": engine_status(),
|
|
725
|
+
"firewall": state.firewall.get_stats(),
|
|
726
|
+
"loop_breaker": state.loop_breaker.get_stats(),
|
|
727
|
+
"pii_redactor": state.pii_redactor.get_stats(),
|
|
728
|
+
"forensic_blackbox": (state.forensic_box.get_stats() if state.forensic_box else {}),
|
|
729
|
+
"compliance": state.compliance.get_stats(),
|
|
730
|
+
"routing": (state.router.get_stats() if state.router.is_multi_upstream else {}),
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
@app.get("/api/events", tags=["admin"], summary="Recent governance events")
|
|
735
|
+
async def get_events(request: Request, limit: int = 50) -> dict[str, Any]:
|
|
736
|
+
"""Retrieve recent governance events from ClickHouse."""
|
|
737
|
+
state = _get_state(request)
|
|
738
|
+
if not state.clickhouse:
|
|
739
|
+
return {"events": [], "error": "ClickHouse not available"}
|
|
740
|
+
_validate_identifier(settings.CLICKHOUSE_DB, "CLICKHOUSE_DB")
|
|
741
|
+
limit = max(1, min(int(limit), 1000))
|
|
742
|
+
try:
|
|
743
|
+
loop = asyncio.get_running_loop()
|
|
744
|
+
ch = state.clickhouse
|
|
745
|
+
result = await loop.run_in_executor(
|
|
746
|
+
None,
|
|
747
|
+
lambda: ch.query(
|
|
748
|
+
f"SELECT * FROM {settings.CLICKHOUSE_DB}.governance_events "
|
|
749
|
+
f"ORDER BY timestamp DESC LIMIT {limit}"
|
|
750
|
+
),
|
|
751
|
+
)
|
|
752
|
+
events = [dict(zip(result.column_names, row)) for row in result.result_rows]
|
|
753
|
+
return {"events": events, "count": len(events)}
|
|
754
|
+
except (OSError, clickhouse_connect.driver.exceptions.DatabaseError) as e:
|
|
755
|
+
return {"events": [], "error": str(e)}
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
# ── EU AI Act API ────────────────────────────────────────────
|
|
759
|
+
@app.post(
|
|
760
|
+
"/api/compliance/classify",
|
|
761
|
+
tags=["compliance"],
|
|
762
|
+
summary="Classify a system under the EU AI Act risk taxonomy",
|
|
763
|
+
)
|
|
764
|
+
async def classify_risk(request: Request, body: dict) -> dict[str, Any]:
|
|
765
|
+
state = _get_state(request)
|
|
766
|
+
result = state.compliance.classify_risk(
|
|
767
|
+
system_description=body.get("description", ""),
|
|
768
|
+
use_case=body.get("use_case", ""),
|
|
769
|
+
data_types=body.get("data_types", []),
|
|
770
|
+
)
|
|
771
|
+
return result
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
@app.post(
|
|
775
|
+
"/api/compliance/gap-analysis",
|
|
776
|
+
tags=["compliance"],
|
|
777
|
+
summary="Compute the compliance gap report for a risk category",
|
|
778
|
+
)
|
|
779
|
+
async def gap_analysis(request: Request, body: dict) -> dict[str, Any]:
|
|
780
|
+
state = _get_state(request)
|
|
781
|
+
result = state.compliance.gap_analysis(
|
|
782
|
+
risk_category=body.get("risk_category", "high"),
|
|
783
|
+
current_compliance=body.get("current_compliance", {}),
|
|
784
|
+
)
|
|
785
|
+
return result
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
@app.post(
|
|
789
|
+
"/api/compliance/report",
|
|
790
|
+
tags=["compliance"],
|
|
791
|
+
summary="Generate a structured EU AI Act compliance report",
|
|
792
|
+
)
|
|
793
|
+
async def generate_compliance_report(request: Request, body: dict) -> dict[str, Any]:
|
|
794
|
+
state = _get_state(request)
|
|
795
|
+
classification = state.compliance.classify_risk(
|
|
796
|
+
body.get("description", ""),
|
|
797
|
+
body.get("use_case", ""),
|
|
798
|
+
body.get("data_types", []),
|
|
799
|
+
)
|
|
800
|
+
gap_result = state.compliance.gap_analysis(
|
|
801
|
+
classification["risk_category"],
|
|
802
|
+
body.get("current_compliance", {}),
|
|
803
|
+
)
|
|
804
|
+
report = state.compliance.generate_report(
|
|
805
|
+
body.get("system_name", "Unknown System"),
|
|
806
|
+
classification,
|
|
807
|
+
gap_result,
|
|
808
|
+
)
|
|
809
|
+
return report
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
# ── NIS2 API ────────────────────────────────────────────────
|
|
813
|
+
@app.get(
|
|
814
|
+
"/api/compliance/nis2/areas",
|
|
815
|
+
tags=["compliance"],
|
|
816
|
+
summary="List NIS2 Art. 21 measure areas and their controls",
|
|
817
|
+
)
|
|
818
|
+
async def nis2_areas(request: Request) -> dict[str, Any]:
|
|
819
|
+
state = _get_state(request)
|
|
820
|
+
return {"areas": state.nis2.list_areas(), "stats": state.nis2.get_stats()}
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
@app.post(
|
|
824
|
+
"/api/compliance/nis2/assess",
|
|
825
|
+
tags=["compliance"],
|
|
826
|
+
summary="Run NIS2 self-assessment (returns coverage score and gaps)",
|
|
827
|
+
)
|
|
828
|
+
async def nis2_assess(request: Request, body: dict) -> dict[str, Any]:
|
|
829
|
+
state = _get_state(request)
|
|
830
|
+
return state.nis2.assess(current_compliance=body.get("current_compliance", {}))
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
# ── GDPR API ────────────────────────────────────────────────
|
|
834
|
+
@app.get(
|
|
835
|
+
"/api/compliance/gdpr/records",
|
|
836
|
+
tags=["compliance"],
|
|
837
|
+
summary="List Art. 30 records of processing activities",
|
|
838
|
+
)
|
|
839
|
+
async def gdpr_list_records(request: Request) -> dict[str, Any]:
|
|
840
|
+
state = _get_state(request)
|
|
841
|
+
return {"records": state.gdpr.list(), "stats": state.gdpr.get_stats()}
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
@app.post(
|
|
845
|
+
"/api/compliance/gdpr/records",
|
|
846
|
+
tags=["compliance"],
|
|
847
|
+
summary="Create a new Art. 30 record",
|
|
848
|
+
)
|
|
849
|
+
async def gdpr_create_record(request: Request, body: dict) -> dict[str, Any]:
|
|
850
|
+
state = _get_state(request)
|
|
851
|
+
return state.gdpr.create(payload=body)
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
@app.get(
|
|
855
|
+
"/api/compliance/gdpr/records/{activity_id}",
|
|
856
|
+
tags=["compliance"],
|
|
857
|
+
summary="Get a single Art. 30 record",
|
|
858
|
+
)
|
|
859
|
+
async def gdpr_get_record(request: Request, activity_id: str) -> dict[str, Any]:
|
|
860
|
+
state = _get_state(request)
|
|
861
|
+
rec = state.gdpr.get(activity_id)
|
|
862
|
+
if rec is None:
|
|
863
|
+
raise HTTPException(status_code=404, detail="record not found")
|
|
864
|
+
return rec
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
@app.put(
|
|
868
|
+
"/api/compliance/gdpr/records/{activity_id}",
|
|
869
|
+
tags=["compliance"],
|
|
870
|
+
summary="Update an Art. 30 record",
|
|
871
|
+
)
|
|
872
|
+
async def gdpr_update_record(request: Request, activity_id: str, body: dict) -> dict[str, Any]:
|
|
873
|
+
state = _get_state(request)
|
|
874
|
+
rec = state.gdpr.update(activity_id, body)
|
|
875
|
+
if rec is None:
|
|
876
|
+
raise HTTPException(status_code=404, detail="record not found")
|
|
877
|
+
return rec
|
|
878
|
+
|
|
879
|
+
|
|
880
|
+
@app.delete(
|
|
881
|
+
"/api/compliance/gdpr/records/{activity_id}",
|
|
882
|
+
tags=["compliance"],
|
|
883
|
+
summary="Delete an Art. 30 record",
|
|
884
|
+
)
|
|
885
|
+
async def gdpr_delete_record(request: Request, activity_id: str) -> dict[str, Any]:
|
|
886
|
+
state = _get_state(request)
|
|
887
|
+
if not state.gdpr.delete(activity_id):
|
|
888
|
+
raise HTTPException(status_code=404, detail="record not found")
|
|
889
|
+
return {"deleted": True, "id": activity_id}
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
@app.post(
|
|
893
|
+
"/api/compliance/gdpr/dpia/template",
|
|
894
|
+
tags=["compliance"],
|
|
895
|
+
summary="Render an Art. 35 DPIA scaffold (Markdown) from operator-supplied facts",
|
|
896
|
+
response_class=Response,
|
|
897
|
+
)
|
|
898
|
+
async def gdpr_dpia_template(body: dict) -> Response:
|
|
899
|
+
from admina.domains.compliance.gdpr import render_dpia_template
|
|
900
|
+
|
|
901
|
+
md = render_dpia_template(body)
|
|
902
|
+
return Response(content=md, media_type="text/markdown; charset=utf-8")
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
# ── Consolidated compliance report ──────────────────────────
|
|
906
|
+
@app.get(
|
|
907
|
+
"/api/compliance/report",
|
|
908
|
+
tags=["compliance"],
|
|
909
|
+
summary="Consolidated compliance snapshot (EU AI Act + NIS2 + GDPR + cross-matrix)",
|
|
910
|
+
)
|
|
911
|
+
async def consolidated_compliance_report(
|
|
912
|
+
request: Request,
|
|
913
|
+
format: str = "json",
|
|
914
|
+
) -> Any:
|
|
915
|
+
"""Consolidated report.
|
|
916
|
+
|
|
917
|
+
Aggregates the latest snapshot from the three compliance domains
|
|
918
|
+
plus the proxy's runtime stats. Three serialisations are supported:
|
|
919
|
+
JSON (default, machine-readable), CSV (one section per file would
|
|
920
|
+
be cleaner — for now it's a flat key/value listing), and Markdown
|
|
921
|
+
(human-readable, ready to paste into a wiki or email).
|
|
922
|
+
"""
|
|
923
|
+
from admina.domains.compliance.cross_regulation import (
|
|
924
|
+
coverage_summary as cross_summary,
|
|
925
|
+
)
|
|
926
|
+
|
|
927
|
+
state = _get_state(request)
|
|
928
|
+
|
|
929
|
+
# Latest snapshots — defensive: each module may not have any data yet
|
|
930
|
+
eu_latest = state.compliance.assessments[-1] if state.compliance.assessments else None
|
|
931
|
+
nis2_latest = state.nis2.assessments[-1] if state.nis2.assessments else None
|
|
932
|
+
|
|
933
|
+
snapshot: dict[str, Any] = {
|
|
934
|
+
"generated_at": datetime.now(UTC).isoformat(),
|
|
935
|
+
"admina_version": __version__,
|
|
936
|
+
"engine": engine_status(),
|
|
937
|
+
"proxy_metrics": state.metrics,
|
|
938
|
+
"eu_ai_act": {
|
|
939
|
+
"stats": state.compliance.get_stats(),
|
|
940
|
+
"latest_assessment": eu_latest,
|
|
941
|
+
},
|
|
942
|
+
"nis2": {
|
|
943
|
+
"stats": state.nis2.get_stats(),
|
|
944
|
+
"latest_assessment": nis2_latest,
|
|
945
|
+
},
|
|
946
|
+
"gdpr": {
|
|
947
|
+
"stats": state.gdpr.get_stats(),
|
|
948
|
+
"records": state.gdpr.list(),
|
|
949
|
+
},
|
|
950
|
+
"cross_regulation": cross_summary(),
|
|
951
|
+
"forensic_blackbox": (state.forensic_box.get_stats() if state.forensic_box else {}),
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
fmt = (format or "json").lower()
|
|
955
|
+
if fmt == "json":
|
|
956
|
+
return snapshot
|
|
957
|
+
|
|
958
|
+
if fmt == "csv":
|
|
959
|
+
import csv as _csv
|
|
960
|
+
import io as _io
|
|
961
|
+
|
|
962
|
+
buf = _io.StringIO()
|
|
963
|
+
w = _csv.writer(buf)
|
|
964
|
+
w.writerow(["section", "key", "value"])
|
|
965
|
+
|
|
966
|
+
def _emit(section: str, obj: Any, prefix: str = "") -> None:
|
|
967
|
+
if isinstance(obj, dict):
|
|
968
|
+
for k, v in obj.items():
|
|
969
|
+
_emit(section, v, f"{prefix}.{k}" if prefix else k)
|
|
970
|
+
elif isinstance(obj, list):
|
|
971
|
+
w.writerow([section, prefix, f"<list:{len(obj)} items>"])
|
|
972
|
+
else:
|
|
973
|
+
w.writerow([section, prefix, str(obj)])
|
|
974
|
+
|
|
975
|
+
for section in (
|
|
976
|
+
"eu_ai_act",
|
|
977
|
+
"nis2",
|
|
978
|
+
"gdpr",
|
|
979
|
+
"cross_regulation",
|
|
980
|
+
"forensic_blackbox",
|
|
981
|
+
"proxy_metrics",
|
|
982
|
+
"engine",
|
|
983
|
+
):
|
|
984
|
+
_emit(section, snapshot.get(section, {}))
|
|
985
|
+
return Response(content=buf.getvalue(), media_type="text/csv")
|
|
986
|
+
|
|
987
|
+
if fmt == "markdown":
|
|
988
|
+
lines = [
|
|
989
|
+
"# Admina compliance report",
|
|
990
|
+
"",
|
|
991
|
+
f"_Generated_: {snapshot['generated_at']}",
|
|
992
|
+
f"_Admina version_: {snapshot['admina_version']}",
|
|
993
|
+
"",
|
|
994
|
+
"## Proxy traffic",
|
|
995
|
+
f"- Total requests: **{snapshot['proxy_metrics'].get('requests_total', 0)}**",
|
|
996
|
+
f"- Blocked: **{snapshot['proxy_metrics'].get('requests_blocked', 0)}**",
|
|
997
|
+
f"- PII redacted: **{snapshot['proxy_metrics'].get('requests_redacted', 0)}**",
|
|
998
|
+
f"- Avg latency (ms): {snapshot['proxy_metrics'].get('avg_latency_ms', 0)}",
|
|
999
|
+
"",
|
|
1000
|
+
"## EU AI Act",
|
|
1001
|
+
]
|
|
1002
|
+
eu_stats = snapshot["eu_ai_act"]["stats"]
|
|
1003
|
+
lines.append(f"- Assessments performed: {eu_stats.get('total_assessments', 0)}")
|
|
1004
|
+
if eu_latest:
|
|
1005
|
+
lines.append(f"- Latest score: **{eu_latest.get('compliance_score', 0)}%**")
|
|
1006
|
+
lines.append(f"- Open gaps: {len(eu_latest.get('gaps', []))}")
|
|
1007
|
+
lines += ["", "## NIS2"]
|
|
1008
|
+
nis2_stats = snapshot["nis2"]["stats"]
|
|
1009
|
+
lines.append(f"- Areas tracked: {nis2_stats.get('areas_count', 0)}")
|
|
1010
|
+
lines.append(f"- Total controls: {nis2_stats.get('controls_count', 0)}")
|
|
1011
|
+
if nis2_latest:
|
|
1012
|
+
lines.append(f"- Latest coverage: **{nis2_latest.get('coverage_score', 0)}%**")
|
|
1013
|
+
lines.append(f"- Open gaps: {len(nis2_latest.get('gaps', []))}")
|
|
1014
|
+
lines += ["", "## GDPR"]
|
|
1015
|
+
gdpr_stats = snapshot["gdpr"]["stats"]
|
|
1016
|
+
lines.append(f"- Processing activities recorded: {gdpr_stats.get('total_activities', 0)}")
|
|
1017
|
+
lines.append(
|
|
1018
|
+
f"- With third-country transfers: {gdpr_stats.get('with_third_country_transfers', 0)}"
|
|
1019
|
+
)
|
|
1020
|
+
lines += ["", "## Cross-regulation coverage"]
|
|
1021
|
+
cr = snapshot["cross_regulation"]
|
|
1022
|
+
lines.append(f"- Controls in matrix: {cr['total_controls']}")
|
|
1023
|
+
for reg, n in cr["controls_per_regulation"].items():
|
|
1024
|
+
lines.append(f" - {reg}: {n} controls")
|
|
1025
|
+
lines += ["", "## Forensic"]
|
|
1026
|
+
fb = snapshot["forensic_blackbox"]
|
|
1027
|
+
if fb:
|
|
1028
|
+
lines.append(f"- Records on chain: **{fb.get('record_count', 0)}**")
|
|
1029
|
+
lines.append(f"- Chain head: `{(fb.get('chain_head') or 'GENESIS')[:16]}...`")
|
|
1030
|
+
else:
|
|
1031
|
+
lines.append("- Forensic backend not configured")
|
|
1032
|
+
lines += [
|
|
1033
|
+
"",
|
|
1034
|
+
"---",
|
|
1035
|
+
"*OSS-tier report. PDF / Excel / branded reporting are not "
|
|
1036
|
+
"included in admina-framework.*",
|
|
1037
|
+
]
|
|
1038
|
+
return Response(
|
|
1039
|
+
content="\n".join(lines) + "\n",
|
|
1040
|
+
media_type="text/markdown; charset=utf-8",
|
|
1041
|
+
)
|
|
1042
|
+
|
|
1043
|
+
raise HTTPException(
|
|
1044
|
+
status_code=400,
|
|
1045
|
+
detail=f"unknown format {format!r}: use json | csv | markdown",
|
|
1046
|
+
)
|
|
1047
|
+
|
|
1048
|
+
|
|
1049
|
+
# ── Cross-regulation matrix API ─────────────────────────────
|
|
1050
|
+
@app.get(
|
|
1051
|
+
"/api/compliance/matrix",
|
|
1052
|
+
tags=["compliance"],
|
|
1053
|
+
summary="Cross-regulation control matrix (AI Act ↔ NIS2 ↔ GDPR)",
|
|
1054
|
+
)
|
|
1055
|
+
async def compliance_matrix(format: str = "json") -> Any:
|
|
1056
|
+
from admina.domains.compliance.cross_regulation import (
|
|
1057
|
+
CROSS_REGULATION_MATRIX,
|
|
1058
|
+
coverage_summary,
|
|
1059
|
+
to_markdown,
|
|
1060
|
+
)
|
|
1061
|
+
|
|
1062
|
+
if format == "markdown":
|
|
1063
|
+
return Response(
|
|
1064
|
+
content=to_markdown(),
|
|
1065
|
+
media_type="text/markdown; charset=utf-8",
|
|
1066
|
+
)
|
|
1067
|
+
return {
|
|
1068
|
+
"summary": coverage_summary(),
|
|
1069
|
+
"controls": CROSS_REGULATION_MATRIX,
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
# ── MCP Proxy Endpoint ──────────────────────────────────────
|
|
1074
|
+
@app.post("/mcp", tags=["proxy"], summary="MCP JSON-RPC governance proxy")
|
|
1075
|
+
@app.post("/mcp/{path:path}", tags=["proxy"], include_in_schema=False)
|
|
1076
|
+
async def mcp_proxy(request: Request, path: str = "") -> JSONResponse:
|
|
1077
|
+
"""
|
|
1078
|
+
Main MCP proxy endpoint.
|
|
1079
|
+
All agent traffic flows through here for governance inspection.
|
|
1080
|
+
"""
|
|
1081
|
+
state = _get_state(request)
|
|
1082
|
+
start_time = time.perf_counter()
|
|
1083
|
+
# Sanitize header values: strip CRLF (Redis key injection) and cap length
|
|
1084
|
+
session_id = re.sub(r"[\r\n]", "", request.headers.get("X-Session-Id", "default"))[:128]
|
|
1085
|
+
agent_id = re.sub(r"[\r\n]", "", request.headers.get("X-Agent-Id", "unknown"))[:128]
|
|
1086
|
+
|
|
1087
|
+
state.inc_metric("requests_total")
|
|
1088
|
+
|
|
1089
|
+
# ─── Rate Limiting (Redis) ─────────────────────────────
|
|
1090
|
+
if state.redis and settings.RATE_LIMIT_MAX_REQUESTS > 0:
|
|
1091
|
+
# Per-session rate limit
|
|
1092
|
+
rl_key = f"admina:ratelimit:{session_id}"
|
|
1093
|
+
try:
|
|
1094
|
+
count = await state.redis.incr(rl_key)
|
|
1095
|
+
if count == 1:
|
|
1096
|
+
await state.redis.expire(rl_key, settings.RATE_LIMIT_WINDOW_SECONDS)
|
|
1097
|
+
if count > settings.RATE_LIMIT_MAX_REQUESTS:
|
|
1098
|
+
state.inc_metric("requests_blocked")
|
|
1099
|
+
return JSONResponse(
|
|
1100
|
+
status_code=429,
|
|
1101
|
+
content={
|
|
1102
|
+
"jsonrpc": "2.0",
|
|
1103
|
+
"id": None,
|
|
1104
|
+
"error": {
|
|
1105
|
+
"code": -32000,
|
|
1106
|
+
"message": "Rate limit exceeded",
|
|
1107
|
+
"data": {
|
|
1108
|
+
"session_id": session_id,
|
|
1109
|
+
"limit": settings.RATE_LIMIT_MAX_REQUESTS,
|
|
1110
|
+
"window_seconds": settings.RATE_LIMIT_WINDOW_SECONDS,
|
|
1111
|
+
},
|
|
1112
|
+
},
|
|
1113
|
+
},
|
|
1114
|
+
)
|
|
1115
|
+
except (OSError, aioredis.RedisError) as e:
|
|
1116
|
+
logger.warning("Rate limit check failed: %s", e)
|
|
1117
|
+
|
|
1118
|
+
# Per-IP rate limit (non-bypassable fallback)
|
|
1119
|
+
client_ip = request.client.host if request.client else "unknown"
|
|
1120
|
+
rl_ip_key = f"admina:ratelimit:ip:{client_ip}"
|
|
1121
|
+
try:
|
|
1122
|
+
ip_count = await state.redis.incr(rl_ip_key)
|
|
1123
|
+
if ip_count == 1:
|
|
1124
|
+
await state.redis.expire(rl_ip_key, settings.RATE_LIMIT_WINDOW_SECONDS)
|
|
1125
|
+
if ip_count > settings.RATE_LIMIT_MAX_REQUESTS * settings.RATE_LIMIT_IP_MULTIPLIER:
|
|
1126
|
+
state.inc_metric("requests_blocked")
|
|
1127
|
+
return JSONResponse(
|
|
1128
|
+
status_code=429,
|
|
1129
|
+
content={
|
|
1130
|
+
"jsonrpc": "2.0",
|
|
1131
|
+
"id": None,
|
|
1132
|
+
"error": {
|
|
1133
|
+
"code": -32000,
|
|
1134
|
+
"message": "Rate limit exceeded (IP)",
|
|
1135
|
+
"data": {
|
|
1136
|
+
"limit": settings.RATE_LIMIT_MAX_REQUESTS
|
|
1137
|
+
* settings.RATE_LIMIT_IP_MULTIPLIER,
|
|
1138
|
+
"window_seconds": settings.RATE_LIMIT_WINDOW_SECONDS,
|
|
1139
|
+
},
|
|
1140
|
+
},
|
|
1141
|
+
},
|
|
1142
|
+
)
|
|
1143
|
+
except (OSError, aioredis.RedisError) as e:
|
|
1144
|
+
logger.warning("IP rate limit check failed: %s", e)
|
|
1145
|
+
|
|
1146
|
+
try:
|
|
1147
|
+
body = await request.json()
|
|
1148
|
+
except (ValueError, UnicodeDecodeError):
|
|
1149
|
+
raise HTTPException(status_code=400, detail="Invalid JSON body")
|
|
1150
|
+
|
|
1151
|
+
gov_request = mcp_transport.parse_request(body, session_id=session_id, agent_id=agent_id)
|
|
1152
|
+
event_id = gov_request.request_id
|
|
1153
|
+
method = gov_request.method
|
|
1154
|
+
params = gov_request.metadata.get("params", {})
|
|
1155
|
+
content_str = gov_request.content
|
|
1156
|
+
|
|
1157
|
+
# ─── Token size guard ─────────────────────────────────────
|
|
1158
|
+
if settings.MAX_REQUEST_TOKENS > 0 and len(content_str) > settings.MAX_REQUEST_TOKENS:
|
|
1159
|
+
state.inc_metric("requests_blocked")
|
|
1160
|
+
return JSONResponse(
|
|
1161
|
+
status_code=413,
|
|
1162
|
+
content={
|
|
1163
|
+
"jsonrpc": "2.0",
|
|
1164
|
+
"id": body.get("id"),
|
|
1165
|
+
"error": {
|
|
1166
|
+
"code": -32000,
|
|
1167
|
+
"message": "Request too large",
|
|
1168
|
+
"data": {
|
|
1169
|
+
"content_length": len(content_str),
|
|
1170
|
+
"max_tokens": settings.MAX_REQUEST_TOKENS,
|
|
1171
|
+
},
|
|
1172
|
+
},
|
|
1173
|
+
},
|
|
1174
|
+
)
|
|
1175
|
+
|
|
1176
|
+
# ─── Governance Pipeline ─────────────────────────────────
|
|
1177
|
+
pipeline_result = await run_pipeline(
|
|
1178
|
+
body=body,
|
|
1179
|
+
content_str=content_str,
|
|
1180
|
+
session_id=session_id,
|
|
1181
|
+
agent_id=agent_id,
|
|
1182
|
+
request_id=event_id,
|
|
1183
|
+
params=params,
|
|
1184
|
+
firewall=state.firewall,
|
|
1185
|
+
pii_redactor=state.pii_redactor,
|
|
1186
|
+
loop_breaker=state.loop_breaker,
|
|
1187
|
+
governance_guards=state.governance_guards,
|
|
1188
|
+
injection_enabled=settings.INJECTION_FAST_PATH_ENABLED,
|
|
1189
|
+
pii_enabled=settings.PII_REDACTION_ENABLED,
|
|
1190
|
+
mode=settings.GOVERNANCE_MODE,
|
|
1191
|
+
)
|
|
1192
|
+
|
|
1193
|
+
governance_result = {
|
|
1194
|
+
"action": pipeline_result.action,
|
|
1195
|
+
"risk_level": pipeline_result.risk_level,
|
|
1196
|
+
"checks": pipeline_result.checks,
|
|
1197
|
+
}
|
|
1198
|
+
redacted_body = pipeline_result.redacted_body
|
|
1199
|
+
governance_latency = pipeline_result.latency_ms
|
|
1200
|
+
gov_response = pipeline_result.gov_response
|
|
1201
|
+
|
|
1202
|
+
if pipeline_result.checks.get("pii_redaction", {}).get("count", 0) > 0:
|
|
1203
|
+
state.inc_metric("requests_redacted")
|
|
1204
|
+
_spawn(
|
|
1205
|
+
governance_bus.emit(
|
|
1206
|
+
BusGovernanceEvent(
|
|
1207
|
+
event_type=EventType.GOVERNANCE_DECISION,
|
|
1208
|
+
session_id=session_id,
|
|
1209
|
+
action=gov_response.action,
|
|
1210
|
+
risk_level=gov_response.risk_level,
|
|
1211
|
+
domain="proxy",
|
|
1212
|
+
metadata=gov_response.to_dict(),
|
|
1213
|
+
)
|
|
1214
|
+
)
|
|
1215
|
+
)
|
|
1216
|
+
|
|
1217
|
+
# ── Fire alerts on block/circuit-break (non-blocking) ─────
|
|
1218
|
+
if (
|
|
1219
|
+
governance_result["action"] in (GovernanceAction.BLOCK, GovernanceAction.CIRCUIT_BREAK)
|
|
1220
|
+
and state.alert_channels
|
|
1221
|
+
):
|
|
1222
|
+
_alert = {
|
|
1223
|
+
"level": gov_response.risk_level,
|
|
1224
|
+
"domain": gov_response.domain,
|
|
1225
|
+
"summary": f"{gov_response.action} — {method} from agent {agent_id}",
|
|
1226
|
+
"details": {k: safe_serialize(v) for k, v in governance_result["checks"].items()},
|
|
1227
|
+
"event_id": event_id,
|
|
1228
|
+
"session_id": session_id,
|
|
1229
|
+
}
|
|
1230
|
+
_spawn(_fire_alerts(state.alert_channels, _alert))
|
|
1231
|
+
|
|
1232
|
+
# ─── Forensic Black Box (non-blocking) ─────────────────────
|
|
1233
|
+
forensic_record = None
|
|
1234
|
+
if state.forensic_box:
|
|
1235
|
+
_loop = asyncio.get_running_loop()
|
|
1236
|
+
forensic_record = await _loop.run_in_executor(
|
|
1237
|
+
None,
|
|
1238
|
+
lambda: state.forensic_box.record(
|
|
1239
|
+
{
|
|
1240
|
+
"event_id": event_id,
|
|
1241
|
+
"event_type": EventType.MCP_REQUEST,
|
|
1242
|
+
"agent_id": agent_id,
|
|
1243
|
+
"session_id": session_id,
|
|
1244
|
+
"method": method,
|
|
1245
|
+
"action": governance_result["action"],
|
|
1246
|
+
"risk_level": governance_result["risk_level"],
|
|
1247
|
+
"governance_latency_ms": round(governance_latency, 2),
|
|
1248
|
+
"checks": {
|
|
1249
|
+
k: safe_serialize(v) for k, v in governance_result["checks"].items()
|
|
1250
|
+
},
|
|
1251
|
+
}
|
|
1252
|
+
),
|
|
1253
|
+
)
|
|
1254
|
+
|
|
1255
|
+
# ─── Store to ClickHouse (fire-and-forget) ─────────────────
|
|
1256
|
+
_spawn(
|
|
1257
|
+
_store_event_async(
|
|
1258
|
+
state.clickhouse,
|
|
1259
|
+
GovernanceEvent(
|
|
1260
|
+
event_id=event_id,
|
|
1261
|
+
timestamp=datetime.now(UTC).isoformat(),
|
|
1262
|
+
event_type=EventType.MCP_REQUEST,
|
|
1263
|
+
agent_id=agent_id,
|
|
1264
|
+
session_id=session_id,
|
|
1265
|
+
method=method,
|
|
1266
|
+
tool_name=params.get("name", "") if isinstance(params, dict) else "",
|
|
1267
|
+
action=governance_result["action"],
|
|
1268
|
+
risk_level=governance_result["risk_level"],
|
|
1269
|
+
details=governance_result["checks"],
|
|
1270
|
+
latency_ms=governance_latency,
|
|
1271
|
+
request_hash=hashlib.sha256(content_str.encode()).hexdigest()[:32],
|
|
1272
|
+
),
|
|
1273
|
+
)
|
|
1274
|
+
)
|
|
1275
|
+
|
|
1276
|
+
# ─── Respond based on governance decision ─────────────────
|
|
1277
|
+
if governance_result["action"] == GovernanceAction.BLOCK:
|
|
1278
|
+
state.inc_metric("requests_blocked")
|
|
1279
|
+
if state.router.is_multi_upstream and path.startswith("route/"):
|
|
1280
|
+
state.router.record_block(path.removeprefix("route/").split("/")[0])
|
|
1281
|
+
logger.warning(
|
|
1282
|
+
"[BLOCKED] request %s: %s (risk=%s)",
|
|
1283
|
+
event_id,
|
|
1284
|
+
method,
|
|
1285
|
+
governance_result["risk_level"],
|
|
1286
|
+
)
|
|
1287
|
+
return JSONResponse(
|
|
1288
|
+
status_code=403,
|
|
1289
|
+
content=mcp_transport.format_block_response(gov_response, body),
|
|
1290
|
+
)
|
|
1291
|
+
|
|
1292
|
+
if governance_result["action"] == GovernanceAction.CIRCUIT_BREAK:
|
|
1293
|
+
state.inc_metric("requests_blocked")
|
|
1294
|
+
if state.router.is_multi_upstream and path.startswith("route/"):
|
|
1295
|
+
state.router.record_block(path.removeprefix("route/").split("/")[0])
|
|
1296
|
+
logger.warning("CIRCUIT BREAK for session %s: reasoning loop detected", session_id)
|
|
1297
|
+
return JSONResponse(
|
|
1298
|
+
status_code=429,
|
|
1299
|
+
content=mcp_transport.format_circuit_break_response(gov_response, body),
|
|
1300
|
+
)
|
|
1301
|
+
|
|
1302
|
+
# ─── Forward to upstream MCP server ───────────────────────
|
|
1303
|
+
state.inc_metric("requests_allowed")
|
|
1304
|
+
try:
|
|
1305
|
+
server_name = None
|
|
1306
|
+
if path.startswith("route/"):
|
|
1307
|
+
server_name = path.removeprefix("route/").split("/")[0]
|
|
1308
|
+
|
|
1309
|
+
# Tool-based routing: resolve server by tool name if in multi-upstream mode
|
|
1310
|
+
if not server_name and state.router.is_multi_upstream:
|
|
1311
|
+
tool_name = params.get("name", "") if isinstance(params, dict) else ""
|
|
1312
|
+
if tool_name:
|
|
1313
|
+
tool_route = state.router.resolve_by_tool(tool_name)
|
|
1314
|
+
if tool_route:
|
|
1315
|
+
server_name = tool_route.name
|
|
1316
|
+
|
|
1317
|
+
if server_name and state.router.is_multi_upstream:
|
|
1318
|
+
upstream_url = state.router.get_upstream_url(server_name)
|
|
1319
|
+
extra_headers = state.router.get_upstream_headers(server_name)
|
|
1320
|
+
logger.debug("Routing to server '%s' -> %s", server_name, upstream_url)
|
|
1321
|
+
else:
|
|
1322
|
+
upstream_url = f"{settings.UPSTREAM_MCP_URL}/mcp"
|
|
1323
|
+
if path and not path.startswith("route/"):
|
|
1324
|
+
upstream_url = f"{settings.UPSTREAM_MCP_URL}/mcp/{path}"
|
|
1325
|
+
extra_headers = {}
|
|
1326
|
+
|
|
1327
|
+
upstream_response = await state.http_client.post(
|
|
1328
|
+
upstream_url,
|
|
1329
|
+
json=redacted_body,
|
|
1330
|
+
headers={
|
|
1331
|
+
"X-Session-Id": session_id,
|
|
1332
|
+
"X-Agent-Id": agent_id,
|
|
1333
|
+
"X-Admina-Event-Id": event_id,
|
|
1334
|
+
**extra_headers,
|
|
1335
|
+
},
|
|
1336
|
+
)
|
|
1337
|
+
|
|
1338
|
+
response_data = upstream_response.json()
|
|
1339
|
+
|
|
1340
|
+
# Redact PII from response too (bidirectional)
|
|
1341
|
+
if isinstance(response_data.get("result"), str):
|
|
1342
|
+
resp_redact = state.pii_redactor.redact(response_data["result"])
|
|
1343
|
+
response_data["result"] = resp_redact["redacted_text"]
|
|
1344
|
+
|
|
1345
|
+
# ─── Governance Guards: inspect response ──────────────
|
|
1346
|
+
if state.governance_guards:
|
|
1347
|
+
resp_payload = {"content": json.dumps(response_data, default=str)}
|
|
1348
|
+
for guard in state.governance_guards:
|
|
1349
|
+
try:
|
|
1350
|
+
guard_result = await guard.inspect_response(resp_payload)
|
|
1351
|
+
if guard_result.get("action") in ("BLOCK", "REDACT"):
|
|
1352
|
+
state.inc_metric("requests_blocked")
|
|
1353
|
+
logger.warning(
|
|
1354
|
+
"Guard %r blocked response for event %s",
|
|
1355
|
+
guard.name,
|
|
1356
|
+
event_id,
|
|
1357
|
+
)
|
|
1358
|
+
return JSONResponse(
|
|
1359
|
+
status_code=403,
|
|
1360
|
+
content=mcp_transport.format_block_response(
|
|
1361
|
+
gov_response,
|
|
1362
|
+
body,
|
|
1363
|
+
),
|
|
1364
|
+
)
|
|
1365
|
+
except (ValueError, RuntimeError, OSError, TypeError) as exc:
|
|
1366
|
+
logger.warning("Guard %r response inspection failed: %s", guard.name, exc)
|
|
1367
|
+
|
|
1368
|
+
total_latency = (time.perf_counter() - start_time) * 1000
|
|
1369
|
+
state.update_avg_latency(total_latency)
|
|
1370
|
+
|
|
1371
|
+
headers = mcp_transport.format_allow_headers(
|
|
1372
|
+
gov_response,
|
|
1373
|
+
forensic_hash=(
|
|
1374
|
+
forensic_record.get("record_hash", "")[:16] if forensic_record else None
|
|
1375
|
+
),
|
|
1376
|
+
)
|
|
1377
|
+
|
|
1378
|
+
return JSONResponse(content=response_data, headers=headers)
|
|
1379
|
+
|
|
1380
|
+
except httpx.ConnectError:
|
|
1381
|
+
logger.error("Upstream MCP server unreachable: %s", settings.UPSTREAM_MCP_URL)
|
|
1382
|
+
return JSONResponse(
|
|
1383
|
+
status_code=502,
|
|
1384
|
+
content={
|
|
1385
|
+
"jsonrpc": "2.0",
|
|
1386
|
+
"id": body.get("id"),
|
|
1387
|
+
"error": {
|
|
1388
|
+
"code": -32603,
|
|
1389
|
+
"message": "Upstream MCP server unreachable",
|
|
1390
|
+
"data": {"event_id": event_id},
|
|
1391
|
+
},
|
|
1392
|
+
},
|
|
1393
|
+
)
|
|
1394
|
+
except (httpx.HTTPError, OSError, ValueError, RuntimeError) as e:
|
|
1395
|
+
logger.error("Proxy error: %s", e)
|
|
1396
|
+
return JSONResponse(
|
|
1397
|
+
status_code=500,
|
|
1398
|
+
content={
|
|
1399
|
+
"jsonrpc": "2.0",
|
|
1400
|
+
"id": body.get("id"),
|
|
1401
|
+
"error": {
|
|
1402
|
+
"code": -32603,
|
|
1403
|
+
"message": "Internal proxy error",
|
|
1404
|
+
"data": {"event_id": event_id},
|
|
1405
|
+
},
|
|
1406
|
+
},
|
|
1407
|
+
)
|
|
1408
|
+
|
|
1409
|
+
|
|
1410
|
+
# ── Helpers ──────────────────────────────────────────────────
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
def _store_event_sync(clickhouse_client, event: GovernanceEvent):
|
|
1414
|
+
"""Store governance event to ClickHouse (synchronous — run in thread pool)."""
|
|
1415
|
+
if not clickhouse_client:
|
|
1416
|
+
return
|
|
1417
|
+
_validate_identifier(settings.CLICKHOUSE_DB, "CLICKHOUSE_DB")
|
|
1418
|
+
try:
|
|
1419
|
+
clickhouse_client.insert(
|
|
1420
|
+
f"{settings.CLICKHOUSE_DB}.governance_events",
|
|
1421
|
+
[
|
|
1422
|
+
[
|
|
1423
|
+
event.event_id,
|
|
1424
|
+
datetime.fromisoformat(event.timestamp),
|
|
1425
|
+
(
|
|
1426
|
+
event.event_type.value
|
|
1427
|
+
if hasattr(event.event_type, "value")
|
|
1428
|
+
else event.event_type
|
|
1429
|
+
),
|
|
1430
|
+
event.agent_id,
|
|
1431
|
+
event.session_id,
|
|
1432
|
+
event.method,
|
|
1433
|
+
event.tool_name,
|
|
1434
|
+
(event.action.value if hasattr(event.action, "value") else event.action),
|
|
1435
|
+
(
|
|
1436
|
+
event.risk_level.value
|
|
1437
|
+
if hasattr(event.risk_level, "value")
|
|
1438
|
+
else event.risk_level
|
|
1439
|
+
),
|
|
1440
|
+
json.dumps(event.details, default=str),
|
|
1441
|
+
event.latency_ms,
|
|
1442
|
+
event.request_hash,
|
|
1443
|
+
event.response_hash,
|
|
1444
|
+
]
|
|
1445
|
+
],
|
|
1446
|
+
column_names=[
|
|
1447
|
+
"event_id",
|
|
1448
|
+
"timestamp",
|
|
1449
|
+
"event_type",
|
|
1450
|
+
"agent_id",
|
|
1451
|
+
"session_id",
|
|
1452
|
+
"method",
|
|
1453
|
+
"tool_name",
|
|
1454
|
+
"action",
|
|
1455
|
+
"risk_level",
|
|
1456
|
+
"details",
|
|
1457
|
+
"latency_ms",
|
|
1458
|
+
"request_hash",
|
|
1459
|
+
"response_hash",
|
|
1460
|
+
],
|
|
1461
|
+
)
|
|
1462
|
+
except (OSError, clickhouse_connect.driver.exceptions.DatabaseError) as e:
|
|
1463
|
+
logger.warning("Failed to store event: %s", e)
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
async def _fire_alerts(channels: list, alert: dict) -> None:
|
|
1467
|
+
"""Dispatch a governance alert to all registered alert channels."""
|
|
1468
|
+
for ch in channels:
|
|
1469
|
+
try:
|
|
1470
|
+
await ch.send_alert(alert)
|
|
1471
|
+
except (OSError, ValueError, RuntimeError) as exc:
|
|
1472
|
+
logger.warning("Alert channel %r failed: %s", getattr(ch, "channel_name", "?"), exc)
|
|
1473
|
+
|
|
1474
|
+
|
|
1475
|
+
async def _store_event_async(clickhouse_client, event: GovernanceEvent):
|
|
1476
|
+
"""Non-blocking wrapper: runs ClickHouse insert in the thread pool."""
|
|
1477
|
+
loop = asyncio.get_running_loop()
|
|
1478
|
+
await loop.run_in_executor(None, _store_event_sync, clickhouse_client, event)
|
|
1479
|
+
|
|
1480
|
+
|
|
1481
|
+
if __name__ == "__main__":
|
|
1482
|
+
import uvicorn
|
|
1483
|
+
|
|
1484
|
+
uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info")
|