gatekeep-mcp 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gatekeep/__init__.py +135 -0
- gatekeep/__main__.py +6 -0
- gatekeep/cli.py +5 -0
- gatekeep/control_plane.py +1043 -0
- gatekeep/decorator.py +137 -0
- gatekeep/gate.py +1816 -0
- gatekeep/ledger.py +1021 -0
- gatekeep/mcp/__init__.py +51 -0
- gatekeep/mcp/__main__.py +8 -0
- gatekeep/mcp/cli.py +603 -0
- gatekeep/mcp/http_server.py +1696 -0
- gatekeep/mcp/onboarding.py +583 -0
- gatekeep/mcp/proxy.py +1858 -0
- gatekeep/middleware.py +359 -0
- gatekeep/persistence.py +1688 -0
- gatekeep/policy.py +260 -0
- gatekeep/redaction.py +206 -0
- gatekeep/registry.py +314 -0
- gatekeep/saga.py +609 -0
- gatekeep/security.py +536 -0
- gatekeep_mcp-0.2.0.dist-info/METADATA +245 -0
- gatekeep_mcp-0.2.0.dist-info/RECORD +27 -0
- gatekeep_mcp-0.2.0.dist-info/WHEEL +5 -0
- gatekeep_mcp-0.2.0.dist-info/entry_points.txt +2 -0
- gatekeep_mcp-0.2.0.dist-info/licenses/LICENSE-APACHE +172 -0
- gatekeep_mcp-0.2.0.dist-info/licenses/LICENSE-MIT +21 -0
- gatekeep_mcp-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1043 @@
|
|
|
1
|
+
"""
|
|
2
|
+
gatekeep/control_plane.py
|
|
3
|
+
|
|
4
|
+
Enterprise Control Plane & Governance Layer for Gatekeep:
|
|
5
|
+
1. `CentralPolicyRegistry` & `ToolPolicySpec`: Dynamically load/reload AST-validated
|
|
6
|
+
policies, prerequisites, risk controls, and declarative Saga compensation rules
|
|
7
|
+
across tools without modifying agent/microservice decorators.
|
|
8
|
+
2. `EscalationRouter` & `EscalationTicket`: Human-in-the-loop (HITL) approval routing
|
|
9
|
+
for `Verdict.ESCALATE`, dispatching Slack/webhook notifications and persisting
|
|
10
|
+
cryptographic approval tokens directly to live or store-backed sessions.
|
|
11
|
+
3. `SagaRecoveryDaemon`: Out-of-band crash sweeper that detects orphaned/stale
|
|
12
|
+
uncommitted Saga transactions across durable `SessionStateStore`s and executes
|
|
13
|
+
deterministic LIFO compensation rollbacks automatically.
|
|
14
|
+
4. `AuditVault`: Multi-session SHA-256 hash-chain verification, compliance querying,
|
|
15
|
+
and JSONL audit export.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import hashlib
|
|
21
|
+
import hmac
|
|
22
|
+
import json
|
|
23
|
+
import logging
|
|
24
|
+
import secrets
|
|
25
|
+
import threading
|
|
26
|
+
import time
|
|
27
|
+
import urllib.request
|
|
28
|
+
import uuid
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from types import SimpleNamespace
|
|
32
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple
|
|
33
|
+
|
|
34
|
+
from gatekeep.policy import PolicySyntaxError, SafePolicyEvaluator
|
|
35
|
+
from gatekeep.redaction import (
|
|
36
|
+
SensitiveDataRedactor,
|
|
37
|
+
canonical_action_fingerprint,
|
|
38
|
+
hash_secret_token,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
logger = logging.getLogger("gatekeep.control_plane")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# ── Central Policy Registry ───────────────────────────────────────────────────
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class ToolPolicySpec:
|
|
49
|
+
"""
|
|
50
|
+
Declarative runtime governance policy for a single tool (or wildcard `"*"`).
|
|
51
|
+
Managed centrally and overlaid onto registered `ToolDefinition`s at evaluation time.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
tool_name: str
|
|
55
|
+
enabled: bool = True
|
|
56
|
+
mode: Optional[str] = None
|
|
57
|
+
provides: Optional[Tuple[str, ...]] = None
|
|
58
|
+
requires: Optional[Tuple[str, ...]] = None
|
|
59
|
+
provenance_args: Optional[Tuple[str, ...]] = None
|
|
60
|
+
policy: Optional[str] = None
|
|
61
|
+
requires_approval: Optional[bool] = None
|
|
62
|
+
compensates_with: Optional[str] = None
|
|
63
|
+
compensation_args: Optional[Dict[str, Any]] = None
|
|
64
|
+
risk_level: Optional[str] = None
|
|
65
|
+
description: Optional[str] = None
|
|
66
|
+
max_calls_per_minute: Optional[int] = None
|
|
67
|
+
max_cumulative_amount: Optional[float] = None
|
|
68
|
+
amount_arg: Optional[str] = None
|
|
69
|
+
global_scope: bool = False
|
|
70
|
+
allow_secrets: bool = False
|
|
71
|
+
allow_private_urls: bool = False
|
|
72
|
+
allow_sensitive_paths: bool = False
|
|
73
|
+
allow_destructive_sql: bool = False
|
|
74
|
+
allow_shell: bool = False
|
|
75
|
+
|
|
76
|
+
def __post_init__(self) -> None:
|
|
77
|
+
if self.policy and self.policy.strip():
|
|
78
|
+
# Fail fast on malformed or unsafe declarative policy syntax
|
|
79
|
+
SafePolicyEvaluator(self.policy)
|
|
80
|
+
if self.max_calls_per_minute is not None and int(self.max_calls_per_minute) < 1:
|
|
81
|
+
raise ValueError("max_calls_per_minute must be >= 1")
|
|
82
|
+
if self.max_cumulative_amount is not None and float(self.max_cumulative_amount) < 0:
|
|
83
|
+
raise ValueError("max_cumulative_amount must be >= 0")
|
|
84
|
+
|
|
85
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
86
|
+
return {
|
|
87
|
+
"tool_name": self.tool_name,
|
|
88
|
+
"enabled": self.enabled,
|
|
89
|
+
"mode": self.mode,
|
|
90
|
+
"provides": list(self.provides) if self.provides is not None else None,
|
|
91
|
+
"requires": list(self.requires) if self.requires is not None else None,
|
|
92
|
+
"provenance_args": list(self.provenance_args) if self.provenance_args is not None else None,
|
|
93
|
+
"policy": self.policy,
|
|
94
|
+
"requires_approval": self.requires_approval,
|
|
95
|
+
"compensates_with": self.compensates_with,
|
|
96
|
+
"compensation_args": dict(self.compensation_args) if self.compensation_args is not None else None,
|
|
97
|
+
"risk_level": self.risk_level,
|
|
98
|
+
"description": self.description,
|
|
99
|
+
"max_calls_per_minute": self.max_calls_per_minute,
|
|
100
|
+
"max_cumulative_amount": self.max_cumulative_amount,
|
|
101
|
+
"amount_arg": self.amount_arg,
|
|
102
|
+
"global_scope": self.global_scope,
|
|
103
|
+
"allow_secrets": self.allow_secrets,
|
|
104
|
+
"allow_private_urls": self.allow_private_urls,
|
|
105
|
+
"allow_sensitive_paths": self.allow_sensitive_paths,
|
|
106
|
+
"allow_destructive_sql": self.allow_destructive_sql,
|
|
107
|
+
"allow_shell": self.allow_shell,
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
@classmethod
|
|
111
|
+
def from_dict(cls, data: Dict[str, Any]) -> "ToolPolicySpec":
|
|
112
|
+
def _opt_tuple(val: Any) -> Optional[Tuple[str, ...]]:
|
|
113
|
+
if val is None:
|
|
114
|
+
return None
|
|
115
|
+
return tuple(str(x) for x in val)
|
|
116
|
+
|
|
117
|
+
comp_args = data.get("compensation_args")
|
|
118
|
+
max_calls = data.get("max_calls_per_minute")
|
|
119
|
+
max_amt = data.get("max_cumulative_amount")
|
|
120
|
+
return cls(
|
|
121
|
+
tool_name=str(data["tool_name"]),
|
|
122
|
+
enabled=bool(data.get("enabled", True)),
|
|
123
|
+
mode=data.get("mode"),
|
|
124
|
+
provides=_opt_tuple(data.get("provides")),
|
|
125
|
+
requires=_opt_tuple(data.get("requires")),
|
|
126
|
+
provenance_args=_opt_tuple(data.get("provenance_args")),
|
|
127
|
+
policy=data.get("policy"),
|
|
128
|
+
requires_approval=data.get("requires_approval"),
|
|
129
|
+
compensates_with=data.get("compensates_with"),
|
|
130
|
+
compensation_args=dict(comp_args) if isinstance(comp_args, dict) else None,
|
|
131
|
+
risk_level=data.get("risk_level"),
|
|
132
|
+
description=data.get("description"),
|
|
133
|
+
max_calls_per_minute=int(max_calls) if max_calls is not None else None,
|
|
134
|
+
max_cumulative_amount=float(max_amt) if max_amt is not None else None,
|
|
135
|
+
amount_arg=data.get("amount_arg"),
|
|
136
|
+
global_scope=bool(data.get("global_scope", False)),
|
|
137
|
+
allow_secrets=bool(data.get("allow_secrets", False)),
|
|
138
|
+
allow_private_urls=bool(data.get("allow_private_urls", False)),
|
|
139
|
+
allow_sensitive_paths=bool(data.get("allow_sensitive_paths", False)),
|
|
140
|
+
allow_destructive_sql=bool(data.get("allow_destructive_sql", False)),
|
|
141
|
+
allow_shell=bool(data.get("allow_shell", False)),
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _merge_unique(first: Any, second: Any) -> Optional[List[str]]:
|
|
146
|
+
if first is None and second is None:
|
|
147
|
+
return None
|
|
148
|
+
merged: List[str] = []
|
|
149
|
+
for src in (first, second):
|
|
150
|
+
if src:
|
|
151
|
+
for item in src:
|
|
152
|
+
s = str(item)
|
|
153
|
+
if s not in merged:
|
|
154
|
+
merged.append(s)
|
|
155
|
+
return merged
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def resolve_effective_tool(tool_def: Any, policy_spec: Optional[ToolPolicySpec]) -> Any:
|
|
159
|
+
"""
|
|
160
|
+
Merge a base `ToolDefinition` with an optional central `ToolPolicySpec` overlay.
|
|
161
|
+
Returns a lightweight namespace carrying merged `requires`, `provides`,
|
|
162
|
+
`provenance_args`, `mode`, `requires_approval`, `compensates_with`, etc.,
|
|
163
|
+
while preserving references to `tool_def.check`, `tool_def.fn`, `tool_def.policy`.
|
|
164
|
+
"""
|
|
165
|
+
if policy_spec is None:
|
|
166
|
+
return tool_def
|
|
167
|
+
|
|
168
|
+
base_requires = getattr(tool_def, "requires", None)
|
|
169
|
+
base_provides = getattr(tool_def, "provides", None)
|
|
170
|
+
base_prov = getattr(tool_def, "provenance_args", None)
|
|
171
|
+
merged_prov = (
|
|
172
|
+
list(policy_spec.provenance_args)
|
|
173
|
+
if policy_spec.provenance_args is not None
|
|
174
|
+
else (list(base_prov) if base_prov is not None else None)
|
|
175
|
+
)
|
|
176
|
+
req_approval = (
|
|
177
|
+
policy_spec.requires_approval
|
|
178
|
+
if policy_spec.requires_approval is not None
|
|
179
|
+
else getattr(tool_def, "requires_approval", False)
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
return SimpleNamespace(
|
|
183
|
+
name=getattr(tool_def, "name", policy_spec.tool_name),
|
|
184
|
+
description=policy_spec.description or getattr(tool_def, "description", ""),
|
|
185
|
+
fn=getattr(tool_def, "fn", None),
|
|
186
|
+
mode=policy_spec.mode or getattr(tool_def, "mode", None),
|
|
187
|
+
provides=_merge_unique(base_provides, policy_spec.provides),
|
|
188
|
+
requires=_merge_unique(base_requires, policy_spec.requires),
|
|
189
|
+
provenance_args=merged_prov,
|
|
190
|
+
check=getattr(tool_def, "check", None),
|
|
191
|
+
policy=getattr(tool_def, "policy", None),
|
|
192
|
+
central_policy=policy_spec.policy,
|
|
193
|
+
requires_approval=bool(req_approval),
|
|
194
|
+
compensates_with=policy_spec.compensates_with or getattr(tool_def, "compensates_with", None),
|
|
195
|
+
compensation_args=(
|
|
196
|
+
policy_spec.compensation_args
|
|
197
|
+
if policy_spec.compensation_args is not None
|
|
198
|
+
else getattr(tool_def, "compensation_args", None)
|
|
199
|
+
),
|
|
200
|
+
on_rollback=getattr(tool_def, "on_rollback", None),
|
|
201
|
+
enabled=policy_spec.enabled,
|
|
202
|
+
risk_level=policy_spec.risk_level or getattr(tool_def, "risk_level", "LOW"),
|
|
203
|
+
max_calls_per_minute=policy_spec.max_calls_per_minute,
|
|
204
|
+
max_cumulative_amount=policy_spec.max_cumulative_amount,
|
|
205
|
+
amount_arg=policy_spec.amount_arg,
|
|
206
|
+
global_scope=policy_spec.global_scope,
|
|
207
|
+
allow_secrets=policy_spec.allow_secrets,
|
|
208
|
+
allow_private_urls=policy_spec.allow_private_urls,
|
|
209
|
+
allow_sensitive_paths=policy_spec.allow_sensitive_paths,
|
|
210
|
+
allow_destructive_sql=policy_spec.allow_destructive_sql,
|
|
211
|
+
allow_shell=policy_spec.allow_shell,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class CentralPolicyRegistry:
|
|
216
|
+
"""
|
|
217
|
+
Thread-safe, hot-reloadable central policy registry for Gatekeep Control Plane.
|
|
218
|
+
Enforces policy validation upfront so bad rules cannot slip into production.
|
|
219
|
+
|
|
220
|
+
When `session_store` is provided, all policy mutations persist atomically via
|
|
221
|
+
`save_control_doc("central_policy_registry", ...)` so policies survive worker
|
|
222
|
+
restarts and synchronize across horizontally scaled replicas.
|
|
223
|
+
"""
|
|
224
|
+
|
|
225
|
+
_CONTROL_NAMESPACE = "central_policy_registry"
|
|
226
|
+
|
|
227
|
+
def __init__(
|
|
228
|
+
self,
|
|
229
|
+
specs: Optional[List[ToolPolicySpec]] = None,
|
|
230
|
+
session_store: Optional[Any] = None,
|
|
231
|
+
) -> None:
|
|
232
|
+
self._lock = threading.RLock()
|
|
233
|
+
self._store = session_store
|
|
234
|
+
self._policies: Dict[str, ToolPolicySpec] = {}
|
|
235
|
+
self._version: int = 1
|
|
236
|
+
loaded = self._load_from_store()
|
|
237
|
+
if not loaded and specs:
|
|
238
|
+
for spec in specs:
|
|
239
|
+
self._policies[spec.tool_name] = spec
|
|
240
|
+
self._save_to_store()
|
|
241
|
+
|
|
242
|
+
def _load_from_store(self) -> bool:
|
|
243
|
+
if self._store is None or not hasattr(self._store, "load_control_doc"):
|
|
244
|
+
return False
|
|
245
|
+
doc = self._store.load_control_doc(self._CONTROL_NAMESPACE)
|
|
246
|
+
if not doc or not isinstance(doc, dict):
|
|
247
|
+
return False
|
|
248
|
+
tools = doc.get("tools", {})
|
|
249
|
+
parsed: Dict[str, ToolPolicySpec] = {}
|
|
250
|
+
for tool_name, item in tools.items():
|
|
251
|
+
payload = dict(item)
|
|
252
|
+
payload.setdefault("tool_name", str(tool_name))
|
|
253
|
+
parsed[str(tool_name)] = ToolPolicySpec.from_dict(payload)
|
|
254
|
+
self._policies = parsed
|
|
255
|
+
self._version = int(doc.get("version", 1))
|
|
256
|
+
return True
|
|
257
|
+
|
|
258
|
+
def _save_to_store(self) -> None:
|
|
259
|
+
if self._store is None or not hasattr(self._store, "save_control_doc"):
|
|
260
|
+
return
|
|
261
|
+
self._store.save_control_doc(
|
|
262
|
+
self._CONTROL_NAMESPACE,
|
|
263
|
+
{
|
|
264
|
+
"version": self._version,
|
|
265
|
+
"tools": {k: v.to_dict() for k, v in sorted(self._policies.items())},
|
|
266
|
+
},
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
@property
|
|
270
|
+
def version(self) -> int:
|
|
271
|
+
with self._lock:
|
|
272
|
+
self._load_from_store()
|
|
273
|
+
return self._version
|
|
274
|
+
|
|
275
|
+
def set_policy(self, spec: ToolPolicySpec) -> int:
|
|
276
|
+
with self._lock:
|
|
277
|
+
self._load_from_store()
|
|
278
|
+
self._policies[spec.tool_name] = spec
|
|
279
|
+
self._version += 1
|
|
280
|
+
self._save_to_store()
|
|
281
|
+
return self._version
|
|
282
|
+
|
|
283
|
+
def remove_policy(self, tool_name: str) -> bool:
|
|
284
|
+
with self._lock:
|
|
285
|
+
self._load_from_store()
|
|
286
|
+
existed = tool_name in self._policies
|
|
287
|
+
self._policies.pop(tool_name, None)
|
|
288
|
+
if existed:
|
|
289
|
+
self._version += 1
|
|
290
|
+
self._save_to_store()
|
|
291
|
+
return existed
|
|
292
|
+
|
|
293
|
+
def get_policy(self, tool_name: str) -> Optional[ToolPolicySpec]:
|
|
294
|
+
with self._lock:
|
|
295
|
+
self._load_from_store()
|
|
296
|
+
return self._policies.get(tool_name) or self._policies.get("*")
|
|
297
|
+
|
|
298
|
+
def list_policies(self) -> List[ToolPolicySpec]:
|
|
299
|
+
with self._lock:
|
|
300
|
+
self._load_from_store()
|
|
301
|
+
return [self._policies[k] for k in sorted(self._policies.keys())]
|
|
302
|
+
|
|
303
|
+
def export_dict(self) -> Dict[str, Any]:
|
|
304
|
+
with self._lock:
|
|
305
|
+
self._load_from_store()
|
|
306
|
+
return {
|
|
307
|
+
"version": self._version,
|
|
308
|
+
"tools": {k: v.to_dict() for k, v in sorted(self._policies.items())},
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
def load_dict(self, config: Dict[str, Any], replace: bool = False) -> int:
|
|
312
|
+
"""
|
|
313
|
+
Load policies atomically from a dict of either:
|
|
314
|
+
`{"tools": {"refund_order": {"policy": "amount <= 500", ...}}}`
|
|
315
|
+
or `{ "refund_order": {...} }`.
|
|
316
|
+
Validates all expressions before swapping state.
|
|
317
|
+
"""
|
|
318
|
+
raw_tools = config.get("tools", config)
|
|
319
|
+
if not isinstance(raw_tools, dict):
|
|
320
|
+
raise ValueError("Policy configuration must map tool names to policy objects.")
|
|
321
|
+
|
|
322
|
+
parsed: Dict[str, ToolPolicySpec] = {}
|
|
323
|
+
for tool_name, item in raw_tools.items():
|
|
324
|
+
if isinstance(item, ToolPolicySpec):
|
|
325
|
+
parsed[tool_name] = item
|
|
326
|
+
continue
|
|
327
|
+
if not isinstance(item, dict):
|
|
328
|
+
raise PolicySyntaxError(f"Invalid policy spec for '{tool_name}': expected dict.")
|
|
329
|
+
payload = dict(item)
|
|
330
|
+
payload.setdefault("tool_name", str(tool_name))
|
|
331
|
+
spec = ToolPolicySpec.from_dict(payload)
|
|
332
|
+
parsed[spec.tool_name] = spec
|
|
333
|
+
|
|
334
|
+
with self._lock:
|
|
335
|
+
self._load_from_store()
|
|
336
|
+
if replace:
|
|
337
|
+
self._policies = parsed
|
|
338
|
+
else:
|
|
339
|
+
self._policies.update(parsed)
|
|
340
|
+
self._version += 1
|
|
341
|
+
self._save_to_store()
|
|
342
|
+
return self._version
|
|
343
|
+
|
|
344
|
+
def load_json(self, raw_json: str, replace: bool = False) -> int:
|
|
345
|
+
return self.load_dict(json.loads(raw_json), replace=replace)
|
|
346
|
+
|
|
347
|
+
def load_file(self, path: str | Path, replace: bool = True) -> int:
|
|
348
|
+
content = Path(path).read_text(encoding="utf-8")
|
|
349
|
+
return self.load_json(content, replace=replace)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
# ── Human-in-the-Loop Escalation Router ───────────────────────────────────────
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
@dataclass
|
|
356
|
+
class EscalationTicket:
|
|
357
|
+
"""Represents a paused tool call awaiting human operator decision."""
|
|
358
|
+
|
|
359
|
+
ticket_id: str
|
|
360
|
+
session_id: str
|
|
361
|
+
tool_name: str
|
|
362
|
+
arguments: Dict[str, Any]
|
|
363
|
+
reason: str
|
|
364
|
+
approval_token: str = ""
|
|
365
|
+
approval_token_hash: str = ""
|
|
366
|
+
action_fingerprint: str = ""
|
|
367
|
+
requested_by: Optional[str] = None
|
|
368
|
+
status: str = "PENDING" # PENDING, APPROVED, REJECTED, EXPIRED
|
|
369
|
+
created_at: float = field(default_factory=time.time)
|
|
370
|
+
expires_at: float = 0.0
|
|
371
|
+
decided_by: Optional[str] = None
|
|
372
|
+
decided_at: Optional[float] = None
|
|
373
|
+
decision_note: str = ""
|
|
374
|
+
|
|
375
|
+
def to_dict(self, include_token: bool = False) -> Dict[str, Any]:
|
|
376
|
+
payload: Dict[str, Any] = {
|
|
377
|
+
"ticket_id": self.ticket_id,
|
|
378
|
+
"session_id": self.session_id,
|
|
379
|
+
"tool_name": self.tool_name,
|
|
380
|
+
"arguments": dict(self.arguments),
|
|
381
|
+
"reason": self.reason,
|
|
382
|
+
"approval_token_hash": self.approval_token_hash,
|
|
383
|
+
"action_fingerprint": self.action_fingerprint,
|
|
384
|
+
"requested_by": self.requested_by,
|
|
385
|
+
"status": self.status,
|
|
386
|
+
"created_at": self.created_at,
|
|
387
|
+
"expires_at": self.expires_at,
|
|
388
|
+
"decided_by": self.decided_by,
|
|
389
|
+
"decided_at": self.decided_at,
|
|
390
|
+
"decision_note": self.decision_note,
|
|
391
|
+
}
|
|
392
|
+
if include_token and self.approval_token:
|
|
393
|
+
payload["approval_token"] = self.approval_token
|
|
394
|
+
return payload
|
|
395
|
+
|
|
396
|
+
@classmethod
|
|
397
|
+
def from_dict(cls, data: Dict[str, Any]) -> "EscalationTicket":
|
|
398
|
+
tok = str(data.get("approval_token", ""))
|
|
399
|
+
tok_hash = str(data.get("approval_token_hash", "")) or (hash_secret_token(tok) if tok else "")
|
|
400
|
+
return cls(
|
|
401
|
+
ticket_id=str(data["ticket_id"]),
|
|
402
|
+
session_id=str(data["session_id"]),
|
|
403
|
+
tool_name=str(data["tool_name"]),
|
|
404
|
+
arguments=dict(data.get("arguments") or {}),
|
|
405
|
+
reason=str(data.get("reason", "")),
|
|
406
|
+
approval_token=tok,
|
|
407
|
+
approval_token_hash=tok_hash,
|
|
408
|
+
action_fingerprint=str(data.get("action_fingerprint", "")),
|
|
409
|
+
requested_by=str(data["requested_by"]) if data.get("requested_by") is not None else None,
|
|
410
|
+
status=str(data.get("status", "PENDING")),
|
|
411
|
+
created_at=float(data.get("created_at", time.time())),
|
|
412
|
+
expires_at=float(data.get("expires_at", 0.0)),
|
|
413
|
+
decided_by=data.get("decided_by"),
|
|
414
|
+
decided_at=float(data["decided_at"]) if data.get("decided_at") is not None else None,
|
|
415
|
+
decision_note=str(data.get("decision_note", "")),
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
class EscalationRouter:
|
|
420
|
+
"""
|
|
421
|
+
Coordinates `Verdict.ESCALATE` events with external reviewers (Slack, PagerDuty,
|
|
422
|
+
custom webhooks) and issues single-use, action-bound approval tokens into Gatekeep sessions.
|
|
423
|
+
|
|
424
|
+
Security & Compliance properties:
|
|
425
|
+
- Never persists plaintext `approval_token` to `session_store` (stores only SHA-256 hash).
|
|
426
|
+
- Enforces Separation of Duties (`requested_by != approver`).
|
|
427
|
+
- Redacts sensitive PII/secrets on stored & notified ticket arguments when `redactor` is set,
|
|
428
|
+
while preserving raw `action_fingerprint` for exact replay binding.
|
|
429
|
+
"""
|
|
430
|
+
|
|
431
|
+
_CONTROL_NAMESPACE = "escalation_tickets"
|
|
432
|
+
|
|
433
|
+
def __init__(
|
|
434
|
+
self,
|
|
435
|
+
default_ttl_seconds: float = 900.0,
|
|
436
|
+
session_store: Optional[Any] = None,
|
|
437
|
+
redactor: Optional[SensitiveDataRedactor] = None,
|
|
438
|
+
) -> None:
|
|
439
|
+
self._lock = threading.RLock()
|
|
440
|
+
self._default_ttl = default_ttl_seconds
|
|
441
|
+
self._store = session_store
|
|
442
|
+
self.redactor: Optional[SensitiveDataRedactor] = redactor
|
|
443
|
+
self._tickets: Dict[str, EscalationTicket] = {}
|
|
444
|
+
self._listeners: List[Callable[[EscalationTicket], None]] = []
|
|
445
|
+
self._load_from_store()
|
|
446
|
+
|
|
447
|
+
def _load_from_store(self) -> None:
|
|
448
|
+
if self._store is None or not hasattr(self._store, "load_control_doc"):
|
|
449
|
+
return
|
|
450
|
+
doc = self._store.load_control_doc(self._CONTROL_NAMESPACE)
|
|
451
|
+
if not doc or not isinstance(doc, dict):
|
|
452
|
+
return
|
|
453
|
+
raw_tickets = doc.get("tickets", {})
|
|
454
|
+
merged: Dict[str, EscalationTicket] = {}
|
|
455
|
+
for tid, tdata in raw_tickets.items():
|
|
456
|
+
ticket = EscalationTicket.from_dict(tdata)
|
|
457
|
+
# Keep in-memory plaintext token on same worker if already generated & not stored on disk
|
|
458
|
+
prev = self._tickets.get(str(tid))
|
|
459
|
+
if prev and prev.approval_token and not ticket.approval_token:
|
|
460
|
+
if not ticket.approval_token_hash or ticket.approval_token_hash == prev.approval_token_hash:
|
|
461
|
+
ticket.approval_token = prev.approval_token
|
|
462
|
+
merged[str(tid)] = ticket
|
|
463
|
+
self._tickets = merged
|
|
464
|
+
|
|
465
|
+
def _save_to_store(self) -> None:
|
|
466
|
+
if self._store is None or not hasattr(self._store, "save_control_doc"):
|
|
467
|
+
return
|
|
468
|
+
# Never persist plaintext approval_token at rest; store only approval_token_hash
|
|
469
|
+
self._store.save_control_doc(
|
|
470
|
+
self._CONTROL_NAMESPACE,
|
|
471
|
+
{
|
|
472
|
+
"tickets": {
|
|
473
|
+
tid: t.to_dict(include_token=False)
|
|
474
|
+
for tid, t in self._tickets.items()
|
|
475
|
+
}
|
|
476
|
+
},
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
def add_listener(self, handler: Callable[[EscalationTicket], None]) -> None:
|
|
480
|
+
"""Register a callback (Slack/webhook notifier) invoked on new escalations."""
|
|
481
|
+
with self._lock:
|
|
482
|
+
self._listeners.append(handler)
|
|
483
|
+
|
|
484
|
+
def create_ticket(
|
|
485
|
+
self,
|
|
486
|
+
session_id: str,
|
|
487
|
+
tool_name: str,
|
|
488
|
+
arguments: Dict[str, Any],
|
|
489
|
+
reason: str,
|
|
490
|
+
ttl_seconds: Optional[float] = None,
|
|
491
|
+
requested_by: Optional[str] = None,
|
|
492
|
+
) -> EscalationTicket:
|
|
493
|
+
now = time.time()
|
|
494
|
+
ttl = ttl_seconds if ttl_seconds is not None else self._default_ttl
|
|
495
|
+
raw_args = dict(arguments)
|
|
496
|
+
action_fp = canonical_action_fingerprint(tool_name, raw_args)
|
|
497
|
+
safe_args = self.redactor.redact_dict(raw_args) if self.redactor is not None else raw_args
|
|
498
|
+
safe_reason = self.redactor.redact_string(reason) if self.redactor is not None else reason
|
|
499
|
+
tok = secrets.token_urlsafe(24)
|
|
500
|
+
|
|
501
|
+
ticket = EscalationTicket(
|
|
502
|
+
ticket_id=f"esc_{uuid.uuid4().hex[:12]}",
|
|
503
|
+
session_id=session_id,
|
|
504
|
+
tool_name=tool_name,
|
|
505
|
+
arguments=safe_args,
|
|
506
|
+
reason=safe_reason,
|
|
507
|
+
approval_token=tok,
|
|
508
|
+
approval_token_hash=hash_secret_token(tok),
|
|
509
|
+
action_fingerprint=action_fp,
|
|
510
|
+
requested_by=requested_by,
|
|
511
|
+
created_at=now,
|
|
512
|
+
expires_at=now + ttl,
|
|
513
|
+
)
|
|
514
|
+
with self._lock:
|
|
515
|
+
self._load_from_store()
|
|
516
|
+
self._tickets[ticket.ticket_id] = ticket
|
|
517
|
+
self._save_to_store()
|
|
518
|
+
listeners = list(self._listeners)
|
|
519
|
+
|
|
520
|
+
for handler in listeners:
|
|
521
|
+
try:
|
|
522
|
+
handler(ticket)
|
|
523
|
+
except Exception as exc:
|
|
524
|
+
logger.warning(f"[EscalationRouter] Listener raised error for {ticket.ticket_id}: {exc}")
|
|
525
|
+
|
|
526
|
+
return ticket
|
|
527
|
+
|
|
528
|
+
def get_ticket(self, ticket_id: str) -> Optional[EscalationTicket]:
|
|
529
|
+
with self._lock:
|
|
530
|
+
self._load_from_store()
|
|
531
|
+
return self._tickets.get(ticket_id)
|
|
532
|
+
|
|
533
|
+
def list_pending(self, session_id: Optional[str] = None) -> List[EscalationTicket]:
|
|
534
|
+
now = time.time()
|
|
535
|
+
with self._lock:
|
|
536
|
+
self._load_from_store()
|
|
537
|
+
pending: List[EscalationTicket] = []
|
|
538
|
+
expired_any = False
|
|
539
|
+
for t in self._tickets.values():
|
|
540
|
+
if t.status == "PENDING" and t.expires_at and now > t.expires_at:
|
|
541
|
+
t.status = "EXPIRED"
|
|
542
|
+
expired_any = True
|
|
543
|
+
continue
|
|
544
|
+
if t.status == "PENDING" and (session_id is None or t.session_id == session_id):
|
|
545
|
+
pending.append(t)
|
|
546
|
+
if expired_any:
|
|
547
|
+
self._save_to_store()
|
|
548
|
+
return pending
|
|
549
|
+
|
|
550
|
+
def approve(
|
|
551
|
+
self,
|
|
552
|
+
ticket_id: str,
|
|
553
|
+
approver: str,
|
|
554
|
+
gate: Optional[Any] = None,
|
|
555
|
+
session_store: Optional[Any] = None,
|
|
556
|
+
note: str = "",
|
|
557
|
+
) -> EscalationTicket:
|
|
558
|
+
"""
|
|
559
|
+
Approve a pending escalation ticket, verify Separation of Duties (`approver != requested_by`),
|
|
560
|
+
and grant a single-use SHA-256 + action-fingerprint bound token on the target session.
|
|
561
|
+
"""
|
|
562
|
+
now = time.time()
|
|
563
|
+
with self._lock:
|
|
564
|
+
self._load_from_store()
|
|
565
|
+
ticket = self._tickets.get(ticket_id)
|
|
566
|
+
if ticket is None:
|
|
567
|
+
raise KeyError(f"Unknown escalation ticket: '{ticket_id}'.")
|
|
568
|
+
if ticket.status == "PENDING" and ticket.expires_at and now > ticket.expires_at:
|
|
569
|
+
ticket.status = "EXPIRED"
|
|
570
|
+
self._save_to_store()
|
|
571
|
+
if ticket.status != "PENDING":
|
|
572
|
+
raise RuntimeError(
|
|
573
|
+
f"Cannot approve ticket '{ticket_id}' in status '{ticket.status}'."
|
|
574
|
+
)
|
|
575
|
+
if ticket.requested_by and approver:
|
|
576
|
+
req_norm = ticket.requested_by.strip().lower()
|
|
577
|
+
app_norm = approver.strip().lower()
|
|
578
|
+
app_bare = app_norm.split(":", 1)[-1] if ":" in app_norm else app_norm
|
|
579
|
+
if req_norm == app_norm or req_norm == app_bare:
|
|
580
|
+
raise PermissionError(
|
|
581
|
+
f"Separation of Duties violation: requester '{ticket.requested_by}' cannot self-approve ticket '{ticket_id}'."
|
|
582
|
+
)
|
|
583
|
+
|
|
584
|
+
# On cross-worker approvals where plaintext token wasn't stored at rest,
|
|
585
|
+
# mint a fresh one-time token whose SHA-256 hash is committed to the gate.
|
|
586
|
+
if not ticket.approval_token:
|
|
587
|
+
ticket.approval_token = secrets.token_urlsafe(24)
|
|
588
|
+
ticket.approval_token_hash = hash_secret_token(ticket.approval_token)
|
|
589
|
+
ticket.status = "APPROVED"
|
|
590
|
+
ticket.decided_by = approver
|
|
591
|
+
ticket.decided_at = now
|
|
592
|
+
ticket.decision_note = note
|
|
593
|
+
self._save_to_store()
|
|
594
|
+
|
|
595
|
+
fp = ticket.action_fingerprint or canonical_action_fingerprint(ticket.tool_name, ticket.arguments)
|
|
596
|
+
target_store = session_store if session_store is not None else self._store
|
|
597
|
+
if gate is not None:
|
|
598
|
+
gate.grant_approval(ticket.approval_token, action_fingerprint=fp)
|
|
599
|
+
elif target_store is not None:
|
|
600
|
+
from gatekeep.gate import PreExecutionGate
|
|
601
|
+
|
|
602
|
+
with target_store.session_lock(ticket.session_id):
|
|
603
|
+
loaded_gate = PreExecutionGate.from_store(
|
|
604
|
+
session_id=ticket.session_id,
|
|
605
|
+
session_store=target_store,
|
|
606
|
+
)
|
|
607
|
+
loaded_gate.grant_approval(ticket.approval_token, action_fingerprint=fp)
|
|
608
|
+
loaded_gate.save()
|
|
609
|
+
|
|
610
|
+
return ticket
|
|
611
|
+
|
|
612
|
+
def reject(self, ticket_id: str, approver: str, reason: str = "") -> EscalationTicket:
|
|
613
|
+
now = time.time()
|
|
614
|
+
with self._lock:
|
|
615
|
+
self._load_from_store()
|
|
616
|
+
ticket = self._tickets.get(ticket_id)
|
|
617
|
+
if ticket is None:
|
|
618
|
+
raise KeyError(f"Unknown escalation ticket: '{ticket_id}'.")
|
|
619
|
+
if ticket.status != "PENDING":
|
|
620
|
+
raise RuntimeError(
|
|
621
|
+
f"Cannot reject ticket '{ticket_id}' in status '{ticket.status}'."
|
|
622
|
+
)
|
|
623
|
+
ticket.status = "REJECTED"
|
|
624
|
+
ticket.decided_by = approver
|
|
625
|
+
ticket.decided_at = now
|
|
626
|
+
ticket.decision_note = reason
|
|
627
|
+
self._save_to_store()
|
|
628
|
+
return ticket
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
class WebhookEscalationNotifier:
|
|
632
|
+
"""
|
|
633
|
+
Production HTTP webhook & Slack Block Kit notifier for `EscalationRouter`.
|
|
634
|
+
|
|
635
|
+
Usage:
|
|
636
|
+
notifier = WebhookEscalationNotifier(
|
|
637
|
+
url="https://hooks.slack.com/services/...",
|
|
638
|
+
signing_secret="whsec_...",
|
|
639
|
+
payload_format="slack",
|
|
640
|
+
approve_base_url="https://gatekeep.internal",
|
|
641
|
+
)
|
|
642
|
+
router.add_listener(notifier)
|
|
643
|
+
"""
|
|
644
|
+
|
|
645
|
+
def __init__(
|
|
646
|
+
self,
|
|
647
|
+
url: str,
|
|
648
|
+
signing_secret: Optional[str] = None,
|
|
649
|
+
payload_format: str = "json",
|
|
650
|
+
timeout_s: float = 5.0,
|
|
651
|
+
approve_base_url: Optional[str] = None,
|
|
652
|
+
extra_headers: Optional[Dict[str, str]] = None,
|
|
653
|
+
interactive_actions: bool = False,
|
|
654
|
+
) -> None:
|
|
655
|
+
if not url.startswith(("http://", "https://")):
|
|
656
|
+
raise ValueError(f"Webhook URL must start with http:// or https://, got '{url}'.")
|
|
657
|
+
fmt = payload_format.lower().strip()
|
|
658
|
+
if fmt not in ("json", "slack"):
|
|
659
|
+
raise ValueError(f"Unsupported payload_format '{payload_format}'. Use 'json' or 'slack'.")
|
|
660
|
+
|
|
661
|
+
self.url = url
|
|
662
|
+
self.signing_secret = signing_secret
|
|
663
|
+
self.payload_format = fmt
|
|
664
|
+
self.timeout_s = float(timeout_s)
|
|
665
|
+
self.approve_base_url = approve_base_url.rstrip("/") if approve_base_url else None
|
|
666
|
+
self.extra_headers = dict(extra_headers or {})
|
|
667
|
+
self.interactive_actions = bool(interactive_actions)
|
|
668
|
+
|
|
669
|
+
@staticmethod
|
|
670
|
+
def compute_signature(secret: str, body: bytes) -> str:
|
|
671
|
+
"""Compute `sha256=<hex>` HMAC signature header for webhook verification."""
|
|
672
|
+
digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
|
|
673
|
+
return f"sha256={digest}"
|
|
674
|
+
|
|
675
|
+
def build_payload(self, ticket: EscalationTicket) -> Dict[str, Any]:
|
|
676
|
+
"""Serialize an `EscalationTicket` into signed JSON or Slack Block Kit format."""
|
|
677
|
+
actions: Dict[str, str] = {}
|
|
678
|
+
if self.approve_base_url:
|
|
679
|
+
actions = {
|
|
680
|
+
"approve_url": f"{self.approve_base_url}/admin/escalations/{ticket.ticket_id}/approve",
|
|
681
|
+
"reject_url": f"{self.approve_base_url}/admin/escalations/{ticket.ticket_id}/reject",
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
if self.payload_format == "slack":
|
|
685
|
+
args_preview = json.dumps(ticket.arguments, default=str)
|
|
686
|
+
if len(args_preview) > 400:
|
|
687
|
+
args_preview = args_preview[:397] + "..."
|
|
688
|
+
blocks: List[Dict[str, Any]] = [
|
|
689
|
+
{
|
|
690
|
+
"type": "header",
|
|
691
|
+
"text": {"type": "plain_text", "text": "Gatekeep Escalation Required"},
|
|
692
|
+
},
|
|
693
|
+
{
|
|
694
|
+
"type": "section",
|
|
695
|
+
"fields": [
|
|
696
|
+
{"type": "mrkdwn", "text": f"*Tool:*\n`{ticket.tool_name}`"},
|
|
697
|
+
{"type": "mrkdwn", "text": f"*Session:*\n`{ticket.session_id}`"},
|
|
698
|
+
{"type": "mrkdwn", "text": f"*Ticket:*\n`{ticket.ticket_id}`"},
|
|
699
|
+
{"type": "mrkdwn", "text": f"*Status:*\n`{ticket.status}`"},
|
|
700
|
+
],
|
|
701
|
+
},
|
|
702
|
+
{
|
|
703
|
+
"type": "section",
|
|
704
|
+
"text": {
|
|
705
|
+
"type": "mrkdwn",
|
|
706
|
+
"text": f"*Reason:* {ticket.reason}\n*Arguments:* `{args_preview}`",
|
|
707
|
+
},
|
|
708
|
+
},
|
|
709
|
+
]
|
|
710
|
+
if self.interactive_actions:
|
|
711
|
+
blocks.append(
|
|
712
|
+
{
|
|
713
|
+
"type": "actions",
|
|
714
|
+
"block_id": f"gatekeep_escalation_{ticket.ticket_id}",
|
|
715
|
+
"elements": [
|
|
716
|
+
{
|
|
717
|
+
"type": "button",
|
|
718
|
+
"text": {"type": "plain_text", "text": "Approve"},
|
|
719
|
+
"style": "primary",
|
|
720
|
+
"action_id": "approve_escalation",
|
|
721
|
+
"value": ticket.ticket_id,
|
|
722
|
+
},
|
|
723
|
+
{
|
|
724
|
+
"type": "button",
|
|
725
|
+
"text": {"type": "plain_text", "text": "Reject"},
|
|
726
|
+
"style": "danger",
|
|
727
|
+
"action_id": "reject_escalation",
|
|
728
|
+
"value": ticket.ticket_id,
|
|
729
|
+
},
|
|
730
|
+
],
|
|
731
|
+
}
|
|
732
|
+
)
|
|
733
|
+
if actions:
|
|
734
|
+
blocks.append(
|
|
735
|
+
{
|
|
736
|
+
"type": "context",
|
|
737
|
+
"elements": [
|
|
738
|
+
{
|
|
739
|
+
"type": "mrkdwn",
|
|
740
|
+
"text": (
|
|
741
|
+
f"Approve: `{actions['approve_url']}` | "
|
|
742
|
+
f"Reject: `{actions['reject_url']}`"
|
|
743
|
+
),
|
|
744
|
+
}
|
|
745
|
+
],
|
|
746
|
+
}
|
|
747
|
+
)
|
|
748
|
+
return {
|
|
749
|
+
"text": f"Gatekeep approval required for '{ticket.tool_name}' (ticket {ticket.ticket_id})",
|
|
750
|
+
"blocks": blocks,
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
payload: Dict[str, Any] = {
|
|
754
|
+
"event": "gatekeep.escalation.created",
|
|
755
|
+
"ticket": ticket.to_dict(),
|
|
756
|
+
}
|
|
757
|
+
if actions:
|
|
758
|
+
payload["actions"] = actions
|
|
759
|
+
return payload
|
|
760
|
+
|
|
761
|
+
def __call__(self, ticket: EscalationTicket) -> int:
|
|
762
|
+
payload = self.build_payload(ticket)
|
|
763
|
+
body = json.dumps(payload, sort_keys=True, default=str).encode("utf-8")
|
|
764
|
+
headers: Dict[str, str] = {
|
|
765
|
+
"Content-Type": "application/json",
|
|
766
|
+
"User-Agent": "gatekeep-mcp/0.2",
|
|
767
|
+
"X-Slim-Timestamp": str(int(time.time())),
|
|
768
|
+
}
|
|
769
|
+
if self.signing_secret:
|
|
770
|
+
headers["X-Slim-Signature"] = self.compute_signature(self.signing_secret, body)
|
|
771
|
+
headers.update(self.extra_headers)
|
|
772
|
+
|
|
773
|
+
req = urllib.request.Request(self.url, data=body, headers=headers, method="POST")
|
|
774
|
+
with urllib.request.urlopen(req, timeout=self.timeout_s) as resp:
|
|
775
|
+
return int(getattr(resp, "status", 200))
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
# ── Out-of-Band Saga Recovery Daemon ──────────────────────────────────────────
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
@dataclass
|
|
782
|
+
class SweepReport:
|
|
783
|
+
"""Result summary produced by a `SagaRecoveryDaemon` sweep pass."""
|
|
784
|
+
|
|
785
|
+
scanned_sessions: int
|
|
786
|
+
stale_sessions_found: int
|
|
787
|
+
recovered_sessions: int
|
|
788
|
+
failed_sessions: int
|
|
789
|
+
details: List[Dict[str, Any]] = field(default_factory=list)
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
class SagaRecoveryDaemon:
|
|
793
|
+
"""
|
|
794
|
+
Out-of-band Saga transaction monitor & crash sweeper.
|
|
795
|
+
Detects abandoned sessions in `SessionStateStore` with uncommitted `SagaStep`s
|
|
796
|
+
older than `stale_after_seconds` and triggers LIFO compensation rollbacks cleanly.
|
|
797
|
+
"""
|
|
798
|
+
|
|
799
|
+
def __init__(
|
|
800
|
+
self,
|
|
801
|
+
session_store: Any,
|
|
802
|
+
registry: Optional[Any] = None,
|
|
803
|
+
policy_registry: Optional[CentralPolicyRegistry] = None,
|
|
804
|
+
stale_after_seconds: float = 300.0,
|
|
805
|
+
poll_interval_seconds: float = 15.0,
|
|
806
|
+
) -> None:
|
|
807
|
+
self.session_store = session_store
|
|
808
|
+
self.registry = registry
|
|
809
|
+
self.policy_registry = policy_registry
|
|
810
|
+
self.stale_after_seconds = stale_after_seconds
|
|
811
|
+
self.poll_interval_seconds = poll_interval_seconds
|
|
812
|
+
self._stop_event = threading.Event()
|
|
813
|
+
self._thread: Optional[threading.Thread] = None
|
|
814
|
+
|
|
815
|
+
def sweep_once(self, now: Optional[float] = None) -> SweepReport:
|
|
816
|
+
"""Scan stored sessions and roll back any stale uncommitted Saga stacks."""
|
|
817
|
+
from gatekeep.gate import PreExecutionGate
|
|
818
|
+
from gatekeep.registry import ToolRegistry
|
|
819
|
+
|
|
820
|
+
current_time = now if now is not None else time.time()
|
|
821
|
+
target_reg = self.registry if self.registry is not None else ToolRegistry.instance()
|
|
822
|
+
session_ids = self.session_store.list_sessions()
|
|
823
|
+
|
|
824
|
+
stale_found = 0
|
|
825
|
+
recovered = 0
|
|
826
|
+
failed = 0
|
|
827
|
+
details: List[Dict[str, Any]] = []
|
|
828
|
+
|
|
829
|
+
for sid in session_ids:
|
|
830
|
+
snap = self.session_store.load(sid)
|
|
831
|
+
if not isinstance(snap, dict):
|
|
832
|
+
continue
|
|
833
|
+
updated_at = float(snap.get("updated_at", current_time))
|
|
834
|
+
saga_snap = (snap.get("ledger") or {}).get("saga", {})
|
|
835
|
+
pending_stack = saga_snap.get("stack", [])
|
|
836
|
+
pending_in_flight = saga_snap.get("in_flight", {})
|
|
837
|
+
if (not pending_stack and not pending_in_flight) or (
|
|
838
|
+
current_time - updated_at
|
|
839
|
+
) < self.stale_after_seconds:
|
|
840
|
+
continue
|
|
841
|
+
|
|
842
|
+
stale_found += 1
|
|
843
|
+
try:
|
|
844
|
+
with self.session_store.session_lock(sid, timeout_s=5.0):
|
|
845
|
+
# Re-check inside lock to avoid racing an active worker that just saved/committed
|
|
846
|
+
latest = self.session_store.load(sid)
|
|
847
|
+
if not isinstance(latest, dict):
|
|
848
|
+
continue
|
|
849
|
+
latest_updated = float(latest.get("updated_at", current_time))
|
|
850
|
+
latest_saga = (latest.get("ledger") or {}).get("saga", {})
|
|
851
|
+
latest_stack = latest_saga.get("stack", [])
|
|
852
|
+
latest_in_flight = latest_saga.get("in_flight", {})
|
|
853
|
+
if (not latest_stack and not latest_in_flight) or (
|
|
854
|
+
current_time - latest_updated
|
|
855
|
+
) < self.stale_after_seconds:
|
|
856
|
+
continue
|
|
857
|
+
|
|
858
|
+
gate = PreExecutionGate.from_store(
|
|
859
|
+
session_id=sid,
|
|
860
|
+
session_store=self.session_store,
|
|
861
|
+
registry=target_reg,
|
|
862
|
+
policy_registry=self.policy_registry,
|
|
863
|
+
)
|
|
864
|
+
summary = gate.rollback()
|
|
865
|
+
if summary.success:
|
|
866
|
+
recovered += 1
|
|
867
|
+
else:
|
|
868
|
+
failed += 1
|
|
869
|
+
details.append({
|
|
870
|
+
"session_id": sid,
|
|
871
|
+
"success": summary.success,
|
|
872
|
+
"compensated_steps": summary.compensated_steps,
|
|
873
|
+
"failed_steps": summary.failed_steps,
|
|
874
|
+
})
|
|
875
|
+
except Exception as exc:
|
|
876
|
+
failed += 1
|
|
877
|
+
logger.error(f"[SagaRecoveryDaemon] Failed recovering session '{sid}': {exc}")
|
|
878
|
+
details.append({
|
|
879
|
+
"session_id": sid,
|
|
880
|
+
"success": False,
|
|
881
|
+
"error": str(exc),
|
|
882
|
+
})
|
|
883
|
+
|
|
884
|
+
return SweepReport(
|
|
885
|
+
scanned_sessions=len(session_ids),
|
|
886
|
+
stale_sessions_found=stale_found,
|
|
887
|
+
recovered_sessions=recovered,
|
|
888
|
+
failed_sessions=failed,
|
|
889
|
+
details=details,
|
|
890
|
+
)
|
|
891
|
+
|
|
892
|
+
def _run_loop(self) -> None:
|
|
893
|
+
while not self._stop_event.wait(timeout=self.poll_interval_seconds):
|
|
894
|
+
try:
|
|
895
|
+
self.sweep_once()
|
|
896
|
+
except Exception as exc:
|
|
897
|
+
logger.error(f"[SagaRecoveryDaemon] Sweep iteration failed: {exc}")
|
|
898
|
+
|
|
899
|
+
def start(self) -> None:
|
|
900
|
+
"""Start background recovery sweeper thread (idempotent)."""
|
|
901
|
+
if self._thread is not None and self._thread.is_alive():
|
|
902
|
+
return
|
|
903
|
+
self._stop_event.clear()
|
|
904
|
+
self._thread = threading.Thread(
|
|
905
|
+
target=self._run_loop,
|
|
906
|
+
name="slim-saga-recovery-daemon",
|
|
907
|
+
daemon=True,
|
|
908
|
+
)
|
|
909
|
+
self._thread.start()
|
|
910
|
+
|
|
911
|
+
def stop(self, timeout_s: float = 5.0) -> None:
|
|
912
|
+
"""Signal background sweeper thread to exit cleanly."""
|
|
913
|
+
self._stop_event.set()
|
|
914
|
+
if self._thread is not None:
|
|
915
|
+
self._thread.join(timeout=timeout_s)
|
|
916
|
+
self._thread = None
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
# ── Audit Vault Query & Verification ──────────────────────────────────────────
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
class AuditVault:
|
|
923
|
+
"""
|
|
924
|
+
Searchable, tamper-evident compliance query & SHA-256 verification facade
|
|
925
|
+
across all persisted Gatekeep sessions and append-only audit streams.
|
|
926
|
+
"""
|
|
927
|
+
|
|
928
|
+
def __init__(self, session_store: Any) -> None:
|
|
929
|
+
self.session_store = session_store
|
|
930
|
+
|
|
931
|
+
@staticmethod
|
|
932
|
+
def _verify_receipt_dicts(receipts: List[Dict[str, Any]]) -> bool:
|
|
933
|
+
from gatekeep.gate import AuditReceipt, PreExecutionGate
|
|
934
|
+
|
|
935
|
+
prev = "0" * 64
|
|
936
|
+
for raw in receipts:
|
|
937
|
+
r = AuditReceipt.from_dict(raw) if isinstance(raw, dict) else raw
|
|
938
|
+
if r.prev_hash != prev:
|
|
939
|
+
return False
|
|
940
|
+
expected = PreExecutionGate._compute_hash(
|
|
941
|
+
timestamp=r.timestamp,
|
|
942
|
+
session_id=r.session_id,
|
|
943
|
+
turn=r.turn,
|
|
944
|
+
tool_name=r.tool_name,
|
|
945
|
+
arguments=r.arguments,
|
|
946
|
+
verdict=r.verdict,
|
|
947
|
+
rule_triggered=r.rule_triggered,
|
|
948
|
+
reason=r.reason,
|
|
949
|
+
prev_hash=prev,
|
|
950
|
+
actor_id=r.actor_id,
|
|
951
|
+
shadow=r.shadow,
|
|
952
|
+
would_verdict=r.would_verdict,
|
|
953
|
+
)
|
|
954
|
+
if r.receipt_hash != expected:
|
|
955
|
+
return False
|
|
956
|
+
prev = r.receipt_hash
|
|
957
|
+
return True
|
|
958
|
+
|
|
959
|
+
def verify_all_chains(self) -> Dict[str, bool]:
|
|
960
|
+
"""
|
|
961
|
+
Verify SHA-256 hash-chain integrity across active sessions AND deleted/archived
|
|
962
|
+
sessions preserved in the store's append-only audit log.
|
|
963
|
+
"""
|
|
964
|
+
from gatekeep.gate import PreExecutionGate
|
|
965
|
+
|
|
966
|
+
results: Dict[str, bool] = {}
|
|
967
|
+
active_sids = self.session_store.list_sessions()
|
|
968
|
+
for sid in active_sids:
|
|
969
|
+
gate = PreExecutionGate.from_store(sid, self.session_store, auto_persist=False)
|
|
970
|
+
results[sid] = gate.verify_audit_chain()
|
|
971
|
+
|
|
972
|
+
if hasattr(self.session_store, "load_audit_receipts"):
|
|
973
|
+
grouped: Dict[str, List[Dict[str, Any]]] = {}
|
|
974
|
+
for item in self.session_store.load_audit_receipts(limit=100_000):
|
|
975
|
+
sid = str(item.get("session_id", "default"))
|
|
976
|
+
grouped.setdefault(sid, []).append(item)
|
|
977
|
+
for sid, items in grouped.items():
|
|
978
|
+
log_ok = self._verify_receipt_dicts(items)
|
|
979
|
+
results[sid] = (results[sid] and log_ok) if sid in results else log_ok
|
|
980
|
+
return results
|
|
981
|
+
|
|
982
|
+
def query(
|
|
983
|
+
self,
|
|
984
|
+
session_id: Optional[str] = None,
|
|
985
|
+
verdict: Optional[Any] = None,
|
|
986
|
+
tool_name: Optional[str] = None,
|
|
987
|
+
rule_triggered: Optional[str] = None,
|
|
988
|
+
actor_id: Optional[str] = None,
|
|
989
|
+
limit: int = 500,
|
|
990
|
+
) -> List[Any]:
|
|
991
|
+
"""Return matching `AuditReceipt` objects across append-only logs & persisted sessions."""
|
|
992
|
+
from gatekeep.gate import AuditReceipt, Verdict
|
|
993
|
+
|
|
994
|
+
target_verdict = (
|
|
995
|
+
verdict.value if isinstance(verdict, Verdict) else (str(verdict).upper() if verdict else None)
|
|
996
|
+
)
|
|
997
|
+
if hasattr(self.session_store, "load_audit_receipts"):
|
|
998
|
+
raw_items = self.session_store.load_audit_receipts(
|
|
999
|
+
session_id=session_id,
|
|
1000
|
+
verdict=target_verdict,
|
|
1001
|
+
tool_name=tool_name,
|
|
1002
|
+
rule_triggered=rule_triggered,
|
|
1003
|
+
actor_id=actor_id,
|
|
1004
|
+
limit=limit,
|
|
1005
|
+
)
|
|
1006
|
+
if raw_items:
|
|
1007
|
+
return [AuditReceipt.from_dict(item) for item in raw_items]
|
|
1008
|
+
|
|
1009
|
+
sids = [session_id] if session_id else self.session_store.list_sessions()
|
|
1010
|
+
matches: List[AuditReceipt] = []
|
|
1011
|
+
|
|
1012
|
+
for sid in sids:
|
|
1013
|
+
snap = self.session_store.load(sid)
|
|
1014
|
+
if not isinstance(snap, dict):
|
|
1015
|
+
continue
|
|
1016
|
+
raw_receipts = (snap.get("gate") or {}).get("receipts", [])
|
|
1017
|
+
for item in raw_receipts:
|
|
1018
|
+
if not isinstance(item, dict):
|
|
1019
|
+
continue
|
|
1020
|
+
if target_verdict and str(item.get("verdict", "")).upper() != target_verdict:
|
|
1021
|
+
continue
|
|
1022
|
+
if tool_name and item.get("tool_name") != tool_name:
|
|
1023
|
+
continue
|
|
1024
|
+
if rule_triggered and item.get("rule_triggered") != rule_triggered:
|
|
1025
|
+
continue
|
|
1026
|
+
if actor_id and item.get("actor_id") != actor_id:
|
|
1027
|
+
continue
|
|
1028
|
+
matches.append(AuditReceipt.from_dict(item))
|
|
1029
|
+
if len(matches) >= limit:
|
|
1030
|
+
return matches
|
|
1031
|
+
return matches
|
|
1032
|
+
|
|
1033
|
+
def export_jsonl(
|
|
1034
|
+
self,
|
|
1035
|
+
output_path: Optional[str | Path] = None,
|
|
1036
|
+
session_id: Optional[str] = None,
|
|
1037
|
+
) -> str:
|
|
1038
|
+
"""Export matching audit receipts as newline-delimited JSON."""
|
|
1039
|
+
receipts = self.query(session_id=session_id, limit=100_000)
|
|
1040
|
+
lines = "\n".join(r.to_json() for r in receipts)
|
|
1041
|
+
if output_path is not None:
|
|
1042
|
+
Path(output_path).write_text(lines + ("\n" if lines else ""), encoding="utf-8")
|
|
1043
|
+
return lines
|