agenthandler 0.92.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agenthandler might be problematic. Click here for more details.
- agenthandler/__init__.py +163 -0
- agenthandler/approval.py +135 -0
- agenthandler/audit.py +144 -0
- agenthandler/auth.py +188 -0
- agenthandler/budget.py +94 -0
- agenthandler/builder.py +178 -0
- agenthandler/circuit_breaker.py +88 -0
- agenthandler/connectors.py +425 -0
- agenthandler/context.py +410 -0
- agenthandler/errors.py +130 -0
- agenthandler/guardrails/__init__.py +79 -0
- agenthandler/guardrails/deterministic.py +472 -0
- agenthandler/guardrails/llm_based.py +549 -0
- agenthandler/guardrails/types.py +83 -0
- agenthandler/observe.py +525 -0
- agenthandler/openclaw.py +370 -0
- agenthandler/pipeline.py +532 -0
- agenthandler/pipeline_poml.py +593 -0
- agenthandler/policy.py +251 -0
- agenthandler/py.typed +0 -0
- agenthandler/redact.py +260 -0
- agenthandler/reflection.py +371 -0
- agenthandler/replay.py +225 -0
- agenthandler/server.py +752 -0
- agenthandler/session.py +391 -0
- agenthandler/skill.py +265 -0
- agenthandler/store.py +341 -0
- agenthandler/streams.py +339 -0
- agenthandler/supervisor.py +882 -0
- agenthandler/triggers.py +471 -0
- agenthandler-0.92.0.dist-info/METADATA +1355 -0
- agenthandler-0.92.0.dist-info/RECORD +34 -0
- agenthandler-0.92.0.dist-info/WHEEL +4 -0
- agenthandler-0.92.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,882 @@
|
|
|
1
|
+
"""Supervisor — the core of agenthandler.
|
|
2
|
+
|
|
3
|
+
Wraps tool calls with timeout, budget, circuit breaker, and audit.
|
|
4
|
+
Doesn't own the agent loop — supervises whatever loop you're running.
|
|
5
|
+
|
|
6
|
+
Usage:
|
|
7
|
+
supervisor = Supervisor(policy)
|
|
8
|
+
|
|
9
|
+
# In your agent loop:
|
|
10
|
+
iteration = supervisor.record_iteration() # raises if over limit
|
|
11
|
+
result = await supervisor.call("web_search", search_fn, query="python")
|
|
12
|
+
|
|
13
|
+
if result.succeeded:
|
|
14
|
+
process(result.output)
|
|
15
|
+
else:
|
|
16
|
+
handle_error(result.error)
|
|
17
|
+
|
|
18
|
+
# When done:
|
|
19
|
+
supervisor.finish()
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import asyncio
|
|
25
|
+
import time
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
from typing import TYPE_CHECKING, Any, Callable, Coroutine, Dict, List, Optional
|
|
29
|
+
|
|
30
|
+
from .audit import AuditLog, AuditOutcome, AuditPhase
|
|
31
|
+
from .budget import BudgetSnapshot, BudgetTracker
|
|
32
|
+
from .circuit_breaker import CircuitBreaker
|
|
33
|
+
from .errors import AgentHandlerError
|
|
34
|
+
from .policy import Policy
|
|
35
|
+
from .redact import Redactor
|
|
36
|
+
|
|
37
|
+
if TYPE_CHECKING:
|
|
38
|
+
from .approval import ApprovalQueue
|
|
39
|
+
from .observe import Observer
|
|
40
|
+
from .store import Checkpoint, StateStore
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class SupervisedResult:
|
|
45
|
+
"""Result of a supervised tool call."""
|
|
46
|
+
|
|
47
|
+
output: Any = None
|
|
48
|
+
error: Optional[AgentHandlerError] = None
|
|
49
|
+
succeeded: bool = False
|
|
50
|
+
duration_ms: int = 0
|
|
51
|
+
budget: Optional[BudgetSnapshot] = None
|
|
52
|
+
tool_name: Optional[str] = None
|
|
53
|
+
|
|
54
|
+
def user_message(self) -> str:
|
|
55
|
+
"""Always returns something — either the output or an error message."""
|
|
56
|
+
if self.succeeded and self.output is not None:
|
|
57
|
+
return str(self.output)
|
|
58
|
+
if self.error:
|
|
59
|
+
return self.error.user_message()
|
|
60
|
+
return "No output (tool returned nothing)"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Supervisor:
|
|
64
|
+
"""Supervises tool calls with policy enforcement.
|
|
65
|
+
|
|
66
|
+
One Supervisor per skill invocation. Tracks budget across multiple
|
|
67
|
+
tool calls within a single request.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
def __init__(
|
|
71
|
+
self,
|
|
72
|
+
policy: Policy,
|
|
73
|
+
audit: Optional[AuditLog] = None,
|
|
74
|
+
store: Optional[StateStore] = None,
|
|
75
|
+
session_id: Optional[str] = None,
|
|
76
|
+
agent_id: Optional[str] = None,
|
|
77
|
+
approval_queue: Optional[ApprovalQueue] = None,
|
|
78
|
+
observer: Optional[Observer] = None,
|
|
79
|
+
pre_guardrails: Optional[List[Any]] = None,
|
|
80
|
+
post_guardrails: Optional[List[Any]] = None,
|
|
81
|
+
):
|
|
82
|
+
self._policy = policy
|
|
83
|
+
self._budget = BudgetTracker.from_policy(policy)
|
|
84
|
+
self._circuit_breakers: Dict[str, CircuitBreaker] = {}
|
|
85
|
+
self._tool_call_times: Dict[str, List[float]] = {}
|
|
86
|
+
self._audit = audit or AuditLog.noop()
|
|
87
|
+
self._pre_guardrails: List[Any] = pre_guardrails or []
|
|
88
|
+
self._post_guardrails: List[Any] = post_guardrails or []
|
|
89
|
+
self._recent_calls: List[Dict[str, Any]] = []
|
|
90
|
+
self._last_call_meta: Dict[str, Any] = {
|
|
91
|
+
"input_tokens": 0,
|
|
92
|
+
"output_tokens": 0,
|
|
93
|
+
"model": "",
|
|
94
|
+
}
|
|
95
|
+
self._start_time = time.monotonic()
|
|
96
|
+
self._last_activity = time.monotonic()
|
|
97
|
+
self._paused = False
|
|
98
|
+
|
|
99
|
+
# Optional durable state
|
|
100
|
+
self._store = store
|
|
101
|
+
self._session_id = session_id
|
|
102
|
+
self._agent_id = agent_id
|
|
103
|
+
self._approval_queue = approval_queue
|
|
104
|
+
self._observer = observer
|
|
105
|
+
|
|
106
|
+
# Privacy shielding
|
|
107
|
+
redact_cats = policy.redact
|
|
108
|
+
self._redactor: Optional[Redactor] = None
|
|
109
|
+
if redact_cats is None or (isinstance(redact_cats, list) and len(redact_cats) > 0):
|
|
110
|
+
self._redactor = Redactor(categories=redact_cats)
|
|
111
|
+
self._redact_direction = policy.redact_direction
|
|
112
|
+
|
|
113
|
+
self._audit.record(
|
|
114
|
+
AuditPhase.REQUEST_START,
|
|
115
|
+
AuditOutcome.INFO,
|
|
116
|
+
detail=f"Supervisor started with policy: max_iter={policy.max_iterations}, "
|
|
117
|
+
f"tool_timeout={policy.tool_timeout}s, token_budget={policy.token_budget}",
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
@classmethod
|
|
121
|
+
def with_defaults(cls) -> "Supervisor":
|
|
122
|
+
"""Create a supervisor with default policy."""
|
|
123
|
+
return cls(Policy())
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def policy(self) -> Policy:
|
|
127
|
+
return self._policy
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def session_id(self) -> Optional[str]:
|
|
131
|
+
return self._session_id
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def agent_id(self) -> Optional[str]:
|
|
135
|
+
return self._agent_id
|
|
136
|
+
|
|
137
|
+
@property
|
|
138
|
+
def paused(self) -> bool:
|
|
139
|
+
return self._paused
|
|
140
|
+
|
|
141
|
+
@paused.setter
|
|
142
|
+
def paused(self, value: bool) -> None:
|
|
143
|
+
self._paused = value
|
|
144
|
+
|
|
145
|
+
def budget(self) -> BudgetSnapshot:
|
|
146
|
+
"""Get current budget snapshot."""
|
|
147
|
+
return self._budget.snapshot()
|
|
148
|
+
|
|
149
|
+
def record_tokens(self, tokens: int) -> int:
|
|
150
|
+
"""Record token usage. Returns new total. Raises AgentHandlerError if over budget."""
|
|
151
|
+
total = self._budget.record_tokens(tokens)
|
|
152
|
+
self._auto_checkpoint()
|
|
153
|
+
return total
|
|
154
|
+
|
|
155
|
+
def record_iteration(self) -> int:
|
|
156
|
+
"""Record an iteration of the agent loop. Returns iteration count.
|
|
157
|
+
Raises AgentHandlerError if max iterations exceeded.
|
|
158
|
+
"""
|
|
159
|
+
self._last_activity = time.monotonic()
|
|
160
|
+
try:
|
|
161
|
+
count = self._budget.record_iteration()
|
|
162
|
+
self._audit.record(
|
|
163
|
+
AuditPhase.ITERATION_CHECK,
|
|
164
|
+
AuditOutcome.ALLOWED,
|
|
165
|
+
detail=f"Iteration {count} / {self._policy.max_iterations}",
|
|
166
|
+
)
|
|
167
|
+
self._auto_checkpoint()
|
|
168
|
+
return count
|
|
169
|
+
except AgentHandlerError as e:
|
|
170
|
+
self._audit.record(
|
|
171
|
+
AuditPhase.ITERATION_CHECK,
|
|
172
|
+
AuditOutcome.DENIED,
|
|
173
|
+
detail=str(e),
|
|
174
|
+
)
|
|
175
|
+
raise
|
|
176
|
+
|
|
177
|
+
def circuit_breaker_states(self) -> Dict[str, str]:
|
|
178
|
+
"""Get current circuit breaker states for all tracked tools."""
|
|
179
|
+
return {name: cb.state.value for name, cb in self._circuit_breakers.items()}
|
|
180
|
+
|
|
181
|
+
def _get_circuit_breaker(self, tool_name: str) -> CircuitBreaker:
|
|
182
|
+
"""Get or create a circuit breaker for a tool."""
|
|
183
|
+
if tool_name not in self._circuit_breakers:
|
|
184
|
+
self._circuit_breakers[tool_name] = CircuitBreaker(
|
|
185
|
+
threshold=self._policy.circuit_breaker_threshold,
|
|
186
|
+
reset_after=self._policy.circuit_breaker_reset,
|
|
187
|
+
)
|
|
188
|
+
return self._circuit_breakers[tool_name]
|
|
189
|
+
|
|
190
|
+
def _check_request_timeout(self) -> None:
|
|
191
|
+
"""Check if the overall request has timed out."""
|
|
192
|
+
elapsed = time.monotonic() - self._start_time
|
|
193
|
+
if elapsed > self._policy.request_timeout:
|
|
194
|
+
raise AgentHandlerError.timeout(int(elapsed * 1000))
|
|
195
|
+
|
|
196
|
+
def _check_silence(self) -> None:
|
|
197
|
+
"""Check the dead man's switch."""
|
|
198
|
+
if self._policy.silence_timeout is None:
|
|
199
|
+
return
|
|
200
|
+
silence = time.monotonic() - self._last_activity
|
|
201
|
+
if silence > self._policy.silence_timeout:
|
|
202
|
+
raise AgentHandlerError.dead_man_switch(int(silence * 1000))
|
|
203
|
+
|
|
204
|
+
def _check_paused(self) -> None:
|
|
205
|
+
"""Check if this supervisor is paused."""
|
|
206
|
+
if self._paused and self._session_id:
|
|
207
|
+
raise AgentHandlerError.agent_paused(self._session_id)
|
|
208
|
+
|
|
209
|
+
def _check_confirm(self, tool_name: str, kwargs: Dict[str, Any]) -> Optional[str]:
|
|
210
|
+
"""Check if this tool requires confirmation.
|
|
211
|
+
|
|
212
|
+
If an approval queue is configured, queues the call and returns an
|
|
213
|
+
approval_id. Otherwise raises AgentHandlerError.
|
|
214
|
+
|
|
215
|
+
Returns:
|
|
216
|
+
approval_id if queued, None if no confirmation needed.
|
|
217
|
+
"""
|
|
218
|
+
if tool_name not in self._policy.require_confirm:
|
|
219
|
+
return None
|
|
220
|
+
if self._approval_queue is not None and self._session_id:
|
|
221
|
+
req = self._approval_queue.submit(tool_name, kwargs, self._session_id)
|
|
222
|
+
return req.approval_id
|
|
223
|
+
raise AgentHandlerError.policy_denied(f"Tool '{tool_name}' requires user confirmation")
|
|
224
|
+
|
|
225
|
+
def _check_scopes(self, tool_name: str, kwargs: Dict[str, Any]) -> None:
|
|
226
|
+
"""Check if tool arguments satisfy scope rules."""
|
|
227
|
+
now = time.monotonic()
|
|
228
|
+
for scope in self._policy.tool_scopes:
|
|
229
|
+
if scope.tool_name == tool_name:
|
|
230
|
+
# Check argument rules
|
|
231
|
+
error_msg = scope.check_args(kwargs)
|
|
232
|
+
if error_msg:
|
|
233
|
+
raise AgentHandlerError.scope_denied(tool_name, error_msg)
|
|
234
|
+
# Check rate limit
|
|
235
|
+
if scope.max_calls_per_minute is not None:
|
|
236
|
+
times = self._tool_call_times.get(tool_name, [])
|
|
237
|
+
cutoff = now - 60.0
|
|
238
|
+
recent = [t for t in times if t > cutoff]
|
|
239
|
+
if len(recent) >= scope.max_calls_per_minute:
|
|
240
|
+
raise AgentHandlerError.rate_limited(tool_name, scope.max_calls_per_minute)
|
|
241
|
+
recent.append(now)
|
|
242
|
+
self._tool_call_times[tool_name] = recent
|
|
243
|
+
|
|
244
|
+
async def call(
|
|
245
|
+
self,
|
|
246
|
+
tool_name: str,
|
|
247
|
+
fn: Callable[..., Coroutine[Any, Any, Any]],
|
|
248
|
+
**kwargs: Any,
|
|
249
|
+
) -> SupervisedResult:
|
|
250
|
+
"""Supervise an async tool call.
|
|
251
|
+
|
|
252
|
+
Args:
|
|
253
|
+
tool_name: Name of the tool being called
|
|
254
|
+
fn: Async callable to execute
|
|
255
|
+
**kwargs: Arguments passed to fn
|
|
256
|
+
|
|
257
|
+
Returns:
|
|
258
|
+
SupervisedResult with output or error (never raises)
|
|
259
|
+
"""
|
|
260
|
+
start = time.monotonic()
|
|
261
|
+
self._last_activity = start
|
|
262
|
+
|
|
263
|
+
# Pre-flight checks
|
|
264
|
+
try:
|
|
265
|
+
self._check_paused()
|
|
266
|
+
self._check_request_timeout()
|
|
267
|
+
self._check_silence()
|
|
268
|
+
approval_id = self._check_confirm(tool_name, kwargs)
|
|
269
|
+
if approval_id is not None:
|
|
270
|
+
self._audit.record(
|
|
271
|
+
AuditPhase.POLICY_CHECK,
|
|
272
|
+
AuditOutcome.DENIED,
|
|
273
|
+
target=tool_name,
|
|
274
|
+
detail=f"Queued for approval: {approval_id}",
|
|
275
|
+
metadata={"approval_id": approval_id},
|
|
276
|
+
)
|
|
277
|
+
return SupervisedResult(
|
|
278
|
+
error=AgentHandlerError.approval_pending(tool_name, approval_id),
|
|
279
|
+
duration_ms=int((time.monotonic() - start) * 1000),
|
|
280
|
+
budget=self._budget.snapshot(),
|
|
281
|
+
tool_name=tool_name,
|
|
282
|
+
)
|
|
283
|
+
self._check_scopes(tool_name, kwargs)
|
|
284
|
+
except AgentHandlerError as e:
|
|
285
|
+
phase = (
|
|
286
|
+
AuditPhase.SCOPE_CHECK
|
|
287
|
+
if e.kind in ("scope_denied", "rate_limited")
|
|
288
|
+
else AuditPhase.POLICY_CHECK
|
|
289
|
+
)
|
|
290
|
+
self._audit.record(
|
|
291
|
+
phase,
|
|
292
|
+
AuditOutcome.DENIED,
|
|
293
|
+
target=tool_name,
|
|
294
|
+
detail=str(e),
|
|
295
|
+
)
|
|
296
|
+
return SupervisedResult(
|
|
297
|
+
error=e,
|
|
298
|
+
duration_ms=int((time.monotonic() - start) * 1000),
|
|
299
|
+
budget=self._budget.snapshot(),
|
|
300
|
+
tool_name=tool_name,
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
# Circuit breaker check
|
|
304
|
+
cb = self._get_circuit_breaker(tool_name)
|
|
305
|
+
try:
|
|
306
|
+
cb.check(tool_name)
|
|
307
|
+
except AgentHandlerError as e:
|
|
308
|
+
self._audit.record(
|
|
309
|
+
AuditPhase.CIRCUIT_BREAKER,
|
|
310
|
+
AuditOutcome.DENIED,
|
|
311
|
+
target=tool_name,
|
|
312
|
+
detail=str(e),
|
|
313
|
+
)
|
|
314
|
+
return SupervisedResult(
|
|
315
|
+
error=e,
|
|
316
|
+
duration_ms=int((time.monotonic() - start) * 1000),
|
|
317
|
+
budget=self._budget.snapshot(),
|
|
318
|
+
tool_name=tool_name,
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
# Pre-call guardrails
|
|
322
|
+
gr_context = self._build_guardrail_context()
|
|
323
|
+
cached_output: Any = None
|
|
324
|
+
for gr in self._pre_guardrails:
|
|
325
|
+
result = await self._run_guardrail(gr, tool_name, kwargs, None, gr_context, "pre")
|
|
326
|
+
if not result.allowed:
|
|
327
|
+
self._audit.record(
|
|
328
|
+
AuditPhase.POLICY_CHECK,
|
|
329
|
+
AuditOutcome.DENIED,
|
|
330
|
+
target=tool_name,
|
|
331
|
+
detail=f"Guardrail '{result.guardrail_name}' blocked: {result.reason}",
|
|
332
|
+
)
|
|
333
|
+
return SupervisedResult(
|
|
334
|
+
error=AgentHandlerError.policy_denied(
|
|
335
|
+
f"{result.guardrail_name}: {result.reason}"
|
|
336
|
+
),
|
|
337
|
+
duration_ms=int((time.monotonic() - start) * 1000),
|
|
338
|
+
budget=self._budget.snapshot(),
|
|
339
|
+
tool_name=tool_name,
|
|
340
|
+
)
|
|
341
|
+
# Guardrail may replace the output (e.g., idempotency cache hit)
|
|
342
|
+
if result.replacement_output is not None:
|
|
343
|
+
cached_output = result.replacement_output
|
|
344
|
+
|
|
345
|
+
# Outbound PII redaction — clean args before they reach the tool
|
|
346
|
+
exec_kwargs = kwargs
|
|
347
|
+
if self._redactor and self._redact_direction in ("outbound", "both"):
|
|
348
|
+
exec_kwargs, redact_count = self._redactor.redact_dict(kwargs)
|
|
349
|
+
if redact_count > 0:
|
|
350
|
+
self._audit.record(
|
|
351
|
+
AuditPhase.POLICY_CHECK,
|
|
352
|
+
AuditOutcome.INFO,
|
|
353
|
+
target=tool_name,
|
|
354
|
+
detail=f"Redacted {redact_count} PII value(s) from outbound args",
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
# Execute with timeout
|
|
358
|
+
self._audit.record(
|
|
359
|
+
AuditPhase.TOOL_CALL,
|
|
360
|
+
AuditOutcome.ALLOWED,
|
|
361
|
+
target=tool_name,
|
|
362
|
+
metadata={"args": {k: str(v)[:100] for k, v in exec_kwargs.items()}},
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
# If a pre-guardrail provided cached output (idempotency), skip execution
|
|
366
|
+
if cached_output is not None:
|
|
367
|
+
output = cached_output
|
|
368
|
+
duration_ms = int((time.monotonic() - start) * 1000)
|
|
369
|
+
cb.record_success()
|
|
370
|
+
self._audit.record(
|
|
371
|
+
AuditPhase.TOOL_CALL,
|
|
372
|
+
AuditOutcome.INFO,
|
|
373
|
+
target=tool_name,
|
|
374
|
+
detail=f"Returned cached result in {duration_ms}ms",
|
|
375
|
+
)
|
|
376
|
+
result = SupervisedResult(
|
|
377
|
+
output=output,
|
|
378
|
+
succeeded=True,
|
|
379
|
+
duration_ms=duration_ms,
|
|
380
|
+
budget=self._budget.snapshot(),
|
|
381
|
+
tool_name=tool_name,
|
|
382
|
+
)
|
|
383
|
+
self._observe_call(result)
|
|
384
|
+
return result
|
|
385
|
+
|
|
386
|
+
try:
|
|
387
|
+
output = await asyncio.wait_for(
|
|
388
|
+
fn(**exec_kwargs),
|
|
389
|
+
timeout=self._policy.tool_timeout,
|
|
390
|
+
)
|
|
391
|
+
except asyncio.TimeoutError:
|
|
392
|
+
cb.record_failure()
|
|
393
|
+
error = AgentHandlerError.timeout(int(self._policy.tool_timeout * 1000))
|
|
394
|
+
self._audit.record(
|
|
395
|
+
AuditPhase.TOOL_CALL,
|
|
396
|
+
AuditOutcome.TIMED_OUT,
|
|
397
|
+
target=tool_name,
|
|
398
|
+
detail=str(error),
|
|
399
|
+
)
|
|
400
|
+
r = SupervisedResult(
|
|
401
|
+
error=error,
|
|
402
|
+
duration_ms=int((time.monotonic() - start) * 1000),
|
|
403
|
+
budget=self._budget.snapshot(),
|
|
404
|
+
tool_name=tool_name,
|
|
405
|
+
)
|
|
406
|
+
self._observe_call(r)
|
|
407
|
+
return r
|
|
408
|
+
except Exception as e:
|
|
409
|
+
cb.record_failure()
|
|
410
|
+
error = AgentHandlerError.tool_error(str(e))
|
|
411
|
+
self._audit.record(
|
|
412
|
+
AuditPhase.TOOL_CALL,
|
|
413
|
+
AuditOutcome.FAILED,
|
|
414
|
+
target=tool_name,
|
|
415
|
+
detail=str(e),
|
|
416
|
+
)
|
|
417
|
+
r = SupervisedResult(
|
|
418
|
+
error=error,
|
|
419
|
+
duration_ms=int((time.monotonic() - start) * 1000),
|
|
420
|
+
budget=self._budget.snapshot(),
|
|
421
|
+
tool_name=tool_name,
|
|
422
|
+
)
|
|
423
|
+
self._observe_call(r)
|
|
424
|
+
return r
|
|
425
|
+
|
|
426
|
+
# Success
|
|
427
|
+
cb.record_success()
|
|
428
|
+
duration_ms = int((time.monotonic() - start) * 1000)
|
|
429
|
+
self._last_activity = time.monotonic()
|
|
430
|
+
|
|
431
|
+
# Post-call guardrails — may block, modify, or allow the output
|
|
432
|
+
for gr in self._post_guardrails:
|
|
433
|
+
result = await self._run_guardrail(
|
|
434
|
+
gr, tool_name, exec_kwargs, output, gr_context, "post"
|
|
435
|
+
)
|
|
436
|
+
if not result.allowed:
|
|
437
|
+
self._audit.record(
|
|
438
|
+
AuditPhase.POLICY_CHECK,
|
|
439
|
+
AuditOutcome.DENIED,
|
|
440
|
+
target=tool_name,
|
|
441
|
+
detail=f"Post-guardrail '{result.guardrail_name}' blocked: {result.reason}",
|
|
442
|
+
)
|
|
443
|
+
return SupervisedResult(
|
|
444
|
+
error=AgentHandlerError.policy_denied(
|
|
445
|
+
f"{result.guardrail_name}: {result.reason}"
|
|
446
|
+
),
|
|
447
|
+
duration_ms=duration_ms,
|
|
448
|
+
budget=self._budget.snapshot(),
|
|
449
|
+
tool_name=tool_name,
|
|
450
|
+
)
|
|
451
|
+
if result.replacement_output is not None:
|
|
452
|
+
output = result.replacement_output
|
|
453
|
+
self._audit.record(
|
|
454
|
+
AuditPhase.POLICY_CHECK,
|
|
455
|
+
AuditOutcome.INFO,
|
|
456
|
+
target=tool_name,
|
|
457
|
+
detail=f"Guardrail '{result.guardrail_name}' replaced output: {result.reason}",
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
# Track this call in recent_calls for anomaly detection
|
|
461
|
+
self._recent_calls.append(
|
|
462
|
+
{
|
|
463
|
+
"tool": tool_name,
|
|
464
|
+
"args_summary": {k: str(v)[:50] for k, v in exec_kwargs.items()},
|
|
465
|
+
"duration_ms": duration_ms,
|
|
466
|
+
}
|
|
467
|
+
)
|
|
468
|
+
if len(self._recent_calls) > 50:
|
|
469
|
+
self._recent_calls = self._recent_calls[-50:]
|
|
470
|
+
|
|
471
|
+
# Call idempotency cache recording on any pre-guardrail that supports it
|
|
472
|
+
for gr in self._pre_guardrails:
|
|
473
|
+
if hasattr(gr, "record"):
|
|
474
|
+
gr.record(tool_name, exec_kwargs, output)
|
|
475
|
+
|
|
476
|
+
# Inbound PII redaction — clean output before agent sees it
|
|
477
|
+
if self._redactor and self._redact_direction in ("inbound", "both"):
|
|
478
|
+
if isinstance(output, str):
|
|
479
|
+
redaction = self._redactor.redact(output)
|
|
480
|
+
if redaction.redacted_count > 0:
|
|
481
|
+
self._audit.record(
|
|
482
|
+
AuditPhase.POLICY_CHECK,
|
|
483
|
+
AuditOutcome.INFO,
|
|
484
|
+
target=tool_name,
|
|
485
|
+
detail=f"Redacted {redaction.redacted_count} PII value(s) from output",
|
|
486
|
+
)
|
|
487
|
+
output = redaction.text
|
|
488
|
+
elif isinstance(output, dict):
|
|
489
|
+
cleaned, count = self._redactor.redact_dict(output)
|
|
490
|
+
if count > 0:
|
|
491
|
+
self._audit.record(
|
|
492
|
+
AuditPhase.POLICY_CHECK,
|
|
493
|
+
AuditOutcome.INFO,
|
|
494
|
+
target=tool_name,
|
|
495
|
+
detail=f"Redacted {count} PII value(s) from output",
|
|
496
|
+
)
|
|
497
|
+
output = cleaned
|
|
498
|
+
|
|
499
|
+
# Record token usage if output includes it.
|
|
500
|
+
# Protocol: tool outputs can include {"tokens_used": N} for legacy
|
|
501
|
+
# single-value budgeting, or {"input_tokens": N, "output_tokens": M,
|
|
502
|
+
# "model": "..."} for full cost attribution.
|
|
503
|
+
tokens = 0
|
|
504
|
+
call_input_tokens = 0
|
|
505
|
+
call_output_tokens = 0
|
|
506
|
+
call_model = ""
|
|
507
|
+
if isinstance(output, dict):
|
|
508
|
+
if "tokens_used" in output:
|
|
509
|
+
tokens = output["tokens_used"]
|
|
510
|
+
if "input_tokens" in output:
|
|
511
|
+
call_input_tokens = output.get("input_tokens", 0) or 0
|
|
512
|
+
if "output_tokens" in output:
|
|
513
|
+
call_output_tokens = output.get("output_tokens", 0) or 0
|
|
514
|
+
if "model" in output:
|
|
515
|
+
call_model = output.get("model", "") or ""
|
|
516
|
+
# If both input/output are provided but tokens_used isn't, compute it
|
|
517
|
+
if tokens == 0 and (call_input_tokens or call_output_tokens):
|
|
518
|
+
tokens = call_input_tokens + call_output_tokens
|
|
519
|
+
|
|
520
|
+
# Stash call-level metadata for _observe_call
|
|
521
|
+
self._last_call_meta = {
|
|
522
|
+
"input_tokens": call_input_tokens,
|
|
523
|
+
"output_tokens": call_output_tokens,
|
|
524
|
+
"model": call_model,
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
if tokens > 0:
|
|
528
|
+
try:
|
|
529
|
+
self._budget.record_tokens(tokens)
|
|
530
|
+
except AgentHandlerError as e:
|
|
531
|
+
self._audit.record(
|
|
532
|
+
AuditPhase.BUDGET_CHECK,
|
|
533
|
+
AuditOutcome.BUDGET_EXCEEDED,
|
|
534
|
+
target=tool_name,
|
|
535
|
+
detail=str(e),
|
|
536
|
+
)
|
|
537
|
+
# Budget exceeded — still return the output (best effort)
|
|
538
|
+
return SupervisedResult(
|
|
539
|
+
output=output,
|
|
540
|
+
error=e,
|
|
541
|
+
succeeded=True, # tool succeeded, budget is over
|
|
542
|
+
duration_ms=duration_ms,
|
|
543
|
+
budget=self._budget.snapshot(),
|
|
544
|
+
tool_name=tool_name,
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
self._audit.record(
|
|
548
|
+
AuditPhase.TOOL_CALL,
|
|
549
|
+
AuditOutcome.INFO,
|
|
550
|
+
target=tool_name,
|
|
551
|
+
detail=f"Completed in {duration_ms}ms",
|
|
552
|
+
)
|
|
553
|
+
|
|
554
|
+
self._auto_checkpoint()
|
|
555
|
+
|
|
556
|
+
result = SupervisedResult(
|
|
557
|
+
output=output,
|
|
558
|
+
succeeded=True,
|
|
559
|
+
duration_ms=duration_ms,
|
|
560
|
+
budget=self._budget.snapshot(),
|
|
561
|
+
tool_name=tool_name,
|
|
562
|
+
)
|
|
563
|
+
self._observe_call(result)
|
|
564
|
+
return result
|
|
565
|
+
|
|
566
|
+
async def execute_approved(
|
|
567
|
+
self,
|
|
568
|
+
approval_id: str,
|
|
569
|
+
fn: Callable[..., Coroutine[Any, Any, Any]],
|
|
570
|
+
) -> SupervisedResult:
|
|
571
|
+
"""Execute a previously-approved tool call.
|
|
572
|
+
|
|
573
|
+
Looks up the approval in the queue, verifies it was approved,
|
|
574
|
+
then executes the tool call bypassing the confirmation check.
|
|
575
|
+
|
|
576
|
+
Args:
|
|
577
|
+
approval_id: The approval ID returned by the original call().
|
|
578
|
+
fn: The async callable to execute (same function the agent
|
|
579
|
+
originally tried to call).
|
|
580
|
+
|
|
581
|
+
Returns:
|
|
582
|
+
SupervisedResult — same as call().
|
|
583
|
+
"""
|
|
584
|
+
if self._approval_queue is None:
|
|
585
|
+
return SupervisedResult(
|
|
586
|
+
error=AgentHandlerError.policy_denied("No approval queue configured"),
|
|
587
|
+
budget=self._budget.snapshot(),
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
from .approval import ApprovalStatus
|
|
591
|
+
|
|
592
|
+
req = self._approval_queue.get(approval_id)
|
|
593
|
+
if req is None:
|
|
594
|
+
return SupervisedResult(
|
|
595
|
+
error=AgentHandlerError.policy_denied(f"Approval {approval_id} not found"),
|
|
596
|
+
budget=self._budget.snapshot(),
|
|
597
|
+
)
|
|
598
|
+
if req.status == ApprovalStatus.DENIED:
|
|
599
|
+
return SupervisedResult(
|
|
600
|
+
error=AgentHandlerError.approval_denied(req.tool_name, req.deny_reason),
|
|
601
|
+
budget=self._budget.snapshot(),
|
|
602
|
+
tool_name=req.tool_name,
|
|
603
|
+
)
|
|
604
|
+
if req.status != ApprovalStatus.APPROVED:
|
|
605
|
+
return SupervisedResult(
|
|
606
|
+
error=AgentHandlerError.approval_pending(req.tool_name, approval_id),
|
|
607
|
+
budget=self._budget.snapshot(),
|
|
608
|
+
tool_name=req.tool_name,
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
# Approved — execute bypassing _check_confirm
|
|
612
|
+
self._audit.record(
|
|
613
|
+
AuditPhase.POLICY_CHECK,
|
|
614
|
+
AuditOutcome.ALLOWED,
|
|
615
|
+
target=req.tool_name,
|
|
616
|
+
detail=f"Approved (id: {approval_id}, by: {req.resolved_by})",
|
|
617
|
+
metadata={"approval_id": approval_id},
|
|
618
|
+
)
|
|
619
|
+
|
|
620
|
+
# Run through the normal execution path but skip confirmation
|
|
621
|
+
# We temporarily remove the tool from require_confirm
|
|
622
|
+
original_confirm = list(self._policy.require_confirm)
|
|
623
|
+
self._policy.require_confirm = [
|
|
624
|
+
t for t in self._policy.require_confirm if t != req.tool_name
|
|
625
|
+
]
|
|
626
|
+
try:
|
|
627
|
+
result = await self.call(req.tool_name, fn, **req.tool_args)
|
|
628
|
+
finally:
|
|
629
|
+
self._policy.require_confirm = original_confirm
|
|
630
|
+
|
|
631
|
+
return result
|
|
632
|
+
|
|
633
|
+
def call_sync(
|
|
634
|
+
self,
|
|
635
|
+
tool_name: str,
|
|
636
|
+
fn: Callable[..., Any],
|
|
637
|
+
**kwargs: Any,
|
|
638
|
+
) -> SupervisedResult:
|
|
639
|
+
"""Supervise a synchronous tool call.
|
|
640
|
+
|
|
641
|
+
Wraps the sync function and runs it through the async supervisor.
|
|
642
|
+
For use when you don't have an event loop.
|
|
643
|
+
"""
|
|
644
|
+
|
|
645
|
+
async def _wrapper(**kw: Any) -> Any:
|
|
646
|
+
return fn(**kw)
|
|
647
|
+
|
|
648
|
+
try:
|
|
649
|
+
loop = asyncio.get_event_loop()
|
|
650
|
+
if loop.is_running():
|
|
651
|
+
# We're inside an async context — can't use run()
|
|
652
|
+
import concurrent.futures
|
|
653
|
+
|
|
654
|
+
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
655
|
+
future = pool.submit(asyncio.run, self.call(tool_name, _wrapper, **kwargs))
|
|
656
|
+
return future.result(timeout=self._policy.tool_timeout + 5)
|
|
657
|
+
else:
|
|
658
|
+
return loop.run_until_complete(self.call(tool_name, _wrapper, **kwargs))
|
|
659
|
+
except RuntimeError:
|
|
660
|
+
return asyncio.run(self.call(tool_name, _wrapper, **kwargs))
|
|
661
|
+
|
|
662
|
+
def finish(self) -> BudgetSnapshot:
|
|
663
|
+
"""Mark the request as complete. Returns final budget snapshot."""
|
|
664
|
+
snap = self._budget.snapshot()
|
|
665
|
+
elapsed = time.monotonic() - self._start_time
|
|
666
|
+
self._audit.record(
|
|
667
|
+
AuditPhase.REQUEST_END,
|
|
668
|
+
AuditOutcome.INFO,
|
|
669
|
+
detail=f"Request complete: {snap.iterations} iterations, "
|
|
670
|
+
f"{snap.tokens_used} tokens, {int(elapsed * 1000)}ms",
|
|
671
|
+
metadata={
|
|
672
|
+
"tokens_used": snap.tokens_used,
|
|
673
|
+
"iterations": snap.iterations,
|
|
674
|
+
"duration_ms": int(elapsed * 1000),
|
|
675
|
+
},
|
|
676
|
+
)
|
|
677
|
+
return snap
|
|
678
|
+
|
|
679
|
+
# ------------------------------------------------------------------
|
|
680
|
+
# Checkpoint support (opt-in via store parameter)
|
|
681
|
+
# ------------------------------------------------------------------
|
|
682
|
+
|
|
683
|
+
def to_checkpoint_data(self) -> Dict[str, Any]:
|
|
684
|
+
"""Export supervisor state as a dict suitable for a Checkpoint."""
|
|
685
|
+
snap = self._budget.snapshot()
|
|
686
|
+
cb_states = {}
|
|
687
|
+
for name, cb in self._circuit_breakers.items():
|
|
688
|
+
cb_states[name] = {
|
|
689
|
+
"state": cb.state.value,
|
|
690
|
+
"consecutive_failures": cb.consecutive_failures,
|
|
691
|
+
}
|
|
692
|
+
return {
|
|
693
|
+
"iterations": snap.iterations,
|
|
694
|
+
"tokens_used": snap.tokens_used,
|
|
695
|
+
"token_limit": snap.token_limit,
|
|
696
|
+
"iteration_limit": snap.iteration_limit,
|
|
697
|
+
"circuit_breaker_states": cb_states,
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
@classmethod
|
|
701
|
+
def restore_from_checkpoint(
|
|
702
|
+
cls,
|
|
703
|
+
checkpoint: Checkpoint,
|
|
704
|
+
audit: Optional[AuditLog] = None,
|
|
705
|
+
store: Optional[StateStore] = None,
|
|
706
|
+
policy_override: Optional[Dict[str, Any]] = None,
|
|
707
|
+
approval_queue: Optional[ApprovalQueue] = None,
|
|
708
|
+
observer: Optional[Observer] = None,
|
|
709
|
+
pre_guardrails: Optional[List[Any]] = None,
|
|
710
|
+
post_guardrails: Optional[List[Any]] = None,
|
|
711
|
+
) -> Supervisor:
|
|
712
|
+
"""Rebuild a Supervisor from a checkpoint.
|
|
713
|
+
|
|
714
|
+
Restores budget counters and circuit breaker states so the agent
|
|
715
|
+
can continue where it left off.
|
|
716
|
+
|
|
717
|
+
Args:
|
|
718
|
+
checkpoint: The checkpoint to restore from.
|
|
719
|
+
audit: Optional audit log for the restored supervisor.
|
|
720
|
+
store: Optional state store for auto-checkpointing.
|
|
721
|
+
policy_override: If provided, use this policy dict instead of the
|
|
722
|
+
checkpoint's. This is the SECURITY mechanism —
|
|
723
|
+
SessionManager always passes the original immutable
|
|
724
|
+
policy here so agents cannot downgrade supervision.
|
|
725
|
+
"""
|
|
726
|
+
from .store import SessionStatus
|
|
727
|
+
|
|
728
|
+
# SECURITY: prefer the override (original immutable policy) over
|
|
729
|
+
# whatever the checkpoint contains.
|
|
730
|
+
policy_dict = policy_override if policy_override is not None else checkpoint.policy_dict
|
|
731
|
+
policy = Policy.from_dict(policy_dict)
|
|
732
|
+
sv = cls(
|
|
733
|
+
policy,
|
|
734
|
+
audit=audit,
|
|
735
|
+
store=store,
|
|
736
|
+
session_id=checkpoint.session_id,
|
|
737
|
+
agent_id=checkpoint.agent_id,
|
|
738
|
+
approval_queue=approval_queue,
|
|
739
|
+
observer=observer,
|
|
740
|
+
pre_guardrails=pre_guardrails,
|
|
741
|
+
post_guardrails=post_guardrails,
|
|
742
|
+
)
|
|
743
|
+
# Restore budget counters
|
|
744
|
+
if checkpoint.tokens_used > 0:
|
|
745
|
+
sv._budget._tokens_used = checkpoint.tokens_used
|
|
746
|
+
if checkpoint.iterations > 0:
|
|
747
|
+
sv._budget._iterations = checkpoint.iterations
|
|
748
|
+
# Restore circuit breaker states
|
|
749
|
+
for name, cb_data in checkpoint.circuit_breaker_states.items():
|
|
750
|
+
from .circuit_breaker import CircuitState
|
|
751
|
+
|
|
752
|
+
cb = sv._get_circuit_breaker(name)
|
|
753
|
+
cb._consecutive_failures = cb_data.get("consecutive_failures", 0)
|
|
754
|
+
state_str = cb_data.get("state", "closed")
|
|
755
|
+
cb._state = CircuitState(state_str)
|
|
756
|
+
if cb._state == CircuitState.OPEN:
|
|
757
|
+
cb._opened_at = time.monotonic()
|
|
758
|
+
# Restore pause state
|
|
759
|
+
sv._paused = checkpoint.status == SessionStatus.PAUSED
|
|
760
|
+
return sv
|
|
761
|
+
|
|
762
|
+
def _build_guardrail_context(self) -> Dict[str, Any]:
|
|
763
|
+
"""Build the context dict passed to guardrails."""
|
|
764
|
+
return {
|
|
765
|
+
"session_id": self._session_id or "",
|
|
766
|
+
"agent_id": self._agent_id or "",
|
|
767
|
+
"recent_calls": list(self._recent_calls),
|
|
768
|
+
"iterations": self._budget.snapshot().iterations,
|
|
769
|
+
"tokens_used": self._budget.snapshot().tokens_used,
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
async def _run_guardrail(
|
|
773
|
+
self,
|
|
774
|
+
guardrail: Any,
|
|
775
|
+
tool_name: str,
|
|
776
|
+
kwargs: Dict[str, Any],
|
|
777
|
+
output: Any,
|
|
778
|
+
context: Dict[str, Any],
|
|
779
|
+
phase: str,
|
|
780
|
+
) -> Any:
|
|
781
|
+
"""Run a guardrail, handling both sync and async check methods."""
|
|
782
|
+
from .guardrails.types import GuardrailResult
|
|
783
|
+
|
|
784
|
+
try:
|
|
785
|
+
# Try async methods first
|
|
786
|
+
if hasattr(guardrail, "check_async"):
|
|
787
|
+
if phase == "pre":
|
|
788
|
+
# Some async checks take phase, some take output
|
|
789
|
+
import inspect
|
|
790
|
+
|
|
791
|
+
sig = inspect.signature(guardrail.check_async)
|
|
792
|
+
if "phase" in sig.parameters:
|
|
793
|
+
return await guardrail.check_async(
|
|
794
|
+
tool_name, kwargs, output, context, phase=phase
|
|
795
|
+
)
|
|
796
|
+
if "output" in sig.parameters:
|
|
797
|
+
return await guardrail.check_async(tool_name, kwargs, output, context)
|
|
798
|
+
return await guardrail.check_async(tool_name, kwargs, context)
|
|
799
|
+
else:
|
|
800
|
+
import inspect
|
|
801
|
+
|
|
802
|
+
sig = inspect.signature(guardrail.check_async)
|
|
803
|
+
if "phase" in sig.parameters:
|
|
804
|
+
return await guardrail.check_async(
|
|
805
|
+
tool_name, kwargs, output, context, phase=phase
|
|
806
|
+
)
|
|
807
|
+
if "output" in sig.parameters:
|
|
808
|
+
return await guardrail.check_async(tool_name, kwargs, output, context)
|
|
809
|
+
return await guardrail.check_async(tool_name, kwargs, context)
|
|
810
|
+
if hasattr(guardrail, "check"):
|
|
811
|
+
# Sync check — some take output, some don't
|
|
812
|
+
import inspect
|
|
813
|
+
|
|
814
|
+
sig = inspect.signature(guardrail.check)
|
|
815
|
+
params = sig.parameters
|
|
816
|
+
if "output" in params:
|
|
817
|
+
return guardrail.check(tool_name, kwargs, output, context)
|
|
818
|
+
return guardrail.check(tool_name, kwargs, context)
|
|
819
|
+
except Exception as e:
|
|
820
|
+
return GuardrailResult(
|
|
821
|
+
allowed=True,
|
|
822
|
+
guardrail_name=getattr(guardrail, "name", "unknown"),
|
|
823
|
+
reason=f"Guardrail error (allowing): {e}",
|
|
824
|
+
)
|
|
825
|
+
return GuardrailResult.allow(getattr(guardrail, "name", "unknown"))
|
|
826
|
+
|
|
827
|
+
def _observe_call(self, result: SupervisedResult) -> None:
|
|
828
|
+
"""Record a tool call with the observer if configured."""
|
|
829
|
+
if self._observer is None or result.tool_name is None:
|
|
830
|
+
return
|
|
831
|
+
meta = self._last_call_meta
|
|
832
|
+
self._observer.record_tool_call(
|
|
833
|
+
tool_name=result.tool_name,
|
|
834
|
+
duration_ms=result.duration_ms,
|
|
835
|
+
succeeded=result.succeeded,
|
|
836
|
+
session_id=self._session_id or "",
|
|
837
|
+
agent_id=self._agent_id or "",
|
|
838
|
+
error_kind=result.error.kind if result.error else "",
|
|
839
|
+
input_tokens=meta.get("input_tokens", 0),
|
|
840
|
+
output_tokens=meta.get("output_tokens", 0),
|
|
841
|
+
model=meta.get("model", ""),
|
|
842
|
+
)
|
|
843
|
+
# Clear for next call
|
|
844
|
+
self._last_call_meta = {"input_tokens": 0, "output_tokens": 0, "model": ""}
|
|
845
|
+
|
|
846
|
+
def _auto_checkpoint(self) -> None:
|
|
847
|
+
"""Save a checkpoint if a store is configured.
|
|
848
|
+
|
|
849
|
+
Only updates SYSTEM-CONTROLLED fields (budget counters, circuit breaker
|
|
850
|
+
states, status, timestamp). Does NOT touch payload (agent-writable) or
|
|
851
|
+
policy_dict (immutable). This preserves the trust boundary between
|
|
852
|
+
supervisor-controlled and agent-controlled data.
|
|
853
|
+
"""
|
|
854
|
+
if self._store is None or self._session_id is None:
|
|
855
|
+
return
|
|
856
|
+
from .store import Checkpoint, SessionStatus
|
|
857
|
+
|
|
858
|
+
# Load existing checkpoint to preserve payload, policy, created_at, audit_log
|
|
859
|
+
existing = self._store.load_checkpoint(self._session_id)
|
|
860
|
+
|
|
861
|
+
status = SessionStatus.PAUSED if self._paused else SessionStatus.RUNNING
|
|
862
|
+
data = self.to_checkpoint_data()
|
|
863
|
+
cp = Checkpoint(
|
|
864
|
+
session_id=self._session_id,
|
|
865
|
+
agent_id=self._agent_id or "",
|
|
866
|
+
status=status,
|
|
867
|
+
iterations=data["iterations"],
|
|
868
|
+
tokens_used=data["tokens_used"],
|
|
869
|
+
token_limit=data["token_limit"],
|
|
870
|
+
iteration_limit=data["iteration_limit"],
|
|
871
|
+
circuit_breaker_states=data["circuit_breaker_states"],
|
|
872
|
+
# Preserve immutable fields from the original checkpoint
|
|
873
|
+
policy_dict=existing.policy_dict if existing else self._policy.to_dict(),
|
|
874
|
+
payload=existing.payload if existing else {},
|
|
875
|
+
timestamp=datetime.now(timezone.utc).isoformat(),
|
|
876
|
+
created_at=existing.created_at if existing else "",
|
|
877
|
+
audit_log=existing.audit_log if existing else [],
|
|
878
|
+
)
|
|
879
|
+
try:
|
|
880
|
+
self._store.save_checkpoint(cp)
|
|
881
|
+
except Exception:
|
|
882
|
+
pass # checkpoint must never crash the request
|