nullrun 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nullrun/__init__.py +282 -0
- nullrun/__version__.py +4 -0
- nullrun/actions.py +455 -0
- nullrun/breaker/__init__.py +27 -0
- nullrun/breaker/circuit_breaker.py +402 -0
- nullrun/breaker/exceptions.py +319 -0
- nullrun/context.py +208 -0
- nullrun/decorators.py +649 -0
- nullrun/instrumentation/__init__.py +23 -0
- nullrun/instrumentation/_safe_patch.py +99 -0
- nullrun/instrumentation/auto.py +1095 -0
- nullrun/instrumentation/auto_requests.py +257 -0
- nullrun/instrumentation/autogen.py +163 -0
- nullrun/instrumentation/crewai.py +140 -0
- nullrun/instrumentation/langgraph.py +412 -0
- nullrun/instrumentation/llama_index.py +110 -0
- nullrun/observability.py +160 -0
- nullrun/py.typed +0 -0
- nullrun/runtime.py +1806 -0
- nullrun/toolbox/__init__.py +20 -0
- nullrun/toolbox/langgraph.py +94 -0
- nullrun/tracing.py +155 -0
- nullrun/transport.py +1509 -0
- nullrun/transport_websocket.py +627 -0
- nullrun-0.4.0.dist-info/METADATA +194 -0
- nullrun-0.4.0.dist-info/RECORD +28 -0
- nullrun-0.4.0.dist-info/WHEEL +4 -0
- nullrun-0.4.0.dist-info/licenses/LICENSE +201 -0
nullrun/decorators.py
ADDED
|
@@ -0,0 +1,649 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Decorators for the NullRun SDK.
|
|
3
|
+
|
|
4
|
+
Public surface (Phase 2 Commit 4): `protect` is the only gate decorator.
|
|
5
|
+
It takes NO parameters — span hierarchy is built automatically from the
|
|
6
|
+
caller's context via contextvars, and the workflow is derived from the
|
|
7
|
+
API key on the backend (the dashboard surfaces the agent's name from
|
|
8
|
+
the key's `name` field).
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
# Basic — auto-init from env, auto-build span tree
|
|
12
|
+
import nullrun
|
|
13
|
+
nullrun.init(api_key="...")
|
|
14
|
+
|
|
15
|
+
@nullrun.protect
|
|
16
|
+
def my_agent(query: str) -> str:
|
|
17
|
+
return call_llm(query)
|
|
18
|
+
|
|
19
|
+
@nullrun.protect
|
|
20
|
+
async def my_async_agent(query: str) -> str:
|
|
21
|
+
return await call_llm_async(query)
|
|
22
|
+
|
|
23
|
+
# Manual: protected functions compose into a tree automatically
|
|
24
|
+
@nullrun.protect
|
|
25
|
+
def orchestrator(q):
|
|
26
|
+
return researcher(q) # researcher is a child span
|
|
27
|
+
|
|
28
|
+
@nullrun.protect
|
|
29
|
+
def researcher(q):
|
|
30
|
+
return get_current_span() # parent's span_id == its parent_span_id
|
|
31
|
+
|
|
32
|
+
`reset` and `get_protected_runtime` are the runtime-lifecycle helpers.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import functools
|
|
38
|
+
import inspect
|
|
39
|
+
import logging
|
|
40
|
+
import os
|
|
41
|
+
from collections.abc import Callable
|
|
42
|
+
from typing import Any, TypeVar
|
|
43
|
+
|
|
44
|
+
from nullrun.breaker.exceptions import (
|
|
45
|
+
NullRunBlockedException,
|
|
46
|
+
WorkflowKilledInterrupt,
|
|
47
|
+
WorkflowPausedException,
|
|
48
|
+
)
|
|
49
|
+
from nullrun.context import get_workflow_id
|
|
50
|
+
from nullrun.runtime import NullRunRuntime, get_runtime
|
|
51
|
+
|
|
52
|
+
# Sentinel used when a gate fires outside a workflow context.
|
|
53
|
+
# Matches the constant in nullrun.runtime so we don't introduce
|
|
54
|
+
# a new magic string in audit logs.
|
|
55
|
+
UNKNOWN_WORKFLOW_ID = "__nullrun_unknown__"
|
|
56
|
+
|
|
57
|
+
from nullrun.tracing import (
|
|
58
|
+
SpanContext,
|
|
59
|
+
create_child_span,
|
|
60
|
+
create_root_span,
|
|
61
|
+
get_current_span,
|
|
62
|
+
reset_span,
|
|
63
|
+
set_span,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
logger = logging.getLogger(__name__)
|
|
67
|
+
|
|
68
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
69
|
+
|
|
70
|
+
# Phase 3: expanded sensitive-arg keys. The original 7-key set
|
|
71
|
+
# missed obvious PII tokens and credential names; ``@sensitive`` and
|
|
72
|
+
# ``_safe_kwargs`` would have shipped them in the audit log.
|
|
73
|
+
# Matching is case-insensitive (see ``_safe_kwargs`` which calls
|
|
74
|
+
# ``.lower()`` on the key).
|
|
75
|
+
SENSITIVE_ARG_KEYS = frozenset({
|
|
76
|
+
# Credentials / secrets
|
|
77
|
+
"password", "passwd", "pwd",
|
|
78
|
+
"token", "secret", "api_key", "apikey",
|
|
79
|
+
"key", "auth", "authorization", "bearer",
|
|
80
|
+
"session", "session_id", "cookie",
|
|
81
|
+
"access_token", "refresh_token", "id_token",
|
|
82
|
+
"private_key", "secret_key",
|
|
83
|
+
# PII
|
|
84
|
+
"email", "phone", "ssn",
|
|
85
|
+
"credit_card", "credit_card_number", "cvv", "cvc", "pin",
|
|
86
|
+
"otp", "mfa",
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _safe_repr(value: object, max_len: int = 50) -> str:
|
|
91
|
+
"""Safe representation of an argument for logging."""
|
|
92
|
+
r = repr(value)
|
|
93
|
+
if len(r) > max_len:
|
|
94
|
+
return r[:max_len] + "...<truncated>"
|
|
95
|
+
return r
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _safe_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
|
|
99
|
+
"""Mask sensitive kwargs (case-insensitive)."""
|
|
100
|
+
return {
|
|
101
|
+
k: "***" if k.lower() in SENSITIVE_ARG_KEYS else _safe_repr(v)
|
|
102
|
+
for k, v in kwargs.items()
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# SEC-29: strip the `details={...}` payload from an exception's
|
|
107
|
+
# string form before it lands in the span_end audit event.
|
|
108
|
+
# Phase 3 replaced the previous one-level regex with a
|
|
109
|
+
# balanced-brace walker that handles nested dicts and dict values
|
|
110
|
+
# that contain `{` / `}` in their string content.
|
|
111
|
+
_DETAILS_REDACTED = "<redacted>" # the payload only — caller prepends "details="
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _strip_details_balanced(text: str) -> str:
|
|
115
|
+
"""Replace every top-level ``details={...}`` substring with
|
|
116
|
+
``details=<redacted>``.
|
|
117
|
+
|
|
118
|
+
Walks the string with a small state machine that tracks
|
|
119
|
+
brace depth and string-literal state. At depth 1 the opening
|
|
120
|
+
``{`` was just consumed; when the depth returns to 0 the
|
|
121
|
+
substring is replaced. The walker tolerates ``{`` and ``}``
|
|
122
|
+
inside string values so it does not under-report nesting.
|
|
123
|
+
|
|
124
|
+
Only ``details={…}`` constructs are redacted; a bare
|
|
125
|
+
``details=foo`` (no opening brace) is left as-is so we
|
|
126
|
+
don't lose the user's free-form text.
|
|
127
|
+
"""
|
|
128
|
+
out: list[str] = []
|
|
129
|
+
i = 0
|
|
130
|
+
n = len(text)
|
|
131
|
+
needle = "details="
|
|
132
|
+
while i < n:
|
|
133
|
+
idx = text.find(needle, i)
|
|
134
|
+
if idx < 0:
|
|
135
|
+
out.append(text[i:])
|
|
136
|
+
break
|
|
137
|
+
out.append(text[i:idx])
|
|
138
|
+
j = idx + len(needle)
|
|
139
|
+
while j < n and text[j] in " \t":
|
|
140
|
+
j += 1
|
|
141
|
+
if j >= n or text[j] != "{":
|
|
142
|
+
end = j
|
|
143
|
+
while end < n and text[end] not in ",)\n":
|
|
144
|
+
end += 1
|
|
145
|
+
out.append(text[idx:end])
|
|
146
|
+
i = end
|
|
147
|
+
continue
|
|
148
|
+
out.append(text[idx:j])
|
|
149
|
+
depth = 0
|
|
150
|
+
in_str: str | None = None
|
|
151
|
+
k = j
|
|
152
|
+
while k < n:
|
|
153
|
+
ch = text[k]
|
|
154
|
+
if in_str is not None:
|
|
155
|
+
if ch == "\\" and k + 1 < n:
|
|
156
|
+
k += 2
|
|
157
|
+
continue
|
|
158
|
+
if ch == in_str:
|
|
159
|
+
in_str = None
|
|
160
|
+
elif ch in ('"', "'"):
|
|
161
|
+
in_str = ch
|
|
162
|
+
elif ch == "{":
|
|
163
|
+
depth += 1
|
|
164
|
+
elif ch == "}":
|
|
165
|
+
depth -= 1
|
|
166
|
+
if depth == 0:
|
|
167
|
+
k += 1
|
|
168
|
+
break
|
|
169
|
+
k += 1
|
|
170
|
+
out.append(_DETAILS_REDACTED)
|
|
171
|
+
i = k
|
|
172
|
+
return "".join(out)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _safe_error_str(error: BaseException | None) -> str | None:
|
|
176
|
+
"""Return a log-safe string for ``error`` (SEC-29, Phase 3)."""
|
|
177
|
+
if error is None:
|
|
178
|
+
return None
|
|
179
|
+
raw = str(error)
|
|
180
|
+
return _strip_details_balanced(raw)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
# Module-level cache for the runtime instance — the @protect decorator needs
|
|
184
|
+
# a runtime to emit span_start/span_end events, but the runtime is normally
|
|
185
|
+
# created via `nullrun.init()`. We lazily instantiate one if @protect is
|
|
186
|
+
# used before init(). The slot is also where tests can inject a noop.
|
|
187
|
+
_runtime: NullRunRuntime | None = None
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _get_or_create_runtime() -> NullRunRuntime:
|
|
191
|
+
"""Lazy initialization of runtime from environment.
|
|
192
|
+
|
|
193
|
+
Order of resolution:
|
|
194
|
+
1. The module-level `_runtime` slot (set by tests or by `init()`)
|
|
195
|
+
2. The global `NullRunRuntime.get_instance()` singleton, which
|
|
196
|
+
reads `NULLRUN_API_KEY` / `NULLRUN_API_URL` from the environment
|
|
197
|
+
and constructs the canonical cloud runtime.
|
|
198
|
+
|
|
199
|
+
FIX-4 (0.3.x): the previous code wrapped `get_instance()` in a
|
|
200
|
+
`try/except` that caught every exception and rebuilt a no-arg
|
|
201
|
+
`NullRunRuntime()` as a "fallback". That fallback was doubly broken
|
|
202
|
+
in 0.3.0: it silently swallowed `NullRunAuthenticationError` raised
|
|
203
|
+
by the env-var-less branch, then crashed with the same error from
|
|
204
|
+
the no-arg `NullRunRuntime()` constructor (which also requires
|
|
205
|
+
`api_key` per T3-S2). The net effect was a delayed crash with a
|
|
206
|
+
worse error message, plus a misleading "we have a runtime" log line.
|
|
207
|
+
|
|
208
|
+
The fix removes the fallback entirely. `get_instance()` propagates
|
|
209
|
+
`NullRunAuthenticationError` to the caller, where it surfaces at
|
|
210
|
+
the first `@protect` invocation — the same fail-loud path that
|
|
211
|
+
`nullrun.init()` uses. This aligns with the T3-S2 invariant that
|
|
212
|
+
the SDK has no local mode: a missing API key must be a hard error,
|
|
213
|
+
not a silent allow-all.
|
|
214
|
+
|
|
215
|
+
Tries to patch OpenAI on first creation so the auto-instrumentation
|
|
216
|
+
path picks up the runtime the user will eventually use.
|
|
217
|
+
"""
|
|
218
|
+
global _runtime
|
|
219
|
+
|
|
220
|
+
if _runtime is not None:
|
|
221
|
+
return _runtime
|
|
222
|
+
|
|
223
|
+
_runtime = NullRunRuntime.get_instance()
|
|
224
|
+
|
|
225
|
+
# The previous OpenAI v0.x auto-patch hook was removed in 0.4.0:
|
|
226
|
+
# openai>=1.0 does not expose ChatCompletion.create as an
|
|
227
|
+
# attribute. All OpenAI v1.0+ traffic is now tracked
|
|
228
|
+
# vendor-independently by the httpx transport hook in
|
|
229
|
+
# nullrun.instrumentation.auto, which is wired by
|
|
230
|
+
# nullrun.init() — not at the lazy-resolve path here.
|
|
231
|
+
logger.info("NullRun runtime initialized: mode=cloud")
|
|
232
|
+
return _runtime
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _next_span() -> SpanContext:
|
|
236
|
+
"""
|
|
237
|
+
Derive the span for a new @protect call.
|
|
238
|
+
|
|
239
|
+
If we're already inside a span (i.e. nested @protect calls), the new
|
|
240
|
+
span is a child of the current one. Otherwise we open a fresh root —
|
|
241
|
+
the dashboard reconstructs the whole tree from the `parent_span_id`
|
|
242
|
+
chain emitted in span_start events.
|
|
243
|
+
"""
|
|
244
|
+
parent = get_current_span()
|
|
245
|
+
if parent is None:
|
|
246
|
+
return create_root_span()
|
|
247
|
+
return create_child_span(parent)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _emit_span_start(runtime: Any, ctx: SpanContext, fn_name: str) -> None:
|
|
251
|
+
"""
|
|
252
|
+
Best-effort emission of a span_start event.
|
|
253
|
+
|
|
254
|
+
A failure here must NEVER block the wrapped function — observability
|
|
255
|
+
is downstream of the user's work. We swallow every exception.
|
|
256
|
+
"""
|
|
257
|
+
try:
|
|
258
|
+
runtime.track_event(
|
|
259
|
+
event_type="span_start",
|
|
260
|
+
trace_id=ctx.trace_id,
|
|
261
|
+
span_id=ctx.span_id,
|
|
262
|
+
parent_span_id=ctx.parent_span_id,
|
|
263
|
+
depth=ctx.depth,
|
|
264
|
+
fn_name=fn_name,
|
|
265
|
+
)
|
|
266
|
+
except Exception as exc: # noqa: BLE001
|
|
267
|
+
logger.debug(f"span_start emission failed: {exc}")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _emit_span_end(
|
|
271
|
+
runtime: Any,
|
|
272
|
+
ctx: SpanContext,
|
|
273
|
+
error: str | None = None,
|
|
274
|
+
) -> None:
|
|
275
|
+
"""
|
|
276
|
+
Best-effort emission of a span_end event. Same contract as
|
|
277
|
+
`_emit_span_start` — never blocks.
|
|
278
|
+
"""
|
|
279
|
+
try:
|
|
280
|
+
runtime.track_event(
|
|
281
|
+
event_type="span_end",
|
|
282
|
+
trace_id=ctx.trace_id,
|
|
283
|
+
span_id=ctx.span_id,
|
|
284
|
+
parent_span_id=ctx.parent_span_id,
|
|
285
|
+
depth=ctx.depth,
|
|
286
|
+
fn_name=getattr(ctx, "fn_name", None),
|
|
287
|
+
error=error,
|
|
288
|
+
)
|
|
289
|
+
except Exception as exc: # noqa: BLE001
|
|
290
|
+
logger.debug(f"span_end emission failed: {exc}")
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def protect(fn: F | None = None) -> F | Callable[[F], F]:
|
|
294
|
+
"""
|
|
295
|
+
Decorator that wraps a function in a NullRun span.
|
|
296
|
+
|
|
297
|
+
Usage:
|
|
298
|
+
@nullrun.protect
|
|
299
|
+
def my_agent(query: str) -> str:
|
|
300
|
+
...
|
|
301
|
+
|
|
302
|
+
@nullrun.protect
|
|
303
|
+
async def my_async_agent(query: str) -> str:
|
|
304
|
+
...
|
|
305
|
+
|
|
306
|
+
The span hierarchy is built automatically from the calling context
|
|
307
|
+
(via `nullrun.tracing.SpanContext` contextvars) — nested `@protect`
|
|
308
|
+
calls become child spans of the outer one. No parameters are needed:
|
|
309
|
+
the workflow is derived from the API key on the backend.
|
|
310
|
+
|
|
311
|
+
## Pre-execution gate order (ADR-008 Rule 4)
|
|
312
|
+
|
|
313
|
+
The wrapper runs three gates in this order. KILL short-circuits:
|
|
314
|
+
|
|
315
|
+
1. `check_control_plane` — KILL/PAUSE is terminal.
|
|
316
|
+
2. `check_workflow_budget` — "any budget left?" via /gate.
|
|
317
|
+
3. `_enforce_sensitive_tool` — per-tool policy (no-op if not
|
|
318
|
+
marked sensitive).
|
|
319
|
+
|
|
320
|
+
Each gate has its own fail-OPEN/CLOSED policy declared in
|
|
321
|
+
`runtime.py`; see ADR-008 Rule 5 for the full table. `span_end`
|
|
322
|
+
is emitted on every path (including KILL/PAUSE) so the dashboard
|
|
323
|
+
can render the kill with span context.
|
|
324
|
+
|
|
325
|
+
`fn` may be omitted to return the decorator itself (the standard
|
|
326
|
+
`@decorator` vs `@decorator()` shape), so this works for both:
|
|
327
|
+
|
|
328
|
+
@nullrun.protect
|
|
329
|
+
def f(): ...
|
|
330
|
+
|
|
331
|
+
@nullrun.protect()
|
|
332
|
+
def g(): ...
|
|
333
|
+
"""
|
|
334
|
+
if fn is None:
|
|
335
|
+
# `@nullrun.protect()` with empty parens — return the decorator
|
|
336
|
+
# bound to itself so the next call wraps the target function.
|
|
337
|
+
return protect
|
|
338
|
+
|
|
339
|
+
if inspect.iscoroutinefunction(fn):
|
|
340
|
+
|
|
341
|
+
@functools.wraps(fn)
|
|
342
|
+
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
343
|
+
runtime = _get_or_create_runtime()
|
|
344
|
+
span = _next_span()
|
|
345
|
+
token = set_span(span)
|
|
346
|
+
|
|
347
|
+
# ADR-008 Rule 4: gate order is
|
|
348
|
+
# control_plane → budget → span_start → sensitive
|
|
349
|
+
# Wrapped in try/except so span_end still emits on KILL/PAUSE.
|
|
350
|
+
error: BaseException | None = None
|
|
351
|
+
try:
|
|
352
|
+
# 1. KILL/PAUSE from the dashboard short-circuits
|
|
353
|
+
# everything else. The resolution order is the
|
|
354
|
+
# user-set contextvar first, then the API-key-bound
|
|
355
|
+
# workflow — same precedence as check_workflow_budget.
|
|
356
|
+
runtime.check_control_plane(get_workflow_id() or None)
|
|
357
|
+
|
|
358
|
+
# 2. Budget pre-flight via /gate. Raises
|
|
359
|
+
# WorkflowKilledInterrupt on real block; fails open
|
|
360
|
+
# on transport error (see runtime.check_workflow_budget).
|
|
361
|
+
runtime.check_workflow_budget()
|
|
362
|
+
|
|
363
|
+
# 3. Span start — best-effort, never blocks.
|
|
364
|
+
_emit_span_start(runtime, span, fn.__name__)
|
|
365
|
+
|
|
366
|
+
# 4. Per-tool policy for @sensitive tools. Fails CLOSED
|
|
367
|
+
# on transport error (see _enforce_sensitive_tool).
|
|
368
|
+
_enforce_sensitive_tool(runtime, fn, args, kwargs)
|
|
369
|
+
|
|
370
|
+
return await fn(*args, **kwargs)
|
|
371
|
+
except BaseException as exc: # noqa: BLE001
|
|
372
|
+
# Capture the error so we can include it in span_end
|
|
373
|
+
# *after* the contextvar is reset. Re-raise so the
|
|
374
|
+
# caller's try/except still sees the original exception.
|
|
375
|
+
error = exc
|
|
376
|
+
raise
|
|
377
|
+
finally:
|
|
378
|
+
reset_span(token)
|
|
379
|
+
_emit_span_end(
|
|
380
|
+
runtime,
|
|
381
|
+
span,
|
|
382
|
+
error=_safe_error_str(error),
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
return async_wrapper # type: ignore[return-value]
|
|
386
|
+
|
|
387
|
+
@functools.wraps(fn)
|
|
388
|
+
def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
389
|
+
runtime = _get_or_create_runtime()
|
|
390
|
+
span = _next_span()
|
|
391
|
+
token = set_span(span)
|
|
392
|
+
|
|
393
|
+
# ADR-008 Rule 4: gate order is
|
|
394
|
+
# control_plane → budget → span_start → sensitive
|
|
395
|
+
# Wrapped in try/except so span_end still emits on KILL/PAUSE.
|
|
396
|
+
error: BaseException | None = None
|
|
397
|
+
try:
|
|
398
|
+
# 1. KILL/PAUSE from the dashboard short-circuits
|
|
399
|
+
# everything else. The resolution order is the
|
|
400
|
+
# user-set contextvar first, then the API-key-bound
|
|
401
|
+
# workflow — same precedence as check_workflow_budget.
|
|
402
|
+
runtime.check_control_plane(get_workflow_id() or None)
|
|
403
|
+
|
|
404
|
+
# 2. Budget pre-flight via /gate. Raises
|
|
405
|
+
# WorkflowKilledInterrupt on real block; fails open
|
|
406
|
+
# on transport error (see runtime.check_workflow_budget).
|
|
407
|
+
runtime.check_workflow_budget()
|
|
408
|
+
|
|
409
|
+
# 3. Span start — best-effort, never blocks.
|
|
410
|
+
_emit_span_start(runtime, span, fn.__name__)
|
|
411
|
+
|
|
412
|
+
# 4. Per-tool policy for @sensitive tools. Fails CLOSED
|
|
413
|
+
# on transport error (see _enforce_sensitive_tool).
|
|
414
|
+
_enforce_sensitive_tool(runtime, fn, args, kwargs)
|
|
415
|
+
|
|
416
|
+
return fn(*args, **kwargs)
|
|
417
|
+
except BaseException as exc: # noqa: BLE001
|
|
418
|
+
error = exc
|
|
419
|
+
# Round 3 (Phase 0.4.0): unify the "blocked" signal at
|
|
420
|
+
# the @protect boundary so callers can catch a single
|
|
421
|
+
# NullRunBlockedException for both policy blocks and
|
|
422
|
+
# sensitive-tool blocks. Direct calls to
|
|
423
|
+
# check_workflow_budget() still raise the original
|
|
424
|
+
# exception type so callers that distinguish hard vs
|
|
425
|
+
# soft blocks keep that signal.
|
|
426
|
+
if isinstance(exc, (WorkflowKilledInterrupt, WorkflowPausedException)):
|
|
427
|
+
raise NullRunBlockedException(
|
|
428
|
+
workflow_id=exc.workflow_id,
|
|
429
|
+
reason=exc.reason,
|
|
430
|
+
) from exc
|
|
431
|
+
raise
|
|
432
|
+
finally:
|
|
433
|
+
reset_span(token)
|
|
434
|
+
_emit_span_end(
|
|
435
|
+
runtime,
|
|
436
|
+
span,
|
|
437
|
+
error=_safe_error_str(error),
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
return sync_wrapper # type: ignore[return-value]
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _enforce_sensitive_tool(
|
|
444
|
+
runtime: Any,
|
|
445
|
+
fn: Callable[..., Any],
|
|
446
|
+
args: tuple[Any, ...],
|
|
447
|
+
kwargs: dict[str, Any],
|
|
448
|
+
) -> None:
|
|
449
|
+
"""
|
|
450
|
+
Pre-execution policy check for sensitive tools.
|
|
451
|
+
|
|
452
|
+
If `fn.__name__` is in the runtime's sensitive-tool set (built-in
|
|
453
|
+
or registered via `add_sensitive_tool` / `@sensitive`), call
|
|
454
|
+
`runtime.execute(...)` BEFORE the body runs. The /execute endpoint
|
|
455
|
+
is the authoritative gate; `NullRunBlockedException` propagates to
|
|
456
|
+
the caller, mirroring the contract of `check_workflow_budget`.
|
|
457
|
+
|
|
458
|
+
kwargs are masked via `SENSITIVE_ARG_KEYS` so passwords / tokens
|
|
459
|
+
never leave the process. The same masking is used for span events.
|
|
460
|
+
|
|
461
|
+
## Fail-OPEN/CLOSED Policy (ADR-008)
|
|
462
|
+
|
|
463
|
+
This gate is **fail-CLOSED**: the body MUST NOT run when the
|
|
464
|
+
policy engine is unreachable, regardless of what /execute returns.
|
|
465
|
+
Two failure paths both result in `NullRunBlockedException`:
|
|
466
|
+
|
|
467
|
+
1. **Transport raises** `NullRunTransportError` (the new
|
|
468
|
+
`on_transport_error="raise"` path): the runtime layer surfaces
|
|
469
|
+
classified NETWORK / GATEWAY / BREAKER-OPEN failures as
|
|
470
|
+
exceptions. The body of this gate catches them and re-raises
|
|
471
|
+
as `NullRunBlockedException` with the source in the reason
|
|
472
|
+
("policy engine unavailable: NETWORK_ERROR" etc.).
|
|
473
|
+
|
|
474
|
+
2. **Transport returns a dict** whose `decision_source` starts
|
|
475
|
+
with `FALLBACK_` (defense in depth — covers the legacy
|
|
476
|
+
`fallback_mode=PERMISSIVE` path and any future regression in
|
|
477
|
+
`runtime.execute` that drops the `on_transport_error="raise"`
|
|
478
|
+
argument). The body of this gate inspects the result and
|
|
479
|
+
re-raises as `NullRunBlockedException` before the wrapped
|
|
480
|
+
function runs.
|
|
481
|
+
|
|
482
|
+
This is the opposite of `check_workflow_budget` /
|
|
483
|
+
`check_control_plane`, which deliberately fail-OPEN — a transient
|
|
484
|
+
backend outage must not freeze the user's agent. Sensitive tools
|
|
485
|
+
have a different threat model: an unblocked `charge_card()` that
|
|
486
|
+
runs when the policy engine is down is worse than a denied
|
|
487
|
+
`charge_card()` during an outage.
|
|
488
|
+
|
|
489
|
+
Opt-out: set `NULLRUN_SENSITIVE_FAIL_OPEN=1` to restore the prior
|
|
490
|
+
fail-OPEN behavior on transport error. Useful in dev / test
|
|
491
|
+
environments where the policy engine is intentionally absent.
|
|
492
|
+
The opt-out is intentionally scoped to the *transport-error*
|
|
493
|
+
case; a real `decision=block` from the gateway is still honored
|
|
494
|
+
and still raises `NullRunBlockedException`.
|
|
495
|
+
"""
|
|
496
|
+
if not runtime.is_sensitive_tool(fn.__name__):
|
|
497
|
+
return
|
|
498
|
+
masked = _safe_kwargs(kwargs)
|
|
499
|
+
|
|
500
|
+
# ADR-008: prefer `on_transport_error` (raise classified
|
|
501
|
+
# NullRunTransportError); fall back to legacy `fallback_mode` for
|
|
502
|
+
# older runtimes that pre-date the rename.
|
|
503
|
+
from nullrun.breaker.exceptions import (
|
|
504
|
+
NullRunBlockedException,
|
|
505
|
+
NullRunTransportError,
|
|
506
|
+
TransportErrorSource,
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
fail_open = os.environ.get("NULLRUN_SENSITIVE_FAIL_OPEN", "").strip() == "1"
|
|
510
|
+
workflow_id = get_workflow_id() or UNKNOWN_WORKFLOW_ID
|
|
511
|
+
|
|
512
|
+
try:
|
|
513
|
+
# Round 3 (Phase 0.4.0): pass on_transport_error="raise" so
|
|
514
|
+
# the transport raises NullRunTransportError on network / 5xx
|
|
515
|
+
# failure instead of returning a synthetic dict. The arm
|
|
516
|
+
# below converts the typed error into NullRunBlockedException
|
|
517
|
+
# so the caller's `except NullRunBlockedException` catches it
|
|
518
|
+
# uniformly.
|
|
519
|
+
result = runtime.execute(
|
|
520
|
+
fn.__name__,
|
|
521
|
+
{"args": list(args), "kwargs": masked},
|
|
522
|
+
on_transport_error="raise",
|
|
523
|
+
)
|
|
524
|
+
except NullRunBlockedException:
|
|
525
|
+
# Real policy-block decision from the gateway — propagate as-is.
|
|
526
|
+
raise
|
|
527
|
+
except NullRunTransportError as exc:
|
|
528
|
+
# ADR-008: classified transport failure. Re-raise as
|
|
529
|
+
# NullRunBlockedException so the caller's existing
|
|
530
|
+
# `except NullRunBlockedException` catches the same way as a
|
|
531
|
+
# real policy block. The body never runs.
|
|
532
|
+
if fail_open:
|
|
533
|
+
logger.warning(
|
|
534
|
+
f"sensitive tool pre-check unavailable for {fn.__name__!r}: "
|
|
535
|
+
f"{exc.source} on /{exc.endpoint}. NULLRUN_SENSITIVE_FAIL_OPEN=1 — body will run."
|
|
536
|
+
)
|
|
537
|
+
return
|
|
538
|
+
raise NullRunBlockedException(
|
|
539
|
+
workflow_id=workflow_id,
|
|
540
|
+
reason=f"policy engine unavailable: {exc.source}",
|
|
541
|
+
tool_name=fn.__name__,
|
|
542
|
+
) from exc
|
|
543
|
+
except Exception as exc: # noqa: BLE001
|
|
544
|
+
# Any other exception is a transport / network / backend
|
|
545
|
+
# failure. Re-raise as NullRunBlockedException so the caller
|
|
546
|
+
# sees a uniform "this tool was denied" signal — they should
|
|
547
|
+
# not need to also catch httpx.ConnectError or similar.
|
|
548
|
+
if fail_open:
|
|
549
|
+
logger.warning(
|
|
550
|
+
f"sensitive tool pre-check unavailable for {fn.__name__!r}: "
|
|
551
|
+
f"{exc}. NULLRUN_SENSITIVE_FAIL_OPEN=1 — body will run."
|
|
552
|
+
)
|
|
553
|
+
return
|
|
554
|
+
raise NullRunBlockedException(
|
|
555
|
+
workflow_id=workflow_id,
|
|
556
|
+
reason=f"policy engine unavailable: {exc}",
|
|
557
|
+
tool_name=fn.__name__,
|
|
558
|
+
) from exc
|
|
559
|
+
|
|
560
|
+
# Defense in depth (ADR-008 Rule 1 + Rule 2): if `runtime.execute`
|
|
561
|
+
# ever returns a dict with `decision_source` indicating a transport
|
|
562
|
+
# failure (legacy `FALLBACK_*` strings OR the typed
|
|
563
|
+
# `TransportErrorSource` enum values), honor the gate's fail-CLOSED
|
|
564
|
+
# policy here. The body still must not run.
|
|
565
|
+
if isinstance(result, dict):
|
|
566
|
+
decision_source = result.get("decision_source", "")
|
|
567
|
+
if isinstance(decision_source, str) and (
|
|
568
|
+
decision_source.startswith("FALLBACK_")
|
|
569
|
+
or decision_source in {
|
|
570
|
+
TransportErrorSource.NETWORK_ERROR,
|
|
571
|
+
TransportErrorSource.GATEWAY_ERROR,
|
|
572
|
+
TransportErrorSource.BREAKER_OPEN,
|
|
573
|
+
TransportErrorSource.AUTH_ERROR,
|
|
574
|
+
}
|
|
575
|
+
):
|
|
576
|
+
if fail_open:
|
|
577
|
+
logger.warning(
|
|
578
|
+
f"sensitive tool pre-check for {fn.__name__!r} returned "
|
|
579
|
+
f"{decision_source}; NULLRUN_SENSITIVE_FAIL_OPEN=1 — body will run."
|
|
580
|
+
)
|
|
581
|
+
return
|
|
582
|
+
raise NullRunBlockedException(
|
|
583
|
+
workflow_id=workflow_id,
|
|
584
|
+
reason=f"policy engine unavailable: {decision_source}",
|
|
585
|
+
tool_name=fn.__name__,
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
# Real `decision=block` from the gateway is already converted to
|
|
589
|
+
# NullRunBlockedException by `runtime.execute` — no second check
|
|
590
|
+
# needed here. A `decision=allow` with `decision_source=GATEWAY`
|
|
591
|
+
# (the happy path) just falls through and the body runs.
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def sensitive(fn: F) -> F:
|
|
595
|
+
"""
|
|
596
|
+
Mark a function as sensitive. `@protect` will pre-check
|
|
597
|
+
`runtime.execute(...)` before the body runs.
|
|
598
|
+
|
|
599
|
+
This is the discoverable alternative to the lower-level
|
|
600
|
+
`runtime.add_sensitive_tool(fn.__name__)`. Chain with `@protect`
|
|
601
|
+
in either order (both work via `functools.wraps`); the
|
|
602
|
+
recommended form is `@sensitive` outside so the name is
|
|
603
|
+
registered before the wrapper is built:
|
|
604
|
+
|
|
605
|
+
@nullrun.sensitive
|
|
606
|
+
@nullrun.protect
|
|
607
|
+
def charge_card(amount: int) -> str:
|
|
608
|
+
...
|
|
609
|
+
"""
|
|
610
|
+
try:
|
|
611
|
+
# Use the same slot the @protect wrapper uses so the
|
|
612
|
+
# registration lands on the same runtime instance the
|
|
613
|
+
# wrapper will consult. Falling back to get_runtime()
|
|
614
|
+
# would hit a different singleton and silently no-op in
|
|
615
|
+
# tests that build a custom runtime.
|
|
616
|
+
rt = _get_or_create_runtime()
|
|
617
|
+
rt.add_sensitive_tool(fn.__name__)
|
|
618
|
+
except Exception as exc: # noqa: BLE001 — never let registration fail the import
|
|
619
|
+
logger.debug(f"@sensitive: failed to register {fn.__name__!r}: {exc}")
|
|
620
|
+
return fn
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def reset() -> None:
|
|
624
|
+
"""
|
|
625
|
+
Reset NullRun runtime. Mainly for testing or when you need to
|
|
626
|
+
reinitialize the global runtime instance.
|
|
627
|
+
"""
|
|
628
|
+
global _runtime
|
|
629
|
+
if _runtime:
|
|
630
|
+
try:
|
|
631
|
+
_runtime.shutdown()
|
|
632
|
+
except Exception as exc: # noqa: BLE001
|
|
633
|
+
logger.debug(f"Runtime shutdown raised: {exc}")
|
|
634
|
+
_runtime = None
|
|
635
|
+
logger.info("NullRun runtime reset")
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def get_protected_runtime() -> NullRunRuntime | None:
|
|
639
|
+
"""Get the current protected runtime (the one `@protect` would use)."""
|
|
640
|
+
global _runtime
|
|
641
|
+
if _runtime is not None:
|
|
642
|
+
return _runtime
|
|
643
|
+
# Fall back to the global singleton if the decorator-level slot is
|
|
644
|
+
# empty — this matches the behaviour of every other helper that
|
|
645
|
+
# reads from `get_runtime()`.
|
|
646
|
+
try:
|
|
647
|
+
return get_runtime()
|
|
648
|
+
except Exception:
|
|
649
|
+
return None
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""
|
|
2
|
+
NullRun instrumentation module.
|
|
3
|
+
|
|
4
|
+
Provides low-level instrumentation primitives for various AI
|
|
5
|
+
frameworks. The user-facing "wrap my compiled app" helpers
|
|
6
|
+
live in `nullrun.toolbox` (e.g. `nullrun.toolbox.langgraph.wrapper`,
|
|
7
|
+
which replaced `nullrun.instrumentation.langgraph.instrument`
|
|
8
|
+
in Phase 1 Commit 6).
|
|
9
|
+
|
|
10
|
+
The v0.x ``openai.ChatCompletion.create`` patcher was removed
|
|
11
|
+
in 0.4.0 — ``openai>=1.0`` does not expose that attribute. All
|
|
12
|
+
OpenAI v1.0+ traffic is now tracked vendor-independently by the
|
|
13
|
+
httpx transport hook in ``nullrun.instrumentation.auto``.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from nullrun.instrumentation.auto import auto_instrument, is_auto_instrumented
|
|
17
|
+
from nullrun.instrumentation.langgraph import NullRunCallback
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"NullRunCallback",
|
|
21
|
+
"auto_instrument",
|
|
22
|
+
"is_auto_instrumented",
|
|
23
|
+
]
|