ledger-runtime 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ledger/__init__.py +81 -0
- ledger/__main__.py +7 -0
- ledger/_proto/__init__.py +0 -0
- ledger/_proto/events_pb2.py +65 -0
- ledger/_proto/events_pb2.pyi +216 -0
- ledger/blame/__init__.py +22 -0
- ledger/blame/budget.py +88 -0
- ledger/blame/delta.py +335 -0
- ledger/cli.py +474 -0
- ledger/core/__init__.py +0 -0
- ledger/core/blobs.py +108 -0
- ledger/core/clock.py +96 -0
- ledger/core/events.py +88 -0
- ledger/core/normalize.py +111 -0
- ledger/core/session.py +280 -0
- ledger/core/store.py +246 -0
- ledger/effects/__init__.py +7 -0
- ledger/effects/idempotency.py +26 -0
- ledger/effects/manager.py +410 -0
- ledger/effects/saga.py +129 -0
- ledger/intercept/__init__.py +0 -0
- ledger/intercept/_common.py +171 -0
- ledger/intercept/langgraph.py +98 -0
- ledger/intercept/llm.py +160 -0
- ledger/intercept/tools.py +119 -0
- ledger/replay/__init__.py +0 -0
- ledger/replay/engine.py +612 -0
- ledger/replay/fork.py +185 -0
- ledger/replay/resume.py +135 -0
- ledger/testing/__init__.py +0 -0
- ledger/testing/fake_llm.py +173 -0
- ledger/testing/fake_stripe.py +355 -0
- ledger/testing/faults.py +137 -0
- ledger_runtime-0.0.1.dist-info/METADATA +540 -0
- ledger_runtime-0.0.1.dist-info/RECORD +37 -0
- ledger_runtime-0.0.1.dist-info/WHEEL +4 -0
- ledger_runtime-0.0.1.dist-info/entry_points.txt +2 -0
ledger/blame/delta.py
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
"""Blame: find the step that broke a failed run by rerunning versions of it.
|
|
2
|
+
|
|
3
|
+
diagnosis = ledger.diagnose(run_id, graph, check=lambda out: out["ok"])
|
|
4
|
+
diagnosis.step # StepInfo of the decisive step, or None
|
|
5
|
+
diagnosis.forks # fork runs it took
|
|
6
|
+
|
|
7
|
+
``check`` says whether a run's output is a success. The failed run is the
|
|
8
|
+
recording ``run_id``; each counterfactual is a fork of it (``ledger.fork``)
|
|
9
|
+
in which chosen calls run live again (``RERUN``) instead of being replayed,
|
|
10
|
+
along with everything downstream of them. The calls before them come from
|
|
11
|
+
the log, so a counterfactual pays only for what it reruns. A run that raises
|
|
12
|
+
counts as failed.
|
|
13
|
+
|
|
14
|
+
**Rerunning from step k** reruns every call at or after k in log order. If
|
|
15
|
+
that fixes the run and rerunning from the next step doesn't, k's recorded
|
|
16
|
+
outcome is needed for the failure: keep it and the run fails even with every
|
|
17
|
+
later call redrawn; redraw it too and the run passes. That makes k the
|
|
18
|
+
decisive step in Who&When's sense, the earliest step whose correction turns
|
|
19
|
+
the failure into a success.
|
|
20
|
+
|
|
21
|
+
With a single bad step f, rerunning from k fixes the run for every k up to f
|
|
22
|
+
and for none after it, so f can be searched for. The search gallops back from
|
|
23
|
+
the end, rerunning the last call, then the last 2, 4, 8 and so on, until a
|
|
24
|
+
rerun fixes the run, then bisects inside the last gap. Live calls are what a
|
|
25
|
+
counterfactual costs, and the first forks rerun only the end of the run, so
|
|
26
|
+
a step d calls from the end takes about 2 log2(d) forks, none of them
|
|
27
|
+
rerunning much more than 2d calls. One more fork checks that rerunning f
|
|
28
|
+
alone (with what depends on it) is enough, unless every call after f depends
|
|
29
|
+
on f, in which case the fork that found f already did. Rerunning everything is only
|
|
30
|
+
tried when nothing shorter fixes the run.
|
|
31
|
+
|
|
32
|
+
**Several bad steps.** If rerunning the first step found alone doesn't fix
|
|
33
|
+
the run, another step is needed too: the search goes on over the steps after
|
|
34
|
+
it, with the ones found so far always rerun, until rerunning just those
|
|
35
|
+
fixes the run. ``diagnosis.steps`` lists them in log order;
|
|
36
|
+
``diagnosis.step`` is the first.
|
|
37
|
+
|
|
38
|
+
**No step.** If rerunning everything still fails, no recorded outcome is to
|
|
39
|
+
blame: the cause is in the code or the input, or the failure comes back
|
|
40
|
+
whenever its step is redrawn (a model that makes the same mistake every
|
|
41
|
+
time). ``diagnosis.step`` is then None.
|
|
42
|
+
|
|
43
|
+
**Noise.** Each counterfactual runs up to ``repeats`` times and counts as a
|
|
44
|
+
fix when more than ``threshold`` of them pass. It stops early once that is
|
|
45
|
+
settled, so a fork that passes or fails every time costs about half of
|
|
46
|
+
``repeats`` with the default threshold of 0.5 (a majority vote). Each
|
|
47
|
+
reports its pass rate with a Wilson interval.
|
|
48
|
+
|
|
49
|
+
The right threshold depends on where the noise is. A majority vote suits
|
|
50
|
+
runs that sometimes pass even with the fault kept. When they essentially
|
|
51
|
+
never do, and noise only makes good runs fail (a flaky call later on, a
|
|
52
|
+
model that repeats its mistake when redrawn), a real fix can pass less than
|
|
53
|
+
half the time; then ``threshold=0``, one pass out of ``repeats``, is right,
|
|
54
|
+
and stops at the first pass. Bisection
|
|
55
|
+
assumes the answers are consistent; when noise makes them contradict each
|
|
56
|
+
other the diagnosis ends ``"inconsistent"`` with the steps found so far.
|
|
57
|
+
|
|
58
|
+
**Budget.** ``budget`` caps forks, live calls or dollars (see
|
|
59
|
+
``ledger.blame.budget``). A diagnosis that runs out ends ``"budget"``: the
|
|
60
|
+
steps found so far, and in ``narrowed`` the candidates the step being
|
|
61
|
+
searched for must be among.
|
|
62
|
+
|
|
63
|
+
Effects are not candidates: rerunning one would repeat it. A counterfactual
|
|
64
|
+
that would run an effect live raises ``LiveEffectRefused`` unless
|
|
65
|
+
``live_effects`` is true.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
from __future__ import annotations
|
|
69
|
+
|
|
70
|
+
from collections.abc import Callable, Sequence
|
|
71
|
+
from dataclasses import dataclass, field
|
|
72
|
+
from typing import Any
|
|
73
|
+
|
|
74
|
+
from ledger.blame.budget import Budget, OutOfBudget, Pricing, Spend, usage, wilson
|
|
75
|
+
from ledger.core.events import body_kind, load_payload
|
|
76
|
+
from ledger.core.store import EventStore, step_key
|
|
77
|
+
from ledger.effects import LiveEffectRefused
|
|
78
|
+
from ledger.replay.engine import CALL_KINDS, RERUN, is_downstream
|
|
79
|
+
from ledger.replay.fork import StepInfo, fork, steps
|
|
80
|
+
|
|
81
|
+
__all__ = ["CANDIDATE_KINDS", "Counterfactual", "Diagnosis", "RunNotFailed", "diagnose"]
|
|
82
|
+
|
|
83
|
+
CANDIDATE_KINDS = ("llm_call", "tool_call", "nondet")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class RunNotFailed(ValueError):
|
|
87
|
+
"""The run passes its check, so there is nothing to blame."""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True)
|
|
91
|
+
class Counterfactual:
|
|
92
|
+
"""One counterfactual: the failed run with some calls rerun live."""
|
|
93
|
+
|
|
94
|
+
# Keys of the calls given RERUN; whatever depends on them reruns too.
|
|
95
|
+
reran: tuple[str, ...]
|
|
96
|
+
# "from <key>": every call from that step on; "only <keys>": those alone.
|
|
97
|
+
label: str
|
|
98
|
+
passed: int
|
|
99
|
+
trials: int
|
|
100
|
+
run_ids: tuple[str, ...]
|
|
101
|
+
# LLM and tool calls made live, over all trials.
|
|
102
|
+
live_calls: int
|
|
103
|
+
# Runs it was allowed (``repeats``); ``trials`` can stop short of it.
|
|
104
|
+
repeats: int = 1
|
|
105
|
+
# It fixes the run when more than this share of ``repeats`` pass.
|
|
106
|
+
threshold: float = 0.5
|
|
107
|
+
tokens_in: int = 0
|
|
108
|
+
tokens_out: int = 0
|
|
109
|
+
dollars: float = 0.0
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def fixed(self) -> bool:
|
|
113
|
+
return self.passed > self.threshold * self.repeats
|
|
114
|
+
|
|
115
|
+
@property
|
|
116
|
+
def rate(self) -> float:
|
|
117
|
+
return self.passed / self.trials if self.trials else 0.0
|
|
118
|
+
|
|
119
|
+
@property
|
|
120
|
+
def interval(self) -> tuple[float, float]:
|
|
121
|
+
"""95% Wilson interval for the pass rate."""
|
|
122
|
+
return wilson(self.passed, self.trials)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@dataclass
|
|
126
|
+
class Diagnosis:
|
|
127
|
+
run_id: str
|
|
128
|
+
# Decisive steps in log order; empty if no rerun fixes the run.
|
|
129
|
+
steps: list[StepInfo]
|
|
130
|
+
# How many recorded calls were candidates.
|
|
131
|
+
candidates: int
|
|
132
|
+
counterfactuals: list[Counterfactual] = field(default_factory=list)
|
|
133
|
+
# "found": rerunning ``steps`` alone fixed the run. "no_step": rerunning
|
|
134
|
+
# everything didn't. "budget": the budget ran out. "inconsistent": noisy
|
|
135
|
+
# forks contradicted each other. "" until the diagnosis ends.
|
|
136
|
+
status: str = ""
|
|
137
|
+
# With "budget": the candidates the step being searched for is among.
|
|
138
|
+
narrowed: list[StepInfo] = field(default_factory=list)
|
|
139
|
+
spent: Spend = field(default_factory=Spend)
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def complete(self) -> bool:
|
|
143
|
+
return self.status == "found"
|
|
144
|
+
|
|
145
|
+
@property
|
|
146
|
+
def step(self) -> StepInfo | None:
|
|
147
|
+
"""The first step found or, if the budget ran out before any, the
|
|
148
|
+
first candidate it was narrowed to (a best guess)."""
|
|
149
|
+
if self.steps:
|
|
150
|
+
return self.steps[0]
|
|
151
|
+
return self.narrowed[0] if self.narrowed else None
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def forks(self) -> int:
|
|
155
|
+
return self.spent.forks
|
|
156
|
+
|
|
157
|
+
@property
|
|
158
|
+
def live_calls(self) -> int:
|
|
159
|
+
return self.spent.live_calls
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def dollars(self) -> float:
|
|
163
|
+
return self.spent.dollars
|
|
164
|
+
|
|
165
|
+
def __str__(self) -> str:
|
|
166
|
+
found = ", ".join(s.key for s in self.steps) or "no step"
|
|
167
|
+
return (f"{self.run_id}: {self.status} {found} ({self.forks} forks, "
|
|
168
|
+
f"{self.live_calls} live calls, ${self.dollars:.4f}, "
|
|
169
|
+
f"{self.candidates} candidates)")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _recording(store: EventStore, run_id: str) -> tuple[Any, bool, Any]:
|
|
173
|
+
"""(input, finished without error, output) of the recorded run."""
|
|
174
|
+
graph_input, ok, output = None, False, None
|
|
175
|
+
for ev in store.events(run_id):
|
|
176
|
+
kind = body_kind(ev)
|
|
177
|
+
if kind == "run_started":
|
|
178
|
+
graph_input = load_payload(ev.run_started.input)
|
|
179
|
+
elif kind == "run_finished":
|
|
180
|
+
ok = ev.run_finished.HasField("output")
|
|
181
|
+
output = load_payload(ev.run_finished.output) if ok else None
|
|
182
|
+
return graph_input, ok, output
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def diagnose(
|
|
186
|
+
run_id: str,
|
|
187
|
+
graph: Any,
|
|
188
|
+
check: Callable[[Any], bool],
|
|
189
|
+
*,
|
|
190
|
+
store: EventStore | None = None,
|
|
191
|
+
inputs: Any = None,
|
|
192
|
+
kinds: Sequence[str] = CANDIDATE_KINDS,
|
|
193
|
+
repeats: int = 1,
|
|
194
|
+
threshold: float = 0.5,
|
|
195
|
+
budget: Budget | None = None,
|
|
196
|
+
pricing: Pricing | None = None,
|
|
197
|
+
live_effects: bool = False,
|
|
198
|
+
) -> Diagnosis:
|
|
199
|
+
"""Name the step(s) of the failed run ``run_id`` that caused the failure.
|
|
200
|
+
|
|
201
|
+
``graph`` is the instrumented graph (anything with ``invoke``) or a
|
|
202
|
+
function of the input; it is run with the recording's input unless
|
|
203
|
+
``inputs`` is given. ``check(output)`` is true for a successful run; for
|
|
204
|
+
the recording it gets the output as logged (JSON). ``kinds`` are the
|
|
205
|
+
call kinds that can be blamed. ``repeats``, ``threshold``, ``budget``
|
|
206
|
+
and ``pricing`` are described in the module docstring. Raises RunNotFailed if the
|
|
207
|
+
recording passes ``check``.
|
|
208
|
+
"""
|
|
209
|
+
if repeats < 1:
|
|
210
|
+
raise ValueError("repeats must be at least 1")
|
|
211
|
+
if not 0 <= threshold < 1:
|
|
212
|
+
raise ValueError("threshold must be in [0, 1)")
|
|
213
|
+
need = threshold * repeats # passes must exceed this
|
|
214
|
+
if budget is not None and budget.dollars is not None and pricing is None:
|
|
215
|
+
raise ValueError("a dollar budget needs pricing")
|
|
216
|
+
invoke = graph.invoke if hasattr(graph, "invoke") else graph
|
|
217
|
+
owns_store = store is None
|
|
218
|
+
store = store or EventStore()
|
|
219
|
+
try:
|
|
220
|
+
candidates = [s for s in steps(run_id, store) if s.kind in kinds]
|
|
221
|
+
recorded_input, finished, output = _recording(store, run_id)
|
|
222
|
+
if inputs is None:
|
|
223
|
+
inputs = recorded_input
|
|
224
|
+
if finished and check(output):
|
|
225
|
+
raise RunNotFailed(f"run {run_id} passes its check: nothing to blame")
|
|
226
|
+
diagnosis = Diagnosis(run_id, [], len(candidates))
|
|
227
|
+
if not candidates:
|
|
228
|
+
diagnosis.status = "no_step"
|
|
229
|
+
return diagnosis
|
|
230
|
+
spent = diagnosis.spent
|
|
231
|
+
|
|
232
|
+
def counterfactual(reran: list[str], label: str) -> Counterfactual:
|
|
233
|
+
passed, run_ids, live, t_in, t_out, cost = 0, [], 0, 0, 0, 0.0
|
|
234
|
+
for trial in range(repeats):
|
|
235
|
+
if passed > need or passed + (repeats - trial) <= need:
|
|
236
|
+
break # settled either way
|
|
237
|
+
if budget is not None and budget.exhausted(spent):
|
|
238
|
+
if trial == 0:
|
|
239
|
+
raise OutOfBudget
|
|
240
|
+
break
|
|
241
|
+
with fork(
|
|
242
|
+
run_id, edits={k: RERUN for k in reran}, store=store,
|
|
243
|
+
on_divergence="live", live_effects=live_effects,
|
|
244
|
+
) as session:
|
|
245
|
+
try:
|
|
246
|
+
out, ok = invoke(inputs), True
|
|
247
|
+
except LiveEffectRefused:
|
|
248
|
+
raise
|
|
249
|
+
except Exception:
|
|
250
|
+
out, ok = None, False
|
|
251
|
+
passed += ok and bool(check(out))
|
|
252
|
+
run_ids.append(session.run_id)
|
|
253
|
+
live += session.stats.live_calls
|
|
254
|
+
tin, tout = usage(store, session.run_id)
|
|
255
|
+
dollars = pricing.dollars(tin, tout) if pricing else 0.0
|
|
256
|
+
t_in, t_out, cost = t_in + tin, t_out + tout, cost + dollars
|
|
257
|
+
spent.forks += 1
|
|
258
|
+
spent.live_calls += session.stats.live_calls
|
|
259
|
+
spent.tokens_in += tin
|
|
260
|
+
spent.tokens_out += tout
|
|
261
|
+
spent.dollars += dollars
|
|
262
|
+
c = Counterfactual(
|
|
263
|
+
tuple(reran), label, passed, len(run_ids), tuple(run_ids), live,
|
|
264
|
+
repeats=repeats, threshold=threshold, tokens_in=t_in, tokens_out=t_out, dollars=cost,
|
|
265
|
+
)
|
|
266
|
+
diagnosis.counterfactuals.append(c)
|
|
267
|
+
return c
|
|
268
|
+
|
|
269
|
+
keys = [s.key for s in candidates]
|
|
270
|
+
n = len(keys)
|
|
271
|
+
refs = {step_key(ev.step): ev.step for ev in store.events(run_id, hydrate=False)
|
|
272
|
+
if body_kind(ev) in CALL_KINDS}
|
|
273
|
+
|
|
274
|
+
def covered(found: list[int], lo: int) -> bool:
|
|
275
|
+
"""Whether every candidate after ``lo`` depends on a found step, so
|
|
276
|
+
rerunning the found steps alone reruns what found + keys[lo:] did."""
|
|
277
|
+
return all(any(is_downstream(refs[keys[k]], refs[keys[g]]) for g in found)
|
|
278
|
+
for k in range(lo + 1, n))
|
|
279
|
+
|
|
280
|
+
def from_(i: int, forced: list[int]) -> bool:
|
|
281
|
+
reran = [keys[j] for j in forced] + keys[i:]
|
|
282
|
+
return counterfactual(reran, f"from {keys[i]}").fixed
|
|
283
|
+
|
|
284
|
+
found: list[int] = []
|
|
285
|
+
lo, hi = 0, n
|
|
286
|
+
searching = True # between finding one step and checking it alone: False
|
|
287
|
+
try:
|
|
288
|
+
# Each round finds the last k >= lo with found + keys[k:] fixing
|
|
289
|
+
# the run; found + keys[n:] (the found steps alone, or in the
|
|
290
|
+
# first round the recording itself) is known not to. In later
|
|
291
|
+
# rounds found + keys[lo:] is known to fix it.
|
|
292
|
+
known = False
|
|
293
|
+
while True:
|
|
294
|
+
hi, width = n, 1
|
|
295
|
+
while True: # gallop back from the end
|
|
296
|
+
k = max(lo, n - width)
|
|
297
|
+
if k == lo and known:
|
|
298
|
+
break
|
|
299
|
+
if from_(k, found):
|
|
300
|
+
lo = k
|
|
301
|
+
break
|
|
302
|
+
hi = k
|
|
303
|
+
if k == lo:
|
|
304
|
+
diagnosis.status = "no_step"
|
|
305
|
+
return diagnosis
|
|
306
|
+
width *= 2
|
|
307
|
+
while hi - lo > 1:
|
|
308
|
+
mid = (lo + hi) // 2
|
|
309
|
+
if from_(mid, found):
|
|
310
|
+
lo = mid
|
|
311
|
+
else:
|
|
312
|
+
hi = mid
|
|
313
|
+
found.append(lo)
|
|
314
|
+
searching = False
|
|
315
|
+
reran = [keys[j] for j in found]
|
|
316
|
+
# found + keys[lo:] is known to fix the run; if that reran
|
|
317
|
+
# nothing beyond what the found steps alone rerun, it was
|
|
318
|
+
# already this check.
|
|
319
|
+
if covered(found, lo) or counterfactual(reran, "only " + ", ".join(reran)).fixed:
|
|
320
|
+
diagnosis.status = "found"
|
|
321
|
+
break
|
|
322
|
+
lo += 1
|
|
323
|
+
searching, known = True, True
|
|
324
|
+
if lo >= n:
|
|
325
|
+
diagnosis.status = "inconsistent"
|
|
326
|
+
break
|
|
327
|
+
except OutOfBudget:
|
|
328
|
+
diagnosis.status = "budget"
|
|
329
|
+
if searching:
|
|
330
|
+
diagnosis.narrowed = candidates[lo:hi]
|
|
331
|
+
diagnosis.steps = [candidates[j] for j in found]
|
|
332
|
+
return diagnosis
|
|
333
|
+
finally:
|
|
334
|
+
if owns_store:
|
|
335
|
+
store.close()
|