ledger-runtime 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ledger/blame/delta.py ADDED
@@ -0,0 +1,335 @@
1
+ """Blame: find the step that broke a failed run by rerunning versions of it.
2
+
3
+ diagnosis = ledger.diagnose(run_id, graph, check=lambda out: out["ok"])
4
+ diagnosis.step # StepInfo of the decisive step, or None
5
+ diagnosis.forks # fork runs it took
6
+
7
+ ``check`` says whether a run's output is a success. The failed run is the
8
+ recording ``run_id``; each counterfactual is a fork of it (``ledger.fork``)
9
+ in which chosen calls run live again (``RERUN``) instead of being replayed,
10
+ along with everything downstream of them. The calls before them come from
11
+ the log, so a counterfactual pays only for what it reruns. A run that raises
12
+ counts as failed.
13
+
14
+ **Rerunning from step k** reruns every call at or after k in log order. If
15
+ that fixes the run and rerunning from the next step doesn't, k's recorded
16
+ outcome is needed for the failure: keep it and the run fails even with every
17
+ later call redrawn; redraw it too and the run passes. That makes k the
18
+ decisive step in Who&When's sense, the earliest step whose correction turns
19
+ the failure into a success.
20
+
21
+ With a single bad step f, rerunning from k fixes the run for every k up to f
22
+ and for none after it, so f can be searched for. The search gallops back from
23
+ the end, rerunning the last call, then the last 2, 4, 8 and so on, until a
24
+ rerun fixes the run, then bisects inside the last gap. Live calls are what a
25
+ counterfactual costs, and the first forks rerun only the end of the run, so
26
+ a step d calls from the end takes about 2 log2(d) forks, none of them
27
+ rerunning much more than 2d calls. One more fork checks that rerunning f
28
+ alone (with what depends on it) is enough, unless every call after f depends
29
+ on f, in which case the fork that found f already did. Rerunning everything is only
30
+ tried when nothing shorter fixes the run.
31
+
32
+ **Several bad steps.** If rerunning the first step found alone doesn't fix
33
+ the run, another step is needed too: the search goes on over the steps after
34
+ it, with the ones found so far always rerun, until rerunning just those
35
+ fixes the run. ``diagnosis.steps`` lists them in log order;
36
+ ``diagnosis.step`` is the first.
37
+
38
+ **No step.** If rerunning everything still fails, no recorded outcome is to
39
+ blame: the cause is in the code or the input, or the failure comes back
40
+ whenever its step is redrawn (a model that makes the same mistake every
41
+ time). ``diagnosis.step`` is then None.
42
+
43
+ **Noise.** Each counterfactual runs up to ``repeats`` times and counts as a
44
+ fix when more than ``threshold`` of them pass. It stops early once that is
45
+ settled, so a fork that passes or fails every time costs about half of
46
+ ``repeats`` with the default threshold of 0.5 (a majority vote). Each
47
+ reports its pass rate with a Wilson interval.
48
+
49
+ The right threshold depends on where the noise is. A majority vote suits
50
+ runs that sometimes pass even with the fault kept. When they essentially
51
+ never do, and noise only makes good runs fail (a flaky call later on, a
52
+ model that repeats its mistake when redrawn), a real fix can pass less than
53
+ half the time; then ``threshold=0``, one pass out of ``repeats``, is right,
54
+ and stops at the first pass. Bisection
55
+ assumes the answers are consistent; when noise makes them contradict each
56
+ other the diagnosis ends ``"inconsistent"`` with the steps found so far.
57
+
58
+ **Budget.** ``budget`` caps forks, live calls or dollars (see
59
+ ``ledger.blame.budget``). A diagnosis that runs out ends ``"budget"``: the
60
+ steps found so far, and in ``narrowed`` the candidates the step being
61
+ searched for must be among.
62
+
63
+ Effects are not candidates: rerunning one would repeat it. A counterfactual
64
+ that would run an effect live raises ``LiveEffectRefused`` unless
65
+ ``live_effects`` is true.
66
+ """
67
+
68
+ from __future__ import annotations
69
+
70
+ from collections.abc import Callable, Sequence
71
+ from dataclasses import dataclass, field
72
+ from typing import Any
73
+
74
+ from ledger.blame.budget import Budget, OutOfBudget, Pricing, Spend, usage, wilson
75
+ from ledger.core.events import body_kind, load_payload
76
+ from ledger.core.store import EventStore, step_key
77
+ from ledger.effects import LiveEffectRefused
78
+ from ledger.replay.engine import CALL_KINDS, RERUN, is_downstream
79
+ from ledger.replay.fork import StepInfo, fork, steps
80
+
81
+ __all__ = ["CANDIDATE_KINDS", "Counterfactual", "Diagnosis", "RunNotFailed", "diagnose"]
82
+
83
+ CANDIDATE_KINDS = ("llm_call", "tool_call", "nondet")
84
+
85
+
86
+ class RunNotFailed(ValueError):
87
+ """The run passes its check, so there is nothing to blame."""
88
+
89
+
90
+ @dataclass(frozen=True)
91
+ class Counterfactual:
92
+ """One counterfactual: the failed run with some calls rerun live."""
93
+
94
+ # Keys of the calls given RERUN; whatever depends on them reruns too.
95
+ reran: tuple[str, ...]
96
+ # "from <key>": every call from that step on; "only <keys>": those alone.
97
+ label: str
98
+ passed: int
99
+ trials: int
100
+ run_ids: tuple[str, ...]
101
+ # LLM and tool calls made live, over all trials.
102
+ live_calls: int
103
+ # Runs it was allowed (``repeats``); ``trials`` can stop short of it.
104
+ repeats: int = 1
105
+ # It fixes the run when more than this share of ``repeats`` pass.
106
+ threshold: float = 0.5
107
+ tokens_in: int = 0
108
+ tokens_out: int = 0
109
+ dollars: float = 0.0
110
+
111
+ @property
112
+ def fixed(self) -> bool:
113
+ return self.passed > self.threshold * self.repeats
114
+
115
+ @property
116
+ def rate(self) -> float:
117
+ return self.passed / self.trials if self.trials else 0.0
118
+
119
+ @property
120
+ def interval(self) -> tuple[float, float]:
121
+ """95% Wilson interval for the pass rate."""
122
+ return wilson(self.passed, self.trials)
123
+
124
+
125
+ @dataclass
126
+ class Diagnosis:
127
+ run_id: str
128
+ # Decisive steps in log order; empty if no rerun fixes the run.
129
+ steps: list[StepInfo]
130
+ # How many recorded calls were candidates.
131
+ candidates: int
132
+ counterfactuals: list[Counterfactual] = field(default_factory=list)
133
+ # "found": rerunning ``steps`` alone fixed the run. "no_step": rerunning
134
+ # everything didn't. "budget": the budget ran out. "inconsistent": noisy
135
+ # forks contradicted each other. "" until the diagnosis ends.
136
+ status: str = ""
137
+ # With "budget": the candidates the step being searched for is among.
138
+ narrowed: list[StepInfo] = field(default_factory=list)
139
+ spent: Spend = field(default_factory=Spend)
140
+
141
+ @property
142
+ def complete(self) -> bool:
143
+ return self.status == "found"
144
+
145
+ @property
146
+ def step(self) -> StepInfo | None:
147
+ """The first step found or, if the budget ran out before any, the
148
+ first candidate it was narrowed to (a best guess)."""
149
+ if self.steps:
150
+ return self.steps[0]
151
+ return self.narrowed[0] if self.narrowed else None
152
+
153
+ @property
154
+ def forks(self) -> int:
155
+ return self.spent.forks
156
+
157
+ @property
158
+ def live_calls(self) -> int:
159
+ return self.spent.live_calls
160
+
161
+ @property
162
+ def dollars(self) -> float:
163
+ return self.spent.dollars
164
+
165
+ def __str__(self) -> str:
166
+ found = ", ".join(s.key for s in self.steps) or "no step"
167
+ return (f"{self.run_id}: {self.status} {found} ({self.forks} forks, "
168
+ f"{self.live_calls} live calls, ${self.dollars:.4f}, "
169
+ f"{self.candidates} candidates)")
170
+
171
+
172
+ def _recording(store: EventStore, run_id: str) -> tuple[Any, bool, Any]:
173
+ """(input, finished without error, output) of the recorded run."""
174
+ graph_input, ok, output = None, False, None
175
+ for ev in store.events(run_id):
176
+ kind = body_kind(ev)
177
+ if kind == "run_started":
178
+ graph_input = load_payload(ev.run_started.input)
179
+ elif kind == "run_finished":
180
+ ok = ev.run_finished.HasField("output")
181
+ output = load_payload(ev.run_finished.output) if ok else None
182
+ return graph_input, ok, output
183
+
184
+
185
+ def diagnose(
186
+ run_id: str,
187
+ graph: Any,
188
+ check: Callable[[Any], bool],
189
+ *,
190
+ store: EventStore | None = None,
191
+ inputs: Any = None,
192
+ kinds: Sequence[str] = CANDIDATE_KINDS,
193
+ repeats: int = 1,
194
+ threshold: float = 0.5,
195
+ budget: Budget | None = None,
196
+ pricing: Pricing | None = None,
197
+ live_effects: bool = False,
198
+ ) -> Diagnosis:
199
+ """Name the step(s) of the failed run ``run_id`` that caused the failure.
200
+
201
+ ``graph`` is the instrumented graph (anything with ``invoke``) or a
202
+ function of the input; it is run with the recording's input unless
203
+ ``inputs`` is given. ``check(output)`` is true for a successful run; for
204
+ the recording it gets the output as logged (JSON). ``kinds`` are the
205
+ call kinds that can be blamed. ``repeats``, ``threshold``, ``budget``
206
+ and ``pricing`` are described in the module docstring. Raises RunNotFailed if the
207
+ recording passes ``check``.
208
+ """
209
+ if repeats < 1:
210
+ raise ValueError("repeats must be at least 1")
211
+ if not 0 <= threshold < 1:
212
+ raise ValueError("threshold must be in [0, 1)")
213
+ need = threshold * repeats # passes must exceed this
214
+ if budget is not None and budget.dollars is not None and pricing is None:
215
+ raise ValueError("a dollar budget needs pricing")
216
+ invoke = graph.invoke if hasattr(graph, "invoke") else graph
217
+ owns_store = store is None
218
+ store = store or EventStore()
219
+ try:
220
+ candidates = [s for s in steps(run_id, store) if s.kind in kinds]
221
+ recorded_input, finished, output = _recording(store, run_id)
222
+ if inputs is None:
223
+ inputs = recorded_input
224
+ if finished and check(output):
225
+ raise RunNotFailed(f"run {run_id} passes its check: nothing to blame")
226
+ diagnosis = Diagnosis(run_id, [], len(candidates))
227
+ if not candidates:
228
+ diagnosis.status = "no_step"
229
+ return diagnosis
230
+ spent = diagnosis.spent
231
+
232
+ def counterfactual(reran: list[str], label: str) -> Counterfactual:
233
+ passed, run_ids, live, t_in, t_out, cost = 0, [], 0, 0, 0, 0.0
234
+ for trial in range(repeats):
235
+ if passed > need or passed + (repeats - trial) <= need:
236
+ break # settled either way
237
+ if budget is not None and budget.exhausted(spent):
238
+ if trial == 0:
239
+ raise OutOfBudget
240
+ break
241
+ with fork(
242
+ run_id, edits={k: RERUN for k in reran}, store=store,
243
+ on_divergence="live", live_effects=live_effects,
244
+ ) as session:
245
+ try:
246
+ out, ok = invoke(inputs), True
247
+ except LiveEffectRefused:
248
+ raise
249
+ except Exception:
250
+ out, ok = None, False
251
+ passed += ok and bool(check(out))
252
+ run_ids.append(session.run_id)
253
+ live += session.stats.live_calls
254
+ tin, tout = usage(store, session.run_id)
255
+ dollars = pricing.dollars(tin, tout) if pricing else 0.0
256
+ t_in, t_out, cost = t_in + tin, t_out + tout, cost + dollars
257
+ spent.forks += 1
258
+ spent.live_calls += session.stats.live_calls
259
+ spent.tokens_in += tin
260
+ spent.tokens_out += tout
261
+ spent.dollars += dollars
262
+ c = Counterfactual(
263
+ tuple(reran), label, passed, len(run_ids), tuple(run_ids), live,
264
+ repeats=repeats, threshold=threshold, tokens_in=t_in, tokens_out=t_out, dollars=cost,
265
+ )
266
+ diagnosis.counterfactuals.append(c)
267
+ return c
268
+
269
+ keys = [s.key for s in candidates]
270
+ n = len(keys)
271
+ refs = {step_key(ev.step): ev.step for ev in store.events(run_id, hydrate=False)
272
+ if body_kind(ev) in CALL_KINDS}
273
+
274
+ def covered(found: list[int], lo: int) -> bool:
275
+ """Whether every candidate after ``lo`` depends on a found step, so
276
+ rerunning the found steps alone reruns what found + keys[lo:] did."""
277
+ return all(any(is_downstream(refs[keys[k]], refs[keys[g]]) for g in found)
278
+ for k in range(lo + 1, n))
279
+
280
+ def from_(i: int, forced: list[int]) -> bool:
281
+ reran = [keys[j] for j in forced] + keys[i:]
282
+ return counterfactual(reran, f"from {keys[i]}").fixed
283
+
284
+ found: list[int] = []
285
+ lo, hi = 0, n
286
+ searching = True # between finding one step and checking it alone: False
287
+ try:
288
+ # Each round finds the last k >= lo with found + keys[k:] fixing
289
+ # the run; found + keys[n:] (the found steps alone, or in the
290
+ # first round the recording itself) is known not to. In later
291
+ # rounds found + keys[lo:] is known to fix it.
292
+ known = False
293
+ while True:
294
+ hi, width = n, 1
295
+ while True: # gallop back from the end
296
+ k = max(lo, n - width)
297
+ if k == lo and known:
298
+ break
299
+ if from_(k, found):
300
+ lo = k
301
+ break
302
+ hi = k
303
+ if k == lo:
304
+ diagnosis.status = "no_step"
305
+ return diagnosis
306
+ width *= 2
307
+ while hi - lo > 1:
308
+ mid = (lo + hi) // 2
309
+ if from_(mid, found):
310
+ lo = mid
311
+ else:
312
+ hi = mid
313
+ found.append(lo)
314
+ searching = False
315
+ reran = [keys[j] for j in found]
316
+ # found + keys[lo:] is known to fix the run; if that reran
317
+ # nothing beyond what the found steps alone rerun, it was
318
+ # already this check.
319
+ if covered(found, lo) or counterfactual(reran, "only " + ", ".join(reran)).fixed:
320
+ diagnosis.status = "found"
321
+ break
322
+ lo += 1
323
+ searching, known = True, True
324
+ if lo >= n:
325
+ diagnosis.status = "inconsistent"
326
+ break
327
+ except OutOfBudget:
328
+ diagnosis.status = "budget"
329
+ if searching:
330
+ diagnosis.narrowed = candidates[lo:hi]
331
+ diagnosis.steps = [candidates[j] for j in found]
332
+ return diagnosis
333
+ finally:
334
+ if owns_store:
335
+ store.close()