ledger-runtime 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ledger/__init__.py +81 -0
- ledger/__main__.py +7 -0
- ledger/_proto/__init__.py +0 -0
- ledger/_proto/events_pb2.py +65 -0
- ledger/_proto/events_pb2.pyi +216 -0
- ledger/blame/__init__.py +22 -0
- ledger/blame/budget.py +88 -0
- ledger/blame/delta.py +335 -0
- ledger/cli.py +474 -0
- ledger/core/__init__.py +0 -0
- ledger/core/blobs.py +108 -0
- ledger/core/clock.py +96 -0
- ledger/core/events.py +88 -0
- ledger/core/normalize.py +111 -0
- ledger/core/session.py +280 -0
- ledger/core/store.py +246 -0
- ledger/effects/__init__.py +7 -0
- ledger/effects/idempotency.py +26 -0
- ledger/effects/manager.py +410 -0
- ledger/effects/saga.py +129 -0
- ledger/intercept/__init__.py +0 -0
- ledger/intercept/_common.py +171 -0
- ledger/intercept/langgraph.py +98 -0
- ledger/intercept/llm.py +160 -0
- ledger/intercept/tools.py +119 -0
- ledger/replay/__init__.py +0 -0
- ledger/replay/engine.py +612 -0
- ledger/replay/fork.py +185 -0
- ledger/replay/resume.py +135 -0
- ledger/testing/__init__.py +0 -0
- ledger/testing/fake_llm.py +173 -0
- ledger/testing/fake_stripe.py +355 -0
- ledger/testing/faults.py +137 -0
- ledger_runtime-0.0.1.dist-info/METADATA +540 -0
- ledger_runtime-0.0.1.dist-info/RECORD +37 -0
- ledger_runtime-0.0.1.dist-info/WHEEL +4 -0
- ledger_runtime-0.0.1.dist-info/entry_points.txt +2 -0
ledger/__init__.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Ledger: replayable, crash-safe agent runs under LangGraph.
|
|
2
|
+
|
|
3
|
+
import ledger
|
|
4
|
+
from openai import OpenAI
|
|
5
|
+
|
|
6
|
+
llm = ledger.wrap(OpenAI())
|
|
7
|
+
graph = ledger.instrument(builder.compile())
|
|
8
|
+
|
|
9
|
+
graph.invoke(inputs) # recorded as a new run
|
|
10
|
+
with ledger.replay(run_id) as session: # answered from the log
|
|
11
|
+
graph.invoke(inputs)
|
|
12
|
+
with ledger.fork(run_id, at=step, edit="other text"): # counterfactual
|
|
13
|
+
graph.invoke(inputs)
|
|
14
|
+
with ledger.resume(run_id): # after a crash: same run, same keys
|
|
15
|
+
graph.invoke(inputs)
|
|
16
|
+
ledger.diagnose(run_id, graph, check) # which step broke a failed run
|
|
17
|
+
|
|
18
|
+
Inside nodes, read the clock, ids and randomness through ``ledger.now()``,
|
|
19
|
+
``ledger.time()``, ``ledger.uuid4()`` and ``ledger.rng()`` so replays see the
|
|
20
|
+
recorded values.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from ledger.blame import Diagnosis, RunNotFailed, diagnose
|
|
24
|
+
from ledger.core.clock import now, rng, time, uuid4
|
|
25
|
+
from ledger.core.normalize import EXACT, Normalizer
|
|
26
|
+
from ledger.core.session import Mode, Session, current_session, record
|
|
27
|
+
from ledger.core.store import EventStore
|
|
28
|
+
from ledger.effects import EffectInDoubt, LiveEffectRefused, SagaIncomplete, effect
|
|
29
|
+
from ledger.intercept.langgraph import instrument
|
|
30
|
+
from ledger.intercept.llm import wrap
|
|
31
|
+
from ledger.intercept.tools import tool
|
|
32
|
+
from ledger.replay.engine import (
|
|
33
|
+
RERUN,
|
|
34
|
+
DivergencePoint,
|
|
35
|
+
DivergenceReason,
|
|
36
|
+
ReplayDivergence,
|
|
37
|
+
ReplayedError,
|
|
38
|
+
ReplayMismatch,
|
|
39
|
+
replay,
|
|
40
|
+
)
|
|
41
|
+
from ledger.replay.fork import StepInfo, fork, steps
|
|
42
|
+
from ledger.replay.resume import RunAlreadyFinished, resumable_runs, resume
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"Diagnosis",
|
|
46
|
+
"DivergencePoint",
|
|
47
|
+
"DivergenceReason",
|
|
48
|
+
"EXACT",
|
|
49
|
+
"EffectInDoubt",
|
|
50
|
+
"RERUN",
|
|
51
|
+
"EventStore",
|
|
52
|
+
"LiveEffectRefused",
|
|
53
|
+
"Mode",
|
|
54
|
+
"Normalizer",
|
|
55
|
+
"ReplayDivergence",
|
|
56
|
+
"ReplayMismatch",
|
|
57
|
+
"ReplayedError",
|
|
58
|
+
"RunAlreadyFinished",
|
|
59
|
+
"RunNotFailed",
|
|
60
|
+
"SagaIncomplete",
|
|
61
|
+
"Session",
|
|
62
|
+
"StepInfo",
|
|
63
|
+
"current_session",
|
|
64
|
+
"diagnose",
|
|
65
|
+
"effect",
|
|
66
|
+
"fork",
|
|
67
|
+
"instrument",
|
|
68
|
+
"now",
|
|
69
|
+
"record",
|
|
70
|
+
"replay",
|
|
71
|
+
"resumable_runs",
|
|
72
|
+
"resume",
|
|
73
|
+
"rng",
|
|
74
|
+
"steps",
|
|
75
|
+
"time",
|
|
76
|
+
"tool",
|
|
77
|
+
"uuid4",
|
|
78
|
+
"wrap",
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
__version__ = "0.0.1"
|
ledger/__main__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
|
+
# NO CHECKED-IN PROTOBUF GENCODE
|
|
4
|
+
# source: events.proto
|
|
5
|
+
# Protobuf Python Version: 7.35.1
|
|
6
|
+
"""Generated protocol buffer code."""
|
|
7
|
+
from google.protobuf import descriptor as _descriptor
|
|
8
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
9
|
+
from google.protobuf import runtime_version as _runtime_version
|
|
10
|
+
from google.protobuf import symbol_database as _symbol_database
|
|
11
|
+
from google.protobuf.internal import builder as _builder
|
|
12
|
+
_runtime_version.ValidateProtobufRuntimeVersion(
|
|
13
|
+
_runtime_version.Domain.PUBLIC,
|
|
14
|
+
7,
|
|
15
|
+
35,
|
|
16
|
+
1,
|
|
17
|
+
'',
|
|
18
|
+
'events.proto'
|
|
19
|
+
)
|
|
20
|
+
# @@protoc_insertion_point(imports)
|
|
21
|
+
|
|
22
|
+
_sym_db = _symbol_database.Default()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x65vents.proto\x12\tledger.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"X\n\x07StepRef\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tsuperstep\x18\x02 \x01(\x03\x12\x0c\n\x04node\x18\x03 \x01(\t\x12\x0c\n\x04path\x18\x04 \x01(\t\x12\x0b\n\x03seq\x18\x05 \x01(\x03\"\'\n\x07Payload\x12\x0e\n\x06sha256\x18\x01 \x01(\t\x12\x0c\n\x04json\x18\x02 \x01(\x0c\"H\n\x05\x45rror\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x0f\n\x07message\x18\x02 \x01(\t\x12 \n\x04\x61rgs\x18\x03 \x01(\x0b\x32\x12.ledger.v1.Payload\"t\n\nRunStarted\x12\x1d\n\x04mode\x18\x01 \x01(\x0e\x32\x0f.ledger.v1.Mode\x12!\n\x05input\x18\x02 \x01(\x0b\x32\x12.ledger.v1.Payload\x12\x15\n\rsource_run_id\x18\x03 \x01(\t\x12\r\n\x05graph\x18\x04 \x01(\t\"G\n\nRunResumed\x12\x0f\n\x07\x61ttempt\x18\x01 \x01(\x05\x12(\n\x0eprevious_error\x18\x02 \x01(\x0b\x32\x10.ledger.v1.Error\"R\n\x0bRunFinished\x12\"\n\x06output\x18\x01 \x01(\x0b\x32\x12.ledger.v1.Payload\x12\x1f\n\x05\x65rror\x18\x02 \x01(\x0b\x32\x10.ledger.v1.Error\"\xd2\x01\n\x07LLMCall\x12\x10\n\x08\x62\x61se_url\x18\x01 \x01(\t\x12\x11\n\toperation\x18\x02 \x01(\t\x12\r\n\x05model\x18\x03 \x01(\t\x12#\n\x07request\x18\x04 \x01(\x0b\x32\x12.ledger.v1.Payload\x12$\n\x08response\x18\x05 \x01(\x0b\x32\x12.ledger.v1.Payload\x12\x1f\n\x05\x65rror\x18\x06 \x01(\x0b\x32\x10.ledger.v1.Error\x12\x12\n\nlatency_us\x18\x07 \x01(\x03\x12\x13\n\x0brequest_key\x18\x08 \x01(\t\"\xa5\x01\n\x08ToolCall\x12\x0c\n\x04name\x18\x01 \x01(\t\x12 \n\x04\x61rgs\x18\x02 \x01(\x0b\x32\x12.ledger.v1.Payload\x12\"\n\x06result\x18\x03 \x01(\x0b\x32\x12.ledger.v1.Payload\x12\x1f\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x10.ledger.v1.Error\x12\x12\n\nlatency_us\x18\x05 \x01(\x03\x12\x10\n\x08\x61rgs_key\x18\x06 \x01(\t\";\n\x06Nondet\x12\x0e\n\x06source\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.ledger.v1.Payload\"\xec\x02\n\nEffectCall\x12*\n\x05phase\x18\x01 \x01(\x0e\x32\x1b.ledger.v1.EffectCall.Phase\x12\x0c\n\x04name\x18\x02 \x01(\t\x12 \n\x04\x61rgs\x18\x03 \x01(\x0b\x32\x12.ledger.v1.Payload\x12\x10\n\x08\x61rgs_key\x18\x04 \x01(\t\x12\x17\n\x0fidempotency_key\x18\x05 \x01(\t\x12\"\n\x06result\x18\x06 \x01(\x0b\x32\x12.ledger.v1.Payload\x12\x1f\n\x05\x65rror\x18\x07 \x01(\x0b\x32\x10.ledger.v1.Error\x12\x12\n\nlatency_us\x18\x08 \x01(\x03\x12\x10\n\x08\x61ttempts\x18\t \x01(\x05\x12\x12\n\nreconciled\x18\n \x01(\x08\"X\n\x05Phase\x12\x15\n\x11PHASE_UNSPECIFIED\x10\x00\x12\x11\n\rPHASE_PENDING\x10\x01\x12\x13\n\x0fPHASE_CONFIRMED\x10\x02\x12\x10\n\x0cPHASE_FAILED\x10\x03\"g\n\x08\x44iverged\x12+\n\x06reason\x18\x01 \x01(\x0e\x32\x1b.ledger.v1.DivergenceReason\x12\x0c\n\x04kind\x18\x02 \x01(\t\x12\x10\n\x08recorded\x18\x03 \x01(\t\x12\x0e\n\x06\x61\x63tual\x18\x04 \x01(\t\"\xe8\x03\n\x05\x45vent\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\r\n\x05index\x18\x02 \x01(\x03\x12/\n\x0brecorded_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12 \n\x04step\x18\x04 \x01(\x0b\x32\x12.ledger.v1.StepRef\x12\x10\n\x08\x66rom_log\x18\x05 \x01(\x08\x12,\n\x0brun_started\x18\n \x01(\x0b\x32\x15.ledger.v1.RunStartedH\x00\x12.\n\x0crun_finished\x18\x0b \x01(\x0b\x32\x16.ledger.v1.RunFinishedH\x00\x12&\n\x08llm_call\x18\x0c \x01(\x0b\x32\x12.ledger.v1.LLMCallH\x00\x12(\n\ttool_call\x18\r \x01(\x0b\x32\x13.ledger.v1.ToolCallH\x00\x12#\n\x06nondet\x18\x0e \x01(\x0b\x32\x11.ledger.v1.NondetH\x00\x12\'\n\x08\x64iverged\x18\x0f \x01(\x0b\x32\x13.ledger.v1.DivergedH\x00\x12\'\n\x06\x65\x66\x66\x65\x63t\x18\x10 \x01(\x0b\x32\x15.ledger.v1.EffectCallH\x00\x12,\n\x0brun_resumed\x18\x11 \x01(\x0b\x32\x15.ledger.v1.RunResumedH\x00\x42\x06\n\x04\x62ody*M\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_RECORD\x10\x01\x12\x0f\n\x0bMODE_REPLAY\x10\x02\x12\r\n\tMODE_FORK\x10\x03*\xe2\x01\n\x10\x44ivergenceReason\x12!\n\x1d\x44IVERGENCE_REASON_UNSPECIFIED\x10\x00\x12%\n!DIVERGENCE_REASON_CHANGED_REQUEST\x10\x01\x12%\n!DIVERGENCE_REASON_UNRECORDED_CALL\x10\x02\x12\"\n\x1e\x44IVERGENCE_REASON_CHANGED_KIND\x10\x03\x12\x1c\n\x18\x44IVERGENCE_REASON_EDITED\x10\x04\x12\x1b\n\x17\x44IVERGENCE_REASON_RERUN\x10\x05\x62\x06proto3')
|
|
29
|
+
|
|
30
|
+
_globals = globals()
|
|
31
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
32
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'events_pb2', _globals)
|
|
33
|
+
if not _descriptor._USE_C_DESCRIPTORS:
|
|
34
|
+
DESCRIPTOR._loaded_options = None
|
|
35
|
+
_globals['_MODE']._serialized_start=1945
|
|
36
|
+
_globals['_MODE']._serialized_end=2022
|
|
37
|
+
_globals['_DIVERGENCEREASON']._serialized_start=2025
|
|
38
|
+
_globals['_DIVERGENCEREASON']._serialized_end=2251
|
|
39
|
+
_globals['_STEPREF']._serialized_start=60
|
|
40
|
+
_globals['_STEPREF']._serialized_end=148
|
|
41
|
+
_globals['_PAYLOAD']._serialized_start=150
|
|
42
|
+
_globals['_PAYLOAD']._serialized_end=189
|
|
43
|
+
_globals['_ERROR']._serialized_start=191
|
|
44
|
+
_globals['_ERROR']._serialized_end=263
|
|
45
|
+
_globals['_RUNSTARTED']._serialized_start=265
|
|
46
|
+
_globals['_RUNSTARTED']._serialized_end=381
|
|
47
|
+
_globals['_RUNRESUMED']._serialized_start=383
|
|
48
|
+
_globals['_RUNRESUMED']._serialized_end=454
|
|
49
|
+
_globals['_RUNFINISHED']._serialized_start=456
|
|
50
|
+
_globals['_RUNFINISHED']._serialized_end=538
|
|
51
|
+
_globals['_LLMCALL']._serialized_start=541
|
|
52
|
+
_globals['_LLMCALL']._serialized_end=751
|
|
53
|
+
_globals['_TOOLCALL']._serialized_start=754
|
|
54
|
+
_globals['_TOOLCALL']._serialized_end=919
|
|
55
|
+
_globals['_NONDET']._serialized_start=921
|
|
56
|
+
_globals['_NONDET']._serialized_end=980
|
|
57
|
+
_globals['_EFFECTCALL']._serialized_start=983
|
|
58
|
+
_globals['_EFFECTCALL']._serialized_end=1347
|
|
59
|
+
_globals['_EFFECTCALL_PHASE']._serialized_start=1259
|
|
60
|
+
_globals['_EFFECTCALL_PHASE']._serialized_end=1347
|
|
61
|
+
_globals['_DIVERGED']._serialized_start=1349
|
|
62
|
+
_globals['_DIVERGED']._serialized_end=1452
|
|
63
|
+
_globals['_EVENT']._serialized_start=1455
|
|
64
|
+
_globals['_EVENT']._serialized_end=1943
|
|
65
|
+
# @@protoc_insertion_point(module_scope)
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
|
|
3
|
+
from google.protobuf import timestamp_pb2 as _timestamp_pb2
|
|
4
|
+
from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
|
|
5
|
+
from google.protobuf import descriptor as _descriptor
|
|
6
|
+
from google.protobuf import message as _message
|
|
7
|
+
from collections.abc import Mapping as _Mapping
|
|
8
|
+
from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
|
|
9
|
+
|
|
10
|
+
DESCRIPTOR: _descriptor.FileDescriptor
|
|
11
|
+
|
|
12
|
+
class Mode(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
|
13
|
+
__slots__ = ()
|
|
14
|
+
MODE_UNSPECIFIED: _ClassVar[Mode]
|
|
15
|
+
MODE_RECORD: _ClassVar[Mode]
|
|
16
|
+
MODE_REPLAY: _ClassVar[Mode]
|
|
17
|
+
MODE_FORK: _ClassVar[Mode]
|
|
18
|
+
|
|
19
|
+
class DivergenceReason(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
|
20
|
+
__slots__ = ()
|
|
21
|
+
DIVERGENCE_REASON_UNSPECIFIED: _ClassVar[DivergenceReason]
|
|
22
|
+
DIVERGENCE_REASON_CHANGED_REQUEST: _ClassVar[DivergenceReason]
|
|
23
|
+
DIVERGENCE_REASON_UNRECORDED_CALL: _ClassVar[DivergenceReason]
|
|
24
|
+
DIVERGENCE_REASON_CHANGED_KIND: _ClassVar[DivergenceReason]
|
|
25
|
+
DIVERGENCE_REASON_EDITED: _ClassVar[DivergenceReason]
|
|
26
|
+
DIVERGENCE_REASON_RERUN: _ClassVar[DivergenceReason]
|
|
27
|
+
MODE_UNSPECIFIED: Mode
|
|
28
|
+
MODE_RECORD: Mode
|
|
29
|
+
MODE_REPLAY: Mode
|
|
30
|
+
MODE_FORK: Mode
|
|
31
|
+
DIVERGENCE_REASON_UNSPECIFIED: DivergenceReason
|
|
32
|
+
DIVERGENCE_REASON_CHANGED_REQUEST: DivergenceReason
|
|
33
|
+
DIVERGENCE_REASON_UNRECORDED_CALL: DivergenceReason
|
|
34
|
+
DIVERGENCE_REASON_CHANGED_KIND: DivergenceReason
|
|
35
|
+
DIVERGENCE_REASON_EDITED: DivergenceReason
|
|
36
|
+
DIVERGENCE_REASON_RERUN: DivergenceReason
|
|
37
|
+
|
|
38
|
+
class StepRef(_message.Message):
|
|
39
|
+
__slots__ = ("namespace", "superstep", "node", "path", "seq")
|
|
40
|
+
NAMESPACE_FIELD_NUMBER: _ClassVar[int]
|
|
41
|
+
SUPERSTEP_FIELD_NUMBER: _ClassVar[int]
|
|
42
|
+
NODE_FIELD_NUMBER: _ClassVar[int]
|
|
43
|
+
PATH_FIELD_NUMBER: _ClassVar[int]
|
|
44
|
+
SEQ_FIELD_NUMBER: _ClassVar[int]
|
|
45
|
+
namespace: str
|
|
46
|
+
superstep: int
|
|
47
|
+
node: str
|
|
48
|
+
path: str
|
|
49
|
+
seq: int
|
|
50
|
+
def __init__(self, namespace: _Optional[str] = ..., superstep: _Optional[int] = ..., node: _Optional[str] = ..., path: _Optional[str] = ..., seq: _Optional[int] = ...) -> None: ...
|
|
51
|
+
|
|
52
|
+
class Payload(_message.Message):
|
|
53
|
+
__slots__ = ("sha256", "json")
|
|
54
|
+
SHA256_FIELD_NUMBER: _ClassVar[int]
|
|
55
|
+
JSON_FIELD_NUMBER: _ClassVar[int]
|
|
56
|
+
sha256: str
|
|
57
|
+
json: bytes
|
|
58
|
+
def __init__(self, sha256: _Optional[str] = ..., json: _Optional[bytes] = ...) -> None: ...
|
|
59
|
+
|
|
60
|
+
class Error(_message.Message):
|
|
61
|
+
__slots__ = ("type", "message", "args")
|
|
62
|
+
TYPE_FIELD_NUMBER: _ClassVar[int]
|
|
63
|
+
MESSAGE_FIELD_NUMBER: _ClassVar[int]
|
|
64
|
+
ARGS_FIELD_NUMBER: _ClassVar[int]
|
|
65
|
+
type: str
|
|
66
|
+
message: str
|
|
67
|
+
args: Payload
|
|
68
|
+
def __init__(self, type: _Optional[str] = ..., message: _Optional[str] = ..., args: _Optional[_Union[Payload, _Mapping]] = ...) -> None: ...
|
|
69
|
+
|
|
70
|
+
class RunStarted(_message.Message):
|
|
71
|
+
__slots__ = ("mode", "input", "source_run_id", "graph")
|
|
72
|
+
MODE_FIELD_NUMBER: _ClassVar[int]
|
|
73
|
+
INPUT_FIELD_NUMBER: _ClassVar[int]
|
|
74
|
+
SOURCE_RUN_ID_FIELD_NUMBER: _ClassVar[int]
|
|
75
|
+
GRAPH_FIELD_NUMBER: _ClassVar[int]
|
|
76
|
+
mode: Mode
|
|
77
|
+
input: Payload
|
|
78
|
+
source_run_id: str
|
|
79
|
+
graph: str
|
|
80
|
+
def __init__(self, mode: _Optional[_Union[Mode, str]] = ..., input: _Optional[_Union[Payload, _Mapping]] = ..., source_run_id: _Optional[str] = ..., graph: _Optional[str] = ...) -> None: ...
|
|
81
|
+
|
|
82
|
+
class RunResumed(_message.Message):
|
|
83
|
+
__slots__ = ("attempt", "previous_error")
|
|
84
|
+
ATTEMPT_FIELD_NUMBER: _ClassVar[int]
|
|
85
|
+
PREVIOUS_ERROR_FIELD_NUMBER: _ClassVar[int]
|
|
86
|
+
attempt: int
|
|
87
|
+
previous_error: Error
|
|
88
|
+
def __init__(self, attempt: _Optional[int] = ..., previous_error: _Optional[_Union[Error, _Mapping]] = ...) -> None: ...
|
|
89
|
+
|
|
90
|
+
class RunFinished(_message.Message):
|
|
91
|
+
__slots__ = ("output", "error")
|
|
92
|
+
OUTPUT_FIELD_NUMBER: _ClassVar[int]
|
|
93
|
+
ERROR_FIELD_NUMBER: _ClassVar[int]
|
|
94
|
+
output: Payload
|
|
95
|
+
error: Error
|
|
96
|
+
def __init__(self, output: _Optional[_Union[Payload, _Mapping]] = ..., error: _Optional[_Union[Error, _Mapping]] = ...) -> None: ...
|
|
97
|
+
|
|
98
|
+
class LLMCall(_message.Message):
|
|
99
|
+
__slots__ = ("base_url", "operation", "model", "request", "response", "error", "latency_us", "request_key")
|
|
100
|
+
BASE_URL_FIELD_NUMBER: _ClassVar[int]
|
|
101
|
+
OPERATION_FIELD_NUMBER: _ClassVar[int]
|
|
102
|
+
MODEL_FIELD_NUMBER: _ClassVar[int]
|
|
103
|
+
REQUEST_FIELD_NUMBER: _ClassVar[int]
|
|
104
|
+
RESPONSE_FIELD_NUMBER: _ClassVar[int]
|
|
105
|
+
ERROR_FIELD_NUMBER: _ClassVar[int]
|
|
106
|
+
LATENCY_US_FIELD_NUMBER: _ClassVar[int]
|
|
107
|
+
REQUEST_KEY_FIELD_NUMBER: _ClassVar[int]
|
|
108
|
+
base_url: str
|
|
109
|
+
operation: str
|
|
110
|
+
model: str
|
|
111
|
+
request: Payload
|
|
112
|
+
response: Payload
|
|
113
|
+
error: Error
|
|
114
|
+
latency_us: int
|
|
115
|
+
request_key: str
|
|
116
|
+
def __init__(self, base_url: _Optional[str] = ..., operation: _Optional[str] = ..., model: _Optional[str] = ..., request: _Optional[_Union[Payload, _Mapping]] = ..., response: _Optional[_Union[Payload, _Mapping]] = ..., error: _Optional[_Union[Error, _Mapping]] = ..., latency_us: _Optional[int] = ..., request_key: _Optional[str] = ...) -> None: ...
|
|
117
|
+
|
|
118
|
+
class ToolCall(_message.Message):
|
|
119
|
+
__slots__ = ("name", "args", "result", "error", "latency_us", "args_key")
|
|
120
|
+
NAME_FIELD_NUMBER: _ClassVar[int]
|
|
121
|
+
ARGS_FIELD_NUMBER: _ClassVar[int]
|
|
122
|
+
RESULT_FIELD_NUMBER: _ClassVar[int]
|
|
123
|
+
ERROR_FIELD_NUMBER: _ClassVar[int]
|
|
124
|
+
LATENCY_US_FIELD_NUMBER: _ClassVar[int]
|
|
125
|
+
ARGS_KEY_FIELD_NUMBER: _ClassVar[int]
|
|
126
|
+
name: str
|
|
127
|
+
args: Payload
|
|
128
|
+
result: Payload
|
|
129
|
+
error: Error
|
|
130
|
+
latency_us: int
|
|
131
|
+
args_key: str
|
|
132
|
+
def __init__(self, name: _Optional[str] = ..., args: _Optional[_Union[Payload, _Mapping]] = ..., result: _Optional[_Union[Payload, _Mapping]] = ..., error: _Optional[_Union[Error, _Mapping]] = ..., latency_us: _Optional[int] = ..., args_key: _Optional[str] = ...) -> None: ...
|
|
133
|
+
|
|
134
|
+
class Nondet(_message.Message):
|
|
135
|
+
__slots__ = ("source", "value")
|
|
136
|
+
SOURCE_FIELD_NUMBER: _ClassVar[int]
|
|
137
|
+
VALUE_FIELD_NUMBER: _ClassVar[int]
|
|
138
|
+
source: str
|
|
139
|
+
value: Payload
|
|
140
|
+
def __init__(self, source: _Optional[str] = ..., value: _Optional[_Union[Payload, _Mapping]] = ...) -> None: ...
|
|
141
|
+
|
|
142
|
+
class EffectCall(_message.Message):
|
|
143
|
+
__slots__ = ("phase", "name", "args", "args_key", "idempotency_key", "result", "error", "latency_us", "attempts", "reconciled")
|
|
144
|
+
class Phase(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
|
145
|
+
__slots__ = ()
|
|
146
|
+
PHASE_UNSPECIFIED: _ClassVar[EffectCall.Phase]
|
|
147
|
+
PHASE_PENDING: _ClassVar[EffectCall.Phase]
|
|
148
|
+
PHASE_CONFIRMED: _ClassVar[EffectCall.Phase]
|
|
149
|
+
PHASE_FAILED: _ClassVar[EffectCall.Phase]
|
|
150
|
+
PHASE_UNSPECIFIED: EffectCall.Phase
|
|
151
|
+
PHASE_PENDING: EffectCall.Phase
|
|
152
|
+
PHASE_CONFIRMED: EffectCall.Phase
|
|
153
|
+
PHASE_FAILED: EffectCall.Phase
|
|
154
|
+
PHASE_FIELD_NUMBER: _ClassVar[int]
|
|
155
|
+
NAME_FIELD_NUMBER: _ClassVar[int]
|
|
156
|
+
ARGS_FIELD_NUMBER: _ClassVar[int]
|
|
157
|
+
ARGS_KEY_FIELD_NUMBER: _ClassVar[int]
|
|
158
|
+
IDEMPOTENCY_KEY_FIELD_NUMBER: _ClassVar[int]
|
|
159
|
+
RESULT_FIELD_NUMBER: _ClassVar[int]
|
|
160
|
+
ERROR_FIELD_NUMBER: _ClassVar[int]
|
|
161
|
+
LATENCY_US_FIELD_NUMBER: _ClassVar[int]
|
|
162
|
+
ATTEMPTS_FIELD_NUMBER: _ClassVar[int]
|
|
163
|
+
RECONCILED_FIELD_NUMBER: _ClassVar[int]
|
|
164
|
+
phase: EffectCall.Phase
|
|
165
|
+
name: str
|
|
166
|
+
args: Payload
|
|
167
|
+
args_key: str
|
|
168
|
+
idempotency_key: str
|
|
169
|
+
result: Payload
|
|
170
|
+
error: Error
|
|
171
|
+
latency_us: int
|
|
172
|
+
attempts: int
|
|
173
|
+
reconciled: bool
|
|
174
|
+
def __init__(self, phase: _Optional[_Union[EffectCall.Phase, str]] = ..., name: _Optional[str] = ..., args: _Optional[_Union[Payload, _Mapping]] = ..., args_key: _Optional[str] = ..., idempotency_key: _Optional[str] = ..., result: _Optional[_Union[Payload, _Mapping]] = ..., error: _Optional[_Union[Error, _Mapping]] = ..., latency_us: _Optional[int] = ..., attempts: _Optional[int] = ..., reconciled: _Optional[bool] = ...) -> None: ...
|
|
175
|
+
|
|
176
|
+
class Diverged(_message.Message):
|
|
177
|
+
__slots__ = ("reason", "kind", "recorded", "actual")
|
|
178
|
+
REASON_FIELD_NUMBER: _ClassVar[int]
|
|
179
|
+
KIND_FIELD_NUMBER: _ClassVar[int]
|
|
180
|
+
RECORDED_FIELD_NUMBER: _ClassVar[int]
|
|
181
|
+
ACTUAL_FIELD_NUMBER: _ClassVar[int]
|
|
182
|
+
reason: DivergenceReason
|
|
183
|
+
kind: str
|
|
184
|
+
recorded: str
|
|
185
|
+
actual: str
|
|
186
|
+
def __init__(self, reason: _Optional[_Union[DivergenceReason, str]] = ..., kind: _Optional[str] = ..., recorded: _Optional[str] = ..., actual: _Optional[str] = ...) -> None: ...
|
|
187
|
+
|
|
188
|
+
class Event(_message.Message):
|
|
189
|
+
__slots__ = ("run_id", "index", "recorded_at", "step", "from_log", "run_started", "run_finished", "llm_call", "tool_call", "nondet", "diverged", "effect", "run_resumed")
|
|
190
|
+
RUN_ID_FIELD_NUMBER: _ClassVar[int]
|
|
191
|
+
INDEX_FIELD_NUMBER: _ClassVar[int]
|
|
192
|
+
RECORDED_AT_FIELD_NUMBER: _ClassVar[int]
|
|
193
|
+
STEP_FIELD_NUMBER: _ClassVar[int]
|
|
194
|
+
FROM_LOG_FIELD_NUMBER: _ClassVar[int]
|
|
195
|
+
RUN_STARTED_FIELD_NUMBER: _ClassVar[int]
|
|
196
|
+
RUN_FINISHED_FIELD_NUMBER: _ClassVar[int]
|
|
197
|
+
LLM_CALL_FIELD_NUMBER: _ClassVar[int]
|
|
198
|
+
TOOL_CALL_FIELD_NUMBER: _ClassVar[int]
|
|
199
|
+
NONDET_FIELD_NUMBER: _ClassVar[int]
|
|
200
|
+
DIVERGED_FIELD_NUMBER: _ClassVar[int]
|
|
201
|
+
EFFECT_FIELD_NUMBER: _ClassVar[int]
|
|
202
|
+
RUN_RESUMED_FIELD_NUMBER: _ClassVar[int]
|
|
203
|
+
run_id: str
|
|
204
|
+
index: int
|
|
205
|
+
recorded_at: _timestamp_pb2.Timestamp
|
|
206
|
+
step: StepRef
|
|
207
|
+
from_log: bool
|
|
208
|
+
run_started: RunStarted
|
|
209
|
+
run_finished: RunFinished
|
|
210
|
+
llm_call: LLMCall
|
|
211
|
+
tool_call: ToolCall
|
|
212
|
+
nondet: Nondet
|
|
213
|
+
diverged: Diverged
|
|
214
|
+
effect: EffectCall
|
|
215
|
+
run_resumed: RunResumed
|
|
216
|
+
def __init__(self, run_id: _Optional[str] = ..., index: _Optional[int] = ..., recorded_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., step: _Optional[_Union[StepRef, _Mapping]] = ..., from_log: _Optional[bool] = ..., run_started: _Optional[_Union[RunStarted, _Mapping]] = ..., run_finished: _Optional[_Union[RunFinished, _Mapping]] = ..., llm_call: _Optional[_Union[LLMCall, _Mapping]] = ..., tool_call: _Optional[_Union[ToolCall, _Mapping]] = ..., nondet: _Optional[_Union[Nondet, _Mapping]] = ..., diverged: _Optional[_Union[Diverged, _Mapping]] = ..., effect: _Optional[_Union[EffectCall, _Mapping]] = ..., run_resumed: _Optional[_Union[RunResumed, _Mapping]] = ...) -> None: ...
|
ledger/blame/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Counterfactual blame: which step of a failed run caused the failure."""
|
|
2
|
+
|
|
3
|
+
from ledger.blame.budget import Budget, Pricing, Spend, wilson
|
|
4
|
+
from ledger.blame.delta import (
|
|
5
|
+
CANDIDATE_KINDS,
|
|
6
|
+
Counterfactual,
|
|
7
|
+
Diagnosis,
|
|
8
|
+
RunNotFailed,
|
|
9
|
+
diagnose,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"Budget",
|
|
14
|
+
"CANDIDATE_KINDS",
|
|
15
|
+
"Counterfactual",
|
|
16
|
+
"Diagnosis",
|
|
17
|
+
"Pricing",
|
|
18
|
+
"RunNotFailed",
|
|
19
|
+
"Spend",
|
|
20
|
+
"diagnose",
|
|
21
|
+
"wilson",
|
|
22
|
+
]
|
ledger/blame/budget.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""What a diagnosis may spend, what it did spend, and how sure each fork is.
|
|
2
|
+
|
|
3
|
+
budget = Budget(forks=20, dollars=0.50)
|
|
4
|
+
pricing = Pricing(input_per_mtok=0.15, output_per_mtok=0.60)
|
|
5
|
+
d = ledger.diagnose(run_id, graph, check, repeats=5, budget=budget, pricing=pricing)
|
|
6
|
+
|
|
7
|
+
A Budget caps forks, live LLM and tool calls, or dollars per diagnosis;
|
|
8
|
+
``None`` leaves a cap off. It is checked before each fork, so the fork that
|
|
9
|
+
crosses a cap still finishes and the total can go over by one fork's cost.
|
|
10
|
+
Dollars need a Pricing, applied to the token usage the live LLM calls
|
|
11
|
+
report.
|
|
12
|
+
|
|
13
|
+
``wilson`` is the Wilson score interval for a success rate, which stays
|
|
14
|
+
inside [0, 1] and is sensible for the handful of trials a counterfactual
|
|
15
|
+
gets.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import math
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
|
|
23
|
+
from ledger.core.events import body_kind, load_payload
|
|
24
|
+
from ledger.core.store import EventStore
|
|
25
|
+
|
|
26
|
+
__all__ = ["Budget", "OutOfBudget", "Pricing", "Spend", "usage", "wilson"]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def wilson(passed: int, trials: int, z: float = 1.96) -> tuple[float, float]:
|
|
30
|
+
"""95% (by default) Wilson score interval for ``passed`` out of ``trials``."""
|
|
31
|
+
if trials <= 0:
|
|
32
|
+
return (0.0, 1.0)
|
|
33
|
+
p = passed / trials
|
|
34
|
+
denom = 1 + z * z / trials
|
|
35
|
+
centre = (p + z * z / (2 * trials)) / denom
|
|
36
|
+
half = z * math.sqrt(p * (1 - p) / trials + z * z / (4 * trials * trials)) / denom
|
|
37
|
+
return (max(0.0, centre - half), min(1.0, centre + half))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Pricing:
|
|
42
|
+
"""Dollars per million input and output tokens."""
|
|
43
|
+
|
|
44
|
+
input_per_mtok: float
|
|
45
|
+
output_per_mtok: float
|
|
46
|
+
|
|
47
|
+
def dollars(self, tokens_in: int, tokens_out: int) -> float:
|
|
48
|
+
return (tokens_in * self.input_per_mtok + tokens_out * self.output_per_mtok) / 1e6
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class Spend:
|
|
53
|
+
forks: int = 0
|
|
54
|
+
live_calls: int = 0
|
|
55
|
+
tokens_in: int = 0
|
|
56
|
+
tokens_out: int = 0
|
|
57
|
+
dollars: float = 0.0
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class OutOfBudget(Exception):
|
|
61
|
+
"""Raised inside a diagnosis when the next fork would start over budget."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class Budget:
|
|
66
|
+
forks: int | None = None
|
|
67
|
+
live_calls: int | None = None
|
|
68
|
+
dollars: float | None = None
|
|
69
|
+
|
|
70
|
+
def exhausted(self, spent: Spend) -> bool:
|
|
71
|
+
return (
|
|
72
|
+
(self.forks is not None and spent.forks >= self.forks)
|
|
73
|
+
or (self.live_calls is not None and spent.live_calls >= self.live_calls)
|
|
74
|
+
or (self.dollars is not None and spent.dollars >= self.dollars)
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def usage(store: EventStore, run_id: str) -> tuple[int, int]:
|
|
79
|
+
"""(prompt, completion) tokens of the LLM calls a run made live."""
|
|
80
|
+
tokens_in = tokens_out = 0
|
|
81
|
+
for ev in store.events(run_id):
|
|
82
|
+
if body_kind(ev) != "llm_call" or ev.from_log or ev.llm_call.HasField("error"):
|
|
83
|
+
continue
|
|
84
|
+
resp = load_payload(ev.llm_call.response)
|
|
85
|
+
u = (resp or {}).get("usage") or {} if isinstance(resp, dict) else {}
|
|
86
|
+
tokens_in += int(u.get("prompt_tokens") or 0)
|
|
87
|
+
tokens_out += int(u.get("completion_tokens") or 0)
|
|
88
|
+
return tokens_in, tokens_out
|