stepledger 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stepledger/__init__.py +25 -0
- stepledger/__main__.py +5 -0
- stepledger/_compat.py +49 -0
- stepledger/_hooks.py +22 -0
- stepledger/canonical.py +53 -0
- stepledger/cli.py +261 -0
- stepledger/config.py +108 -0
- stepledger/effects/__init__.py +1 -0
- stepledger/effects/once.py +159 -0
- stepledger/errors.py +64 -0
- stepledger/headers.py +72 -0
- stepledger/keys.py +54 -0
- stepledger/ledger/__init__.py +1 -0
- stepledger/ledger/activity_interceptor.py +201 -0
- stepledger/ledger/context.py +38 -0
- stepledger/ledger/history.py +88 -0
- stepledger/ledger/reconcile.py +255 -0
- stepledger/ledger/schema.sql +82 -0
- stepledger/ledger/seal.py +40 -0
- stepledger/ledger/store.py +417 -0
- stepledger/ledger/workflow_interceptor.py +174 -0
- stepledger/llm/__init__.py +2 -0
- stepledger/llm/journal.py +162 -0
- stepledger/llm/meter.py +114 -0
- stepledger/plugin.py +150 -0
- stepledger/py.typed +0 -0
- stepledger/read/__init__.py +1 -0
- stepledger/read/ledger.py +105 -0
- stepledger/read/materialize.py +246 -0
- stepledger/read/views.sql +62 -0
- stepledger/storage/__init__.py +29 -0
- stepledger/storage/backends.py +259 -0
- stepledger/storage/chunking.py +30 -0
- stepledger/storage/driver.py +114 -0
- stepledger/storage/gc.py +191 -0
- stepledger/testing/__init__.py +4 -0
- stepledger/testing/fake_llm.py +92 -0
- stepledger/testing/faults.py +208 -0
- stepledger/testing/history.py +164 -0
- stepledger-0.1.0.dist-info/METADATA +165 -0
- stepledger-0.1.0.dist-info/RECORD +44 -0
- stepledger-0.1.0.dist-info/WHEEL +4 -0
- stepledger-0.1.0.dist-info/entry_points.txt +2 -0
- stepledger-0.1.0.dist-info/licenses/LICENSE +201 -0
stepledger/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Stepledger: a fenced, committed Postgres ledger for LangGraph nodes on Temporal."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
|
|
9
|
+
__all__ = ["JournaledChatModel", "StepledgerPlugin", "__version__", "materialize", "once"]
|
|
10
|
+
|
|
11
|
+
_LAZY = {
|
|
12
|
+
"StepledgerPlugin": ("stepledger.plugin", "StepledgerPlugin"),
|
|
13
|
+
"once": ("stepledger.effects.once", "once"),
|
|
14
|
+
"materialize": ("stepledger.read.materialize", "materialize"),
|
|
15
|
+
"JournaledChatModel": ("stepledger.llm.journal", "JournaledChatModel"),
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def __getattr__(name: str) -> Any:
|
|
20
|
+
if name in _LAZY:
|
|
21
|
+
import importlib
|
|
22
|
+
|
|
23
|
+
module, attr = _LAZY[name]
|
|
24
|
+
return getattr(importlib.import_module(module), attr)
|
|
25
|
+
raise AttributeError(f"module 'stepledger' has no attribute {name!r}")
|
stepledger/__main__.py
ADDED
stepledger/_compat.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Every private Temporal SDK or LangGraph symbol Stepledger uses, in one place (Hard Rule 5, D2).
|
|
2
|
+
|
|
3
|
+
Pinned against temporalio 1.33.x and langgraph 1.2.x. tests/unit/test_compat.py fails loudly if
|
|
4
|
+
an upgrade moves or changes any of them. Nothing else in the package imports a private module.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import temporalio.activity
|
|
13
|
+
from langgraph._internal._typing import MISSING # sentinel for channel.from_checkpoint
|
|
14
|
+
from langgraph.pregel._algo import task_path_str # LangGraph's sortable task-path string
|
|
15
|
+
from temporalio.contrib.langgraph._activity import ( # the plugin's node Activity I/O
|
|
16
|
+
ActivityInput,
|
|
17
|
+
ActivityOutput,
|
|
18
|
+
)
|
|
19
|
+
from temporalio.contrib.langgraph._task_cache import get_task_cache as _get_task_cache
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"MISSING",
|
|
23
|
+
"ActivityInput",
|
|
24
|
+
"ActivityOutput",
|
|
25
|
+
"activity_name",
|
|
26
|
+
"langgraph_used_in_this_run",
|
|
27
|
+
"plugin_activity_names",
|
|
28
|
+
"task_path_str",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def activity_name(fn: Callable[..., Any]) -> str:
|
|
33
|
+
"""The registered name of an @activity.defn function."""
|
|
34
|
+
name = temporalio.activity._Definition.must_from_callable(fn).name
|
|
35
|
+
if name is None:
|
|
36
|
+
raise ValueError(f"{fn!r} is a dynamic activity; it has no fixed name to track")
|
|
37
|
+
return name
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def plugin_activity_names(langgraph_plugin: Any) -> frozenset[str]:
|
|
41
|
+
"""Names of every Activity a LangGraphPlugin registered (its node and task Activities)."""
|
|
42
|
+
return frozenset(activity_name(a) for a in langgraph_plugin.activities)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def langgraph_used_in_this_run() -> bool:
|
|
46
|
+
"""True inside workflow code once graph() / entrypoint() ran: both install the plugin's task
|
|
47
|
+
cache (a context variable), even when every node is then served from it and no Activity is
|
|
48
|
+
scheduled. Pure in-memory state, so safe to read from the deterministic workflow side."""
|
|
49
|
+
return _get_task_cache() is not None
|
stepledger/_hooks.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Fault-injection hook points inside Stepledger's own commit path (F3, F4, F5).
|
|
2
|
+
|
|
3
|
+
A no-op unless `stepledger.testing.faults.install()` registers an injector, which only chaos
|
|
4
|
+
workers do. Production code never imports stepledger.testing.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Awaitable, Callable
|
|
10
|
+
|
|
11
|
+
Injector = Callable[..., Awaitable[None]]
|
|
12
|
+
_injector: Injector | None = None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def set_injector(fn: Injector | None) -> None:
|
|
16
|
+
global _injector
|
|
17
|
+
_injector = fn
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
async def fault(point: str, **where: object) -> None:
|
|
21
|
+
if _injector is not None:
|
|
22
|
+
await _injector(point, **where)
|
stepledger/canonical.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Canonical JSON and content hashes, used only for hashing and equality.
|
|
2
|
+
|
|
3
|
+
Values are first turned into plain JSON by the worker's own payload converter, so a hash here
|
|
4
|
+
equals the hash of what Temporal recorded for the same value.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from temporalio.api.common.v1 import Payload
|
|
15
|
+
from temporalio.converter import PayloadConverter
|
|
16
|
+
|
|
17
|
+
JSON_PLAIN = b"json/plain"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def canonical_json(obj: Any) -> str:
|
|
21
|
+
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def chash(obj: Any) -> str:
|
|
25
|
+
"""sha256 hex of the canonical JSON of a plain-JSON value."""
|
|
26
|
+
return hashlib.sha256(canonical_json(obj).encode("utf-8")).hexdigest()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Serialized:
|
|
31
|
+
"""A value as the payload converter serialized it."""
|
|
32
|
+
|
|
33
|
+
encoding: str
|
|
34
|
+
data: bytes
|
|
35
|
+
plain: Any # the decoded JSON value when encoding is json/plain, else None
|
|
36
|
+
is_json: bool
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def hash(self) -> str:
|
|
40
|
+
if self.is_json:
|
|
41
|
+
return chash(self.plain)
|
|
42
|
+
return hashlib.sha256(self.data).hexdigest()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def serialize(value: Any, converter: PayloadConverter) -> Serialized:
|
|
46
|
+
return from_payload(converter.to_payload(value))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def from_payload(payload: Payload) -> Serialized:
|
|
50
|
+
encoding = payload.metadata.get("encoding", b"").decode()
|
|
51
|
+
if encoding == JSON_PLAIN.decode():
|
|
52
|
+
return Serialized(encoding, payload.data, json.loads(payload.data), True)
|
|
53
|
+
return Serialized(encoding, payload.data, None, False)
|
stepledger/cli.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""The `stepledger` command line."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from stepledger import __version__
|
|
10
|
+
from stepledger.config import resolve_dsn
|
|
11
|
+
|
|
12
|
+
app = typer.Typer(
|
|
13
|
+
name="stepledger",
|
|
14
|
+
help="Stepledger: a fenced, committed Postgres ledger for LangGraph nodes on Temporal.",
|
|
15
|
+
no_args_is_help=True,
|
|
16
|
+
add_completion=False,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _version(value: bool) -> None:
|
|
21
|
+
if value:
|
|
22
|
+
typer.echo(f"stepledger {__version__}")
|
|
23
|
+
raise typer.Exit()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@app.callback()
|
|
27
|
+
def main(
|
|
28
|
+
version: bool = typer.Option(
|
|
29
|
+
False, "--version", callback=_version, is_eager=True, help="Print the version and exit."
|
|
30
|
+
),
|
|
31
|
+
) -> None:
|
|
32
|
+
"""Stepledger command line."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
DsnOption = typer.Option(None, "--dsn", envvar="STEPLEDGER_DSN", help="Postgres DSN.")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@app.command("init-db")
|
|
39
|
+
def init_db_cmd(dsn: str | None = DsnOption) -> None:
|
|
40
|
+
"""Create the Stepledger tables and views. Safe to run repeatedly."""
|
|
41
|
+
from stepledger.ledger.store import init_db
|
|
42
|
+
|
|
43
|
+
init_db(resolve_dsn(dsn))
|
|
44
|
+
typer.echo("stepledger schema applied")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
AddressOption = typer.Option("localhost:7233", "--address", envvar="TEMPORAL_ADDRESS")
|
|
48
|
+
NamespaceOption = typer.Option("default", "--namespace", "-n", envvar="TEMPORAL_NAMESPACE")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _run(coro: Any) -> Any:
|
|
52
|
+
import asyncio
|
|
53
|
+
|
|
54
|
+
return asyncio.run(coro)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@app.command("reconcile")
|
|
58
|
+
def reconcile_cmd(
|
|
59
|
+
workflow_id: str | None = typer.Argument(None, help="Reconcile every run of this workflow."),
|
|
60
|
+
all_open: bool = typer.Option(False, "--all-open", help="Every run the ledger has not sealed."),
|
|
61
|
+
dsn: str | None = DsnOption,
|
|
62
|
+
address: str = AddressOption,
|
|
63
|
+
namespace: str = NamespaceOption,
|
|
64
|
+
) -> None:
|
|
65
|
+
"""Repair the ledger from Temporal's history (terminations, timeouts, degraded runs)."""
|
|
66
|
+
if not workflow_id and not all_open:
|
|
67
|
+
raise typer.BadParameter("give a workflow id or --all-open")
|
|
68
|
+
|
|
69
|
+
async def go() -> None:
|
|
70
|
+
from temporalio.client import Client
|
|
71
|
+
|
|
72
|
+
from stepledger.ledger.reconcile import reconcile
|
|
73
|
+
from stepledger.ledger.store import LedgerStore
|
|
74
|
+
|
|
75
|
+
client = await Client.connect(address, namespace=namespace)
|
|
76
|
+
store = LedgerStore(resolve_dsn(dsn))
|
|
77
|
+
try:
|
|
78
|
+
reports = await reconcile(client, store, workflow_id, all_open=all_open)
|
|
79
|
+
finally:
|
|
80
|
+
await store.close()
|
|
81
|
+
if not reports:
|
|
82
|
+
typer.echo("nothing to reconcile")
|
|
83
|
+
for r in reports:
|
|
84
|
+
typer.echo(
|
|
85
|
+
f"{r.workflow_id} {r.run_id} {r.run_status}: {r.action}"
|
|
86
|
+
f" committed={r.committed} abandoned={r.abandoned} inserted={r.inserted}"
|
|
87
|
+
f" divergence_repaired={r.divergence_repaired}"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
_run(go())
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@app.command("resolve")
|
|
94
|
+
def resolve_cmd(
|
|
95
|
+
key: str = typer.Argument(..., help="The effect key from the UnknownEffectOutcome error."),
|
|
96
|
+
outcome: str = typer.Option(..., "--outcome", help="done | not-done"),
|
|
97
|
+
result: str | None = typer.Option(None, "--result", help="JSON result when done."),
|
|
98
|
+
dsn: str | None = DsnOption,
|
|
99
|
+
) -> None:
|
|
100
|
+
"""Record a human's verdict on an effect whose outcome is unknown."""
|
|
101
|
+
import json
|
|
102
|
+
|
|
103
|
+
async def go() -> None:
|
|
104
|
+
from stepledger.effects.once import resolve
|
|
105
|
+
from stepledger.ledger.store import LedgerStore
|
|
106
|
+
|
|
107
|
+
store = LedgerStore(resolve_dsn(dsn))
|
|
108
|
+
try:
|
|
109
|
+
state = await resolve(store, key, outcome, json.loads(result) if result else None)
|
|
110
|
+
finally:
|
|
111
|
+
await store.close()
|
|
112
|
+
typer.echo(f"{key}: {state}")
|
|
113
|
+
|
|
114
|
+
_run(go())
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@app.command("ledger")
|
|
118
|
+
def ledger_cmd(
|
|
119
|
+
workflow_id: str = typer.Argument(...),
|
|
120
|
+
run_id: str | None = typer.Option(None, "--run-id"),
|
|
121
|
+
dsn: str | None = DsnOption,
|
|
122
|
+
) -> None:
|
|
123
|
+
"""Print the run ledger: one line per node execution, with attempts, effects and waste."""
|
|
124
|
+
from stepledger.read.ledger import run_ledger
|
|
125
|
+
|
|
126
|
+
typer.echo(run_ledger(resolve_dsn(dsn), workflow_id, run_id))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@app.command("status")
|
|
130
|
+
def status_cmd(
|
|
131
|
+
limit: int = typer.Option(20, "--limit"),
|
|
132
|
+
dsn: str | None = DsnOption,
|
|
133
|
+
) -> None:
|
|
134
|
+
"""Recent runs in the ledger."""
|
|
135
|
+
import psycopg
|
|
136
|
+
|
|
137
|
+
with psycopg.connect(resolve_dsn(dsn)) as conn:
|
|
138
|
+
rows = conn.execute(
|
|
139
|
+
"SELECT workflow_id, left(run_id, 12), status, sealed_at IS NOT NULL, committed_count,"
|
|
140
|
+
" abandoned_count, degraded FROM sl_runs ORDER BY first_seen_at DESC LIMIT %s",
|
|
141
|
+
(limit,),
|
|
142
|
+
).fetchall()
|
|
143
|
+
typer.echo(f"{'workflow_id':<36} {'run':<13} {'status':<17} sealed committed abandoned")
|
|
144
|
+
for wf, run, status, sealed, c, a, degraded in rows:
|
|
145
|
+
flag = " DEGRADED" if degraded else ""
|
|
146
|
+
typer.echo(
|
|
147
|
+
f"{wf:<36} {run:<13} {status:<17} {'yes' if sealed else 'no ':<6} {c or 0:>9}"
|
|
148
|
+
f" {a or 0:>9}{flag}"
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@app.command("gc")
|
|
153
|
+
def gc_cmd(
|
|
154
|
+
execute: bool = typer.Option(False, "--execute", help="Delete for real (default: dry run)."),
|
|
155
|
+
retention_days: float | None = typer.Option(None, "--retention-days"),
|
|
156
|
+
margin_days: float | None = typer.Option(None, "--margin-days"),
|
|
157
|
+
grace_hours: float | None = typer.Option(None, "--grace-hours"),
|
|
158
|
+
orphan_ref_days: float | None = typer.Option(None, "--orphan-ref-days"),
|
|
159
|
+
dsn: str | None = DsnOption,
|
|
160
|
+
address: str = AddressOption,
|
|
161
|
+
namespace: str = NamespaceOption,
|
|
162
|
+
) -> None:
|
|
163
|
+
"""Mark and sweep the dedup store. Dry run unless --execute."""
|
|
164
|
+
from datetime import timedelta
|
|
165
|
+
|
|
166
|
+
from stepledger.config import load_settings
|
|
167
|
+
|
|
168
|
+
cfg = load_settings().gc
|
|
169
|
+
retention = timedelta(days=retention_days if retention_days is not None else cfg.retention_days)
|
|
170
|
+
margin = timedelta(days=margin_days if margin_days is not None else cfg.margin_days)
|
|
171
|
+
grace = timedelta(hours=grace_hours if grace_hours is not None else cfg.grace_hours)
|
|
172
|
+
orphan = timedelta(days=orphan_ref_days if orphan_ref_days is not None else cfg.orphan_ref_days)
|
|
173
|
+
|
|
174
|
+
async def go() -> None:
|
|
175
|
+
from temporalio.client import Client
|
|
176
|
+
|
|
177
|
+
from stepledger.storage.gc import check_retention, namespace_retention, summary, sweep
|
|
178
|
+
|
|
179
|
+
client = await Client.connect(address, namespace=namespace)
|
|
180
|
+
problem = check_retention(retention, await namespace_retention(client))
|
|
181
|
+
if problem:
|
|
182
|
+
typer.echo(f"refusing to run: {problem}", err=True)
|
|
183
|
+
raise typer.Exit(2)
|
|
184
|
+
report = await sweep(
|
|
185
|
+
client,
|
|
186
|
+
resolve_dsn(dsn),
|
|
187
|
+
retention=retention,
|
|
188
|
+
margin=margin,
|
|
189
|
+
grace=grace,
|
|
190
|
+
orphan_ref_age=orphan,
|
|
191
|
+
dry_run=not execute,
|
|
192
|
+
)
|
|
193
|
+
for k, v in summary(report).items():
|
|
194
|
+
if k != "details":
|
|
195
|
+
typer.echo(f"{k}: {v}")
|
|
196
|
+
|
|
197
|
+
_run(go())
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
@app.command("materialize")
|
|
201
|
+
def materialize_cmd(
|
|
202
|
+
workflow_id: str = typer.Argument(...),
|
|
203
|
+
graph: str = typer.Option(
|
|
204
|
+
..., "--graph", help="module:attr of a StateGraph, a compiled graph, or a factory for one"
|
|
205
|
+
),
|
|
206
|
+
run_id: str | None = typer.Option(None, "--run-id"),
|
|
207
|
+
chain: bool = typer.Option(False, "--chain", help="Fill task-cache gaps from earlier runs."),
|
|
208
|
+
include_provisional: bool = typer.Option(False, "--include-provisional"),
|
|
209
|
+
show_state: bool = typer.Option(False, "--state", help="Print the rebuilt state as JSON."),
|
|
210
|
+
dsn: str | None = DsnOption,
|
|
211
|
+
) -> None:
|
|
212
|
+
"""Rebuild a run's graph state from the ledger and say whether it is EXACT."""
|
|
213
|
+
import importlib
|
|
214
|
+
import json
|
|
215
|
+
import os
|
|
216
|
+
import sys
|
|
217
|
+
|
|
218
|
+
from stepledger.read.materialize import materialize
|
|
219
|
+
|
|
220
|
+
if os.getcwd() not in sys.path: # like uvicorn: --graph resolves from the working directory
|
|
221
|
+
sys.path.insert(0, os.getcwd())
|
|
222
|
+
module, _, attr = graph.partition(":")
|
|
223
|
+
obj: Any = getattr(importlib.import_module(module), attr)
|
|
224
|
+
if callable(obj) and not hasattr(obj, "compile") and not hasattr(obj, "channels"):
|
|
225
|
+
obj = obj()
|
|
226
|
+
compiled = obj.compile() if hasattr(obj, "compile") else obj
|
|
227
|
+
result = _run(
|
|
228
|
+
materialize(
|
|
229
|
+
resolve_dsn(dsn),
|
|
230
|
+
compiled,
|
|
231
|
+
workflow_id,
|
|
232
|
+
run_id,
|
|
233
|
+
include_provisional=include_provisional,
|
|
234
|
+
chain=chain,
|
|
235
|
+
)
|
|
236
|
+
)
|
|
237
|
+
typer.echo(f"{result.completeness}: {result.reason}")
|
|
238
|
+
typer.echo(f"rows used: {result.rows_used} chain rows used: {result.chain_rows_used}")
|
|
239
|
+
for p in result.positions:
|
|
240
|
+
typer.echo(f" gap: {p}")
|
|
241
|
+
if show_state:
|
|
242
|
+
typer.echo(json.dumps(result.state, indent=1, default=str))
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@app.command("bench")
|
|
246
|
+
def bench_cmd(
|
|
247
|
+
demo: str = typer.Argument(..., help="cliff | chaos | growth | materialize | cost"),
|
|
248
|
+
) -> None:
|
|
249
|
+
"""Run a demo from a repository checkout (the bench is not part of the wheel)."""
|
|
250
|
+
import subprocess
|
|
251
|
+
import sys
|
|
252
|
+
from pathlib import Path
|
|
253
|
+
|
|
254
|
+
if not Path("bench/demo.py").is_file():
|
|
255
|
+
typer.echo("run this from a stepledger repository checkout (bench/ is not installed)")
|
|
256
|
+
raise typer.Exit(2)
|
|
257
|
+
raise typer.Exit(subprocess.call([sys.executable, "bench/demo.py", "--demo", demo]))
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
if __name__ == "__main__":
|
|
261
|
+
app()
|
stepledger/config.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Configuration: `stepledger.yaml`, then environment variables (`STEPLEDGER_DSN`, ...)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Literal
|
|
9
|
+
|
|
10
|
+
import yaml
|
|
11
|
+
from pydantic import BaseModel, Field
|
|
12
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
13
|
+
|
|
14
|
+
DEFAULT_DSN = "postgresql://stepledger:stepledger@localhost:5432/stepledger"
|
|
15
|
+
SCHEMA_VERSION = 1
|
|
16
|
+
|
|
17
|
+
_ENV_REF = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class LedgerConfig(BaseModel):
|
|
21
|
+
store_outputs: Literal["full", "hash_only"] = "full"
|
|
22
|
+
on_ledger_error: Literal["fail", "warn"] = "fail"
|
|
23
|
+
snapshot_first_input: bool = True
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ChunkConfig(BaseModel):
|
|
27
|
+
min: int = 4096
|
|
28
|
+
avg: int = 16384
|
|
29
|
+
max: int = 65536
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class StorageConfig(BaseModel):
|
|
33
|
+
enabled: bool = True
|
|
34
|
+
payload_size_threshold: int = 64 * 1024
|
|
35
|
+
dedupe: bool = True
|
|
36
|
+
chunk: ChunkConfig = Field(default_factory=ChunkConfig)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class GcConfig(BaseModel):
|
|
40
|
+
retention_days: int = 30
|
|
41
|
+
margin_days: int = 7
|
|
42
|
+
grace_hours: float = 1.0
|
|
43
|
+
orphan_ref_days: int = 90
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Price(BaseModel):
|
|
47
|
+
input: float
|
|
48
|
+
output: float
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _default_prices() -> dict[str, Price]:
|
|
52
|
+
# USD per 1M tokens, Anthropic first-party API rates as published at
|
|
53
|
+
# https://platform.claude.com/docs/en/about-claude/pricing (checked 2026-09-26).
|
|
54
|
+
# Prices change: set `prices:` in stepledger.yaml for the models you actually use.
|
|
55
|
+
return {
|
|
56
|
+
"claude-haiku-4-5": Price(input=1.0, output=5.0),
|
|
57
|
+
"claude-sonnet-5": Price(input=2.0, output=10.0),
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Settings(BaseSettings):
|
|
62
|
+
model_config = SettingsConfigDict(
|
|
63
|
+
env_prefix="STEPLEDGER_", env_nested_delimiter="__", extra="ignore"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
schema_version: int = SCHEMA_VERSION
|
|
67
|
+
dsn: str = DEFAULT_DSN
|
|
68
|
+
ledger: LedgerConfig = Field(default_factory=LedgerConfig)
|
|
69
|
+
storage: StorageConfig = Field(default_factory=StorageConfig)
|
|
70
|
+
gc: GcConfig = Field(default_factory=GcConfig)
|
|
71
|
+
prices: dict[str, Price] = Field(default_factory=_default_prices)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _expand(value: Any) -> Any:
|
|
75
|
+
if isinstance(value, str):
|
|
76
|
+
return _ENV_REF.sub(lambda m: os.environ.get(m.group(1), ""), value)
|
|
77
|
+
if isinstance(value, dict):
|
|
78
|
+
return {k: _expand(v) for k, v in value.items()}
|
|
79
|
+
if isinstance(value, list):
|
|
80
|
+
return [_expand(v) for v in value]
|
|
81
|
+
return value
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def load_settings(path: str | Path | None = None) -> Settings:
|
|
85
|
+
"""Load `stepledger.yaml` (or `$STEPLEDGER_CONFIG`), then let environment variables win."""
|
|
86
|
+
candidate = Path(path or os.environ.get("STEPLEDGER_CONFIG", "stepledger.yaml"))
|
|
87
|
+
file_values: dict[str, Any] = {}
|
|
88
|
+
if candidate.is_file():
|
|
89
|
+
loaded = yaml.safe_load(candidate.read_text()) or {}
|
|
90
|
+
if not isinstance(loaded, dict):
|
|
91
|
+
raise ValueError(f"{candidate}: expected a mapping at the top level")
|
|
92
|
+
file_values = _expand(loaded)
|
|
93
|
+
version = file_values.get("schema_version", SCHEMA_VERSION)
|
|
94
|
+
if version != SCHEMA_VERSION:
|
|
95
|
+
raise ValueError(f"{candidate}: schema_version {version} != {SCHEMA_VERSION}")
|
|
96
|
+
if not file_values.get("dsn"):
|
|
97
|
+
file_values.pop("dsn", None)
|
|
98
|
+
env = Settings()
|
|
99
|
+
merged = Settings.model_validate(file_values).model_dump()
|
|
100
|
+
# Environment variables override the file, field by field.
|
|
101
|
+
for name in Settings.model_fields:
|
|
102
|
+
if env.model_fields_set and name in env.model_fields_set:
|
|
103
|
+
merged[name] = env.model_dump()[name]
|
|
104
|
+
return Settings.model_validate(merged)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def resolve_dsn(dsn: str | None = None) -> str:
|
|
108
|
+
return dsn or load_settings().dsn
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""External side effects with a stable idempotency key and a journal: once()."""
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""once(): at-least-once external effects with dedupe, never silently repeated.
|
|
2
|
+
|
|
3
|
+
ticket = await once("open_ticket", lambda key: jira.create(req, idempotency_key=key),
|
|
4
|
+
request=req)
|
|
5
|
+
|
|
6
|
+
Inside a tracked node Activity, the key is sha256(namespace, workflow, run, seq, name, idx), where
|
|
7
|
+
idx counts once() calls with this name in this node execution, so it is stable across retries
|
|
8
|
+
of the node. `fn` receives the key so the tool can pass it upstream as its own idempotency key.
|
|
9
|
+
|
|
10
|
+
no journal row -> record STARTED, call fn(key), record DONE with the result
|
|
11
|
+
DONE, same request -> return the recorded result (a duplicate prevented, counted)
|
|
12
|
+
other request -> EffectDivergence (non-retryable)
|
|
13
|
+
STARTED / UNKNOWN -> the outcome is unknown: ask `reconcile(key)`. It returns the effect's
|
|
14
|
+
result if the tool shows it happened, NOT_DONE if the tool shows it did
|
|
15
|
+
not (then fn is called again), or None if it cannot tell: then mark
|
|
16
|
+
UNKNOWN and raise UnknownEffectOutcome, which waits (retrying slowly)
|
|
17
|
+
for `stepledger resolve <key> --outcome done|not-done`
|
|
18
|
+
RESOLVED not-done -> the effect did not happen: claim it again and call fn(key)
|
|
19
|
+
|
|
20
|
+
Results are stored as JSON, so fn must return a JSON-serializable value. This is at-least-once
|
|
21
|
+
delivery with dedupe, not exactly-once: an effect whose tool ignores the key can repeat if the
|
|
22
|
+
worker dies between the call and the DONE write, which is why that case stops for a human.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from collections.abc import Awaitable, Callable
|
|
28
|
+
from datetime import timedelta
|
|
29
|
+
from typing import Any, TypeVar
|
|
30
|
+
|
|
31
|
+
from psycopg.types.json import Jsonb
|
|
32
|
+
|
|
33
|
+
from stepledger._hooks import fault
|
|
34
|
+
from stepledger.canonical import chash
|
|
35
|
+
from stepledger.errors import EffectDivergence, NotInTrackedNode, UnknownEffectOutcome
|
|
36
|
+
from stepledger.ledger.context import current_node
|
|
37
|
+
|
|
38
|
+
T = TypeVar("T")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class _NotDone:
|
|
42
|
+
def __repr__(self) -> str:
|
|
43
|
+
return "NOT_DONE"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
NOT_DONE: Any = _NotDone()
|
|
47
|
+
"""Returned by a reconcile callback when the tool shows the effect did not happen."""
|
|
48
|
+
|
|
49
|
+
UNKNOWN_RETRY_DELAY = timedelta(seconds=30)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def once(
|
|
53
|
+
name: str,
|
|
54
|
+
fn: Callable[[str], Awaitable[T]],
|
|
55
|
+
*,
|
|
56
|
+
request: Any,
|
|
57
|
+
reconcile: Callable[[str], Awaitable[T | None]] | None = None,
|
|
58
|
+
) -> T:
|
|
59
|
+
node = current_node()
|
|
60
|
+
if node is None:
|
|
61
|
+
raise NotInTrackedNode("once() must be called inside a node Activity tracked by Stepledger")
|
|
62
|
+
idx = node.next_effect_idx(name)
|
|
63
|
+
key = node.key.effect_key(name, idx)
|
|
64
|
+
request_hash = chash(request)
|
|
65
|
+
k = node.key
|
|
66
|
+
|
|
67
|
+
async with node.store.tx() as tx:
|
|
68
|
+
cur = await tx.conn.execute(
|
|
69
|
+
"SELECT status, request_hash, result, resolution FROM sl_effects WHERE key = %s"
|
|
70
|
+
" FOR UPDATE",
|
|
71
|
+
(key,),
|
|
72
|
+
)
|
|
73
|
+
row = await cur.fetchone()
|
|
74
|
+
if row is None:
|
|
75
|
+
await tx.conn.execute(
|
|
76
|
+
"INSERT INTO sl_effects (key, namespace, workflow_id, run_id, seq, name, idx,"
|
|
77
|
+
" request_hash, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'STARTED')",
|
|
78
|
+
(key, k.namespace, k.workflow_id, k.run_id, k.seq, name, idx, request_hash),
|
|
79
|
+
)
|
|
80
|
+
claimed = True
|
|
81
|
+
else:
|
|
82
|
+
status, stored_hash, result, resolution = row
|
|
83
|
+
if stored_hash != request_hash:
|
|
84
|
+
raise EffectDivergence(key, name)
|
|
85
|
+
if status == "DONE":
|
|
86
|
+
await tx.conn.execute(
|
|
87
|
+
"UPDATE sl_effects SET duplicates_prevented = duplicates_prevented + 1,"
|
|
88
|
+
" attempts = attempts + 1 WHERE key = %s",
|
|
89
|
+
(key,),
|
|
90
|
+
)
|
|
91
|
+
return result # type: ignore[no-any-return]
|
|
92
|
+
if status == "RESOLVED" and resolution == "not-done":
|
|
93
|
+
await tx.conn.execute(
|
|
94
|
+
"UPDATE sl_effects SET status = 'STARTED', attempts = attempts + 1,"
|
|
95
|
+
" resolution = NULL WHERE key = %s",
|
|
96
|
+
(key,),
|
|
97
|
+
)
|
|
98
|
+
claimed = True
|
|
99
|
+
else:
|
|
100
|
+
claimed = False # STARTED or UNKNOWN: an earlier attempt's outcome is unknown
|
|
101
|
+
|
|
102
|
+
if claimed:
|
|
103
|
+
result = await fn(key)
|
|
104
|
+
await fault("FE", effect=name) # chaos only: a crash between the call and its record
|
|
105
|
+
await _done(node.store, key, result)
|
|
106
|
+
return result
|
|
107
|
+
|
|
108
|
+
if reconcile is not None:
|
|
109
|
+
found = await reconcile(key)
|
|
110
|
+
if found is NOT_DONE:
|
|
111
|
+
result = await fn(key)
|
|
112
|
+
await _done(node.store, key, result, resolution="reconciled:not-done")
|
|
113
|
+
return result
|
|
114
|
+
if found is not None:
|
|
115
|
+
await _done(node.store, key, found, resolution="reconciled:done")
|
|
116
|
+
return found
|
|
117
|
+
async with node.store.tx() as tx:
|
|
118
|
+
await tx.conn.execute(
|
|
119
|
+
"UPDATE sl_effects SET status = 'UNKNOWN', attempts = attempts + 1 WHERE key = %s"
|
|
120
|
+
" AND status IN ('STARTED', 'UNKNOWN')",
|
|
121
|
+
(key,),
|
|
122
|
+
)
|
|
123
|
+
raise UnknownEffectOutcome(key, name, next_retry_delay=UNKNOWN_RETRY_DELAY)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
async def _done(store: Any, key: str, result: Any, resolution: str | None = None) -> None:
|
|
127
|
+
async with store.tx() as tx:
|
|
128
|
+
await tx.conn.execute(
|
|
129
|
+
"UPDATE sl_effects SET status = 'DONE', result = %s, done_at = now(),"
|
|
130
|
+
" resolution = coalesce(%s, resolution) WHERE key = %s",
|
|
131
|
+
(Jsonb(result), resolution, key),
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
async def resolve(store: Any, key: str, outcome: str, result: Any = None) -> str:
|
|
136
|
+
"""Record a human's verdict on an effect with an unknown outcome."""
|
|
137
|
+
if outcome not in ("done", "not-done"):
|
|
138
|
+
raise ValueError("outcome must be 'done' or 'not-done'")
|
|
139
|
+
async with store.tx() as tx:
|
|
140
|
+
cur = await tx.conn.execute(
|
|
141
|
+
"SELECT status FROM sl_effects WHERE key = %s FOR UPDATE", (key,)
|
|
142
|
+
)
|
|
143
|
+
row = await cur.fetchone()
|
|
144
|
+
if row is None:
|
|
145
|
+
raise KeyError(f"no effect with key {key}")
|
|
146
|
+
if row[0] == "DONE":
|
|
147
|
+
return "already DONE"
|
|
148
|
+
if outcome == "done":
|
|
149
|
+
await tx.conn.execute(
|
|
150
|
+
"UPDATE sl_effects SET status = 'DONE', result = %s, done_at = now(),"
|
|
151
|
+
" resolution = 'manual:done' WHERE key = %s",
|
|
152
|
+
(Jsonb(result), key),
|
|
153
|
+
)
|
|
154
|
+
return "DONE"
|
|
155
|
+
await tx.conn.execute(
|
|
156
|
+
"UPDATE sl_effects SET status = 'RESOLVED', resolution = 'not-done' WHERE key = %s",
|
|
157
|
+
(key,),
|
|
158
|
+
)
|
|
159
|
+
return "RESOLVED not-done"
|