python-durable 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- durable/__init__.py +51 -0
- durable/backoff.py +40 -0
- durable/context.py +39 -0
- durable/store.py +198 -0
- durable/workflow.py +338 -0
- python_durable-0.1.0.dist-info/METADATA +133 -0
- python_durable-0.1.0.dist-info/RECORD +9 -0
- python_durable-0.1.0.dist-info/WHEEL +4 -0
- python_durable-0.1.0.dist-info/licenses/LICENSE +21 -0
durable/__init__.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""
|
|
2
|
+
durable — lightweight workflow durability for Python.
|
|
3
|
+
|
|
4
|
+
Make any async workflow resumable after crashes with just a decorator.
|
|
5
|
+
Backed by SQLite out of the box; swap in any Store subclass for production.
|
|
6
|
+
|
|
7
|
+
Quick start:
|
|
8
|
+
|
|
9
|
+
from durable import Workflow
|
|
10
|
+
from durable.backoff import exponential
|
|
11
|
+
|
|
12
|
+
wf = Workflow("my-app")
|
|
13
|
+
|
|
14
|
+
@wf.task(retries=3, backoff=exponential(base=2, max=60))
|
|
15
|
+
async def fetch_data(url: str) -> dict:
|
|
16
|
+
async with httpx.AsyncClient() as client:
|
|
17
|
+
return (await client.get(url)).json()
|
|
18
|
+
|
|
19
|
+
@wf.task
|
|
20
|
+
async def save_result(data: dict) -> None:
|
|
21
|
+
await db.insert(data)
|
|
22
|
+
|
|
23
|
+
@wf.workflow(id="pipeline-{source}")
|
|
24
|
+
async def run_pipeline(source: str) -> None:
|
|
25
|
+
data = await fetch_data(f"https://api.example.com/{source}")
|
|
26
|
+
await save_result(data)
|
|
27
|
+
|
|
28
|
+
# First call: runs all steps and checkpoints each one.
|
|
29
|
+
# If it crashes and you call it again with the same args,
|
|
30
|
+
# completed steps are replayed from SQLite instantly.
|
|
31
|
+
await run_pipeline(source="users")
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from .backoff import BackoffStrategy, constant, exponential, linear
|
|
35
|
+
from .context import RunContext
|
|
36
|
+
from .store import InMemoryStore, SQLiteStore, Store
|
|
37
|
+
from .workflow import Workflow
|
|
38
|
+
|
|
39
|
+
__all__ = [
|
|
40
|
+
"Workflow",
|
|
41
|
+
"Store",
|
|
42
|
+
"SQLiteStore",
|
|
43
|
+
"InMemoryStore",
|
|
44
|
+
"RunContext",
|
|
45
|
+
"BackoffStrategy",
|
|
46
|
+
"exponential",
|
|
47
|
+
"constant",
|
|
48
|
+
"linear",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
__version__ = "0.1.0"
|
durable/backoff.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Backoff strategies for task retries.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
exponential(base=2, max=60) → 2s, 4s, 8s, 16s... capped at 60s
|
|
6
|
+
constant(5) → always 5s
|
|
7
|
+
linear(start=2, step=3) → 2s, 5s, 8s, 11s...
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from typing import Callable
|
|
11
|
+
|
|
12
|
+
# A strategy takes the attempt number (0-indexed) and returns seconds to wait.
|
|
13
|
+
BackoffStrategy = Callable[[int], float]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def exponential(base: float = 2.0, max: float = 60.0) -> BackoffStrategy:
|
|
17
|
+
"""Exponential backoff: base^attempt, capped at max seconds."""
|
|
18
|
+
|
|
19
|
+
def strategy(attempt: int) -> float:
|
|
20
|
+
return min(base**attempt, max)
|
|
21
|
+
|
|
22
|
+
return strategy
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def constant(seconds: float) -> BackoffStrategy:
|
|
26
|
+
"""Always wait the same number of seconds."""
|
|
27
|
+
|
|
28
|
+
def strategy(attempt: int) -> float:
|
|
29
|
+
return seconds
|
|
30
|
+
|
|
31
|
+
return strategy
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def linear(start: float = 1.0, step: float = 1.0) -> BackoffStrategy:
|
|
35
|
+
"""Linearly increasing wait: start, start+step, start+2*step..."""
|
|
36
|
+
|
|
37
|
+
def strategy(attempt: int) -> float:
|
|
38
|
+
return start + step * attempt
|
|
39
|
+
|
|
40
|
+
return strategy
|
durable/context.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
RunContext holds the state of a single workflow execution.
|
|
3
|
+
|
|
4
|
+
It's stored in a ContextVar so it propagates implicitly through async call chains —
|
|
5
|
+
tasks can read it without the caller threading it through manually (same pattern
|
|
6
|
+
as pydantic-ai's RunContext / dependency injection).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections import defaultdict
|
|
12
|
+
from contextvars import ContextVar
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import TYPE_CHECKING
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from .store import Store
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class RunContext:
|
|
22
|
+
run_id: str
|
|
23
|
+
workflow_id: str
|
|
24
|
+
store: "Store"
|
|
25
|
+
|
|
26
|
+
# Tracks how many times each step name has been used so we can
|
|
27
|
+
# auto-generate unique, deterministic step IDs without the user needing
|
|
28
|
+
# to think about it. e.g. fetch_user called twice → "fetch_user", "fetch_user#1"
|
|
29
|
+
_counters: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
30
|
+
|
|
31
|
+
def next_step_id(self, name: str) -> str:
|
|
32
|
+
count = self._counters[name]
|
|
33
|
+
self._counters[name] += 1
|
|
34
|
+
return name if count == 0 else f"{name}#{count}"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
# The single active run for the current async task / coroutine tree.
|
|
38
|
+
# set() returns a Token you can use to reset later (important for nesting).
|
|
39
|
+
_active_run: ContextVar[RunContext | None] = ContextVar("_active_run", default=None)
|
durable/store.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Storage backends for durable step checkpoints.
|
|
3
|
+
|
|
4
|
+
Default: SQLiteStore (zero deps beyond aiosqlite).
|
|
5
|
+
Override by passing any Store subclass to Workflow(db=...).
|
|
6
|
+
|
|
7
|
+
Schema is intentionally minimal — two tables:
|
|
8
|
+
runs → track lifecycle of a workflow execution
|
|
9
|
+
steps → store serialized results keyed by (run_id, step_id)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from abc import ABC, abstractmethod
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
import aiosqlite
|
|
19
|
+
|
|
20
|
+
# We wrap every stored value in {"v": ...} so we can distinguish
|
|
21
|
+
# "step not found" (None) from "step completed and returned None" ({"v": null}).
|
|
22
|
+
_WRAP_KEY = "v"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _wrap(value: Any) -> str:
|
|
26
|
+
return json.dumps({_WRAP_KEY: value})
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _unwrap(raw: str) -> Any:
|
|
30
|
+
return json.loads(raw)[_WRAP_KEY]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Store(ABC):
|
|
34
|
+
"""Abstract base — implement this to use Postgres, Redis, etc."""
|
|
35
|
+
|
|
36
|
+
@abstractmethod
|
|
37
|
+
async def setup(self) -> None:
|
|
38
|
+
"""Called once before the store is used. Create tables, connect pools, etc."""
|
|
39
|
+
|
|
40
|
+
@abstractmethod
|
|
41
|
+
async def get_step(self, run_id: str, step_id: str) -> tuple[bool, Any]:
|
|
42
|
+
"""Return (found, value). found=False means the step hasn't run yet."""
|
|
43
|
+
|
|
44
|
+
@abstractmethod
|
|
45
|
+
async def set_step(
|
|
46
|
+
self, run_id: str, step_id: str, result: Any, attempt: int = 1
|
|
47
|
+
) -> None:
|
|
48
|
+
"""Persist a completed step result."""
|
|
49
|
+
|
|
50
|
+
@abstractmethod
|
|
51
|
+
async def mark_run_done(self, run_id: str) -> None:
|
|
52
|
+
"""Mark the whole workflow run as successfully completed."""
|
|
53
|
+
|
|
54
|
+
@abstractmethod
|
|
55
|
+
async def mark_run_failed(self, run_id: str, error: str) -> None:
|
|
56
|
+
"""Mark the run as failed with an error message."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
_SENTINEL = object()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class InMemoryStore(Store):
|
|
63
|
+
"""
|
|
64
|
+
Dict-backed store — no persistence, no dependencies.
|
|
65
|
+
|
|
66
|
+
Useful for testing, short-lived scripts, and development where you don't
|
|
67
|
+
need crash recovery across process restarts.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
def __init__(self) -> None:
|
|
71
|
+
self._steps: dict[tuple[str, str], Any] = {}
|
|
72
|
+
self._runs: dict[str, str] = {}
|
|
73
|
+
|
|
74
|
+
async def setup(self) -> None:
|
|
75
|
+
pass
|
|
76
|
+
|
|
77
|
+
async def get_step(self, run_id: str, step_id: str) -> tuple[bool, Any]:
|
|
78
|
+
value = self._steps.get((run_id, step_id), _SENTINEL)
|
|
79
|
+
if value is _SENTINEL:
|
|
80
|
+
return False, None
|
|
81
|
+
return True, value
|
|
82
|
+
|
|
83
|
+
async def set_step(
|
|
84
|
+
self, run_id: str, step_id: str, result: Any, attempt: int = 1
|
|
85
|
+
) -> None:
|
|
86
|
+
self._steps[(run_id, step_id)] = result
|
|
87
|
+
|
|
88
|
+
async def mark_run_done(self, run_id: str) -> None:
|
|
89
|
+
self._runs[run_id] = "done"
|
|
90
|
+
|
|
91
|
+
async def mark_run_failed(self, run_id: str, error: str) -> None:
|
|
92
|
+
self._runs[run_id] = "failed"
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class SQLiteStore(Store):
|
|
96
|
+
"""
|
|
97
|
+
Default store backed by a local SQLite file via aiosqlite.
|
|
98
|
+
|
|
99
|
+
Works great for local dev, single-process services, CLIs, and scripts.
|
|
100
|
+
For production or multi-process workloads, swap in a Postgres/Redis store.
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
def __init__(self, path: str = "durable.db") -> None:
|
|
104
|
+
self.path = path
|
|
105
|
+
self._ready = False
|
|
106
|
+
|
|
107
|
+
async def setup(self) -> None:
|
|
108
|
+
if self._ready:
|
|
109
|
+
return
|
|
110
|
+
async with aiosqlite.connect(self.path) as db:
|
|
111
|
+
await db.executescript("""
|
|
112
|
+
CREATE TABLE IF NOT EXISTS runs (
|
|
113
|
+
run_id TEXT PRIMARY KEY,
|
|
114
|
+
workflow_id TEXT NOT NULL,
|
|
115
|
+
status TEXT NOT NULL DEFAULT 'running',
|
|
116
|
+
error TEXT,
|
|
117
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
118
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
CREATE TABLE IF NOT EXISTS steps (
|
|
122
|
+
run_id TEXT NOT NULL,
|
|
123
|
+
step_id TEXT NOT NULL,
|
|
124
|
+
result TEXT NOT NULL,
|
|
125
|
+
attempt INTEGER NOT NULL DEFAULT 1,
|
|
126
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
127
|
+
PRIMARY KEY (run_id, step_id)
|
|
128
|
+
);
|
|
129
|
+
""")
|
|
130
|
+
await db.commit()
|
|
131
|
+
self._ready = True
|
|
132
|
+
|
|
133
|
+
async def get_step(self, run_id: str, step_id: str) -> tuple[bool, Any]:
|
|
134
|
+
async with aiosqlite.connect(self.path) as db:
|
|
135
|
+
async with db.execute(
|
|
136
|
+
"SELECT result FROM steps WHERE run_id = ? AND step_id = ?",
|
|
137
|
+
(run_id, step_id),
|
|
138
|
+
) as cursor:
|
|
139
|
+
row = await cursor.fetchone()
|
|
140
|
+
if row is None:
|
|
141
|
+
return False, None
|
|
142
|
+
return True, _unwrap(row[0])
|
|
143
|
+
|
|
144
|
+
async def set_step(
|
|
145
|
+
self, run_id: str, step_id: str, result: Any, attempt: int = 1
|
|
146
|
+
) -> None:
|
|
147
|
+
async with aiosqlite.connect(self.path) as db:
|
|
148
|
+
await db.execute(
|
|
149
|
+
"""
|
|
150
|
+
INSERT INTO steps (run_id, step_id, result, attempt)
|
|
151
|
+
VALUES (?, ?, ?, ?)
|
|
152
|
+
ON CONFLICT(run_id, step_id) DO UPDATE
|
|
153
|
+
SET result = excluded.result, attempt = excluded.attempt
|
|
154
|
+
""",
|
|
155
|
+
(run_id, step_id, _wrap(result), attempt),
|
|
156
|
+
)
|
|
157
|
+
await db.commit()
|
|
158
|
+
|
|
159
|
+
async def _upsert_run(
|
|
160
|
+
self, run_id: str, workflow_id: str, status: str, error: str | None = None
|
|
161
|
+
) -> None:
|
|
162
|
+
async with aiosqlite.connect(self.path) as db:
|
|
163
|
+
await db.execute(
|
|
164
|
+
"""
|
|
165
|
+
INSERT INTO runs (run_id, workflow_id, status, error)
|
|
166
|
+
VALUES (?, ?, ?, ?)
|
|
167
|
+
ON CONFLICT(run_id) DO UPDATE
|
|
168
|
+
SET status = excluded.status,
|
|
169
|
+
error = excluded.error,
|
|
170
|
+
updated_at = CURRENT_TIMESTAMP
|
|
171
|
+
""",
|
|
172
|
+
(run_id, workflow_id, status, error),
|
|
173
|
+
)
|
|
174
|
+
await db.commit()
|
|
175
|
+
|
|
176
|
+
async def mark_run_done(self, run_id: str) -> None:
|
|
177
|
+
async with aiosqlite.connect(self.path) as db:
|
|
178
|
+
await db.execute(
|
|
179
|
+
"UPDATE runs SET status = 'done', updated_at = CURRENT_TIMESTAMP WHERE run_id = ?",
|
|
180
|
+
(run_id,),
|
|
181
|
+
)
|
|
182
|
+
await db.commit()
|
|
183
|
+
|
|
184
|
+
async def mark_run_failed(self, run_id: str, error: str) -> None:
|
|
185
|
+
async with aiosqlite.connect(self.path) as db:
|
|
186
|
+
await db.execute(
|
|
187
|
+
"UPDATE runs SET status = 'failed', error = ?, updated_at = CURRENT_TIMESTAMP WHERE run_id = ?",
|
|
188
|
+
(error, run_id),
|
|
189
|
+
)
|
|
190
|
+
await db.commit()
|
|
191
|
+
|
|
192
|
+
async def ensure_run(self, run_id: str, workflow_id: str) -> None:
|
|
193
|
+
async with aiosqlite.connect(self.path) as db:
|
|
194
|
+
await db.execute(
|
|
195
|
+
"INSERT OR IGNORE INTO runs (run_id, workflow_id, status) VALUES (?, ?, 'running')",
|
|
196
|
+
(run_id, workflow_id),
|
|
197
|
+
)
|
|
198
|
+
await db.commit()
|
durable/workflow.py
ADDED
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Core Workflow class — the single object developers interact with.
|
|
3
|
+
|
|
4
|
+
wf = Workflow("my-app") # defaults to SQLite at ./durable.db
|
|
5
|
+
wf = Workflow("my-app", db="sqlite:///jobs.db")
|
|
6
|
+
wf = Workflow("my-app", db=MyPostgresStore())
|
|
7
|
+
|
|
8
|
+
Then decorate tasks and workflows:
|
|
9
|
+
|
|
10
|
+
@wf.task
|
|
11
|
+
async def my_task(x: int) -> str: ...
|
|
12
|
+
|
|
13
|
+
@wf.task(retries=5, backoff=exponential(base=2, max=30))
|
|
14
|
+
async def flaky_task(url: str) -> dict: ...
|
|
15
|
+
|
|
16
|
+
@wf.workflow(id="job-{job_id}")
|
|
17
|
+
async def my_workflow(job_id: str) -> None:
|
|
18
|
+
result = await my_task(42)
|
|
19
|
+
...
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import asyncio
|
|
25
|
+
import functools
|
|
26
|
+
import inspect
|
|
27
|
+
import logging
|
|
28
|
+
import re
|
|
29
|
+
from typing import Any, Callable, ParamSpec, TypeVar, overload
|
|
30
|
+
|
|
31
|
+
from .backoff import BackoffStrategy, exponential
|
|
32
|
+
from .context import RunContext, _active_run
|
|
33
|
+
from .store import SQLiteStore, Store
|
|
34
|
+
|
|
35
|
+
log = logging.getLogger("durable")
|
|
36
|
+
|
|
37
|
+
P = ParamSpec("P")
|
|
38
|
+
R = TypeVar("R")
|
|
39
|
+
|
|
40
|
+
# Matches "{param_name}" in workflow id templates
|
|
41
|
+
_TEMPLATE_RE = re.compile(r"\{(\w+)\}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _format_run_id(template: str, bound: inspect.BoundArguments) -> str:
|
|
45
|
+
"""Replace {param} placeholders with actual argument values."""
|
|
46
|
+
args = {**bound.arguments}
|
|
47
|
+
# Flatten **kwargs into the top-level dict if present
|
|
48
|
+
for k, v in list(args.items()):
|
|
49
|
+
if isinstance(v, dict) and k not in _TEMPLATE_RE.findall(template):
|
|
50
|
+
args.update(v)
|
|
51
|
+
return template.format_map(args)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class _TaskWrapper:
|
|
55
|
+
"""
|
|
56
|
+
Wraps an async function with checkpoint + retry logic.
|
|
57
|
+
|
|
58
|
+
Behaves like the original coroutine function — fully typed, awaitable,
|
|
59
|
+
introspectable. The durable magic only activates when called inside an
|
|
60
|
+
active workflow run (i.e. a RunContext is set in the ContextVar).
|
|
61
|
+
|
|
62
|
+
Outside a workflow, it just executes normally — great for testing.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
def __init__(
|
|
66
|
+
self,
|
|
67
|
+
fn: Callable[..., Any],
|
|
68
|
+
*,
|
|
69
|
+
step_name: str,
|
|
70
|
+
retries: int,
|
|
71
|
+
backoff: BackoffStrategy,
|
|
72
|
+
) -> None:
|
|
73
|
+
self._fn = fn
|
|
74
|
+
self._step_name = step_name
|
|
75
|
+
self._retries = retries
|
|
76
|
+
self._backoff = backoff
|
|
77
|
+
# Preserve the original function's metadata for IDE / tooling support
|
|
78
|
+
functools.update_wrapper(self, fn)
|
|
79
|
+
|
|
80
|
+
async def __call__(
|
|
81
|
+
self, *args: Any, step_id: str | None = None, **kwargs: Any
|
|
82
|
+
) -> Any:
|
|
83
|
+
ctx = _active_run.get()
|
|
84
|
+
|
|
85
|
+
if ctx is None:
|
|
86
|
+
# Called outside a workflow — run as a plain async function.
|
|
87
|
+
log.debug(
|
|
88
|
+
"[durable] %s called outside workflow context, running directly",
|
|
89
|
+
self._step_name,
|
|
90
|
+
)
|
|
91
|
+
return await self._fn(*args, **kwargs)
|
|
92
|
+
|
|
93
|
+
sid = step_id or ctx.next_step_id(self._step_name)
|
|
94
|
+
found, cached = await ctx.store.get_step(ctx.run_id, sid)
|
|
95
|
+
|
|
96
|
+
if found:
|
|
97
|
+
log.debug(
|
|
98
|
+
"[durable] ↩ %s (step=%s) — replayed from store", self._step_name, sid
|
|
99
|
+
)
|
|
100
|
+
return cached
|
|
101
|
+
|
|
102
|
+
return await self._execute_with_retry(ctx, sid, args, kwargs)
|
|
103
|
+
|
|
104
|
+
async def _execute_with_retry(
|
|
105
|
+
self, ctx: RunContext, sid: str, args: tuple, kwargs: dict
|
|
106
|
+
) -> Any:
|
|
107
|
+
last_exc: Exception | None = None
|
|
108
|
+
|
|
109
|
+
for attempt in range(self._retries + 1):
|
|
110
|
+
if attempt > 0:
|
|
111
|
+
wait = self._backoff(attempt - 1)
|
|
112
|
+
log.warning(
|
|
113
|
+
"[durable] ↻ %s (step=%s) attempt %d/%d — retrying in %.1fs",
|
|
114
|
+
self._step_name,
|
|
115
|
+
sid,
|
|
116
|
+
attempt,
|
|
117
|
+
self._retries,
|
|
118
|
+
wait,
|
|
119
|
+
)
|
|
120
|
+
await asyncio.sleep(wait)
|
|
121
|
+
|
|
122
|
+
try:
|
|
123
|
+
result = await self._fn(*args, **kwargs)
|
|
124
|
+
await ctx.store.set_step(ctx.run_id, sid, result, attempt + 1)
|
|
125
|
+
log.debug("[durable] ✓ %s (step=%s)", self._step_name, sid)
|
|
126
|
+
return result
|
|
127
|
+
|
|
128
|
+
except Exception as exc:
|
|
129
|
+
last_exc = exc
|
|
130
|
+
log.warning(
|
|
131
|
+
"[durable] ✗ %s (step=%s) attempt %d failed: %s",
|
|
132
|
+
self._step_name,
|
|
133
|
+
sid,
|
|
134
|
+
attempt + 1,
|
|
135
|
+
exc,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
raise last_exc # type: ignore[misc]
|
|
139
|
+
|
|
140
|
+
def __repr__(self) -> str:
|
|
141
|
+
return f"<DurableTask '{self._step_name}' retries={self._retries}>"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class Workflow:
|
|
145
|
+
"""
|
|
146
|
+
The main entry point for the durable library.
|
|
147
|
+
|
|
148
|
+
Create one per application (or one per logical domain):
|
|
149
|
+
|
|
150
|
+
wf = Workflow("orders", db="sqlite:///orders.db")
|
|
151
|
+
|
|
152
|
+
Then use @wf.task and @wf.workflow to make your code durable.
|
|
153
|
+
"""
|
|
154
|
+
|
|
155
|
+
def __init__(
|
|
156
|
+
self,
|
|
157
|
+
name: str,
|
|
158
|
+
db: str | Store = "durable.db",
|
|
159
|
+
default_retries: int = 3,
|
|
160
|
+
default_backoff: BackoffStrategy = exponential(),
|
|
161
|
+
) -> None:
|
|
162
|
+
self.name = name
|
|
163
|
+
self._default_retries = default_retries
|
|
164
|
+
self._default_backoff = default_backoff
|
|
165
|
+
self._store = self._build_store(db)
|
|
166
|
+
self._initialized = False
|
|
167
|
+
|
|
168
|
+
# ------------------------------------------------------------------
|
|
169
|
+
# @wf.task — can be used bare or with arguments
|
|
170
|
+
# ------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
@overload
|
|
173
|
+
def task(self, fn: Callable[P, R]) -> _TaskWrapper: ...
|
|
174
|
+
|
|
175
|
+
@overload
|
|
176
|
+
def task(
|
|
177
|
+
self,
|
|
178
|
+
fn: None = None,
|
|
179
|
+
*,
|
|
180
|
+
name: str | None = None,
|
|
181
|
+
retries: int | None = None,
|
|
182
|
+
backoff: BackoffStrategy | None = None,
|
|
183
|
+
) -> Callable[[Callable[P, R]], _TaskWrapper]: ...
|
|
184
|
+
|
|
185
|
+
def task(
|
|
186
|
+
self,
|
|
187
|
+
fn: Callable | None = None,
|
|
188
|
+
*,
|
|
189
|
+
name: str | None = None,
|
|
190
|
+
retries: int | None = None,
|
|
191
|
+
backoff: BackoffStrategy | None = None,
|
|
192
|
+
) -> _TaskWrapper | Callable:
|
|
193
|
+
"""
|
|
194
|
+
Decorate an async function to make it a durable task.
|
|
195
|
+
|
|
196
|
+
Usage (bare):
|
|
197
|
+
@wf.task
|
|
198
|
+
async def fetch_user(user_id: str) -> User: ...
|
|
199
|
+
|
|
200
|
+
Usage (with options):
|
|
201
|
+
@wf.task(retries=5, backoff=exponential(base=2, max=120))
|
|
202
|
+
async def call_api(url: str) -> dict: ...
|
|
203
|
+
|
|
204
|
+
@wf.task(name="send-welcome-email")
|
|
205
|
+
async def send_email(user: User) -> None: ...
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
def decorator(func: Callable[..., Any]) -> _TaskWrapper:
|
|
209
|
+
fn_name = getattr(func, "__name__", repr(func))
|
|
210
|
+
if not asyncio.iscoroutinefunction(func):
|
|
211
|
+
raise TypeError(
|
|
212
|
+
f"@wf.task requires an async function, got: {fn_name!r}"
|
|
213
|
+
)
|
|
214
|
+
return _TaskWrapper(
|
|
215
|
+
func,
|
|
216
|
+
step_name=name or fn_name,
|
|
217
|
+
retries=retries if retries is not None else self._default_retries,
|
|
218
|
+
backoff=backoff or self._default_backoff,
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
if fn is not None:
|
|
222
|
+
# Used bare: @wf.task
|
|
223
|
+
return decorator(fn)
|
|
224
|
+
# Used with args: @wf.task(retries=5)
|
|
225
|
+
return decorator
|
|
226
|
+
|
|
227
|
+
# ------------------------------------------------------------------
|
|
228
|
+
# @wf.workflow — entry point for a durable run
|
|
229
|
+
# ------------------------------------------------------------------
|
|
230
|
+
|
|
231
|
+
def workflow(
|
|
232
|
+
self,
|
|
233
|
+
fn: Callable | None = None,
|
|
234
|
+
*,
|
|
235
|
+
id: str | None = None, # noqa: A002 (shadows builtin intentionally)
|
|
236
|
+
) -> Callable:
|
|
237
|
+
"""
|
|
238
|
+
Decorate an async function as a durable workflow entry point.
|
|
239
|
+
|
|
240
|
+
The `id` parameter is a template string resolved from the function's
|
|
241
|
+
arguments at call time:
|
|
242
|
+
|
|
243
|
+
@wf.workflow(id="process-order-{order_id}")
|
|
244
|
+
async def process_order(order_id: str) -> None: ...
|
|
245
|
+
|
|
246
|
+
await process_order(order_id="ord-99")
|
|
247
|
+
# run_id → "process-order-ord-99"
|
|
248
|
+
|
|
249
|
+
If `id` is omitted, the run_id is "{workflow_name}-{fn_name}-{all_args_joined}".
|
|
250
|
+
|
|
251
|
+
You can also call `.run(run_id, ...)` to supply an explicit run ID:
|
|
252
|
+
|
|
253
|
+
await process_order.run("my-run-123", order_id="ord-99")
|
|
254
|
+
"""
|
|
255
|
+
|
|
256
|
+
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
|
|
257
|
+
fn_name = getattr(func, "__name__", repr(func))
|
|
258
|
+
if not asyncio.iscoroutinefunction(func):
|
|
259
|
+
raise TypeError(
|
|
260
|
+
f"@wf.workflow requires an async function, got: {fn_name!r}"
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
id_template = id # capture for closure
|
|
264
|
+
sig = inspect.signature(func)
|
|
265
|
+
|
|
266
|
+
async def _run_with_id(run_id: str, *args: Any, **kwargs: Any) -> Any:
|
|
267
|
+
await self._ensure_initialized()
|
|
268
|
+
|
|
269
|
+
# Create a new RunContext and install it in the ContextVar.
|
|
270
|
+
ctx = RunContext(
|
|
271
|
+
run_id=run_id,
|
|
272
|
+
workflow_id=f"{self.name}.{fn_name}",
|
|
273
|
+
store=self._store,
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
if isinstance(self._store, SQLiteStore):
|
|
277
|
+
await self._store.ensure_run(run_id, ctx.workflow_id)
|
|
278
|
+
|
|
279
|
+
token = _active_run.set(ctx)
|
|
280
|
+
try:
|
|
281
|
+
result = await func(*args, **kwargs)
|
|
282
|
+
await self._store.mark_run_done(run_id)
|
|
283
|
+
log.info("[durable] ✓✓ workflow %s (%s) completed", fn_name, run_id)
|
|
284
|
+
return result
|
|
285
|
+
except Exception as exc:
|
|
286
|
+
await self._store.mark_run_failed(run_id, str(exc))
|
|
287
|
+
log.error(
|
|
288
|
+
"[durable] ✗✗ workflow %s (%s) failed: %s",
|
|
289
|
+
fn_name,
|
|
290
|
+
run_id,
|
|
291
|
+
exc,
|
|
292
|
+
)
|
|
293
|
+
raise
|
|
294
|
+
finally:
|
|
295
|
+
_active_run.reset(token)
|
|
296
|
+
|
|
297
|
+
@functools.wraps(func)
|
|
298
|
+
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
299
|
+
bound = sig.bind(*args, **kwargs)
|
|
300
|
+
bound.apply_defaults()
|
|
301
|
+
|
|
302
|
+
if id_template:
|
|
303
|
+
run_id = _format_run_id(id_template, bound)
|
|
304
|
+
else:
|
|
305
|
+
parts = [self.name, fn_name] + [
|
|
306
|
+
str(v) for v in bound.arguments.values()
|
|
307
|
+
]
|
|
308
|
+
run_id = "-".join(parts)
|
|
309
|
+
|
|
310
|
+
return await _run_with_id(run_id, *args, **kwargs)
|
|
311
|
+
|
|
312
|
+
async def run(run_id: str, *args: Any, **kwargs: Any) -> Any:
|
|
313
|
+
"""Call with an explicit run ID instead of the template-derived one."""
|
|
314
|
+
return await _run_with_id(run_id, *args, **kwargs)
|
|
315
|
+
|
|
316
|
+
wrapper.run = run # type: ignore[attr-defined]
|
|
317
|
+
return wrapper
|
|
318
|
+
|
|
319
|
+
if fn is not None:
|
|
320
|
+
return decorator(fn)
|
|
321
|
+
return decorator
|
|
322
|
+
|
|
323
|
+
# ------------------------------------------------------------------
|
|
324
|
+
# Internal helpers
|
|
325
|
+
# ------------------------------------------------------------------
|
|
326
|
+
|
|
327
|
+
async def _ensure_initialized(self) -> None:
|
|
328
|
+
if not self._initialized:
|
|
329
|
+
await self._store.setup()
|
|
330
|
+
self._initialized = True
|
|
331
|
+
|
|
332
|
+
@staticmethod
|
|
333
|
+
def _build_store(db: str | Store) -> Store:
|
|
334
|
+
if isinstance(db, Store):
|
|
335
|
+
return db
|
|
336
|
+
# Accept both "sqlite:///path.db" and bare "path.db"
|
|
337
|
+
path = db.removeprefix("sqlite:///")
|
|
338
|
+
return SQLiteStore(path)
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: python-durable
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Lightweight workflow durability for Python — make any async workflow resumable after crashes with just a decorator.
|
|
5
|
+
Project-URL: Repository, https://github.com/WillemDeGroef/python-durable
|
|
6
|
+
Author: Willem
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Framework :: AsyncIO
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Requires-Dist: aiosqlite>=0.20
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
21
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
22
|
+
Requires-Dist: ruff>=0.9; extra == 'dev'
|
|
23
|
+
Requires-Dist: ty>=0.0.1a7; extra == 'dev'
|
|
24
|
+
Provides-Extra: examples
|
|
25
|
+
Requires-Dist: pydantic-ai>=0.1; extra == 'examples'
|
|
26
|
+
Requires-Dist: pydantic>=2.0; extra == 'examples'
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# durable
|
|
30
|
+
|
|
31
|
+
Lightweight workflow durability for Python. Make any async workflow resumable after crashes with just a decorator.
|
|
32
|
+
|
|
33
|
+
Backed by SQLite out of the box; swap in any `Store` subclass for production.
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install python-durable
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Quick start
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
from durable import Workflow
|
|
45
|
+
from durable.backoff import exponential
|
|
46
|
+
|
|
47
|
+
wf = Workflow("my-app")
|
|
48
|
+
|
|
49
|
+
@wf.task(retries=3, backoff=exponential(base=2, max=60))
|
|
50
|
+
async def fetch_data(url: str) -> dict:
|
|
51
|
+
async with httpx.AsyncClient() as client:
|
|
52
|
+
return (await client.get(url)).json()
|
|
53
|
+
|
|
54
|
+
@wf.task
|
|
55
|
+
async def save_result(data: dict) -> None:
|
|
56
|
+
await db.insert(data)
|
|
57
|
+
|
|
58
|
+
@wf.workflow(id="pipeline-{source}")
|
|
59
|
+
async def run_pipeline(source: str) -> None:
|
|
60
|
+
data = await fetch_data(f"https://api.example.com/{source}")
|
|
61
|
+
await save_result(data)
|
|
62
|
+
|
|
63
|
+
# First call: runs all steps and checkpoints each one.
|
|
64
|
+
# If it crashes and you call it again with the same args,
|
|
65
|
+
# completed steps are replayed from SQLite instantly.
|
|
66
|
+
await run_pipeline(source="users")
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## How it works
|
|
70
|
+
|
|
71
|
+
1. **`@wf.task`** wraps an async function with checkpoint + retry logic. When called inside a workflow, results are persisted to the store. On re-run, completed steps return their cached result without re-executing.
|
|
72
|
+
|
|
73
|
+
2. **`@wf.workflow`** marks the entry point of a durable run. It manages a `RunContext` (via `ContextVar`) so tasks automatically know which run they belong to. The `id` parameter is a template string resolved from function arguments at call time.
|
|
74
|
+
|
|
75
|
+
3. **`Store`** is the persistence backend. `SQLiteStore` is the default (zero config, backed by aiosqlite). Subclass `Store` to use Postgres, Redis, or anything else.
|
|
76
|
+
|
|
77
|
+
## Features
|
|
78
|
+
|
|
79
|
+
- **Crash recovery** — completed steps are never re-executed after a restart
|
|
80
|
+
- **Automatic retries** — configurable per-task with `exponential`, `linear`, or `constant` backoff
|
|
81
|
+
- **Loop support** — use `step_id` to checkpoint each iteration independently
|
|
82
|
+
- **Zero magic outside workflows** — tasks work as plain async functions when called without a workflow context
|
|
83
|
+
- **Pluggable storage** — SQLite by default, bring your own `Store` for production
|
|
84
|
+
|
|
85
|
+
## Backoff strategies
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from durable.backoff import exponential, linear, constant
|
|
89
|
+
|
|
90
|
+
@wf.task(retries=5, backoff=exponential(base=2, max=60)) # 2s, 4s, 8s, 16s, 32s
|
|
91
|
+
async def exp_task(): ...
|
|
92
|
+
|
|
93
|
+
@wf.task(retries=3, backoff=linear(start=2, step=3)) # 2s, 5s, 8s
|
|
94
|
+
async def linear_task(): ...
|
|
95
|
+
|
|
96
|
+
@wf.task(retries=3, backoff=constant(5)) # 5s, 5s, 5s
|
|
97
|
+
async def const_task(): ...
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Loops with step_id
|
|
101
|
+
|
|
102
|
+
When calling the same task in a loop, pass `step_id` so each iteration is checkpointed independently:
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
@wf.workflow(id="batch-{batch_id}")
|
|
106
|
+
async def process_batch(batch_id: str) -> None:
|
|
107
|
+
for i, item in enumerate(items):
|
|
108
|
+
await process_item(item, step_id=f"item-{i}")
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
If the workflow crashes mid-loop, only the remaining items are processed on restart.
|
|
112
|
+
|
|
113
|
+
## Important: JSON serialization
|
|
114
|
+
|
|
115
|
+
Task return values must be JSON-serializable (dicts, lists, strings, numbers, booleans, `None`). The store uses `json.dumps` internally.
|
|
116
|
+
|
|
117
|
+
For Pydantic models, return `.model_dump()` from tasks and reconstruct with `.model_validate()` downstream:
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
@wf.task
|
|
121
|
+
async def validate_invoice(draft: InvoiceDraft) -> dict:
|
|
122
|
+
validated = ValidatedInvoice(...)
|
|
123
|
+
return validated.model_dump()
|
|
124
|
+
|
|
125
|
+
@wf.task
|
|
126
|
+
async def book_invoice(data: dict) -> dict:
|
|
127
|
+
invoice = ValidatedInvoice.model_validate(data)
|
|
128
|
+
...
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
MIT
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
durable/__init__.py,sha256=o1jPcXSMbTbLeHotN2ysDECiguIhSlCnANeCYOVTE9s,1423
|
|
2
|
+
durable/backoff.py,sha256=o5p3hXfJ1YMwmEzjzukqW6inYezYGYnEi3_1YwnmDcU,1056
|
|
3
|
+
durable/context.py,sha256=greEK4jRz9RmVc5kHlmBjU3fisrsl2RCDDtGUPEiQM4,1324
|
|
4
|
+
durable/store.py,sha256=LXSE5h6W4ph9Sb4wN0XiBLfb2f5Q41IkaaWf8EUnWgo,6786
|
|
5
|
+
durable/workflow.py,sha256=qYiOvbGHrcxnifx1sCJCC1CONo8O4BhB2KQ8QdaArkg,11303
|
|
6
|
+
python_durable-0.1.0.dist-info/METADATA,sha256=nD8qU-FfCXb7SccCKDY4rAyyFBE-2bVyoHsfWHb0Qo0,4620
|
|
7
|
+
python_durable-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
8
|
+
python_durable-0.1.0.dist-info/licenses/LICENSE,sha256=S5JKY7biEEYA0tC7Qr2hO-ppopDVnD8muKbaviRFqLk,1084
|
|
9
|
+
python_durable-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 python-durable contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|