python-durable 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of python-durable might be problematic. Click here for more details.
- durable/__init__.py +27 -1
- durable/pydantic_ai.py +328 -0
- durable/redis_store.py +1 -1
- python_durable-0.2.0.dist-info/METADATA +290 -0
- python_durable-0.2.0.dist-info/RECORD +11 -0
- python_durable-0.1.1.dist-info/METADATA +0 -137
- python_durable-0.1.1.dist-info/RECORD +0 -10
- {python_durable-0.1.1.dist-info → python_durable-0.2.0.dist-info}/WHEEL +0 -0
- {python_durable-0.1.1.dist-info → python_durable-0.2.0.dist-info}/licenses/LICENSE +0 -0
durable/__init__.py
CHANGED
|
@@ -37,6 +37,27 @@ from .redis_store import RedisStore
|
|
|
37
37
|
from .store import InMemoryStore, SQLiteStore, Store
|
|
38
38
|
from .workflow import Workflow
|
|
39
39
|
|
|
40
|
+
_PYDANTIC_AI_NAMES = {"DurableAgent", "TaskConfig", "durable_tool", "durable_pipeline"}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def __getattr__(name: str):
|
|
44
|
+
if name in _PYDANTIC_AI_NAMES:
|
|
45
|
+
from .pydantic_ai import (
|
|
46
|
+
DurableAgent,
|
|
47
|
+
TaskConfig,
|
|
48
|
+
durable_pipeline,
|
|
49
|
+
durable_tool,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
return {
|
|
53
|
+
"DurableAgent": DurableAgent,
|
|
54
|
+
"TaskConfig": TaskConfig,
|
|
55
|
+
"durable_tool": durable_tool,
|
|
56
|
+
"durable_pipeline": durable_pipeline,
|
|
57
|
+
}[name]
|
|
58
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
59
|
+
|
|
60
|
+
|
|
40
61
|
__all__ = [
|
|
41
62
|
"Workflow",
|
|
42
63
|
"Store",
|
|
@@ -48,6 +69,11 @@ __all__ = [
|
|
|
48
69
|
"exponential",
|
|
49
70
|
"constant",
|
|
50
71
|
"linear",
|
|
72
|
+
# pydantic-ai integration (optional)
|
|
73
|
+
"DurableAgent",
|
|
74
|
+
"TaskConfig",
|
|
75
|
+
"durable_tool",
|
|
76
|
+
"durable_pipeline",
|
|
51
77
|
]
|
|
52
78
|
|
|
53
|
-
__version__ = "0.
|
|
79
|
+
__version__ = "0.2.0"
|
durable/pydantic_ai.py
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""Pydantic AI integration for python-durable.
|
|
2
|
+
|
|
3
|
+
Provides DurableAgent — a wrapper that makes any pydantic-ai Agent durable,
|
|
4
|
+
automatically checkpointing model requests and tool calls to the store.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import logging
|
|
11
|
+
from typing import Any, Generic, TypeVar
|
|
12
|
+
|
|
13
|
+
from .backoff import BackoffStrategy, exponential
|
|
14
|
+
from .workflow import Workflow
|
|
15
|
+
|
|
16
|
+
log = logging.getLogger("durable.pydantic_ai")
|
|
17
|
+
|
|
18
|
+
AgentDepsT = TypeVar("AgentDepsT")
|
|
19
|
+
OutputT = TypeVar("OutputT")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _serialize_messages(messages: list[Any]) -> list[dict]:
|
|
23
|
+
"""Convert pydantic-ai message objects to JSON-serializable dicts."""
|
|
24
|
+
result = []
|
|
25
|
+
for msg in messages:
|
|
26
|
+
if hasattr(msg, "model_dump"):
|
|
27
|
+
d = msg.model_dump(mode="json")
|
|
28
|
+
d["__type__"] = type(msg).__name__
|
|
29
|
+
result.append(d)
|
|
30
|
+
elif isinstance(msg, dict):
|
|
31
|
+
result.append(msg)
|
|
32
|
+
else:
|
|
33
|
+
result.append({"__repr__": repr(msg)})
|
|
34
|
+
return result
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _deserialize_messages(data: list[dict]) -> list[Any]:
|
|
38
|
+
"""Reconstruct pydantic-ai message objects from serialized dicts.
|
|
39
|
+
|
|
40
|
+
Falls back to returning raw dicts if the classes aren't available.
|
|
41
|
+
"""
|
|
42
|
+
try:
|
|
43
|
+
from pydantic_ai.messages import (
|
|
44
|
+
ModelRequest,
|
|
45
|
+
ModelResponse,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
_type_map: dict[str, type[Any]] = {
|
|
49
|
+
"ModelRequest": ModelRequest,
|
|
50
|
+
"ModelResponse": ModelResponse,
|
|
51
|
+
}
|
|
52
|
+
except ImportError:
|
|
53
|
+
_type_map = {}
|
|
54
|
+
|
|
55
|
+
result = []
|
|
56
|
+
for item in data:
|
|
57
|
+
type_name = item.pop("__type__", None) if isinstance(item, dict) else None
|
|
58
|
+
if type_name and type_name in _type_map:
|
|
59
|
+
try:
|
|
60
|
+
cls = _type_map[type_name]
|
|
61
|
+
result.append(cls(**item))
|
|
62
|
+
except Exception:
|
|
63
|
+
item["__type__"] = type_name
|
|
64
|
+
result.append(item)
|
|
65
|
+
else:
|
|
66
|
+
if type_name:
|
|
67
|
+
item["__type__"] = type_name
|
|
68
|
+
result.append(item)
|
|
69
|
+
return result
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _run_id_for_agent(agent_name: str, prompt: str, run_id: str | None) -> str:
|
|
73
|
+
"""Generate a deterministic run ID from the agent name and prompt."""
|
|
74
|
+
if run_id:
|
|
75
|
+
return run_id
|
|
76
|
+
prompt_hash = hashlib.sha256(prompt.encode()).hexdigest()[:12]
|
|
77
|
+
return f"agent-{agent_name}-{prompt_hash}"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class TaskConfig:
|
|
81
|
+
"""Retry/backoff configuration for model requests or tool calls."""
|
|
82
|
+
|
|
83
|
+
def __init__(
|
|
84
|
+
self,
|
|
85
|
+
retries: int | None = None,
|
|
86
|
+
backoff: BackoffStrategy | None = None,
|
|
87
|
+
) -> None:
|
|
88
|
+
self.retries = retries
|
|
89
|
+
self.backoff = backoff
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class DurableAgent(Generic[AgentDepsT, OutputT]):
|
|
93
|
+
"""Wrap a pydantic-ai Agent for durable execution with python-durable.
|
|
94
|
+
|
|
95
|
+
Wraps each ``agent.run()`` call as a ``@wf.workflow`` and checkpoints
|
|
96
|
+
model requests and tool calls as ``@wf.task`` steps.
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
def __init__(
|
|
100
|
+
self,
|
|
101
|
+
agent: Any, # pydantic_ai.Agent[AgentDepsT, OutputT]
|
|
102
|
+
wf: Workflow,
|
|
103
|
+
*,
|
|
104
|
+
name: str | None = None,
|
|
105
|
+
model_task_config: TaskConfig | None = None,
|
|
106
|
+
tool_task_config: TaskConfig | None = None,
|
|
107
|
+
) -> None:
|
|
108
|
+
self.agent = agent
|
|
109
|
+
self.wf = wf
|
|
110
|
+
self.name = name or getattr(agent, "name", None) or "agent"
|
|
111
|
+
|
|
112
|
+
self._model_retries = (
|
|
113
|
+
model_task_config.retries
|
|
114
|
+
if model_task_config and model_task_config.retries is not None
|
|
115
|
+
else 3
|
|
116
|
+
)
|
|
117
|
+
self._model_backoff = (
|
|
118
|
+
model_task_config.backoff
|
|
119
|
+
if model_task_config and model_task_config.backoff
|
|
120
|
+
else exponential(base=2, max=60)
|
|
121
|
+
)
|
|
122
|
+
self._tool_retries = (
|
|
123
|
+
tool_task_config.retries
|
|
124
|
+
if tool_task_config and tool_task_config.retries is not None
|
|
125
|
+
else 2
|
|
126
|
+
)
|
|
127
|
+
self._tool_backoff = (
|
|
128
|
+
tool_task_config.backoff
|
|
129
|
+
if tool_task_config and tool_task_config.backoff
|
|
130
|
+
else exponential(base=2, max=30)
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
self._model_request_task = wf.task(
|
|
134
|
+
name=f"{self.name}.model_request",
|
|
135
|
+
retries=self._model_retries,
|
|
136
|
+
backoff=self._model_backoff,
|
|
137
|
+
)(self._do_model_request)
|
|
138
|
+
|
|
139
|
+
self._tool_call_task = wf.task(
|
|
140
|
+
name=f"{self.name}.tool_call",
|
|
141
|
+
retries=self._tool_retries,
|
|
142
|
+
backoff=self._tool_backoff,
|
|
143
|
+
)(self._do_tool_call)
|
|
144
|
+
|
|
145
|
+
async def run(
|
|
146
|
+
self,
|
|
147
|
+
prompt: str,
|
|
148
|
+
*,
|
|
149
|
+
deps: Any = None,
|
|
150
|
+
message_history: list[Any] | None = None,
|
|
151
|
+
run_id: str | None = None,
|
|
152
|
+
**kwargs: Any,
|
|
153
|
+
) -> Any:
|
|
154
|
+
"""Run the agent durably.
|
|
155
|
+
|
|
156
|
+
Works like ``Agent.run()`` but the result is checkpointed. If the
|
|
157
|
+
process crashes and you call ``run()`` again with the same ``run_id``
|
|
158
|
+
(or same prompt), the cached result is returned without re-executing.
|
|
159
|
+
"""
|
|
160
|
+
rid = _run_id_for_agent(self.name, prompt, run_id)
|
|
161
|
+
|
|
162
|
+
@self.wf.workflow(id=rid)
|
|
163
|
+
async def _durable_run() -> Any:
|
|
164
|
+
return await self._execute_agent_loop(
|
|
165
|
+
prompt=prompt,
|
|
166
|
+
deps=deps,
|
|
167
|
+
message_history=message_history,
|
|
168
|
+
**kwargs,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
return await _durable_run.run(rid)
|
|
172
|
+
|
|
173
|
+
def run_sync(
|
|
174
|
+
self,
|
|
175
|
+
prompt: str,
|
|
176
|
+
*,
|
|
177
|
+
deps: Any = None,
|
|
178
|
+
message_history: list[Any] | None = None,
|
|
179
|
+
run_id: str | None = None,
|
|
180
|
+
**kwargs: Any,
|
|
181
|
+
) -> Any:
|
|
182
|
+
"""Synchronous version of :meth:`run`."""
|
|
183
|
+
import asyncio
|
|
184
|
+
|
|
185
|
+
return asyncio.run(
|
|
186
|
+
self.run(
|
|
187
|
+
prompt,
|
|
188
|
+
deps=deps,
|
|
189
|
+
message_history=message_history,
|
|
190
|
+
run_id=run_id,
|
|
191
|
+
**kwargs,
|
|
192
|
+
)
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
async def _execute_agent_loop(
|
|
196
|
+
self,
|
|
197
|
+
prompt: str,
|
|
198
|
+
deps: Any = None,
|
|
199
|
+
message_history: list[Any] | None = None,
|
|
200
|
+
**kwargs: Any,
|
|
201
|
+
) -> Any:
|
|
202
|
+
"""Wrap the full agent.run() as a single checkpointed task."""
|
|
203
|
+
result = await self._model_request_task(
|
|
204
|
+
prompt,
|
|
205
|
+
deps=deps,
|
|
206
|
+
message_history=message_history,
|
|
207
|
+
step_id="agent-run",
|
|
208
|
+
**kwargs,
|
|
209
|
+
)
|
|
210
|
+
return result
|
|
211
|
+
|
|
212
|
+
async def _do_model_request(
|
|
213
|
+
self,
|
|
214
|
+
prompt: str,
|
|
215
|
+
deps: Any = None,
|
|
216
|
+
message_history: list[Any] | None = None,
|
|
217
|
+
**kwargs: Any,
|
|
218
|
+
) -> Any:
|
|
219
|
+
"""Execute the actual agent.run() call."""
|
|
220
|
+
run_kwargs: dict[str, Any] = {}
|
|
221
|
+
if deps is not None:
|
|
222
|
+
run_kwargs["deps"] = deps
|
|
223
|
+
if message_history is not None:
|
|
224
|
+
run_kwargs["message_history"] = message_history
|
|
225
|
+
run_kwargs.update(kwargs)
|
|
226
|
+
|
|
227
|
+
result = await self.agent.run(prompt, **run_kwargs)
|
|
228
|
+
return _AgentRunResult(result)
|
|
229
|
+
|
|
230
|
+
async def _do_tool_call(
|
|
231
|
+
self,
|
|
232
|
+
tool_name: str,
|
|
233
|
+
tool_args: dict[str, Any],
|
|
234
|
+
) -> Any:
|
|
235
|
+
"""Execute a single tool call."""
|
|
236
|
+
tools = getattr(self.agent, "_function_tools", {})
|
|
237
|
+
if tool_name not in tools:
|
|
238
|
+
raise ValueError(
|
|
239
|
+
f"Tool {tool_name!r} not found on agent {self.name!r}. "
|
|
240
|
+
f"Available: {list(tools.keys())}"
|
|
241
|
+
)
|
|
242
|
+
tool = tools[tool_name]
|
|
243
|
+
return await tool.run(tool_args)
|
|
244
|
+
|
|
245
|
+
async def tool(
|
|
246
|
+
self,
|
|
247
|
+
tool_name: str,
|
|
248
|
+
tool_args: dict[str, Any],
|
|
249
|
+
*,
|
|
250
|
+
step_id: str | None = None,
|
|
251
|
+
) -> Any:
|
|
252
|
+
"""Durably execute a tool call with checkpointing."""
|
|
253
|
+
return await self._tool_call_task(tool_name, tool_args, step_id=step_id)
|
|
254
|
+
|
|
255
|
+
async def signal(self, name: str, *, poll: float = 2.0) -> Any:
|
|
256
|
+
"""Durably wait for an external signal (e.g., human approval)."""
|
|
257
|
+
return await self.wf.signal(name, poll=poll)
|
|
258
|
+
|
|
259
|
+
def __repr__(self) -> str:
|
|
260
|
+
return (
|
|
261
|
+
f"<DurableAgent name={self.name!r} "
|
|
262
|
+
f"model_retries={self._model_retries} "
|
|
263
|
+
f"tool_retries={self._tool_retries}>"
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
class _AgentRunResult:
|
|
268
|
+
"""Wrapper that holds an agent RunResult and handles both live and
|
|
269
|
+
deserialized (dict) results transparently."""
|
|
270
|
+
|
|
271
|
+
def __init__(self, result: Any) -> None:
|
|
272
|
+
self._result = result
|
|
273
|
+
|
|
274
|
+
@property
|
|
275
|
+
def output(self) -> Any:
|
|
276
|
+
if hasattr(self._result, "output"):
|
|
277
|
+
return self._result.output
|
|
278
|
+
if isinstance(self._result, dict):
|
|
279
|
+
return self._result.get("output")
|
|
280
|
+
return self._result
|
|
281
|
+
|
|
282
|
+
@property
|
|
283
|
+
def all_messages(self) -> list[Any]:
|
|
284
|
+
if hasattr(self._result, "all_messages"):
|
|
285
|
+
return self._result.all_messages()
|
|
286
|
+
if isinstance(self._result, dict):
|
|
287
|
+
return self._result.get("all_messages", [])
|
|
288
|
+
return []
|
|
289
|
+
|
|
290
|
+
@property
|
|
291
|
+
def usage(self) -> Any:
|
|
292
|
+
if hasattr(self._result, "usage"):
|
|
293
|
+
return self._result.usage()
|
|
294
|
+
if isinstance(self._result, dict):
|
|
295
|
+
return self._result.get("usage")
|
|
296
|
+
return None
|
|
297
|
+
|
|
298
|
+
def __repr__(self) -> str:
|
|
299
|
+
return f"<DurableRunResult output={self.output!r}>"
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def durable_tool(
|
|
303
|
+
wf: Workflow,
|
|
304
|
+
*,
|
|
305
|
+
name: str | None = None,
|
|
306
|
+
retries: int = 2,
|
|
307
|
+
backoff: BackoffStrategy | None = None,
|
|
308
|
+
):
|
|
309
|
+
"""Decorator to make a tool function durable (checkpointed as a @wf.task)."""
|
|
310
|
+
_backoff = backoff or exponential(base=2, max=30)
|
|
311
|
+
|
|
312
|
+
def decorator(fn):
|
|
313
|
+
return wf.task(
|
|
314
|
+
name=name or getattr(fn, "__name__", "tool"),
|
|
315
|
+
retries=retries,
|
|
316
|
+
backoff=_backoff,
|
|
317
|
+
)(fn)
|
|
318
|
+
|
|
319
|
+
return decorator
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def durable_pipeline(
|
|
323
|
+
wf: Workflow,
|
|
324
|
+
*,
|
|
325
|
+
id: str, # noqa: A002
|
|
326
|
+
):
|
|
327
|
+
"""Syntactic sugar for ``@wf.workflow`` in multi-agent pipelines."""
|
|
328
|
+
return wf.workflow(id=id)
|
durable/redis_store.py
CHANGED
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: python-durable
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Lightweight workflow durability for Python — make any async workflow resumable after crashes with just a decorator.
|
|
5
|
+
Project-URL: Repository, https://github.com/WillemDeGroef/python-durable
|
|
6
|
+
Author: Willem
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Framework :: AsyncIO
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Requires-Dist: aiosqlite>=0.20
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: fakeredis>=2.26; extra == 'dev'
|
|
21
|
+
Requires-Dist: pydantic-ai>=0.1; extra == 'dev'
|
|
22
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: redis>=5.0; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff>=0.9; extra == 'dev'
|
|
26
|
+
Requires-Dist: ty>=0.0.1a7; extra == 'dev'
|
|
27
|
+
Provides-Extra: examples
|
|
28
|
+
Requires-Dist: pydantic-ai>=0.1; extra == 'examples'
|
|
29
|
+
Requires-Dist: pydantic>=2.0; extra == 'examples'
|
|
30
|
+
Provides-Extra: pydantic-ai
|
|
31
|
+
Requires-Dist: pydantic-ai>=0.1; extra == 'pydantic-ai'
|
|
32
|
+
Provides-Extra: redis
|
|
33
|
+
Requires-Dist: redis>=5.0; extra == 'redis'
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
# durable
|
|
37
|
+
|
|
38
|
+
Lightweight workflow durability for Python. Make any async workflow resumable after crashes with just a decorator.
|
|
39
|
+
|
|
40
|
+
Backed by SQLite out of the box; swap in Redis or any `Store` subclass for production.
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install python-durable
|
|
46
|
+
|
|
47
|
+
# With Redis support
|
|
48
|
+
pip install python-durable[redis]
|
|
49
|
+
|
|
50
|
+
# With Pydantic AI integration
|
|
51
|
+
pip install python-durable[pydantic-ai]
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Quick start
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from durable import Workflow
|
|
58
|
+
from durable.backoff import exponential
|
|
59
|
+
|
|
60
|
+
wf = Workflow("my-app")
|
|
61
|
+
|
|
62
|
+
@wf.task(retries=3, backoff=exponential(base=2, max=60))
|
|
63
|
+
async def fetch_data(url: str) -> dict:
|
|
64
|
+
async with httpx.AsyncClient() as client:
|
|
65
|
+
return (await client.get(url)).json()
|
|
66
|
+
|
|
67
|
+
@wf.task
|
|
68
|
+
async def save_result(data: dict) -> None:
|
|
69
|
+
await db.insert(data)
|
|
70
|
+
|
|
71
|
+
@wf.workflow(id="pipeline-{source}")
|
|
72
|
+
async def run_pipeline(source: str) -> None:
|
|
73
|
+
data = await fetch_data(f"https://api.example.com/{source}")
|
|
74
|
+
await save_result(data)
|
|
75
|
+
|
|
76
|
+
# First call: runs all steps and checkpoints each one.
|
|
77
|
+
# If it crashes and you call it again with the same args,
|
|
78
|
+
# completed steps are replayed from SQLite instantly.
|
|
79
|
+
await run_pipeline(source="users")
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## How it works
|
|
83
|
+
|
|
84
|
+
1. **`@wf.task`** wraps an async function with checkpoint + retry logic. When called inside a workflow, results are persisted to the store. On re-run, completed steps return their cached result without re-executing.
|
|
85
|
+
|
|
86
|
+
2. **`@wf.workflow`** marks the entry point of a durable run. It manages a `RunContext` (via `ContextVar`) so tasks automatically know which run they belong to. The `id` parameter is a template string resolved from function arguments at call time.
|
|
87
|
+
|
|
88
|
+
3. **`Store`** is the persistence backend. `SQLiteStore` is the default (zero config, backed by aiosqlite). `RedisStore` is available for distributed setups. Subclass `Store` to use Postgres or anything else.
|
|
89
|
+
|
|
90
|
+
## Features
|
|
91
|
+
|
|
92
|
+
- **Crash recovery** — completed steps are never re-executed after a restart
|
|
93
|
+
- **Automatic retries** — configurable per-task with `exponential`, `linear`, or `constant` backoff
|
|
94
|
+
- **Signals** — durably wait for external input (approvals, webhooks, human-in-the-loop)
|
|
95
|
+
- **Loop support** — use `step_id` to checkpoint each iteration independently
|
|
96
|
+
- **Zero magic outside workflows** — tasks work as plain async functions when called without a workflow context
|
|
97
|
+
- **Pluggable storage** — SQLite by default, Redis built-in, or bring your own `Store`
|
|
98
|
+
|
|
99
|
+
## Signals
|
|
100
|
+
|
|
101
|
+
Workflows can pause and wait for external input using signals:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
@wf.workflow(id="order-{order_id}")
|
|
105
|
+
async def process_order(order_id: str) -> None:
|
|
106
|
+
await prepare_order(order_id)
|
|
107
|
+
approval = await wf.signal("manager-approval") # pauses here
|
|
108
|
+
if approval["approved"]:
|
|
109
|
+
await ship_order(order_id)
|
|
110
|
+
|
|
111
|
+
# From the outside (e.g. a web handler):
|
|
112
|
+
await wf.complete("order-42", "manager-approval", {"approved": True})
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Signals are durable — if the workflow crashes and restarts, a previously delivered signal replays instantly.
|
|
116
|
+
|
|
117
|
+
## Redis store
|
|
118
|
+
|
|
119
|
+
For distributed or multi-process setups, use `RedisStore` instead of the default SQLite:
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
from durable import Workflow, RedisStore
|
|
123
|
+
|
|
124
|
+
store = RedisStore(url="redis://localhost:6379/0", ttl=86400)
|
|
125
|
+
wf = Workflow("my-app", db=store)
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Keys auto-expire based on `ttl` (default: 24 hours).
|
|
129
|
+
|
|
130
|
+
## Backoff strategies
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from durable.backoff import exponential, linear, constant
|
|
134
|
+
|
|
135
|
+
@wf.task(retries=5, backoff=exponential(base=2, max=60)) # 2s, 4s, 8s, 16s, 32s
|
|
136
|
+
async def exp_task(): ...
|
|
137
|
+
|
|
138
|
+
@wf.task(retries=3, backoff=linear(start=2, step=3)) # 2s, 5s, 8s
|
|
139
|
+
async def linear_task(): ...
|
|
140
|
+
|
|
141
|
+
@wf.task(retries=3, backoff=constant(5)) # 5s, 5s, 5s
|
|
142
|
+
async def const_task(): ...
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Loops with step_id
|
|
146
|
+
|
|
147
|
+
When calling the same task in a loop, pass `step_id` so each iteration is checkpointed independently:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
@wf.workflow(id="batch-{batch_id}")
|
|
151
|
+
async def process_batch(batch_id: str) -> None:
|
|
152
|
+
for i, item in enumerate(items):
|
|
153
|
+
await process_item(item, step_id=f"item-{i}")
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
If the workflow crashes mid-loop, only the remaining items are processed on restart.
|
|
157
|
+
|
|
158
|
+
## Pydantic AI integration
|
|
159
|
+
|
|
160
|
+
Make any [pydantic-ai](https://ai.pydantic.dev) agent durable with **zero infrastructure** — no Temporal server, no Prefect cloud, no Postgres. Just decorators and a SQLite file.
|
|
161
|
+
|
|
162
|
+
Pydantic AI natively supports three durable execution backends: **Temporal**, **DBOS**, and **Prefect**. All three require external infrastructure. `python-durable` is a fourth option that trades scale for simplicity:
|
|
163
|
+
|
|
164
|
+
| Feature | Temporal | DBOS | Prefect | **python-durable** |
|
|
165
|
+
|---------|----------|------|---------|-------------------|
|
|
166
|
+
| Infrastructure | Server + Worker | Postgres | Cloud/Server | **SQLite file** |
|
|
167
|
+
| Setup | Complex | Moderate | Moderate | **`pip install`** |
|
|
168
|
+
| Lines to wrap an agent | ~20 | ~10 | ~10 | **1** |
|
|
169
|
+
| Crash recovery | Yes | Yes | Yes | Yes |
|
|
170
|
+
| Retries + backoff | Yes | Yes | Yes | Yes |
|
|
171
|
+
| Human-in-the-loop signals | Yes | No | No | Yes |
|
|
172
|
+
| Multi-process / distributed | Yes | Yes | Yes | No (single process) |
|
|
173
|
+
| Production scale | Enterprise | Production | Production | **Dev / SME / CLI** |
|
|
174
|
+
|
|
175
|
+
**Best for:** prototyping, CLI tools, single-process services, SME deployments, and any situation where you want durable agents without ops overhead.
|
|
176
|
+
|
|
177
|
+
### DurableAgent
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
from pydantic_ai import Agent
|
|
181
|
+
from durable import Workflow
|
|
182
|
+
from durable.pydantic_ai import DurableAgent, TaskConfig
|
|
183
|
+
from durable.backoff import exponential
|
|
184
|
+
|
|
185
|
+
wf = Workflow("my-app")
|
|
186
|
+
agent = Agent("openai:gpt-5.2", instructions="Be helpful.", name="assistant")
|
|
187
|
+
|
|
188
|
+
durable_agent = DurableAgent(agent, wf)
|
|
189
|
+
|
|
190
|
+
result = await durable_agent.run("What is the capital of France?")
|
|
191
|
+
print(result.output) # Paris
|
|
192
|
+
|
|
193
|
+
# Same run_id after crash → replayed from SQLite, no LLM call
|
|
194
|
+
result = await durable_agent.run("What is the capital of France?", run_id="same-id")
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
With custom retry config:
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
durable_agent = DurableAgent(
|
|
201
|
+
agent,
|
|
202
|
+
wf,
|
|
203
|
+
model_task_config=TaskConfig(retries=5, backoff=exponential(base=2, max=120)),
|
|
204
|
+
tool_task_config=TaskConfig(retries=3),
|
|
205
|
+
)
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
### @durable_tool
|
|
209
|
+
|
|
210
|
+
Make individual tool functions durable:
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
from durable.pydantic_ai import durable_tool
|
|
214
|
+
|
|
215
|
+
@durable_tool(wf, retries=3, backoff=exponential(base=2, max=60))
|
|
216
|
+
async def web_search(query: str) -> str:
|
|
217
|
+
async with httpx.AsyncClient() as client:
|
|
218
|
+
return (await client.get(f"https://api.search.com?q={query}")).text
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
### @durable_pipeline
|
|
222
|
+
|
|
223
|
+
Multi-agent workflows with per-step checkpointing. On crash, completed steps replay from the store and only remaining work executes:
|
|
224
|
+
|
|
225
|
+
```python
|
|
226
|
+
from durable.pydantic_ai import durable_pipeline
|
|
227
|
+
|
|
228
|
+
@durable_pipeline(wf, id="research-{topic_id}")
|
|
229
|
+
async def research(topic_id: str, topic: str) -> str:
|
|
230
|
+
plan = await plan_research(topic)
|
|
231
|
+
findings = []
|
|
232
|
+
for i, query in enumerate(plan["queries"]):
|
|
233
|
+
r = await search(query, step_id=f"q-{i}")
|
|
234
|
+
findings.append(r)
|
|
235
|
+
return await summarize(findings)
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
### Comparison with Temporal
|
|
239
|
+
|
|
240
|
+
```python
|
|
241
|
+
# Temporal — requires server + worker + plugin
|
|
242
|
+
from temporalio import workflow
|
|
243
|
+
from pydantic_ai.durable_exec.temporal import TemporalAgent
|
|
244
|
+
|
|
245
|
+
temporal_agent = TemporalAgent(agent)
|
|
246
|
+
|
|
247
|
+
@workflow.defn
|
|
248
|
+
class MyWorkflow:
|
|
249
|
+
@workflow.run
|
|
250
|
+
async def run(self, prompt: str):
|
|
251
|
+
return await temporal_agent.run(prompt)
|
|
252
|
+
|
|
253
|
+
# python-durable
|
|
254
|
+
from durable import Workflow
|
|
255
|
+
from durable.pydantic_ai import DurableAgent
|
|
256
|
+
|
|
257
|
+
wf = Workflow("my-app")
|
|
258
|
+
durable_agent = DurableAgent(agent, wf)
|
|
259
|
+
result = await durable_agent.run("Hello")
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
### Caveats
|
|
263
|
+
|
|
264
|
+
- **Tool functions** registered on the pydantic-ai agent are NOT automatically wrapped. If they perform I/O, decorate them with `@durable_tool(wf)` or `@wf.task`.
|
|
265
|
+
- **Streaming** (`agent.run_stream()`) is not supported in durable mode (same limitation as DBOS). Use `agent.run()`.
|
|
266
|
+
- **Single process** — unlike Temporal/DBOS, python-durable runs in-process. For distributed workloads, use the Redis store.
|
|
267
|
+
|
|
268
|
+
See [`examples/pydantic_ai_example.py`](examples/pydantic_ai_example.py) for five complete patterns.
|
|
269
|
+
|
|
270
|
+
## Important: JSON serialization
|
|
271
|
+
|
|
272
|
+
Task return values must be JSON-serializable (dicts, lists, strings, numbers, booleans, `None`). The store uses `json.dumps` internally.
|
|
273
|
+
|
|
274
|
+
For Pydantic models, return `.model_dump()` from tasks and reconstruct with `.model_validate()` downstream:
|
|
275
|
+
|
|
276
|
+
```python
|
|
277
|
+
@wf.task
|
|
278
|
+
async def validate_invoice(draft: InvoiceDraft) -> dict:
|
|
279
|
+
validated = ValidatedInvoice(...)
|
|
280
|
+
return validated.model_dump()
|
|
281
|
+
|
|
282
|
+
@wf.task
|
|
283
|
+
async def book_invoice(data: dict) -> dict:
|
|
284
|
+
invoice = ValidatedInvoice.model_validate(data)
|
|
285
|
+
...
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
## License
|
|
289
|
+
|
|
290
|
+
MIT
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
durable/__init__.py,sha256=CMNNJpWOyJHDKK9H2UidatvyDAb76UR4sbhEKIpBOCQ,2187
|
|
2
|
+
durable/backoff.py,sha256=o5p3hXfJ1YMwmEzjzukqW6inYezYGYnEi3_1YwnmDcU,1056
|
|
3
|
+
durable/context.py,sha256=greEK4jRz9RmVc5kHlmBjU3fisrsl2RCDDtGUPEiQM4,1324
|
|
4
|
+
durable/pydantic_ai.py,sha256=3g4mmxJ0X_Kw8LsRZxeZiwho9G7kZR7QTrCD4LgZtUU,9761
|
|
5
|
+
durable/redis_store.py,sha256=ShzNblDOoxTm5rEonbcwotEZmfZ8DYrCfx-7fMTUxAE,3536
|
|
6
|
+
durable/store.py,sha256=YvrVNFzYQGSlLR-TKnXS0b1tJzpY5zr_tkkKDH5sR1k,8769
|
|
7
|
+
durable/workflow.py,sha256=Q0boxwnquNJMueE8LeRlWg8yLxFr6m1I6RzOO3kmdB8,13192
|
|
8
|
+
python_durable-0.2.0.dist-info/METADATA,sha256=N_ygSggefBkxcAutcvZumKfkBFMYVdyXeDyrZ6Q0iwA,10029
|
|
9
|
+
python_durable-0.2.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
10
|
+
python_durable-0.2.0.dist-info/licenses/LICENSE,sha256=S5JKY7biEEYA0tC7Qr2hO-ppopDVnD8muKbaviRFqLk,1084
|
|
11
|
+
python_durable-0.2.0.dist-info/RECORD,,
|
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: python-durable
|
|
3
|
-
Version: 0.1.1
|
|
4
|
-
Summary: Lightweight workflow durability for Python — make any async workflow resumable after crashes with just a decorator.
|
|
5
|
-
Project-URL: Repository, https://github.com/WillemDeGroef/python-durable
|
|
6
|
-
Author: Willem
|
|
7
|
-
License-Expression: MIT
|
|
8
|
-
License-File: LICENSE
|
|
9
|
-
Classifier: Development Status :: 3 - Alpha
|
|
10
|
-
Classifier: Framework :: AsyncIO
|
|
11
|
-
Classifier: Intended Audience :: Developers
|
|
12
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
-
Classifier: Programming Language :: Python :: 3
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
-
Classifier: Typing :: Typed
|
|
17
|
-
Requires-Python: >=3.12
|
|
18
|
-
Requires-Dist: aiosqlite>=0.20
|
|
19
|
-
Provides-Extra: dev
|
|
20
|
-
Requires-Dist: fakeredis>=2.26; extra == 'dev'
|
|
21
|
-
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
22
|
-
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
23
|
-
Requires-Dist: redis>=5.0; extra == 'dev'
|
|
24
|
-
Requires-Dist: ruff>=0.9; extra == 'dev'
|
|
25
|
-
Requires-Dist: ty>=0.0.1a7; extra == 'dev'
|
|
26
|
-
Provides-Extra: examples
|
|
27
|
-
Requires-Dist: pydantic-ai>=0.1; extra == 'examples'
|
|
28
|
-
Requires-Dist: pydantic>=2.0; extra == 'examples'
|
|
29
|
-
Provides-Extra: redis
|
|
30
|
-
Requires-Dist: redis>=5.0; extra == 'redis'
|
|
31
|
-
Description-Content-Type: text/markdown
|
|
32
|
-
|
|
33
|
-
# durable
|
|
34
|
-
|
|
35
|
-
Lightweight workflow durability for Python. Make any async workflow resumable after crashes with just a decorator.
|
|
36
|
-
|
|
37
|
-
Backed by SQLite out of the box; swap in any `Store` subclass for production.
|
|
38
|
-
|
|
39
|
-
## Install
|
|
40
|
-
|
|
41
|
-
```bash
|
|
42
|
-
pip install python-durable
|
|
43
|
-
```
|
|
44
|
-
|
|
45
|
-
## Quick start
|
|
46
|
-
|
|
47
|
-
```python
|
|
48
|
-
from durable import Workflow
|
|
49
|
-
from durable.backoff import exponential
|
|
50
|
-
|
|
51
|
-
wf = Workflow("my-app")
|
|
52
|
-
|
|
53
|
-
@wf.task(retries=3, backoff=exponential(base=2, max=60))
|
|
54
|
-
async def fetch_data(url: str) -> dict:
|
|
55
|
-
async with httpx.AsyncClient() as client:
|
|
56
|
-
return (await client.get(url)).json()
|
|
57
|
-
|
|
58
|
-
@wf.task
|
|
59
|
-
async def save_result(data: dict) -> None:
|
|
60
|
-
await db.insert(data)
|
|
61
|
-
|
|
62
|
-
@wf.workflow(id="pipeline-{source}")
|
|
63
|
-
async def run_pipeline(source: str) -> None:
|
|
64
|
-
data = await fetch_data(f"https://api.example.com/{source}")
|
|
65
|
-
await save_result(data)
|
|
66
|
-
|
|
67
|
-
# First call: runs all steps and checkpoints each one.
|
|
68
|
-
# If it crashes and you call it again with the same args,
|
|
69
|
-
# completed steps are replayed from SQLite instantly.
|
|
70
|
-
await run_pipeline(source="users")
|
|
71
|
-
```
|
|
72
|
-
|
|
73
|
-
## How it works
|
|
74
|
-
|
|
75
|
-
1. **`@wf.task`** wraps an async function with checkpoint + retry logic. When called inside a workflow, results are persisted to the store. On re-run, completed steps return their cached result without re-executing.
|
|
76
|
-
|
|
77
|
-
2. **`@wf.workflow`** marks the entry point of a durable run. It manages a `RunContext` (via `ContextVar`) so tasks automatically know which run they belong to. The `id` parameter is a template string resolved from function arguments at call time.
|
|
78
|
-
|
|
79
|
-
3. **`Store`** is the persistence backend. `SQLiteStore` is the default (zero config, backed by aiosqlite). Subclass `Store` to use Postgres, Redis, or anything else.
|
|
80
|
-
|
|
81
|
-
## Features
|
|
82
|
-
|
|
83
|
-
- **Crash recovery** — completed steps are never re-executed after a restart
|
|
84
|
-
- **Automatic retries** — configurable per-task with `exponential`, `linear`, or `constant` backoff
|
|
85
|
-
- **Loop support** — use `step_id` to checkpoint each iteration independently
|
|
86
|
-
- **Zero magic outside workflows** — tasks work as plain async functions when called without a workflow context
|
|
87
|
-
- **Pluggable storage** — SQLite by default, bring your own `Store` for production
|
|
88
|
-
|
|
89
|
-
## Backoff strategies
|
|
90
|
-
|
|
91
|
-
```python
|
|
92
|
-
from durable.backoff import exponential, linear, constant
|
|
93
|
-
|
|
94
|
-
@wf.task(retries=5, backoff=exponential(base=2, max=60)) # 2s, 4s, 8s, 16s, 32s
|
|
95
|
-
async def exp_task(): ...
|
|
96
|
-
|
|
97
|
-
@wf.task(retries=3, backoff=linear(start=2, step=3)) # 2s, 5s, 8s
|
|
98
|
-
async def linear_task(): ...
|
|
99
|
-
|
|
100
|
-
@wf.task(retries=3, backoff=constant(5)) # 5s, 5s, 5s
|
|
101
|
-
async def const_task(): ...
|
|
102
|
-
```
|
|
103
|
-
|
|
104
|
-
## Loops with step_id
|
|
105
|
-
|
|
106
|
-
When calling the same task in a loop, pass `step_id` so each iteration is checkpointed independently:
|
|
107
|
-
|
|
108
|
-
```python
|
|
109
|
-
@wf.workflow(id="batch-{batch_id}")
|
|
110
|
-
async def process_batch(batch_id: str) -> None:
|
|
111
|
-
for i, item in enumerate(items):
|
|
112
|
-
await process_item(item, step_id=f"item-{i}")
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
If the workflow crashes mid-loop, only the remaining items are processed on restart.
|
|
116
|
-
|
|
117
|
-
## Important: JSON serialization
|
|
118
|
-
|
|
119
|
-
Task return values must be JSON-serializable (dicts, lists, strings, numbers, booleans, `None`). The store uses `json.dumps` internally.
|
|
120
|
-
|
|
121
|
-
For Pydantic models, return `.model_dump()` from tasks and reconstruct with `.model_validate()` downstream:
|
|
122
|
-
|
|
123
|
-
```python
|
|
124
|
-
@wf.task
|
|
125
|
-
async def validate_invoice(draft: InvoiceDraft) -> dict:
|
|
126
|
-
validated = ValidatedInvoice(...)
|
|
127
|
-
return validated.model_dump()
|
|
128
|
-
|
|
129
|
-
@wf.task
|
|
130
|
-
async def book_invoice(data: dict) -> dict:
|
|
131
|
-
invoice = ValidatedInvoice.model_validate(data)
|
|
132
|
-
...
|
|
133
|
-
```
|
|
134
|
-
|
|
135
|
-
## License
|
|
136
|
-
|
|
137
|
-
MIT
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
durable/__init__.py,sha256=IPi-f0B49fps6zZ5Hik1AP5Dp-thp3MoQXCEVikes44,1477
|
|
2
|
-
durable/backoff.py,sha256=o5p3hXfJ1YMwmEzjzukqW6inYezYGYnEi3_1YwnmDcU,1056
|
|
3
|
-
durable/context.py,sha256=greEK4jRz9RmVc5kHlmBjU3fisrsl2RCDDtGUPEiQM4,1324
|
|
4
|
-
durable/redis_store.py,sha256=gYMRqgEzLh5K4EC3dWry515l4eJ8OfZqRGaY-1eRrDA,3548
|
|
5
|
-
durable/store.py,sha256=YvrVNFzYQGSlLR-TKnXS0b1tJzpY5zr_tkkKDH5sR1k,8769
|
|
6
|
-
durable/workflow.py,sha256=Q0boxwnquNJMueE8LeRlWg8yLxFr6m1I6RzOO3kmdB8,13192
|
|
7
|
-
python_durable-0.1.1.dist-info/METADATA,sha256=-ig6huL8l_xGmrNqfQolT_uvcMStHFUbeoKqDmEbx9s,4775
|
|
8
|
-
python_durable-0.1.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
9
|
-
python_durable-0.1.1.dist-info/licenses/LICENSE,sha256=S5JKY7biEEYA0tC7Qr2hO-ppopDVnD8muKbaviRFqLk,1084
|
|
10
|
-
python_durable-0.1.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|