durably 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- durably/__init__.py +26 -0
- durably/activity_context.py +55 -0
- durably/cli.py +41 -0
- durably/client.py +104 -0
- durably/context.py +276 -0
- durably/db.py +46 -0
- durably/errors.py +30 -0
- durably/failpoints.py +42 -0
- durably/py.typed +0 -0
- durably/registry.py +44 -0
- durably/retry.py +24 -0
- durably/schema.sql +74 -0
- durably/store.py +347 -0
- durably/testing.py +117 -0
- durably/worker.py +247 -0
- durably-0.1.0.dist-info/METADATA +343 -0
- durably-0.1.0.dist-info/RECORD +20 -0
- durably-0.1.0.dist-info/WHEEL +4 -0
- durably-0.1.0.dist-info/entry_points.txt +2 -0
- durably-0.1.0.dist-info/licenses/LICENSE +21 -0
durably/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""durably: durable execution for AI agents, with just Postgres."""
|
|
2
|
+
|
|
3
|
+
from durably.activity_context import ActivityInfo, activity_info, heartbeat
|
|
4
|
+
from durably.client import Client
|
|
5
|
+
from durably.context import Context
|
|
6
|
+
from durably.errors import (
|
|
7
|
+
ActivityError, NonDeterminismError, StaleWorkerError, WorkflowFailedError, WorkflowNotRunningError,
|
|
8
|
+
)
|
|
9
|
+
from durably.registry import activity, workflow
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def __getattr__(name: str):
|
|
15
|
+
# Imported lazily so `python -m durably.worker` does not load the module twice.
|
|
16
|
+
if name in ("ActivityWorker", "WorkflowWorker"):
|
|
17
|
+
from durably import worker
|
|
18
|
+
return getattr(worker, name)
|
|
19
|
+
raise AttributeError(f"module 'durably' has no attribute {name!r}")
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"ActivityError", "ActivityInfo", "ActivityWorker", "Client", "Context", "NonDeterminismError",
|
|
23
|
+
"StaleWorkerError", "WorkflowFailedError", "WorkflowNotRunningError", "WorkflowWorker",
|
|
24
|
+
"activity", "activity_info",
|
|
25
|
+
"heartbeat", "workflow",
|
|
26
|
+
]
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""What a running activity can see and do: its own info, and heartbeats."""
|
|
2
|
+
|
|
3
|
+
from contextvars import ContextVar
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import asyncpg
|
|
8
|
+
|
|
9
|
+
from durably import store
|
|
10
|
+
from durably.errors import StaleWorkerError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class ActivityInfo:
|
|
15
|
+
workflow_id: str
|
|
16
|
+
name: str
|
|
17
|
+
idempotency_key: str
|
|
18
|
+
attempt: int
|
|
19
|
+
heartbeat_details: Any # last details saved by a previous attempt, or None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class _Running:
|
|
24
|
+
info: ActivityInfo
|
|
25
|
+
pool: asyncpg.Pool
|
|
26
|
+
claim: store.Claim
|
|
27
|
+
worker_id: str
|
|
28
|
+
lease_s: float
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
_current: ContextVar[_Running] = ContextVar("durably_activity")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _running() -> _Running:
|
|
35
|
+
try:
|
|
36
|
+
return _current.get()
|
|
37
|
+
except LookupError:
|
|
38
|
+
raise RuntimeError("only available inside a running activity") from None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def activity_info() -> ActivityInfo:
|
|
42
|
+
return _running().info
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
async def heartbeat(details: Any = None) -> None:
|
|
46
|
+
"""Tell durably this activity is still alive, optionally saving progress.
|
|
47
|
+
|
|
48
|
+
Extends the lease by the activity's heartbeat timeout. If the activity is retried,
|
|
49
|
+
the next attempt sees the last `details` in `activity_info().heartbeat_details`.
|
|
50
|
+
Raises StaleWorkerError if the task was already handed to another worker; stop work.
|
|
51
|
+
"""
|
|
52
|
+
run = _running()
|
|
53
|
+
async with run.pool.acquire() as conn:
|
|
54
|
+
if not await store.heartbeat(conn, run.claim, run.worker_id, run.lease_s, details):
|
|
55
|
+
raise StaleWorkerError("activity lease lost: another worker owns this task now")
|
durably/cli.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""`durably` command-line entry point."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import asyncio
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
from durably import worker
|
|
8
|
+
from durably.db import connect, init_db
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
async def _init_db(url: str | None) -> None:
|
|
12
|
+
conn = await connect(url)
|
|
13
|
+
try:
|
|
14
|
+
await init_db(conn)
|
|
15
|
+
finally:
|
|
16
|
+
await conn.close()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def main(argv: list[str] | None = None) -> None:
|
|
20
|
+
parser = argparse.ArgumentParser(prog="durably")
|
|
21
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
22
|
+
|
|
23
|
+
init = sub.add_parser("init-db", help="create durably's tables in Postgres")
|
|
24
|
+
init.add_argument("--database-url", default=None, help="defaults to $DATABASE_URL")
|
|
25
|
+
|
|
26
|
+
worker.add_arguments(sub.add_parser("worker", help="run a workflow or activity worker"))
|
|
27
|
+
|
|
28
|
+
args = parser.parse_args(argv)
|
|
29
|
+
if args.command == "init-db":
|
|
30
|
+
asyncio.run(_init_db(args.database_url))
|
|
31
|
+
print("durably: schema ready")
|
|
32
|
+
elif args.command == "worker":
|
|
33
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
|
34
|
+
try:
|
|
35
|
+
asyncio.run(worker.serve(args))
|
|
36
|
+
except KeyboardInterrupt:
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
if __name__ == "__main__":
|
|
41
|
+
main()
|
durably/client.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Client API: start workflows and wait for their results."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import asyncpg
|
|
8
|
+
|
|
9
|
+
from durably import registry, store
|
|
10
|
+
from durably.db import create_pool
|
|
11
|
+
from durably.errors import WorkflowFailedError, WorkflowNotRunningError
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Client:
|
|
15
|
+
def __init__(self, pool: asyncpg.Pool):
|
|
16
|
+
self.pool = pool
|
|
17
|
+
self._owns_pool = False
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
async def connect(cls, url: str | None = None) -> "Client":
|
|
21
|
+
client = cls(await create_pool(url))
|
|
22
|
+
client._owns_pool = True
|
|
23
|
+
return client
|
|
24
|
+
|
|
25
|
+
async def close(self) -> None:
|
|
26
|
+
if self._owns_pool:
|
|
27
|
+
await self.pool.close()
|
|
28
|
+
|
|
29
|
+
async def start(self, workflow: Callable | str, input: Any = None, idempotency_key: str | None = None) -> str:
|
|
30
|
+
"""Start a workflow and return its id.
|
|
31
|
+
|
|
32
|
+
A second start with the same idempotency key returns the existing workflow's
|
|
33
|
+
id and starts nothing, even if `input` differs.
|
|
34
|
+
"""
|
|
35
|
+
async with self.pool.acquire() as conn, conn.transaction():
|
|
36
|
+
wf_id = await store.create_workflow(conn, registry.name_of(workflow), input, idempotency_key=idempotency_key)
|
|
37
|
+
if wf_id is None:
|
|
38
|
+
return str(await conn.fetchval("SELECT id FROM workflows WHERE idempotency_key = $1", idempotency_key))
|
|
39
|
+
return str(wf_id)
|
|
40
|
+
|
|
41
|
+
async def signal(self, workflow_id: str, name: str, payload: Any = None) -> None:
|
|
42
|
+
"""Send a signal to a running workflow; `ctx.wait_signal(name)` receives it.
|
|
43
|
+
|
|
44
|
+
The signal is recorded in the workflow's history before this returns, so it is not
|
|
45
|
+
lost if no worker is running. If the workflow continued as new, the signal goes to
|
|
46
|
+
its latest run. Raises WorkflowNotRunningError if the workflow has finished or does
|
|
47
|
+
not exist.
|
|
48
|
+
"""
|
|
49
|
+
async with self.pool.acquire() as conn:
|
|
50
|
+
while True:
|
|
51
|
+
run = await store.latest_run(conn, workflow_id)
|
|
52
|
+
if run is None or run["status"] != "running":
|
|
53
|
+
raise WorkflowNotRunningError(f"workflow {workflow_id} is not running")
|
|
54
|
+
tx = conn.transaction()
|
|
55
|
+
await tx.start()
|
|
56
|
+
if await store.deliver(conn, run["id"], "SignalReceived", {"name": name, "payload": payload}):
|
|
57
|
+
await tx.commit()
|
|
58
|
+
return
|
|
59
|
+
# The run finished or continued as new in the meantime: roll back the wake-up
|
|
60
|
+
# and look up the latest run again.
|
|
61
|
+
await tx.rollback()
|
|
62
|
+
|
|
63
|
+
async def describe(self, workflow_id: str) -> dict:
|
|
64
|
+
async with self.pool.acquire() as conn:
|
|
65
|
+
row = await conn.fetchrow(
|
|
66
|
+
"SELECT id, type, status, epoch, result, error, continued_as, created_at FROM workflows WHERE id = $1",
|
|
67
|
+
workflow_id,
|
|
68
|
+
)
|
|
69
|
+
if row is None:
|
|
70
|
+
raise LookupError(f"no workflow {workflow_id}")
|
|
71
|
+
return dict(row)
|
|
72
|
+
|
|
73
|
+
async def history(self, workflow_id: str) -> list[store.Event]:
|
|
74
|
+
async with self.pool.acquire() as conn:
|
|
75
|
+
return await store.load_history(conn, workflow_id)
|
|
76
|
+
|
|
77
|
+
async def runs(self, workflow_id: str) -> list[str]:
|
|
78
|
+
"""Ids of every run of a workflow, oldest first (more than one after Continue-As-New)."""
|
|
79
|
+
async with self.pool.acquire() as conn:
|
|
80
|
+
ids, current = [], workflow_id
|
|
81
|
+
while current is not None:
|
|
82
|
+
ids.append(str(current))
|
|
83
|
+
current = await conn.fetchval("SELECT continued_as FROM workflows WHERE id = $1", current)
|
|
84
|
+
return ids
|
|
85
|
+
|
|
86
|
+
async def result(self, workflow_id: str, timeout: float | None = None, poll: float = 0.05) -> Any:
|
|
87
|
+
"""Wait for the workflow to finish and return its result.
|
|
88
|
+
|
|
89
|
+
Follows Continue-As-New to the final run. Raises WorkflowFailedError if it failed,
|
|
90
|
+
TimeoutError if `timeout` passes first.
|
|
91
|
+
"""
|
|
92
|
+
async def wait() -> Any:
|
|
93
|
+
while True:
|
|
94
|
+
async with self.pool.acquire() as conn:
|
|
95
|
+
run = await store.latest_run(conn, workflow_id)
|
|
96
|
+
if run is None:
|
|
97
|
+
raise LookupError(f"no workflow {workflow_id}")
|
|
98
|
+
if run["status"] == "completed":
|
|
99
|
+
return run["result"]
|
|
100
|
+
if run["status"] in ("failed", "cancelled"):
|
|
101
|
+
raise WorkflowFailedError(f"workflow {workflow_id} {run['status']}: {run['error']}")
|
|
102
|
+
await asyncio.sleep(poll)
|
|
103
|
+
|
|
104
|
+
return await asyncio.wait_for(wait(), timeout)
|
durably/context.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
"""The replay engine.
|
|
2
|
+
|
|
3
|
+
A workflow task reruns the workflow function from the top against its history. Each
|
|
4
|
+
`ctx.*` call takes the next command position:
|
|
5
|
+
|
|
6
|
+
- history has a command there: it must match the call (else NonDeterminismError); if
|
|
7
|
+
its result is recorded, return it without running anything (G3);
|
|
8
|
+
- history is exhausted: this is a new command; record it. Commands that need a result
|
|
9
|
+
(activities, timers) then suspend; markers (`now`, `random`, `uuid4`) do not, so one
|
|
10
|
+
workflow task can record several new commands.
|
|
11
|
+
|
|
12
|
+
Suspending raises `_Suspend`, which unwinds the function. Nothing stays parked in memory:
|
|
13
|
+
the next event wakes a fresh replay.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import random
|
|
18
|
+
import uuid
|
|
19
|
+
from collections import Counter, defaultdict
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from datetime import datetime, timezone
|
|
22
|
+
from typing import Any, NoReturn
|
|
23
|
+
from uuid import UUID
|
|
24
|
+
|
|
25
|
+
from durably.errors import ActivityError, NonDeterminismError
|
|
26
|
+
from durably.store import Event
|
|
27
|
+
|
|
28
|
+
COMMAND_TYPES = {"ActivityScheduled", "TimerStarted", "MarkerRecorded"}
|
|
29
|
+
# Result event type -> payload field naming the command it answers.
|
|
30
|
+
RESULT_TYPES = {"ActivityCompleted": "scheduled_seq", "ActivityFailed": "scheduled_seq", "TimerFired": "started_seq"}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class _ContinueAsNew(BaseException):
|
|
34
|
+
def __init__(self, input: Any):
|
|
35
|
+
self.input = input
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class _Suspend(BaseException):
|
|
39
|
+
"""Unwinds the workflow when it needs a result that is not in history yet.
|
|
40
|
+
|
|
41
|
+
A BaseException so that `except Exception` in workflow code does not swallow it.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class Command:
|
|
47
|
+
type: str
|
|
48
|
+
payload: dict
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _normalize(value: Any) -> Any:
|
|
52
|
+
"""What a value looks like after a round trip through jsonb, for comparing with history."""
|
|
53
|
+
return json.loads(json.dumps(value))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
DEFAULT_MAX_HISTORY = 1000
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Context:
|
|
60
|
+
def __init__(self, workflow_id: UUID, history: list[Event], max_history: int = DEFAULT_MAX_HISTORY):
|
|
61
|
+
self.workflow_id = workflow_id
|
|
62
|
+
self.max_history = max_history
|
|
63
|
+
self._history_length = len(history)
|
|
64
|
+
self._last_seq = history[-1].seq if history else 0
|
|
65
|
+
self._commands = [e for e in history if e.type in COMMAND_TYPES]
|
|
66
|
+
self._results = {e.payload[field]: e for e in history if (field := RESULT_TYPES.get(e.type))}
|
|
67
|
+
self._signals: dict[str, list[Event]] = defaultdict(list)
|
|
68
|
+
for e in history:
|
|
69
|
+
if e.type == "SignalReceived":
|
|
70
|
+
self._signals[e.payload["name"]].append(e)
|
|
71
|
+
self._signals_taken: Counter[str] = Counter()
|
|
72
|
+
self._pos = 0
|
|
73
|
+
self.new_commands: list[Command] = []
|
|
74
|
+
self.nondeterminism: NonDeterminismError | None = None
|
|
75
|
+
|
|
76
|
+
def _command(self, type: str, payload: dict, identity: tuple[str, ...]) -> Event:
|
|
77
|
+
"""Like `_take`, but a new command suspends the workflow until its result arrives."""
|
|
78
|
+
recorded = self._take(type, payload, identity)
|
|
79
|
+
if recorded is None:
|
|
80
|
+
raise _Suspend()
|
|
81
|
+
return recorded
|
|
82
|
+
|
|
83
|
+
def _take(self, type: str, payload: dict, identity: tuple[str, ...]) -> Event | None:
|
|
84
|
+
"""Take the next command position and return the event history recorded there.
|
|
85
|
+
|
|
86
|
+
If history is exhausted, record `payload` as a new command and return None. If
|
|
87
|
+
history holds a different command (type or `identity` fields differ), the code
|
|
88
|
+
changed under a running workflow: raise NonDeterminismError (G5).
|
|
89
|
+
"""
|
|
90
|
+
if self._pos >= len(self._commands):
|
|
91
|
+
self._pos += 1
|
|
92
|
+
self.new_commands.append(Command(type, payload))
|
|
93
|
+
return None
|
|
94
|
+
recorded = self._commands[self._pos]
|
|
95
|
+
expected = {k: recorded.payload.get(k) for k in identity}
|
|
96
|
+
actual = {k: payload[k] for k in identity}
|
|
97
|
+
if recorded.type != type or expected != actual:
|
|
98
|
+
# Remembered on the context too, so workflow code that catches broad
|
|
99
|
+
# exceptions cannot hide it.
|
|
100
|
+
self.nondeterminism = NonDeterminismError(
|
|
101
|
+
f"command {self._pos}: history has {recorded.type} "
|
|
102
|
+
f"{ {k: recorded.payload.get(k) for k in ('name', 'args', 'seconds', 'kind') if k in recorded.payload} }, "
|
|
103
|
+
f"code issued {type} {actual}"
|
|
104
|
+
)
|
|
105
|
+
raise self.nondeterminism
|
|
106
|
+
self._pos += 1
|
|
107
|
+
return recorded
|
|
108
|
+
|
|
109
|
+
async def activity(self, name: str, args: list | tuple = (), *, retry: dict | None = None,
|
|
110
|
+
timeout: float | None = None, heartbeat_timeout: float | None = None) -> Any:
|
|
111
|
+
"""Run an activity at least once and return its result (or raise ActivityError).
|
|
112
|
+
|
|
113
|
+
`timeout` limits one attempt. With `heartbeat_timeout`, an attempt that goes that
|
|
114
|
+
long without calling `durably.heartbeat()` is treated as hung and retried.
|
|
115
|
+
"""
|
|
116
|
+
recorded = self._command("ActivityScheduled", {
|
|
117
|
+
"name": name, "args": _normalize(list(args)), "retry": retry, "timeout": timeout,
|
|
118
|
+
"heartbeat_timeout": heartbeat_timeout,
|
|
119
|
+
}, identity=("name", "args"))
|
|
120
|
+
outcome = self._results.get(recorded.seq)
|
|
121
|
+
if outcome is None:
|
|
122
|
+
raise _Suspend() # scheduled, still running
|
|
123
|
+
if outcome.type == "ActivityFailed":
|
|
124
|
+
raise ActivityError(name, outcome.payload["error"])
|
|
125
|
+
return outcome.payload["result"]
|
|
126
|
+
|
|
127
|
+
async def sleep(self, seconds: float) -> None:
|
|
128
|
+
"""Sleep durably: survives crashes, restarts and deploys.
|
|
129
|
+
|
|
130
|
+
The wake-up time is fixed when the timer is first recorded, on the database clock.
|
|
131
|
+
"""
|
|
132
|
+
recorded = self._command("TimerStarted", {"seconds": float(seconds), "for_signal": None},
|
|
133
|
+
identity=("seconds", "for_signal"))
|
|
134
|
+
if recorded.seq not in self._results:
|
|
135
|
+
raise _Suspend()
|
|
136
|
+
|
|
137
|
+
def _marker(self, kind: str, make) -> Any:
|
|
138
|
+
"""Return the value recorded for this marker, or record a fresh one from `make()`."""
|
|
139
|
+
recorded = self._take("MarkerRecorded", {"kind": kind, "value": None}, identity=("kind",))
|
|
140
|
+
if recorded is not None:
|
|
141
|
+
return recorded.payload["value"]
|
|
142
|
+
value = _normalize(make())
|
|
143
|
+
self.new_commands[-1].payload["value"] = value
|
|
144
|
+
return value
|
|
145
|
+
|
|
146
|
+
def now(self) -> datetime:
|
|
147
|
+
"""The current UTC time, recorded the first time so every replay sees the same value."""
|
|
148
|
+
return datetime.fromisoformat(self._marker("now", lambda: datetime.now(timezone.utc).isoformat()))
|
|
149
|
+
|
|
150
|
+
def random(self) -> float:
|
|
151
|
+
"""A random float in [0, 1), recorded so every replay sees the same value."""
|
|
152
|
+
return self._marker("random", random.random)
|
|
153
|
+
|
|
154
|
+
def uuid4(self) -> UUID:
|
|
155
|
+
"""A random UUID, recorded so every replay sees the same value."""
|
|
156
|
+
return UUID(self._marker("uuid4", lambda: str(uuid.uuid4())))
|
|
157
|
+
|
|
158
|
+
def should_continue_as_new(self) -> bool:
|
|
159
|
+
"""True once this run's history has reached `max_history` events.
|
|
160
|
+
|
|
161
|
+
Check it at a safe point (for example once per loop iteration) and call
|
|
162
|
+
`continue_as_new` when it is True, so replay time stays bounded. A True answer is
|
|
163
|
+
recorded as a marker, so replays answer exactly as the original run did.
|
|
164
|
+
"""
|
|
165
|
+
nxt = self._commands[self._pos] if self._pos < len(self._commands) else None
|
|
166
|
+
if nxt is not None: # replaying: True only where the original run recorded it
|
|
167
|
+
if nxt.type == "MarkerRecorded" and nxt.payload.get("kind") == "continue_as_new_suggested":
|
|
168
|
+
self._pos += 1
|
|
169
|
+
return True
|
|
170
|
+
return False
|
|
171
|
+
if self._history_length + len(self.new_commands) < self.max_history:
|
|
172
|
+
return False
|
|
173
|
+
self._take("MarkerRecorded", {"kind": "continue_as_new_suggested", "value": True}, identity=("kind",))
|
|
174
|
+
return True
|
|
175
|
+
|
|
176
|
+
def continue_as_new(self, input: Any) -> NoReturn:
|
|
177
|
+
"""End this run and start a fresh one with `input` and an empty history.
|
|
178
|
+
|
|
179
|
+
Signals this run received but did not take are carried over to the new run, in
|
|
180
|
+
order. Clients holding the original id follow the chain automatically.
|
|
181
|
+
"""
|
|
182
|
+
raise _ContinueAsNew(_normalize(input))
|
|
183
|
+
|
|
184
|
+
def _untaken_signals(self) -> list[dict]:
|
|
185
|
+
untaken = [e for name, events in self._signals.items() for e in events[self._signals_taken[name]:]]
|
|
186
|
+
return [e.payload for e in sorted(untaken, key=lambda e: e.seq)]
|
|
187
|
+
|
|
188
|
+
async def wait_signal(self, name: str, timeout: float | None = None) -> Any:
|
|
189
|
+
"""Wait for the next signal called `name` and return its payload.
|
|
190
|
+
|
|
191
|
+
Signals are taken in the order they were sent, each exactly once (G8): the n-th
|
|
192
|
+
wait for `name` gets the n-th signal, on every replay. With `timeout`, returns None
|
|
193
|
+
if no signal arrives in time; a later signal then goes to the next wait.
|
|
194
|
+
"""
|
|
195
|
+
n = self._signals_taken[name]
|
|
196
|
+
signals = self._signals[name]
|
|
197
|
+
signal = signals[n] if n < len(signals) else None
|
|
198
|
+
|
|
199
|
+
# A timed wait that found no signal yet started a timer. On replay, tell that apart
|
|
200
|
+
# from a wait that got its signal straight away: the timer is the next command and
|
|
201
|
+
# the signal (if any) arrived after it.
|
|
202
|
+
nxt = self._commands[self._pos] if self._pos < len(self._commands) else None
|
|
203
|
+
timer_was_started = (
|
|
204
|
+
timeout is not None and nxt is not None and nxt.type == "TimerStarted"
|
|
205
|
+
and nxt.payload.get("for_signal") == name and (signal is None or signal.seq > nxt.seq)
|
|
206
|
+
)
|
|
207
|
+
if not timer_was_started:
|
|
208
|
+
if signal is not None:
|
|
209
|
+
self._signals_taken[name] += 1
|
|
210
|
+
return signal.payload["payload"]
|
|
211
|
+
if timeout is None:
|
|
212
|
+
raise _Suspend() # nothing to schedule; the signal itself wakes the workflow
|
|
213
|
+
assert timeout is not None # a timer exists only for timed waits
|
|
214
|
+
timer = self._command("TimerStarted", {"seconds": float(timeout), "for_signal": name},
|
|
215
|
+
identity=("seconds", "for_signal"))
|
|
216
|
+
fired = self._results.get(timer.seq)
|
|
217
|
+
if signal is not None and (fired is None or signal.seq < fired.seq):
|
|
218
|
+
self._signals_taken[name] += 1
|
|
219
|
+
return signal.payload["payload"]
|
|
220
|
+
if fired is not None:
|
|
221
|
+
return None # timed out; this signal (if it comes) is left for the next wait
|
|
222
|
+
raise _Suspend()
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
# Every outcome but NonDeterministic carries the new commands to append, in order.
|
|
226
|
+
@dataclass
|
|
227
|
+
class Completed:
|
|
228
|
+
result: Any
|
|
229
|
+
commands: list[Command] = field(default_factory=list)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
@dataclass
|
|
233
|
+
class Failed:
|
|
234
|
+
error: str
|
|
235
|
+
commands: list[Command] = field(default_factory=list)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@dataclass
|
|
239
|
+
class Suspended:
|
|
240
|
+
commands: list[Command] = field(default_factory=list) # empty: waiting on something already recorded
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@dataclass
|
|
244
|
+
class ContinuedAsNew:
|
|
245
|
+
input: Any
|
|
246
|
+
signals: list[dict] # untaken signal payloads to carry over, in order
|
|
247
|
+
seen_through: int # last seq this replay saw; later signals are carried too
|
|
248
|
+
commands: list[Command] = field(default_factory=list)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@dataclass
|
|
252
|
+
class NonDeterministic:
|
|
253
|
+
error: str
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
async def replay(fn, ctx: Context, input: Any) -> Completed | Failed | Suspended | ContinuedAsNew | NonDeterministic:
|
|
257
|
+
"""Run the workflow function against history once and report how it stopped."""
|
|
258
|
+
outcome: Completed | Failed | Suspended | ContinuedAsNew | None
|
|
259
|
+
try:
|
|
260
|
+
result = await fn(ctx, input)
|
|
261
|
+
except _Suspend:
|
|
262
|
+
outcome = Suspended(ctx.new_commands)
|
|
263
|
+
except _ContinueAsNew as e:
|
|
264
|
+
outcome = ContinuedAsNew(e.input, ctx._untaken_signals(), ctx._last_seq, ctx.new_commands)
|
|
265
|
+
except Exception as e:
|
|
266
|
+
outcome = Failed(f"{type(e).__name__}: {e}", ctx.new_commands)
|
|
267
|
+
else:
|
|
268
|
+
outcome = None
|
|
269
|
+
if ctx.nondeterminism is not None:
|
|
270
|
+
return NonDeterministic(str(ctx.nondeterminism))
|
|
271
|
+
if outcome is not None:
|
|
272
|
+
return outcome
|
|
273
|
+
try:
|
|
274
|
+
return Completed(_normalize(result), ctx.new_commands)
|
|
275
|
+
except (TypeError, ValueError) as e:
|
|
276
|
+
return Failed(f"workflow result is not JSON-serializable: {e}", ctx.new_commands)
|
durably/db.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Database access: connection settings and schema setup."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
from importlib.resources import files
|
|
6
|
+
|
|
7
|
+
import asyncpg
|
|
8
|
+
|
|
9
|
+
DEFAULT_URL = "postgres://durably:durably@localhost:5433/durably"
|
|
10
|
+
|
|
11
|
+
# No database call waits forever: a connection that silently stops answering (a network
|
|
12
|
+
# partition, a failed-over primary) raises TimeoutError, and the caller reconnects.
|
|
13
|
+
DEFAULT_COMMAND_TIMEOUT_S = 60.0
|
|
14
|
+
DEFAULT_CONNECT_TIMEOUT_S = 10.0
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def database_url() -> str:
|
|
18
|
+
return os.environ.get("DATABASE_URL", DEFAULT_URL)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def schema_sql() -> str:
|
|
22
|
+
return files("durably").joinpath("schema.sql").read_text(encoding="utf-8")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
async def _init_connection(conn: asyncpg.Connection) -> None:
|
|
26
|
+
# jsonb columns round-trip as Python dicts/lists.
|
|
27
|
+
await conn.set_type_codec("jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
async def connect(url: str | None = None, *, command_timeout: float = DEFAULT_COMMAND_TIMEOUT_S) -> asyncpg.Connection:
|
|
31
|
+
conn = await asyncpg.connect(url or database_url(), command_timeout=command_timeout,
|
|
32
|
+
timeout=DEFAULT_CONNECT_TIMEOUT_S)
|
|
33
|
+
await _init_connection(conn)
|
|
34
|
+
return conn
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
async def create_pool(url: str | None = None, *, command_timeout: float = DEFAULT_COMMAND_TIMEOUT_S,
|
|
38
|
+
**kwargs) -> asyncpg.Pool:
|
|
39
|
+
"""A connection pool whose statements time out after `command_timeout` seconds."""
|
|
40
|
+
return await asyncpg.create_pool(url or database_url(), init=_init_connection, command_timeout=command_timeout,
|
|
41
|
+
timeout=min(command_timeout, DEFAULT_CONNECT_TIMEOUT_S), **kwargs)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
async def init_db(conn: asyncpg.Connection) -> None:
|
|
45
|
+
"""Create tables, constraints and triggers. Idempotent."""
|
|
46
|
+
await conn.execute(schema_sql())
|
durably/errors.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Exceptions raised by durably."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class DurablyError(Exception):
|
|
5
|
+
pass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class StaleWorkerError(DurablyError):
|
|
9
|
+
"""This worker's epoch is no longer current; another worker owns the workflow."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class NonDeterminismError(DurablyError):
|
|
13
|
+
"""Workflow code issued a different command than history recorded at this position."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ActivityError(DurablyError):
|
|
17
|
+
"""An activity failed after exhausting its retries. Raised inside the workflow."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, name: str, message: str):
|
|
20
|
+
super().__init__(f"activity {name!r} failed: {message}")
|
|
21
|
+
self.name = name
|
|
22
|
+
self.message = message
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class WorkflowFailedError(DurablyError):
|
|
26
|
+
"""Raised by `Client.result` when the workflow ended in failure."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class WorkflowNotRunningError(DurablyError):
|
|
30
|
+
"""Raised by `Client.signal` when the workflow does not exist or has already finished."""
|
durably/failpoints.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Failpoints: make a worker die or freeze at a named point, for crash tests.
|
|
2
|
+
|
|
3
|
+
`FAILPOINT=<name>` exits the process abruptly (no cleanup, no flush) the first time it
|
|
4
|
+
reaches `<name>`; `FAILPOINT=<name>@<n>` waits for the n-th time. Adding `:pause`
|
|
5
|
+
(`<name>:pause`, `<name>@<n>:pause`) stops it with SIGSTOP instead, so a test can let its
|
|
6
|
+
lease expire and resume it later with SIGCONT (POSIX only).
|
|
7
|
+
|
|
8
|
+
Points: `before_claim`, `after_claim`, `before_activity`, `after_effect_before_record`,
|
|
9
|
+
`before_append`, `after_record_before_ack`, `during_timer_fire`.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import signal
|
|
14
|
+
from collections import Counter
|
|
15
|
+
|
|
16
|
+
EXIT_CODE = 86
|
|
17
|
+
|
|
18
|
+
_hits: Counter[str] = Counter()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def parse(spec: str) -> tuple[str, int, str]:
|
|
22
|
+
"""`name[@n][:action]` -> (name, n, action)."""
|
|
23
|
+
target, _, action = spec.partition(":")
|
|
24
|
+
name, _, nth = target.partition("@")
|
|
25
|
+
return name, int(nth or 1), action or "exit"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def failpoint(name: str) -> None:
|
|
29
|
+
spec = os.environ.get("FAILPOINT")
|
|
30
|
+
if not spec:
|
|
31
|
+
return
|
|
32
|
+
target, nth, action = parse(spec)
|
|
33
|
+
if target != name:
|
|
34
|
+
return
|
|
35
|
+
_hits[name] += 1
|
|
36
|
+
if _hits[name] != nth: # fire once, so a resumed worker carries on normally
|
|
37
|
+
return
|
|
38
|
+
os.write(2, f"durably: failpoint {name}@{nth} ({action}) pid {os.getpid()}\n".encode())
|
|
39
|
+
if action == "pause":
|
|
40
|
+
os.kill(os.getpid(), signal.SIGSTOP)
|
|
41
|
+
else:
|
|
42
|
+
os._exit(EXIT_CODE)
|
durably/py.typed
ADDED
|
File without changes
|
durably/registry.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""`@workflow` and `@activity` decorators and the name -> function registry.
|
|
2
|
+
|
|
3
|
+
Workers look functions up by name, so a workflow started by one process can be run by
|
|
4
|
+
another that imported the same module.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
|
|
9
|
+
_workflows: dict[str, Callable] = {}
|
|
10
|
+
_activities: dict[str, Callable] = {}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _register(table: dict[str, Callable], name: str | Callable | None):
|
|
14
|
+
def decorate(fn: Callable) -> Callable:
|
|
15
|
+
key = name if isinstance(name, str) else fn.__name__
|
|
16
|
+
setattr(fn, "__durably_name__", key)
|
|
17
|
+
table[key] = fn
|
|
18
|
+
return fn
|
|
19
|
+
|
|
20
|
+
if callable(name): # used bare: @workflow
|
|
21
|
+
return decorate(name)
|
|
22
|
+
return decorate
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def workflow(name: str | Callable | None = None):
|
|
26
|
+
"""Register `async def fn(ctx, input)` as a workflow."""
|
|
27
|
+
return _register(_workflows, name)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def activity(name: str | Callable | None = None):
|
|
31
|
+
"""Register `async def fn(*args, idempotency_key)` as an activity."""
|
|
32
|
+
return _register(_activities, name)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def name_of(fn_or_name: Callable | str) -> str:
|
|
36
|
+
return fn_or_name if isinstance(fn_or_name, str) else getattr(fn_or_name, "__durably_name__")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def get_workflow(name: str) -> Callable | None:
|
|
40
|
+
return _workflows.get(name)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def get_activity(name: str) -> Callable | None:
|
|
44
|
+
return _activities.get(name)
|