synadia-durable 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synadia_durable/__init__.py +119 -0
- synadia_durable/_engine/__init__.py +6 -0
- synadia_durable/_engine/blobs.py +277 -0
- synadia_durable/_engine/client.py +305 -0
- synadia_durable/_engine/errors.py +219 -0
- synadia_durable/_engine/names.py +39 -0
- synadia_durable/_engine/payload.py +203 -0
- synadia_durable/_engine/transport.py +231 -0
- synadia_durable/_engine/worker.py +1020 -0
- synadia_durable/_internal/__init__.py +1 -0
- synadia_durable/_internal/backoff.py +22 -0
- synadia_durable/_internal/invariant.py +43 -0
- synadia_durable/_internal/json.py +76 -0
- synadia_durable/_internal/logger.py +53 -0
- synadia_durable/_internal/metrics.py +94 -0
- synadia_durable/_internal/token.py +29 -0
- synadia_durable/_nats/__init__.py +6 -0
- synadia_durable/_nats/connect.py +29 -0
- synadia_durable/_nats/protocol.py +106 -0
- synadia_durable/core/__init__.py +6 -0
- synadia_durable/core/children.py +54 -0
- synadia_durable/core/ctx.py +1297 -0
- synadia_durable/core/execute.py +268 -0
- synadia_durable/core/inbound.py +194 -0
- synadia_durable/core/registry.py +65 -0
- synadia_durable/core/scheduler.py +36 -0
- synadia_durable/core/work_item.py +57 -0
- synadia_durable/core/workflow.py +61 -0
- synadia_durable/errors.py +228 -0
- synadia_durable/journal/__init__.py +1 -0
- synadia_durable/journal/blobs.py +551 -0
- synadia_durable/journal/codec.py +103 -0
- synadia_durable/journal/entry.py +372 -0
- synadia_durable/journal/payload_codec.py +98 -0
- synadia_durable/journal/store.py +182 -0
- synadia_durable/py.typed +0 -0
- synadia_durable/types.py +258 -0
- synadia_durable/version.py +26 -0
- synadia_durable-0.2.0.dist-info/METADATA +179 -0
- synadia_durable-0.2.0.dist-info/RECORD +42 -0
- synadia_durable-0.2.0.dist-info/WHEEL +4 -0
- synadia_durable-0.2.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"""synadia-durable — the client: start runs, observe them, signal and cancel them, through the
|
|
2
|
+
engine's five client ops (the engine's ``API.md``: start, result, status, signal,
|
|
3
|
+
cancel). Mirrors ``src/engine/client.ts``.
|
|
4
|
+
|
|
5
|
+
The library's ``Client`` and ``RunHandle``, unchanged in shape; what was the shell's (the journal
|
|
6
|
+
reads, the status hint subscription, provisioning, offload) is the engine's now, but for a value too
|
|
7
|
+
large for one request or reply, which goes up in ``put`` parts and comes down in ``get`` parts
|
|
8
|
+
(``blobs.py``).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import contextlib
|
|
15
|
+
import time
|
|
16
|
+
import uuid
|
|
17
|
+
from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast
|
|
18
|
+
|
|
19
|
+
from synadia_durable._engine.blobs import EngineBlobs, Sizes
|
|
20
|
+
from synadia_durable._engine.errors import EngineUnavailableError
|
|
21
|
+
from synadia_durable._engine.names import DEFAULT_PREFIX
|
|
22
|
+
from synadia_durable._engine.payload import fit_slot, unref, unslot
|
|
23
|
+
from synadia_durable._engine.transport import Transport
|
|
24
|
+
from synadia_durable._internal.logger import Logger, SilentLogger
|
|
25
|
+
from synadia_durable._internal.token import assert_token
|
|
26
|
+
from synadia_durable.errors import (
|
|
27
|
+
CancelledError,
|
|
28
|
+
ConnectionClosedError,
|
|
29
|
+
DurableTimeoutError,
|
|
30
|
+
WorkflowFailure,
|
|
31
|
+
)
|
|
32
|
+
from synadia_durable.types import RunStatus, StartOptions, Workflow
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from nats.aio.client import Client as NatsClient
|
|
36
|
+
|
|
37
|
+
from synadia_durable._internal.metrics import Metrics
|
|
38
|
+
from synadia_durable.journal.blobs import OffloadOptions
|
|
39
|
+
from synadia_durable.journal.payload_codec import PayloadCodec
|
|
40
|
+
|
|
41
|
+
__all__ = ["Client", "RunHandle", "create_client"]
|
|
42
|
+
|
|
43
|
+
OutT = TypeVar("OutT")
|
|
44
|
+
|
|
45
|
+
#: How long one ``result`` request asks the engine to wait (the engine caps it at its
|
|
46
|
+
#: ``max_poll_wait_ms``).
|
|
47
|
+
RESULT_WAIT_MS = 30_000
|
|
48
|
+
|
|
49
|
+
#: Back-off between ``result`` requests after a transient refusal (no engine, a timeout).
|
|
50
|
+
RESULT_RETRY_S = 0.5
|
|
51
|
+
|
|
52
|
+
#: How often a waiting ``result()`` looks at the connection's reconnect count.
|
|
53
|
+
_RECONNECT_TICK_S = 0.1
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class RunHandle(Generic[OutT]):
|
|
57
|
+
"""A handle to a started run. Satisfies :class:`~synadia_durable.types.Handle`."""
|
|
58
|
+
|
|
59
|
+
def __init__(self, client: Client, run_id: str) -> None:
|
|
60
|
+
self._client = client
|
|
61
|
+
self._run_id = run_id
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def run_id(self) -> str:
|
|
65
|
+
return self._run_id
|
|
66
|
+
|
|
67
|
+
async def status(self) -> RunStatus:
|
|
68
|
+
r = await self._client._request.request("status", {"runId": self._run_id})
|
|
69
|
+
state = r.get("status", "pending")
|
|
70
|
+
if state == "pending" and "wf" not in r:
|
|
71
|
+
return RunStatus(state="pending", workflow="", version=0, updated_ms=0)
|
|
72
|
+
return RunStatus(
|
|
73
|
+
state=state,
|
|
74
|
+
workflow=r.get("wf", ""),
|
|
75
|
+
version=r.get("version", 0),
|
|
76
|
+
updated_ms=r.get("updatedMs", 0),
|
|
77
|
+
next=r.get("next"),
|
|
78
|
+
detail=r.get("detail"),
|
|
79
|
+
due_ms=r.get("dueMs"),
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
async def result(self, timeout_ms: float = 30_000) -> OutT:
|
|
83
|
+
"""Wait for the run's terminal and return its value.
|
|
84
|
+
|
|
85
|
+
The engine follows ``continued`` itself and answers the generation that returned: the client
|
|
86
|
+
re-asks with ``waitMs`` until a terminal or the caller's deadline, nothing more.
|
|
87
|
+
"""
|
|
88
|
+
c = self._client
|
|
89
|
+
deadline = time.monotonic() + timeout_ms / 1000
|
|
90
|
+
while True:
|
|
91
|
+
if c._nc.is_closed:
|
|
92
|
+
raise self._closed_error()
|
|
93
|
+
remaining_ms = (deadline - time.monotonic()) * 1000
|
|
94
|
+
wait_ms = int(max(0, min(RESULT_WAIT_MS, remaining_ms)))
|
|
95
|
+
r: dict[str, Any] | None = None
|
|
96
|
+
try:
|
|
97
|
+
r = await c._result_request(self._run_id, wait_ms, deadline)
|
|
98
|
+
except ConnectionClosedError:
|
|
99
|
+
raise self._closed_error() from None
|
|
100
|
+
except EngineUnavailableError as e:
|
|
101
|
+
c._log.debug("durable: result() request failed; retrying", run_id=self._run_id, error=str(e)) # fmt: skip
|
|
102
|
+
left = deadline - time.monotonic()
|
|
103
|
+
if left > 0:
|
|
104
|
+
await asyncio.sleep(min(RESULT_RETRY_S, left))
|
|
105
|
+
if r is not None and not r.get("pending") and "kind" in r:
|
|
106
|
+
return cast("OutT", await self._terminal(r))
|
|
107
|
+
if time.monotonic() >= deadline:
|
|
108
|
+
if c._nc.is_closed:
|
|
109
|
+
raise self._closed_error()
|
|
110
|
+
raise DurableTimeoutError(f"run {self._run_id}: result timed out")
|
|
111
|
+
|
|
112
|
+
async def _terminal(self, r: dict[str, Any]) -> Any:
|
|
113
|
+
codec = self._client._payload_codec
|
|
114
|
+
gen = r.get("runId") or self._run_id
|
|
115
|
+
kind = r["kind"]
|
|
116
|
+
if kind == "cancelled":
|
|
117
|
+
raise CancelledError(gen)
|
|
118
|
+
blobs = self._client._blobs
|
|
119
|
+
enc = r.get("payloadEnc") is True
|
|
120
|
+
if kind == "result":
|
|
121
|
+
if "valueRef" in r:
|
|
122
|
+
return await unref(gen, "value", r["valueRef"], enc, codec, blobs)
|
|
123
|
+
if "valueEnc" in r:
|
|
124
|
+
return await unslot(gen, r["valueEnc"], codec)
|
|
125
|
+
return r.get("value")
|
|
126
|
+
if "errorRef" in r:
|
|
127
|
+
err = await unref(gen, "error", r["errorRef"], enc, codec, blobs)
|
|
128
|
+
elif "errorEnc" in r:
|
|
129
|
+
err = await unslot(gen, r["errorEnc"], codec)
|
|
130
|
+
else:
|
|
131
|
+
err = r.get("error") or {"name": "Error", "message": "run failed"}
|
|
132
|
+
raise WorkflowFailure(gen, f"{err['name']}: {err['message']}")
|
|
133
|
+
|
|
134
|
+
def _closed_error(self) -> ConnectionClosedError:
|
|
135
|
+
return ConnectionClosedError(
|
|
136
|
+
f"run {self._run_id}: the NATS connection closed while waiting for the result — the "
|
|
137
|
+
f"run itself is unaffected; call client.get({self._run_id!r}).result() on a live "
|
|
138
|
+
"connection"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class Client:
|
|
143
|
+
"""Starts, observes, signals and cancels runs. Build one with :func:`create_client`."""
|
|
144
|
+
|
|
145
|
+
def __init__(
|
|
146
|
+
self,
|
|
147
|
+
nc: NatsClient,
|
|
148
|
+
*,
|
|
149
|
+
prefix: str = DEFAULT_PREFIX,
|
|
150
|
+
payload_codec: PayloadCodec | None = None,
|
|
151
|
+
logger: Logger | None = None,
|
|
152
|
+
metrics: Metrics | None = None,
|
|
153
|
+
offload: bool = True,
|
|
154
|
+
) -> None:
|
|
155
|
+
self._nc = nc
|
|
156
|
+
self._prefix = prefix
|
|
157
|
+
self._payload_codec = payload_codec
|
|
158
|
+
self._log: Logger = logger or SilentLogger()
|
|
159
|
+
self._request = Transport(nc, prefix, self._log)
|
|
160
|
+
sizes = Sizes(self._request, lambda: nc.max_payload if nc.is_connected else 0)
|
|
161
|
+
#: ``None`` under ``offload=False``: a value too large for a request fails loud.
|
|
162
|
+
self._sizes: Sizes | None = sizes if offload else None
|
|
163
|
+
self._blobs = EngineBlobs(self._request.request, sizes, metrics=metrics, rtt=nc.rtt)
|
|
164
|
+
|
|
165
|
+
async def _result_request(
|
|
166
|
+
self, run_id: str, wait_ms: int, deadline: float
|
|
167
|
+
) -> dict[str, Any] | None:
|
|
168
|
+
"""One ``result`` request, settled with its reply, or with ``None`` at the caller's
|
|
169
|
+
deadline or at the connection's next reconnect (the caller asks again or times out).
|
|
170
|
+
|
|
171
|
+
A request in flight across a disconnect loses its reply: the engine answers an inbox nobody
|
|
172
|
+
is subscribed to for that moment. So the wait is cut short at the next reconnect and the
|
|
173
|
+
engine asked again, instead of waiting out the request's timeout.
|
|
174
|
+
"""
|
|
175
|
+
ask = asyncio.ensure_future(
|
|
176
|
+
self._request.request("result", {"runId": run_id, "waitMs": wait_ms}, wait_ms=wait_ms)
|
|
177
|
+
)
|
|
178
|
+
cut = asyncio.ensure_future(_next_reconnect(self._nc))
|
|
179
|
+
try:
|
|
180
|
+
timeout = max(0.0, deadline - time.monotonic()) + 0.05
|
|
181
|
+
await asyncio.wait({ask, cut}, timeout=timeout, return_when=asyncio.FIRST_COMPLETED)
|
|
182
|
+
if ask.done():
|
|
183
|
+
return ask.result()
|
|
184
|
+
return None
|
|
185
|
+
finally:
|
|
186
|
+
for t in (ask, cut):
|
|
187
|
+
if not t.done():
|
|
188
|
+
t.cancel()
|
|
189
|
+
with contextlib.suppress(asyncio.CancelledError, Exception):
|
|
190
|
+
await t
|
|
191
|
+
|
|
192
|
+
async def start(
|
|
193
|
+
self,
|
|
194
|
+
workflow: Workflow[Any, OutT],
|
|
195
|
+
input: Any,
|
|
196
|
+
opts: StartOptions | None = None,
|
|
197
|
+
) -> RunHandle[OutT]:
|
|
198
|
+
"""Start a run idempotently. The runId is the idempotency key (PROTOCOL §11.1); without
|
|
199
|
+
one the engine mints it, unless the input is too large for a request: it is then ``put``
|
|
200
|
+
under a runId minted here (the engine's form, a UUID), which the start names."""
|
|
201
|
+
run_id = opts.run_id if opts is not None and opts.run_id else None
|
|
202
|
+
# Identifiers are checked here, so a bad one fails before a round trip.
|
|
203
|
+
if run_id is not None:
|
|
204
|
+
assert_token("runId", run_id)
|
|
205
|
+
assert_token("workflow name", workflow.name)
|
|
206
|
+
rid = run_id or str(uuid.uuid4())
|
|
207
|
+
head: dict[str, Any] = {"runId": rid, "wf": workflow.name, "version": workflow.version}
|
|
208
|
+
value = await fit_slot(
|
|
209
|
+
"input", input, self._payload_codec, rid, head, self._sizes, self._blobs
|
|
210
|
+
)
|
|
211
|
+
body: dict[str, Any] = {} if run_id is None and "inputRef" not in value else {"runId": rid}
|
|
212
|
+
body |= {"wf": workflow.name, "version": workflow.version}
|
|
213
|
+
body |= value
|
|
214
|
+
r = await self._request.request("start", body)
|
|
215
|
+
return RunHandle(self, str(r["runId"]))
|
|
216
|
+
|
|
217
|
+
def get(self, run_id: str) -> RunHandle[Any]:
|
|
218
|
+
"""A handle to an existing run, without starting anything."""
|
|
219
|
+
return RunHandle(self, run_id)
|
|
220
|
+
|
|
221
|
+
async def signal(self, run_id: str, name: str, data: Any, *, id: str | None = None) -> None:
|
|
222
|
+
"""Deliver an external signal to a run, for ``ctx.await(name)`` (PROTOCOL §8). The engine
|
|
223
|
+
journals it before it answers, so two calls, each after the other's return, are journaled
|
|
224
|
+
in that order. An unknown run raises
|
|
225
|
+
:class:`~synadia_durable._engine.errors.RunNotFoundError`, a :class:`LookupError`.
|
|
226
|
+
|
|
227
|
+
``id`` is the signal's idempotency key (``[A-Za-z0-9_-]+``, checked before any request); a
|
|
228
|
+
UUID is generated when it is ``None``. **A retry passes the same id:** a caller that sends
|
|
229
|
+
the signal again after an ``EngineUnavailableError``, a timeout, or a restart of its own
|
|
230
|
+
process passes the id of the first try, and the engine then answers ``ok`` and journals
|
|
231
|
+
nothing more. Without an id each call is a new signal. The id is looked for in two journals
|
|
232
|
+
only: the run the request names and the generation current when the request arrives. A
|
|
233
|
+
signal journaled in any other generation (one that was current at the first try and has
|
|
234
|
+
continued since) is not recognised, and is journaled again. The id dedups one signal: two
|
|
235
|
+
signals with the same name and different ids are two.
|
|
236
|
+
"""
|
|
237
|
+
assert_token("runId", run_id)
|
|
238
|
+
if id is not None:
|
|
239
|
+
assert_token("signal id", id)
|
|
240
|
+
# An id always goes with the request, so a retry of this call can pass it.
|
|
241
|
+
body: dict[str, Any] = {"runId": run_id, "name": name, "id": id or str(uuid.uuid4())}
|
|
242
|
+
body |= await fit_slot(
|
|
243
|
+
"data", data, self._payload_codec, run_id, body, self._sizes, self._blobs
|
|
244
|
+
)
|
|
245
|
+
await self._request.request("signal", body)
|
|
246
|
+
|
|
247
|
+
async def cancel(self, run_id: str, reason: str | None = None) -> None:
|
|
248
|
+
"""Request cancellation of a run (idempotent; a no-op on an already-terminal run).
|
|
249
|
+
|
|
250
|
+
Cancellation is an uncatchable hard-stop at the run's next COMMAND BOUNDARY: recorded
|
|
251
|
+
commands still replay, but the next fresh ``ctx.*`` op — or the resume of a step that began
|
|
252
|
+
and never finished — halts before executing. Pure compute BETWEEN commands is not preempted:
|
|
253
|
+
a run that returns without crossing a stop point completes instead (PROTOCOL §11.7).
|
|
254
|
+
"""
|
|
255
|
+
assert_token("runId", run_id)
|
|
256
|
+
body: dict[str, Any] = {"runId": run_id}
|
|
257
|
+
if reason is not None:
|
|
258
|
+
body["reason"] = reason
|
|
259
|
+
await self._request.request("cancel", body)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
async def _next_reconnect(nc: NatsClient) -> None:
|
|
263
|
+
"""Settle at the connection's next reconnect, or when it closes for good.
|
|
264
|
+
|
|
265
|
+
nats-py has no status iterator to wait on and its reconnect callback is fixed at connect time,
|
|
266
|
+
so this watches the client's own reconnect count (``nc.stats["reconnects"]``).
|
|
267
|
+
"""
|
|
268
|
+
base = nc.stats["reconnects"]
|
|
269
|
+
while nc.stats["reconnects"] == base and not nc.is_closed:
|
|
270
|
+
await asyncio.sleep(_RECONNECT_TICK_S)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def create_client(
|
|
274
|
+
nc: NatsClient,
|
|
275
|
+
*,
|
|
276
|
+
prefix: str = DEFAULT_PREFIX,
|
|
277
|
+
stream_max_bytes: int | None = None,
|
|
278
|
+
offload: bool | OffloadOptions = True,
|
|
279
|
+
payload_codec: PayloadCodec | None = None,
|
|
280
|
+
metrics: Metrics | None = None,
|
|
281
|
+
logger: Logger | None = None,
|
|
282
|
+
) -> Client:
|
|
283
|
+
"""Build a client on a caller-owned NATS connection (bring your own connection).
|
|
284
|
+
|
|
285
|
+
Requests go to ``{prefix}.api.v1.<op>``, answered by ``durable-engine serve`` on that prefix.
|
|
286
|
+
``payload_codec`` is encryption at rest, as the library's: every worker and client on a prefix
|
|
287
|
+
must hold the SAME codec. Under one, inputs and signal data travel as ``inputEnc`` /
|
|
288
|
+
``dataEnc``, results come back as ``valueEnc`` / ``errorEnc`` and are decoded here, and
|
|
289
|
+
``status()`` carries no payload (the engine cannot read one).
|
|
290
|
+
|
|
291
|
+
``stream_max_bytes`` is the engine's now (``durable-engine serve --stream-max-bytes``): accepted
|
|
292
|
+
and ignored. So is ``offload``'s tuning (``info``'s ``offload_threshold``, ``serve
|
|
293
|
+
--offload-threshold``); ``offload=False`` keeps the library's meaning, nothing goes up in parts
|
|
294
|
+
and a value too large for a request fails loud. ``metrics`` sees ``blob.put`` and ``blob.get``,
|
|
295
|
+
for a value too large for one request or reply.
|
|
296
|
+
"""
|
|
297
|
+
_ = stream_max_bytes
|
|
298
|
+
return Client(
|
|
299
|
+
nc,
|
|
300
|
+
prefix=prefix,
|
|
301
|
+
payload_codec=payload_codec,
|
|
302
|
+
logger=logger,
|
|
303
|
+
metrics=metrics,
|
|
304
|
+
offload=offload is not False,
|
|
305
|
+
)
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""synadia-durable — the errors the engine's API adds to the library's. Mirrors
|
|
2
|
+
``src/engine/errors.ts``.
|
|
3
|
+
|
|
4
|
+
A refusal is ``{"error": {"code", "message"}}`` from a closed set (the engine's
|
|
5
|
+
``API.md`` "Errors"); the transport turns each code into one of these, or into a library error
|
|
6
|
+
where the library already has the class: ``fenced`` → :class:`FencedError`, ``too_large`` →
|
|
7
|
+
:class:`PayloadTooLargeError`, ``blob_gone`` → :class:`BlobMissingError` (or
|
|
8
|
+
:class:`BlobIntegrityError` when the bytes are not the digest's). ``tenant_limit`` and
|
|
9
|
+
``licence_limit`` are refusals of their own (:class:`LimitError`), never retried. One is this
|
|
10
|
+
SDK's alone, no engine's code: :class:`UnsupportedEngineError`, an engine older than the floor
|
|
11
|
+
(:data:`ENGINE_FLOOR`), read from ``info.capabilities``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from typing import Any, Literal
|
|
17
|
+
|
|
18
|
+
from synadia_durable.errors import DurableError, FencedError
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"ENGINE_ERROR_CODES",
|
|
22
|
+
"ENGINE_FLOOR_CAPABILITIES",
|
|
23
|
+
"ENGINE_FLOOR_SINCE",
|
|
24
|
+
"EngineError",
|
|
25
|
+
"EngineErrorCode",
|
|
26
|
+
"EngineFencedError",
|
|
27
|
+
"EngineUnavailableError",
|
|
28
|
+
"InvalidRequestError",
|
|
29
|
+
"LeaseExpiredError",
|
|
30
|
+
"LimitError",
|
|
31
|
+
"RunNotFoundError",
|
|
32
|
+
"UnsupportedEngineError",
|
|
33
|
+
"check_engine_floor",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
#: The closed set of refusal codes of API v1.
|
|
37
|
+
EngineErrorCode = Literal[
|
|
38
|
+
"invalid",
|
|
39
|
+
"fenced",
|
|
40
|
+
"expired",
|
|
41
|
+
"not_found",
|
|
42
|
+
"too_large",
|
|
43
|
+
"unavailable",
|
|
44
|
+
"blob_gone",
|
|
45
|
+
"internal",
|
|
46
|
+
"tenant_limit",
|
|
47
|
+
"licence_limit",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
#: The codes table of the engine's ``API.md`` "Errors", mirrored: each code and the class
|
|
51
|
+
#: a refusal with it becomes (``invalid`` on an ``append`` or a ``child`` is a
|
|
52
|
+
#: :class:`SerializationError` instead). ``tests/test_engine_errors.py`` checks it against the table
|
|
53
|
+
#: at ``engine.pin``, so a code the engine adds fails a test here rather than reaching a caller as
|
|
54
|
+
#: ``unavailable``.
|
|
55
|
+
ENGINE_ERROR_CODES: dict[str, str] = {
|
|
56
|
+
"invalid": "InvalidRequestError",
|
|
57
|
+
"fenced": "FencedError",
|
|
58
|
+
"expired": "LeaseExpiredError",
|
|
59
|
+
"not_found": "RunNotFoundError",
|
|
60
|
+
"too_large": "PayloadTooLargeError",
|
|
61
|
+
"unavailable": "EngineUnavailableError",
|
|
62
|
+
"blob_gone": "BlobMissingError",
|
|
63
|
+
"internal": "EngineUnavailableError",
|
|
64
|
+
"tenant_limit": "LimitError",
|
|
65
|
+
"licence_limit": "LimitError",
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class EngineError(DurableError):
|
|
70
|
+
"""A refusal (or a failure to reach the engine) on one op."""
|
|
71
|
+
|
|
72
|
+
def __init__(
|
|
73
|
+
self, op: str, code: EngineErrorCode | Literal["unsupported"], message: str
|
|
74
|
+
) -> None:
|
|
75
|
+
super().__init__(message)
|
|
76
|
+
self.op = op
|
|
77
|
+
#: The API's code, or ``unavailable`` for no responders and a request timeout;
|
|
78
|
+
#: ``unsupported`` (this SDK's, never the engine's) for an engine older than the floor.
|
|
79
|
+
self.code: EngineErrorCode | Literal["unsupported"] = code
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class LeaseExpiredError(EngineError):
|
|
83
|
+
"""The item's lease is not the engine's any more (``expired``): the item is done, past
|
|
84
|
+
``max_work_ms``, minted by an instance that is gone, or (for ``extend``) lapsed. The worker
|
|
85
|
+
stops executing that item and sends nothing more under its lease; the item redelivers to a
|
|
86
|
+
poll."""
|
|
87
|
+
|
|
88
|
+
def __init__(self, op: str, message: str) -> None:
|
|
89
|
+
super().__init__(op, "expired", message)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class InvalidRequestError(EngineError):
|
|
93
|
+
"""A malformed request (``invalid``) outside a journal append: a bug or a bad argument. Never
|
|
94
|
+
retried."""
|
|
95
|
+
|
|
96
|
+
def __init__(self, op: str, message: str) -> None:
|
|
97
|
+
super().__init__(op, "invalid", message)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class EngineUnavailableError(EngineError):
|
|
101
|
+
"""Transient: no engine answered (NATS no responders), the request timed out, or the engine
|
|
102
|
+
said ``unavailable`` or ``internal``. Inside an execution the pure core halts with ``retry`` on
|
|
103
|
+
it and the worker reports ``done retry``; a client call may simply be repeated."""
|
|
104
|
+
|
|
105
|
+
def __init__(
|
|
106
|
+
self,
|
|
107
|
+
op: str,
|
|
108
|
+
code: Literal["unavailable", "internal"],
|
|
109
|
+
message: str,
|
|
110
|
+
*,
|
|
111
|
+
timed_out: bool = False,
|
|
112
|
+
) -> None:
|
|
113
|
+
super().__init__(op, code, message)
|
|
114
|
+
#: No answer within the request's timeout (rather than no responders or a refusal).
|
|
115
|
+
self.timed_out = timed_out
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class LimitError(EngineError):
|
|
119
|
+
"""A refusal at a limit: ``tenant_limit`` (the fabric mode: the caller's tenant is at one of its
|
|
120
|
+
limits) or ``licence_limit`` (standalone: the engine's licence bounds what was asked).
|
|
121
|
+
``limit`` names which one, as the engine says it: ``workflows``, ``workers``, ``runs``,
|
|
122
|
+
``runsPerDay``, ``tenants``, ``expired`` (the licence's), ``bytes``, ``runs``, ``workflows``,
|
|
123
|
+
``polls``, ``rate`` (the tenant's).
|
|
124
|
+
|
|
125
|
+
Not a fault of the request and not transient: it clears when runs end, an operator raises the
|
|
126
|
+
limit or installs a renewed licence (``rate`` within a second). So no call of this package
|
|
127
|
+
retries it: a client's call raises it, a worker's refused poll is logged at error naming the
|
|
128
|
+
limit and backs off, ``worker.start()`` raises it when its first poll is refused, and inside an
|
|
129
|
+
execution the item goes back (``done retry``), logged at error.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
def __init__(
|
|
133
|
+
self, op: str, code: Literal["tenant_limit", "licence_limit"], limit: str, message: str
|
|
134
|
+
) -> None:
|
|
135
|
+
super().__init__(op, code, message)
|
|
136
|
+
#: Which limit, as the engine names it (``""`` if a refusal named none).
|
|
137
|
+
self.limit = limit
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
#: What this SDK needs of the engine, read from ``info.capabilities`` (the engine's
|
|
141
|
+
#: ``API.md`` ``info``, its F57), never from ``info.engine``: the repo has no tag, so every
|
|
142
|
+
#: build says ``0.2.0-dev``. ``withdrawn-remembered`` (its F53): a worker withdrawn by
|
|
143
|
+
#: ``cancelPoll`` is remembered 5 s on every instance, so ``stop()`` sends one ``cancelPoll``.
|
|
144
|
+
ENGINE_FLOOR_CAPABILITIES: tuple[str, ...] = ("withdrawn-remembered",)
|
|
145
|
+
#: The engine that first announces them.
|
|
146
|
+
ENGINE_FLOOR_SINCE = "commit 16cb3da"
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class UnsupportedEngineError(EngineError):
|
|
150
|
+
"""The engine that answered ``info`` is older than this SDK's floor
|
|
151
|
+
(:data:`ENGINE_FLOOR_CAPABILITIES`): its ``capabilities`` are absent or lack one this SDK
|
|
152
|
+
needs. ``worker.start()`` raises it; a worker whose start found no engine checks at its first
|
|
153
|
+
answered poll, and stops, logged at error. ``engine`` and ``commit`` are as ``info`` gave them
|
|
154
|
+
(``""`` when absent). Not retried: the engine must be upgraded."""
|
|
155
|
+
|
|
156
|
+
def __init__(self, prefix: str, info: dict[str, Any]) -> None:
|
|
157
|
+
engine = info.get("engine")
|
|
158
|
+
commit = info.get("commit")
|
|
159
|
+
raw = info.get("capabilities")
|
|
160
|
+
caps = [c for c in raw if isinstance(c, str)] if isinstance(raw, list) else None
|
|
161
|
+
missing = [c for c in ENGINE_FLOOR_CAPABILITIES if c not in (caps or [])]
|
|
162
|
+
#: ``info.engine``, the build's version (``0.2.0-dev`` for every build so far).
|
|
163
|
+
self.engine = engine if isinstance(engine, str) else ""
|
|
164
|
+
#: ``info.commit``, the build's commit.
|
|
165
|
+
self.commit = commit if isinstance(commit, str) else ""
|
|
166
|
+
#: ``info.capabilities``, or ``None`` when the reply had none.
|
|
167
|
+
self.capabilities = caps
|
|
168
|
+
#: The capabilities this SDK needs that the engine lacks.
|
|
169
|
+
self.missing = missing
|
|
170
|
+
lacks = "announces no capabilities" if caps is None else f"lacks {', '.join(missing)}"
|
|
171
|
+
super().__init__(
|
|
172
|
+
"info",
|
|
173
|
+
"unsupported",
|
|
174
|
+
f'the engine on prefix "{prefix}" (engine {self.engine or "?"}, commit '
|
|
175
|
+
f"{self.commit or '?'}) {lacks}: this SDK needs the engine at or after "
|
|
176
|
+
f"{ENGINE_FLOOR_SINCE}, which announces {', '.join(ENGINE_FLOOR_CAPABILITIES)} in "
|
|
177
|
+
"info.capabilities",
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def check_engine_floor(prefix: str, info: dict[str, Any]) -> UnsupportedEngineError | None:
|
|
182
|
+
"""The engine's ``info`` reply meets the floor (``None``), or the error that says why not."""
|
|
183
|
+
caps = info.get("capabilities")
|
|
184
|
+
if isinstance(caps, list) and all(c in caps for c in ENGINE_FLOOR_CAPABILITIES):
|
|
185
|
+
return None
|
|
186
|
+
return UnsupportedEngineError(prefix, info)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
class RunNotFoundError(EngineError, LookupError):
|
|
190
|
+
"""``signal`` or ``cancel`` of a run that was never started (``not_found``).
|
|
191
|
+
|
|
192
|
+
Also a :class:`LookupError`, which is what the library's ``client.signal`` and
|
|
193
|
+
``client.cancel`` raised for an unknown run, so a caller's ``except LookupError`` still holds.
|
|
194
|
+
"""
|
|
195
|
+
|
|
196
|
+
def __init__(self, op: str, run_id: str) -> None:
|
|
197
|
+
super().__init__(op, "not_found", f'client.{op}: run "{run_id}" not found')
|
|
198
|
+
self.run_id = run_id
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class EngineFencedError(FencedError):
|
|
202
|
+
"""The library's :class:`FencedError`, as an ``append`` refusal carries it.
|
|
203
|
+
|
|
204
|
+
``expect_last_seq`` is what the append said, ``server_code`` what the broker answered
|
|
205
|
+
underneath (10071; 10164 or 10158 on a replicated journal; ``None`` when the engine fenced a
|
|
206
|
+
duplicate ack itself). Both are for logs only: a fence is a reply, and this package maps no
|
|
207
|
+
broker code of its own.
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
def __init__(
|
|
211
|
+
self,
|
|
212
|
+
run_id: str,
|
|
213
|
+
message: str,
|
|
214
|
+
expect_last_seq: int | None = None,
|
|
215
|
+
server_code: int | None = None,
|
|
216
|
+
) -> None:
|
|
217
|
+
super().__init__(run_id, message)
|
|
218
|
+
self.expect_last_seq = expect_last_seq
|
|
219
|
+
self.server_code = server_code
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""synadia-durable — the engine's API subjects. Mirrors ``src/engine/names.ts``.
|
|
2
|
+
|
|
3
|
+
The SDK knows the prefix and one subject pattern, ``{prefix}.api.v1.<op>`` (the engine's
|
|
4
|
+
``API.md`` "Transport"); the streams, buckets and consumers behind it are the engine's, and
|
|
5
|
+
their names appear nowhere in this package.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Literal
|
|
11
|
+
|
|
12
|
+
__all__ = ["DEFAULT_PREFIX", "Op", "api_subject"]
|
|
13
|
+
|
|
14
|
+
#: The default prefix: the engine's and the library's (``de``).
|
|
15
|
+
DEFAULT_PREFIX = "de"
|
|
16
|
+
|
|
17
|
+
#: The 15 ops of API v1.
|
|
18
|
+
Op = Literal[
|
|
19
|
+
"start",
|
|
20
|
+
"result",
|
|
21
|
+
"status",
|
|
22
|
+
"signal",
|
|
23
|
+
"cancel",
|
|
24
|
+
"poll",
|
|
25
|
+
"load",
|
|
26
|
+
"append",
|
|
27
|
+
"wake",
|
|
28
|
+
"child",
|
|
29
|
+
"done",
|
|
30
|
+
"extend",
|
|
31
|
+
"info",
|
|
32
|
+
"put",
|
|
33
|
+
"get",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def api_subject(prefix: str, op: Op) -> str:
|
|
38
|
+
"""The subject a request for ``op`` is published to."""
|
|
39
|
+
return f"{prefix}.api.v1.{op}"
|