taskferry 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskferry/__init__.py +211 -0
- taskferry/aio.py +486 -0
- taskferry/backends/__init__.py +38 -0
- taskferry/backends/inline.py +235 -0
- taskferry/backends/process.py +292 -0
- taskferry/backends/subprocess.py +390 -0
- taskferry/backends/thread.py +351 -0
- taskferry/capabilities.py +90 -0
- taskferry/cli.py +445 -0
- taskferry/config.py +360 -0
- taskferry/contract/__init__.py +56 -0
- taskferry/contract/base.py +179 -0
- taskferry/contract/inline.py +89 -0
- taskferry/contract/job.py +91 -0
- taskferry/contract/task.py +91 -0
- taskferry/core/__init__.py +130 -0
- taskferry/core/capabilities.py +89 -0
- taskferry/core/config.py +167 -0
- taskferry/core/correlation.py +120 -0
- taskferry/core/delivery.py +36 -0
- taskferry/core/errors.py +55 -0
- taskferry/core/ids.py +37 -0
- taskferry/core/observability.py +136 -0
- taskferry/core/otel.py +83 -0
- taskferry/core/provider.py +50 -0
- taskferry/core/py.typed +0 -0
- taskferry/core/registry.py +92 -0
- taskferry/core/serialization.py +79 -0
- taskferry/core/typing.py +16 -0
- taskferry/envelope.py +197 -0
- taskferry/errors.py +144 -0
- taskferry/execution.py +239 -0
- taskferry/functions.py +290 -0
- taskferry/handle.py +186 -0
- taskferry/hooks.py +238 -0
- taskferry/plugins.py +183 -0
- taskferry/ports.py +356 -0
- taskferry/py.typed +0 -0
- taskferry/retry.py +205 -0
- taskferry/router.py +160 -0
- taskferry/runtime.py +609 -0
- taskferry/specs.py +353 -0
- taskferry/tracking.py +129 -0
- taskferry-0.2.0.dist-info/METADATA +109 -0
- taskferry-0.2.0.dist-info/RECORD +48 -0
- taskferry-0.2.0.dist-info/WHEEL +4 -0
- taskferry-0.2.0.dist-info/entry_points.txt +2 -0
- taskferry-0.2.0.dist-info/licenses/LICENSE +201 -0
taskferry/config.py
ADDED
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"""Typed configuration — one shape, several sources.
|
|
2
|
+
|
|
3
|
+
```mermaid
|
|
4
|
+
flowchart TD
|
|
5
|
+
ENV["Environment<br/>TASKFERRY_*"]
|
|
6
|
+
PY["Python<br/>TaskferryConfig(...)"]
|
|
7
|
+
MAP["Mapping<br/>from_mapping(...)"]
|
|
8
|
+
DJ["Django settings<br/>(taskferry_django)"]
|
|
9
|
+
FILE["TOML / YAML<br/>(application's job)"]
|
|
10
|
+
|
|
11
|
+
CONFIG["TaskferryConfig"]
|
|
12
|
+
RUNTIME["Taskferry runtime"]
|
|
13
|
+
|
|
14
|
+
ENV --> CONFIG
|
|
15
|
+
PY --> CONFIG
|
|
16
|
+
MAP --> CONFIG
|
|
17
|
+
DJ --> CONFIG
|
|
18
|
+
FILE --> CONFIG
|
|
19
|
+
CONFIG --> RUNTIME
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Environment variables are an important *source* — they are how a Twelve-Factor
|
|
23
|
+
deployment configures anything — but they are never the internal model. The
|
|
24
|
+
internal model is :class:`TaskferryConfig`: typed, immutable, validated once at
|
|
25
|
+
construction, and inspectable. Everything else is a loader that produces one.
|
|
26
|
+
|
|
27
|
+
The core never reads ``django.conf.settings``, a config file path, or a
|
|
28
|
+
framework's registry. ``taskferry_django.config_from_settings()`` builds a
|
|
29
|
+
``TaskferryConfig`` from ``settings.TASKFERRY`` and hands it over — the dependency
|
|
30
|
+
points inward, always.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import json
|
|
36
|
+
import os
|
|
37
|
+
from collections.abc import Mapping, Sequence
|
|
38
|
+
from dataclasses import dataclass, field, replace
|
|
39
|
+
from types import MappingProxyType
|
|
40
|
+
from typing import Any
|
|
41
|
+
|
|
42
|
+
from .core.errors import ConfigurationError
|
|
43
|
+
from .execution import ExecutionKind
|
|
44
|
+
from .router import Route, Router
|
|
45
|
+
|
|
46
|
+
ENV_PREFIX = "TASKFERRY_"
|
|
47
|
+
|
|
48
|
+
DEFAULT_BACKENDS: Mapping[str, str] = MappingProxyType(
|
|
49
|
+
{
|
|
50
|
+
"inline": "taskferry.backends.inline:InlineExecutionBackend",
|
|
51
|
+
"thread": "taskferry.backends.thread:ThreadTaskBackend",
|
|
52
|
+
"process": "taskferry.backends.process:ProcessTaskBackend",
|
|
53
|
+
"subprocess": "taskferry.backends.subprocess:SubprocessJobBackend",
|
|
54
|
+
}
|
|
55
|
+
)
|
|
56
|
+
"""The built-in backends, always available without any plugin installed."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True, slots=True)
|
|
60
|
+
class BackendConfig:
|
|
61
|
+
"""How to build one backend.
|
|
62
|
+
|
|
63
|
+
Attributes:
|
|
64
|
+
factory: Either a plugin name registered under the ``taskferry.backends``
|
|
65
|
+
entry-point group (``"procrastinate"``, ``"cloudrun"``), or an
|
|
66
|
+
explicit ``"module.path:callable"`` import string. Names are tried as
|
|
67
|
+
plugins first, then as the built-in aliases.
|
|
68
|
+
options: Keyword arguments handed to the factory. Provider-specific by
|
|
69
|
+
nature — this is the one place provider vocabulary is expected.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
factory: str
|
|
73
|
+
options: Mapping[str, Any] = field(default_factory=dict)
|
|
74
|
+
|
|
75
|
+
def __post_init__(self) -> None:
|
|
76
|
+
if not self.factory:
|
|
77
|
+
raise ConfigurationError("BackendConfig.factory must not be empty")
|
|
78
|
+
object.__setattr__(self, "options", MappingProxyType(dict(self.options)))
|
|
79
|
+
|
|
80
|
+
@classmethod
|
|
81
|
+
def from_mapping(cls, data: Mapping[str, Any]) -> BackendConfig:
|
|
82
|
+
payload = dict(data)
|
|
83
|
+
factory = payload.pop("factory", None) or payload.pop("backend", None)
|
|
84
|
+
if not isinstance(factory, str):
|
|
85
|
+
raise ConfigurationError(
|
|
86
|
+
"backend configuration needs a 'factory' (plugin name or 'module:callable')"
|
|
87
|
+
)
|
|
88
|
+
options = payload.pop("options", None)
|
|
89
|
+
if options is None:
|
|
90
|
+
options = payload # allow flat form: {"factory": ..., "project": ...}
|
|
91
|
+
elif payload:
|
|
92
|
+
raise ConfigurationError(
|
|
93
|
+
f"backend configuration mixes 'options' with extra keys: {sorted(payload)}"
|
|
94
|
+
)
|
|
95
|
+
if not isinstance(options, Mapping):
|
|
96
|
+
raise ConfigurationError("backend 'options' must be a mapping")
|
|
97
|
+
return cls(factory=factory, options=dict(options))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True, slots=True)
|
|
101
|
+
class TaskferryConfig:
|
|
102
|
+
"""Everything the runtime needs, resolved and validated.
|
|
103
|
+
|
|
104
|
+
Attributes:
|
|
105
|
+
backends: Named backend definitions. Instantiated lazily, so configuring
|
|
106
|
+
a Cloud Run backend costs nothing until a job is actually routed to
|
|
107
|
+
it, and importing the Google SDK never happens in a web process that
|
|
108
|
+
only enqueues tasks.
|
|
109
|
+
routes: Ordered routing rules. See :class:`~taskferry.router.Route`.
|
|
110
|
+
defaults: Fallback backend per execution kind.
|
|
111
|
+
allow_import: Whether unregistered task names may be imported dynamically.
|
|
112
|
+
allowed_modules: Import allowlist for task names. Empty means "any
|
|
113
|
+
module" and is only appropriate when task names are trusted.
|
|
114
|
+
max_tracked_executions: How many ``id -> backend`` pairs the runtime
|
|
115
|
+
remembers so ``runtime.get(id)`` can find the right backend without
|
|
116
|
+
being told. Bounded on purpose — a long-lived producer must not grow
|
|
117
|
+
a map forever.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
backends: Mapping[str, BackendConfig] = field(default_factory=dict)
|
|
121
|
+
routes: Sequence[Route] = ()
|
|
122
|
+
defaults: Mapping[ExecutionKind, str] = field(default_factory=dict)
|
|
123
|
+
allow_import: bool = True
|
|
124
|
+
allowed_modules: Sequence[str] = ()
|
|
125
|
+
max_tracked_executions: int = 10_000
|
|
126
|
+
|
|
127
|
+
def __post_init__(self) -> None:
|
|
128
|
+
object.__setattr__(self, "backends", MappingProxyType(dict(self.backends)))
|
|
129
|
+
object.__setattr__(self, "routes", tuple(self.routes))
|
|
130
|
+
object.__setattr__(self, "defaults", MappingProxyType(dict(self.defaults)))
|
|
131
|
+
object.__setattr__(self, "allowed_modules", tuple(self.allowed_modules))
|
|
132
|
+
if self.max_tracked_executions < 0:
|
|
133
|
+
raise ConfigurationError("max_tracked_executions must be >= 0")
|
|
134
|
+
self._validate_references()
|
|
135
|
+
|
|
136
|
+
def _validate_references(self) -> None:
|
|
137
|
+
"""Fail at construction when a route names a backend that is not defined.
|
|
138
|
+
|
|
139
|
+
Catching this here means a typo in a route surfaces at startup rather
|
|
140
|
+
than the first time that queue happens to be used in production.
|
|
141
|
+
"""
|
|
142
|
+
known = set(self.backends)
|
|
143
|
+
for route in self.routes:
|
|
144
|
+
if route.backend not in known:
|
|
145
|
+
raise ConfigurationError(
|
|
146
|
+
f"route {route.describe()!r} names unknown backend {route.backend!r} "
|
|
147
|
+
f"(defined: {', '.join(sorted(known)) or '<none>'})"
|
|
148
|
+
)
|
|
149
|
+
for kind, backend in self.defaults.items():
|
|
150
|
+
if backend not in known:
|
|
151
|
+
raise ConfigurationError(
|
|
152
|
+
f"default backend for {kind.value!r} is {backend!r}, which is not defined "
|
|
153
|
+
f"(defined: {', '.join(sorted(known)) or '<none>'})"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
# -- derived ------------------------------------------------------------ #
|
|
157
|
+
def router(self) -> Router:
|
|
158
|
+
"""Build the :class:`~taskferry.router.Router` this configuration describes."""
|
|
159
|
+
return Router(self.routes, defaults=self.defaults)
|
|
160
|
+
|
|
161
|
+
def evolve(self, **changes: Any) -> TaskferryConfig:
|
|
162
|
+
"""Return a modified copy. Configs are immutable."""
|
|
163
|
+
return replace(self, **changes)
|
|
164
|
+
|
|
165
|
+
def with_backend(self, name: str, config: BackendConfig) -> TaskferryConfig:
|
|
166
|
+
"""Return a copy with one more backend defined."""
|
|
167
|
+
return self.evolve(backends={**self.backends, name: config})
|
|
168
|
+
|
|
169
|
+
# -- loaders ------------------------------------------------------------ #
|
|
170
|
+
@classmethod
|
|
171
|
+
def local(cls) -> TaskferryConfig:
|
|
172
|
+
"""The zero-infrastructure configuration.
|
|
173
|
+
|
|
174
|
+
Inline runs in this process, tasks run on a thread pool, jobs run as
|
|
175
|
+
subprocesses. No queue, no database, no cloud, no worker. This is what
|
|
176
|
+
:meth:`taskferry.Taskferry.local` uses, and it is a genuinely useful
|
|
177
|
+
production configuration for a CLI or a single-process tool.
|
|
178
|
+
"""
|
|
179
|
+
return cls(
|
|
180
|
+
backends={
|
|
181
|
+
"inline": BackendConfig(factory="inline"),
|
|
182
|
+
"thread": BackendConfig(factory="thread"),
|
|
183
|
+
"subprocess": BackendConfig(factory="subprocess"),
|
|
184
|
+
},
|
|
185
|
+
defaults={
|
|
186
|
+
ExecutionKind.INLINE: "inline",
|
|
187
|
+
ExecutionKind.TASK: "thread",
|
|
188
|
+
ExecutionKind.JOB: "subprocess",
|
|
189
|
+
},
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
@classmethod
|
|
193
|
+
def from_mapping(cls, data: Mapping[str, Any]) -> TaskferryConfig:
|
|
194
|
+
"""Build from a plain mapping — the shape a TOML/YAML/settings loader produces.
|
|
195
|
+
|
|
196
|
+
::
|
|
197
|
+
|
|
198
|
+
{
|
|
199
|
+
"backends": {
|
|
200
|
+
"metadata": {"factory": "procrastinate", "app": "myapp.tasks:app"},
|
|
201
|
+
"heavy": {"factory": "cloudrun", "project": "p", "location": "eu"},
|
|
202
|
+
},
|
|
203
|
+
"routes": [
|
|
204
|
+
{"kind": "task", "queue": "metadata", "backend": "metadata"},
|
|
205
|
+
{"kind": "job", "profile": "heavy", "backend": "heavy"},
|
|
206
|
+
],
|
|
207
|
+
"defaults": {"inline": "inline", "task": "metadata", "job": "heavy"},
|
|
208
|
+
}
|
|
209
|
+
"""
|
|
210
|
+
backends_raw = data.get("backends") or {}
|
|
211
|
+
if not isinstance(backends_raw, Mapping):
|
|
212
|
+
raise ConfigurationError("'backends' must be a mapping of name -> configuration")
|
|
213
|
+
backends = {
|
|
214
|
+
str(name): BackendConfig.from_mapping(entry)
|
|
215
|
+
if isinstance(entry, Mapping)
|
|
216
|
+
else BackendConfig(factory=str(entry))
|
|
217
|
+
for name, entry in backends_raw.items()
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
routes_raw = data.get("routes") or ()
|
|
221
|
+
if isinstance(routes_raw, Mapping):
|
|
222
|
+
raise ConfigurationError("'routes' must be an ordered sequence, not a mapping")
|
|
223
|
+
routes = tuple(_route_from_mapping(entry) for entry in routes_raw)
|
|
224
|
+
|
|
225
|
+
defaults_raw = data.get("defaults") or {}
|
|
226
|
+
if not isinstance(defaults_raw, Mapping):
|
|
227
|
+
raise ConfigurationError("'defaults' must be a mapping of kind -> backend name")
|
|
228
|
+
defaults = {_parse_kind(key): str(value) for key, value in defaults_raw.items()}
|
|
229
|
+
|
|
230
|
+
return cls(
|
|
231
|
+
backends=backends,
|
|
232
|
+
routes=routes,
|
|
233
|
+
defaults=defaults,
|
|
234
|
+
allow_import=bool(data.get("allow_import", True)),
|
|
235
|
+
allowed_modules=tuple(str(m) for m in (data.get("allowed_modules") or ())),
|
|
236
|
+
max_tracked_executions=int(data.get("max_tracked_executions", 10_000)),
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
@classmethod
|
|
240
|
+
def from_env(cls, environ: Mapping[str, str] | None = None) -> TaskferryConfig:
|
|
241
|
+
"""Build from ``TASKFERRY_*`` environment variables.
|
|
242
|
+
|
|
243
|
+
Two variables carry the structure, both JSON so that nested provider
|
|
244
|
+
options survive intact:
|
|
245
|
+
|
|
246
|
+
``TASKFERRY_BACKENDS``
|
|
247
|
+
``{"metadata": {"factory": "procrastinate", "app": "myapp:app"}}``
|
|
248
|
+
|
|
249
|
+
``TASKFERRY_ROUTES``
|
|
250
|
+
``[{"kind": "task", "queue": "metadata", "backend": "metadata"}]``
|
|
251
|
+
|
|
252
|
+
And the scalars:
|
|
253
|
+
|
|
254
|
+
``TASKFERRY_DEFAULT_INLINE`` · ``TASKFERRY_DEFAULT_TASK`` ·
|
|
255
|
+
``TASKFERRY_DEFAULT_JOB`` · ``TASKFERRY_ALLOW_IMPORT`` ·
|
|
256
|
+
``TASKFERRY_ALLOWED_MODULES`` (comma-separated).
|
|
257
|
+
|
|
258
|
+
With nothing set at all this returns :meth:`local`, so a container that
|
|
259
|
+
forgot to configure Taskferry still starts and still runs work — locally,
|
|
260
|
+
visibly, and without pretending to have reached a queue.
|
|
261
|
+
"""
|
|
262
|
+
source = environ if environ is not None else os.environ
|
|
263
|
+
raw_backends = source.get(f"{ENV_PREFIX}BACKENDS")
|
|
264
|
+
raw_routes = source.get(f"{ENV_PREFIX}ROUTES")
|
|
265
|
+
|
|
266
|
+
payload: dict[str, Any] = {}
|
|
267
|
+
if raw_backends:
|
|
268
|
+
payload["backends"] = _json_env(f"{ENV_PREFIX}BACKENDS", raw_backends, Mapping)
|
|
269
|
+
if raw_routes:
|
|
270
|
+
payload["routes"] = _json_env(f"{ENV_PREFIX}ROUTES", raw_routes, list)
|
|
271
|
+
|
|
272
|
+
defaults = {
|
|
273
|
+
kind.value: source[f"{ENV_PREFIX}DEFAULT_{kind.value.upper()}"]
|
|
274
|
+
for kind in ExecutionKind
|
|
275
|
+
if source.get(f"{ENV_PREFIX}DEFAULT_{kind.value.upper()}")
|
|
276
|
+
}
|
|
277
|
+
if defaults:
|
|
278
|
+
payload["defaults"] = defaults
|
|
279
|
+
|
|
280
|
+
if not payload:
|
|
281
|
+
return cls.local()
|
|
282
|
+
|
|
283
|
+
allow_import = source.get(f"{ENV_PREFIX}ALLOW_IMPORT")
|
|
284
|
+
if allow_import is not None:
|
|
285
|
+
falsey = {"0", "false", "no", "off"}
|
|
286
|
+
payload["allow_import"] = allow_import.strip().lower() not in falsey
|
|
287
|
+
allowed = source.get(f"{ENV_PREFIX}ALLOWED_MODULES")
|
|
288
|
+
if allowed:
|
|
289
|
+
payload["allowed_modules"] = [m.strip() for m in allowed.split(",") if m.strip()]
|
|
290
|
+
tracked = source.get(f"{ENV_PREFIX}MAX_TRACKED_EXECUTIONS")
|
|
291
|
+
if tracked:
|
|
292
|
+
payload["max_tracked_executions"] = _int_env(
|
|
293
|
+
f"{ENV_PREFIX}MAX_TRACKED_EXECUTIONS", tracked
|
|
294
|
+
)
|
|
295
|
+
return cls.from_mapping(payload)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _json_env(key: str, raw: str, expected: type) -> Any:
|
|
299
|
+
try:
|
|
300
|
+
value = json.loads(raw)
|
|
301
|
+
except ValueError as exc:
|
|
302
|
+
raise ConfigurationError(f"{key} must contain valid JSON: {exc}") from exc
|
|
303
|
+
if not isinstance(value, expected):
|
|
304
|
+
raise ConfigurationError(f"{key} must be a JSON {expected.__name__.lower()}")
|
|
305
|
+
return value
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _int_env(key: str, raw: str) -> int:
|
|
309
|
+
try:
|
|
310
|
+
return int(raw)
|
|
311
|
+
except ValueError as exc:
|
|
312
|
+
raise ConfigurationError(f"{key} must be an integer, got {raw!r}") from exc
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _parse_kind(value: object) -> ExecutionKind:
|
|
316
|
+
if isinstance(value, ExecutionKind):
|
|
317
|
+
return value
|
|
318
|
+
try:
|
|
319
|
+
return ExecutionKind(str(value))
|
|
320
|
+
except ValueError as exc:
|
|
321
|
+
known = ", ".join(k.value for k in ExecutionKind)
|
|
322
|
+
raise ConfigurationError(f"unknown execution kind {value!r} (known: {known})") from exc
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _route_from_mapping(entry: object) -> Route:
|
|
326
|
+
if isinstance(entry, Route):
|
|
327
|
+
return entry
|
|
328
|
+
if not isinstance(entry, Mapping):
|
|
329
|
+
raise ConfigurationError(f"route entries must be mappings, got {type(entry).__name__}")
|
|
330
|
+
data = dict(entry)
|
|
331
|
+
backend = data.pop("backend", None)
|
|
332
|
+
if not isinstance(backend, str):
|
|
333
|
+
raise ConfigurationError("each route needs a 'backend' name")
|
|
334
|
+
kind = data.pop("kind", None)
|
|
335
|
+
labels = data.pop("labels", None) or {}
|
|
336
|
+
if not isinstance(labels, Mapping):
|
|
337
|
+
raise ConfigurationError("route 'labels' must be a mapping")
|
|
338
|
+
unknown = set(data) - {"queue", "profile", "name"}
|
|
339
|
+
if unknown:
|
|
340
|
+
raise ConfigurationError(f"unknown route keys: {', '.join(sorted(unknown))}")
|
|
341
|
+
return Route(
|
|
342
|
+
backend=backend,
|
|
343
|
+
kind=_parse_kind(kind) if kind is not None else None,
|
|
344
|
+
queue=_opt_str(data.get("queue")),
|
|
345
|
+
profile=_opt_str(data.get("profile")),
|
|
346
|
+
name=_opt_str(data.get("name")),
|
|
347
|
+
labels={str(k): str(v) for k, v in labels.items()},
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _opt_str(value: object) -> str | None:
|
|
352
|
+
return None if value is None else str(value)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
__all__ = [
|
|
356
|
+
"DEFAULT_BACKENDS",
|
|
357
|
+
"ENV_PREFIX",
|
|
358
|
+
"BackendConfig",
|
|
359
|
+
"TaskferryConfig",
|
|
360
|
+
]
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Reusable contract suites every adapter must pass.
|
|
2
|
+
|
|
3
|
+
An adapter's job is to be *interchangeable*. That is only true if every adapter
|
|
4
|
+
behaves the same way at the port, so the port ships its own test suite and each
|
|
5
|
+
adapter runs it:
|
|
6
|
+
|
|
7
|
+
```mermaid
|
|
8
|
+
flowchart TD
|
|
9
|
+
SUITE["taskferry.contract<br/>TaskBackendContract · JobBackendContract"]
|
|
10
|
+
|
|
11
|
+
PRO["taskferry-procrastinate tests"]
|
|
12
|
+
CT["taskferry-cloudtasks tests"]
|
|
13
|
+
CR["taskferry-cloudrun tests"]
|
|
14
|
+
LOCAL["built-in backend tests"]
|
|
15
|
+
|
|
16
|
+
SUITE --> PRO
|
|
17
|
+
SUITE --> CT
|
|
18
|
+
SUITE --> CR
|
|
19
|
+
SUITE --> LOCAL
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The suites are **capability-driven**. A backend that does not advertise ``CANCEL``
|
|
23
|
+
is not skipped — it is asserted to *reject* cancellation with
|
|
24
|
+
:class:`~taskferry.errors.UnsupportedCapability`. That is the assertion that
|
|
25
|
+
matters, because the failure mode this design exists to prevent is a backend
|
|
26
|
+
quietly pretending.
|
|
27
|
+
|
|
28
|
+
Usage in an adapter's test module::
|
|
29
|
+
|
|
30
|
+
from taskferry.contract import TaskBackendContract
|
|
31
|
+
|
|
32
|
+
class TestMyBackend(TaskBackendContract):
|
|
33
|
+
def make_backend(self):
|
|
34
|
+
return MyTaskBackend(client=FakeClient())
|
|
35
|
+
|
|
36
|
+
def success_spec(self):
|
|
37
|
+
return TaskSpec(task="tests.tasks:ok")
|
|
38
|
+
|
|
39
|
+
This module imports :mod:`pytest`, so it belongs in test environments. It is
|
|
40
|
+
shipped inside the ``taskferry`` distribution anyway: an adapter released
|
|
41
|
+
separately must be able to import the suite it is required to pass.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
from __future__ import annotations
|
|
45
|
+
|
|
46
|
+
from .base import ExecutionBackendContract
|
|
47
|
+
from .inline import InlineBackendContract
|
|
48
|
+
from .job import JobBackendContract
|
|
49
|
+
from .task import TaskBackendContract
|
|
50
|
+
|
|
51
|
+
__all__ = [
|
|
52
|
+
"ExecutionBackendContract",
|
|
53
|
+
"InlineBackendContract",
|
|
54
|
+
"JobBackendContract",
|
|
55
|
+
"TaskBackendContract",
|
|
56
|
+
]
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Invariants that hold for every backend, of every kind.
|
|
2
|
+
|
|
3
|
+
Subclassed by the per-kind suites; adapters normally use those rather than this
|
|
4
|
+
one directly.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import time
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
from ..capabilities import Capability, CapabilitySet
|
|
16
|
+
from ..errors import ExecutionNotFound, UnsupportedCapability
|
|
17
|
+
from ..execution import Execution, ExecutionId, ExecutionState
|
|
18
|
+
from ..ports import ExecutionBackend
|
|
19
|
+
from ..specs import AnySpec
|
|
20
|
+
|
|
21
|
+
WAIT_TIMEOUT = 30.0
|
|
22
|
+
"""How long the shared helpers wait for a real backend to reach a terminal state."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ExecutionBackendContract(ABC):
|
|
26
|
+
"""Assertions every :class:`~taskferry.ports.ExecutionBackend` must satisfy."""
|
|
27
|
+
|
|
28
|
+
#: Set True when the backend really executes work, so the suite can drive a
|
|
29
|
+
#: full submit → poll → terminal lifecycle. Leave False for adapters tested
|
|
30
|
+
#: against a fake provider client that never completes anything.
|
|
31
|
+
reaches_terminal_state: bool = False
|
|
32
|
+
|
|
33
|
+
@abstractmethod
|
|
34
|
+
def make_backend(self) -> ExecutionBackend:
|
|
35
|
+
"""Return a fresh backend under test. Called once per test."""
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
def success_spec(self) -> AnySpec:
|
|
39
|
+
"""Return a spec this backend should accept and (if it executes) succeed on."""
|
|
40
|
+
|
|
41
|
+
# -- identity ------------------------------------------------------------- #
|
|
42
|
+
def test_declares_a_name_a_kind_and_capabilities(self) -> None:
|
|
43
|
+
backend = self.make_backend()
|
|
44
|
+
assert isinstance(backend.name, str) and backend.name, "a backend needs a stable name"
|
|
45
|
+
caps = backend.capabilities
|
|
46
|
+
assert isinstance(caps, CapabilitySet)
|
|
47
|
+
assert caps.provider == backend.name, "capabilities must be attributed to the backend"
|
|
48
|
+
assert Capability.SUBMIT in caps, "every backend can submit; say so explicitly"
|
|
49
|
+
|
|
50
|
+
def test_kind_matches_the_specs_it_accepts(self) -> None:
|
|
51
|
+
backend = self.make_backend()
|
|
52
|
+
assert backend.kind is self.success_spec().kind
|
|
53
|
+
|
|
54
|
+
# -- submission ------------------------------------------------------------ #
|
|
55
|
+
def test_submit_returns_a_well_formed_execution(self) -> None:
|
|
56
|
+
backend = self.make_backend()
|
|
57
|
+
execution = backend.submit(self.success_spec())
|
|
58
|
+
assert isinstance(execution, Execution)
|
|
59
|
+
assert execution.id, "an execution must carry a Taskferry-owned id"
|
|
60
|
+
assert execution.id.startswith(f"{backend.kind.value}_"), (
|
|
61
|
+
"execution ids are minted by taskferry and prefixed with the kind"
|
|
62
|
+
)
|
|
63
|
+
assert execution.backend == backend.name
|
|
64
|
+
assert execution.kind is backend.kind
|
|
65
|
+
assert isinstance(execution.state, ExecutionState)
|
|
66
|
+
|
|
67
|
+
def test_submit_rejects_a_spec_of_the_wrong_kind(self) -> None:
|
|
68
|
+
backend = self.make_backend()
|
|
69
|
+
wrong = _spec_of_another_kind(self.success_spec())
|
|
70
|
+
with pytest.raises(TypeError):
|
|
71
|
+
backend.submit(wrong)
|
|
72
|
+
|
|
73
|
+
def test_provider_id_is_not_reused_as_the_taskferry_id(self) -> None:
|
|
74
|
+
"""A Taskferry id is Taskferry's; the engine's id lives in ``external_id``."""
|
|
75
|
+
backend = self.make_backend()
|
|
76
|
+
execution = backend.submit(self.success_spec())
|
|
77
|
+
if execution.external_id is not None:
|
|
78
|
+
assert execution.external_id != execution.id
|
|
79
|
+
|
|
80
|
+
# -- capabilities are honest ------------------------------------------------ #
|
|
81
|
+
def test_state_capability_matches_get(self) -> None:
|
|
82
|
+
backend = self.make_backend()
|
|
83
|
+
execution = backend.submit(self.success_spec())
|
|
84
|
+
if Capability.STATE in backend.capabilities:
|
|
85
|
+
fetched = backend.get(execution.id)
|
|
86
|
+
assert fetched.id == execution.id
|
|
87
|
+
assert execution.transitions_to(fetched.state), (
|
|
88
|
+
f"{execution.state} -> {fetched.state} is not a legal transition"
|
|
89
|
+
)
|
|
90
|
+
else:
|
|
91
|
+
with pytest.raises(UnsupportedCapability):
|
|
92
|
+
backend.get(execution.id)
|
|
93
|
+
|
|
94
|
+
def test_cancel_capability_matches_cancel(self) -> None:
|
|
95
|
+
backend = self.make_backend()
|
|
96
|
+
execution = backend.submit(self.success_spec())
|
|
97
|
+
if Capability.CANCEL in backend.capabilities:
|
|
98
|
+
cancelled = backend.cancel(execution.id)
|
|
99
|
+
assert cancelled.id == execution.id
|
|
100
|
+
else:
|
|
101
|
+
with pytest.raises(UnsupportedCapability):
|
|
102
|
+
backend.cancel(execution.id)
|
|
103
|
+
|
|
104
|
+
def test_result_capability_matches_result(self) -> None:
|
|
105
|
+
backend = self.make_backend()
|
|
106
|
+
execution = backend.submit(self.success_spec())
|
|
107
|
+
if Capability.RESULT not in backend.capabilities:
|
|
108
|
+
with pytest.raises(UnsupportedCapability):
|
|
109
|
+
backend.result(execution.id)
|
|
110
|
+
elif self.reaches_terminal_state:
|
|
111
|
+
assert backend.result(execution.id, timeout=WAIT_TIMEOUT) is not None
|
|
112
|
+
|
|
113
|
+
def test_unknown_id_raises_execution_not_found(self) -> None:
|
|
114
|
+
backend = self.make_backend()
|
|
115
|
+
if Capability.STATE not in backend.capabilities:
|
|
116
|
+
pytest.skip("backend cannot report state, so there is nothing to look up")
|
|
117
|
+
with pytest.raises(ExecutionNotFound):
|
|
118
|
+
backend.get(ExecutionId("task_does_not_exist_0000"))
|
|
119
|
+
|
|
120
|
+
# -- lifecycle (only where work really runs) -------------------------------- #
|
|
121
|
+
def test_reaches_a_terminal_state(self) -> None:
|
|
122
|
+
if not self.reaches_terminal_state:
|
|
123
|
+
pytest.skip("backend does not execute work in this test environment")
|
|
124
|
+
backend = self.make_backend()
|
|
125
|
+
execution = backend.submit(self.success_spec())
|
|
126
|
+
final = self.wait_for_terminal(backend, execution)
|
|
127
|
+
assert final.is_terminal
|
|
128
|
+
assert final.state is ExecutionState.SUCCEEDED, (
|
|
129
|
+
f"success_spec() should succeed, got {final.state}: "
|
|
130
|
+
f"{final.result.error if final.result else '<no result>'}"
|
|
131
|
+
)
|
|
132
|
+
assert final.finished_at is not None, "a terminal execution should report when it finished"
|
|
133
|
+
|
|
134
|
+
def test_terminal_state_is_stable(self) -> None:
|
|
135
|
+
"""Re-reading a finished execution must not change it."""
|
|
136
|
+
if not self.reaches_terminal_state:
|
|
137
|
+
pytest.skip("backend does not execute work in this test environment")
|
|
138
|
+
backend = self.make_backend()
|
|
139
|
+
execution = backend.submit(self.success_spec())
|
|
140
|
+
final = self.wait_for_terminal(backend, execution)
|
|
141
|
+
again = backend.get(final.id)
|
|
142
|
+
assert again.state is final.state
|
|
143
|
+
|
|
144
|
+
# -- helper ------------------------------------------------------------------ #
|
|
145
|
+
def wait_for_terminal(
|
|
146
|
+
self, backend: ExecutionBackend, execution: Execution, timeout: float = WAIT_TIMEOUT
|
|
147
|
+
) -> Execution:
|
|
148
|
+
"""Poll until terminal, failing the test rather than hanging."""
|
|
149
|
+
if execution.is_terminal:
|
|
150
|
+
return execution
|
|
151
|
+
deadline = time.monotonic() + timeout
|
|
152
|
+
current = execution
|
|
153
|
+
while time.monotonic() < deadline:
|
|
154
|
+
current = backend.get(execution.id)
|
|
155
|
+
if current.is_terminal:
|
|
156
|
+
return current
|
|
157
|
+
time.sleep(0.02)
|
|
158
|
+
pytest.fail(f"{execution.id} never reached a terminal state (last: {current.state})")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _spec_of_another_kind(spec: AnySpec) -> AnySpec:
|
|
162
|
+
"""Build a spec of a different kind, to prove the backend rejects it."""
|
|
163
|
+
from ..execution import ExecutionKind
|
|
164
|
+
from ..specs import InlineSpec, JobSpec, TaskSpec
|
|
165
|
+
|
|
166
|
+
others: dict[ExecutionKind, Callable[[], AnySpec]] = {
|
|
167
|
+
ExecutionKind.INLINE: lambda: InlineSpec(func=_noop),
|
|
168
|
+
ExecutionKind.TASK: lambda: TaskSpec(task="taskferry.contract.base:_noop"),
|
|
169
|
+
ExecutionKind.JOB: lambda: JobSpec(job="wrong-kind", command=["true"]),
|
|
170
|
+
}
|
|
171
|
+
wrong_kind = next(kind for kind in others if kind is not spec.kind)
|
|
172
|
+
return others[wrong_kind]()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _noop() -> None:
|
|
176
|
+
"""Referenced by :func:`_spec_of_another_kind` so the reference resolves."""
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
__all__ = ["WAIT_TIMEOUT", "ExecutionBackendContract"]
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""The contract every :class:`~taskferry.ports.InlineBackend` must satisfy.
|
|
2
|
+
|
|
3
|
+
Inline is the strictest port: it runs in the caller's process, so there is
|
|
4
|
+
nowhere to hide. By the time ``submit`` returns, the work is finished.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from abc import abstractmethod
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
12
|
+
|
|
13
|
+
from ..capabilities import Capability
|
|
14
|
+
from ..execution import ExecutionKind, ExecutionState
|
|
15
|
+
from ..ports import ExecutionBackend
|
|
16
|
+
from ..specs import InlineSpec
|
|
17
|
+
from .base import ExecutionBackendContract
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _echo(value: object = "ok") -> object:
|
|
21
|
+
return value
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _explode() -> None:
|
|
25
|
+
raise ValueError("contract failure probe")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class InlineBackendContract(ExecutionBackendContract):
|
|
29
|
+
"""Subclass in a test module and implement :meth:`make_backend`."""
|
|
30
|
+
|
|
31
|
+
reaches_terminal_state = True
|
|
32
|
+
|
|
33
|
+
@abstractmethod
|
|
34
|
+
def make_backend(self) -> ExecutionBackend: ...
|
|
35
|
+
|
|
36
|
+
def success_spec(self) -> InlineSpec:
|
|
37
|
+
return InlineSpec(func=_echo, args=("ok",))
|
|
38
|
+
|
|
39
|
+
def test_is_an_inline_backend(self) -> None:
|
|
40
|
+
assert self.make_backend().kind is ExecutionKind.INLINE
|
|
41
|
+
|
|
42
|
+
def test_submit_is_already_terminal(self) -> None:
|
|
43
|
+
"""Inline has no queue: submission and completion are the same moment."""
|
|
44
|
+
backend = self.make_backend()
|
|
45
|
+
execution = backend.submit(self.success_spec())
|
|
46
|
+
assert execution.is_terminal
|
|
47
|
+
assert execution.state is ExecutionState.SUCCEEDED
|
|
48
|
+
|
|
49
|
+
def test_the_value_comes_back(self) -> None:
|
|
50
|
+
backend = self.make_backend()
|
|
51
|
+
execution = backend.submit(InlineSpec(func=_echo, args=(42,)))
|
|
52
|
+
assert execution.result is not None
|
|
53
|
+
assert execution.result.value == 42
|
|
54
|
+
|
|
55
|
+
def test_a_failure_is_recorded_not_raised(self) -> None:
|
|
56
|
+
"""A failed execution is data, not an exception at the submit call site.
|
|
57
|
+
|
|
58
|
+
The exception surfaces from ``handle.result()``, so callers choose when
|
|
59
|
+
to care — the same as for a task that fails on a worker.
|
|
60
|
+
"""
|
|
61
|
+
backend = self.make_backend()
|
|
62
|
+
execution = backend.submit(InlineSpec(func=_explode))
|
|
63
|
+
assert execution.state is ExecutionState.FAILED
|
|
64
|
+
assert execution.result is not None
|
|
65
|
+
assert execution.result.error_type == "ValueError"
|
|
66
|
+
assert execution.result.traceback
|
|
67
|
+
|
|
68
|
+
def test_closures_and_lambdas_are_accepted(self) -> None:
|
|
69
|
+
"""The point of inline: no importable name is required."""
|
|
70
|
+
backend = self.make_backend()
|
|
71
|
+
offset = 5
|
|
72
|
+
execution = backend.submit(InlineSpec(func=lambda x: x + offset, args=(1,)))
|
|
73
|
+
assert execution.result is not None
|
|
74
|
+
assert execution.result.value == 6
|
|
75
|
+
|
|
76
|
+
async def test_async_callables_are_awaited(self) -> None:
|
|
77
|
+
backend = self.make_backend()
|
|
78
|
+
if Capability.ASYNC_CALLABLE not in backend.capabilities:
|
|
79
|
+
pytest.skip("backend does not claim to run async callables")
|
|
80
|
+
|
|
81
|
+
async def coro(value: int) -> int:
|
|
82
|
+
return value * 2
|
|
83
|
+
|
|
84
|
+
execution = backend.submit(InlineSpec(func=coro, args=(21,)))
|
|
85
|
+
assert execution.result is not None
|
|
86
|
+
assert execution.result.value == 42
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
__all__ = ["InlineBackendContract"]
|