taskferry 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskferry/__init__.py +211 -0
- taskferry/aio.py +486 -0
- taskferry/backends/__init__.py +38 -0
- taskferry/backends/inline.py +235 -0
- taskferry/backends/process.py +292 -0
- taskferry/backends/subprocess.py +390 -0
- taskferry/backends/thread.py +351 -0
- taskferry/capabilities.py +90 -0
- taskferry/cli.py +445 -0
- taskferry/config.py +360 -0
- taskferry/contract/__init__.py +56 -0
- taskferry/contract/base.py +179 -0
- taskferry/contract/inline.py +89 -0
- taskferry/contract/job.py +91 -0
- taskferry/contract/task.py +91 -0
- taskferry/core/__init__.py +130 -0
- taskferry/core/capabilities.py +89 -0
- taskferry/core/config.py +167 -0
- taskferry/core/correlation.py +120 -0
- taskferry/core/delivery.py +36 -0
- taskferry/core/errors.py +55 -0
- taskferry/core/ids.py +37 -0
- taskferry/core/observability.py +136 -0
- taskferry/core/otel.py +83 -0
- taskferry/core/provider.py +50 -0
- taskferry/core/py.typed +0 -0
- taskferry/core/registry.py +92 -0
- taskferry/core/serialization.py +79 -0
- taskferry/core/typing.py +16 -0
- taskferry/envelope.py +197 -0
- taskferry/errors.py +144 -0
- taskferry/execution.py +239 -0
- taskferry/functions.py +290 -0
- taskferry/handle.py +186 -0
- taskferry/hooks.py +238 -0
- taskferry/plugins.py +183 -0
- taskferry/ports.py +356 -0
- taskferry/py.typed +0 -0
- taskferry/retry.py +205 -0
- taskferry/router.py +160 -0
- taskferry/runtime.py +609 -0
- taskferry/specs.py +353 -0
- taskferry/tracking.py +129 -0
- taskferry-0.2.0.dist-info/METADATA +109 -0
- taskferry-0.2.0.dist-info/RECORD +48 -0
- taskferry-0.2.0.dist-info/WHEEL +4 -0
- taskferry-0.2.0.dist-info/entry_points.txt +2 -0
- taskferry-0.2.0.dist-info/licenses/LICENSE +201 -0
taskferry/specs.py
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
"""What to execute — the portable request objects.
|
|
2
|
+
|
|
3
|
+
```mermaid
|
|
4
|
+
flowchart TD
|
|
5
|
+
ES["ExecutionSpec<br/>(shared: name · queue · profile · retry · timeout · metadata)"]
|
|
6
|
+
IS["InlineSpec<br/>a callable, right now"]
|
|
7
|
+
TS["TaskSpec<br/>a named function, on a task engine"]
|
|
8
|
+
JS["JobSpec<br/>a container/process, on a batch runtime"]
|
|
9
|
+
|
|
10
|
+
ES --> IS
|
|
11
|
+
ES --> TS
|
|
12
|
+
ES --> JS
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Two rules shape every field here.
|
|
16
|
+
|
|
17
|
+
**Only genuinely portable things get a field.** ``queue``, ``priority``,
|
|
18
|
+
``retry``, ``timeout``, ``image``, ``resources`` mean something on every engine
|
|
19
|
+
that has the corresponding capability. ``procrastinate_lock``,
|
|
20
|
+
``celery_acks_late``, ``cloud_tasks_dispatch_deadline`` and
|
|
21
|
+
``kubernetes_node_selector`` do not, and are therefore *not* fields — they go
|
|
22
|
+
into :class:`~taskferry.core.config.ProviderOptions`, namespaced by backend:
|
|
23
|
+
|
|
24
|
+
TaskSpec(
|
|
25
|
+
task="myapp.tasks.refresh_metadata",
|
|
26
|
+
queue="metadata",
|
|
27
|
+
backend_options=BackendOptions({"procrastinate": {"lock": "meta-42"}}),
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
An adapter reads only its own namespace and ignores the rest, so one spec travels
|
|
31
|
+
unchanged from Procrastinate to Cloud Tasks and only the ignored extras differ.
|
|
32
|
+
|
|
33
|
+
**Every spec declares what it needs.** ``required_capabilities()`` turns the
|
|
34
|
+
fields that were actually set into the capabilities a backend must advertise. The
|
|
35
|
+
base backend classes check that before submitting, so asking a thread pool for a
|
|
36
|
+
GPU fails loudly at submit time rather than quietly producing a CPU run.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
42
|
+
from dataclasses import dataclass, field, replace
|
|
43
|
+
from datetime import datetime, timedelta
|
|
44
|
+
from types import MappingProxyType
|
|
45
|
+
from typing import Any
|
|
46
|
+
|
|
47
|
+
from taskferry.core.config import ProviderOptions
|
|
48
|
+
from taskferry.core.correlation import Correlation
|
|
49
|
+
from taskferry.core.serialization import ensure_json_serializable
|
|
50
|
+
from taskferry.core.typing import JSONObject, JSONValue
|
|
51
|
+
|
|
52
|
+
from .capabilities import Capability
|
|
53
|
+
from .execution import ExecutionKind
|
|
54
|
+
from .functions import FunctionRef
|
|
55
|
+
from .retry import NO_RETRY, NO_TIMEOUT, RetryPolicy, TimeoutPolicy
|
|
56
|
+
|
|
57
|
+
BackendOptions = ProviderOptions
|
|
58
|
+
"""Backend-specific escape hatch, namespaced by backend name.
|
|
59
|
+
|
|
60
|
+
Aliased from :class:`taskferry.core.config.ProviderOptions` — same object, a name
|
|
61
|
+
that reads correctly next to :class:`TaskSpec`."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True, slots=True)
|
|
65
|
+
class Resources:
|
|
66
|
+
"""Compute a job asks for.
|
|
67
|
+
|
|
68
|
+
``cpu`` and ``memory`` use Kubernetes quantity syntax (``"1000m"``,
|
|
69
|
+
``"512Mi"``) because every batch runtime can translate from it and it is
|
|
70
|
+
unambiguous about units. ``gpu`` is a count; ``gpu_type`` is the accelerator
|
|
71
|
+
name when the runtime needs one (``"nvidia-tesla-t4"``).
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
cpu: str | None = None
|
|
75
|
+
memory: str | None = None
|
|
76
|
+
gpu: int = 0
|
|
77
|
+
gpu_type: str | None = None
|
|
78
|
+
|
|
79
|
+
def __post_init__(self) -> None:
|
|
80
|
+
if self.gpu < 0:
|
|
81
|
+
raise ValueError("gpu count must be >= 0")
|
|
82
|
+
if self.gpu_type and not self.gpu:
|
|
83
|
+
raise ValueError("gpu_type was set but gpu count is 0")
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def is_empty(self) -> bool:
|
|
87
|
+
return not (self.cpu or self.memory or self.gpu)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
91
|
+
class ExecutionSpec:
|
|
92
|
+
"""Fields common to every kind of execution.
|
|
93
|
+
|
|
94
|
+
Attributes:
|
|
95
|
+
name: Human-readable label. Defaults to something sensible per subclass.
|
|
96
|
+
queue: Logical queue/lane. The primary routing key for tasks.
|
|
97
|
+
profile: Logical resource profile (``"heavy"``, ``"gpu"``). The primary
|
|
98
|
+
routing key for jobs. Never a provider name — that is the router's
|
|
99
|
+
job to decide.
|
|
100
|
+
retry: Portable retry intent. See :mod:`taskferry.retry`.
|
|
101
|
+
timeout: Portable wall-clock timeout intent.
|
|
102
|
+
idempotency_key: Caller-supplied deduplication key. Backends advertising
|
|
103
|
+
``DEDUPLICATION`` collapse duplicate submissions carrying the same
|
|
104
|
+
key. Taskferry makes **no** exactly-once promise; this is a tool for
|
|
105
|
+
building idempotency, not a guarantee. See ADR-0010.
|
|
106
|
+
correlation: Correlation to propagate. Defaults to the ambient one at
|
|
107
|
+
submit time when left unset.
|
|
108
|
+
labels: Arbitrary metadata forwarded to the engine where supported.
|
|
109
|
+
backend_options: Per-backend escape hatch. See the module docstring.
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
name: str = ""
|
|
113
|
+
queue: str = "default"
|
|
114
|
+
profile: str = "default"
|
|
115
|
+
retry: RetryPolicy = NO_RETRY
|
|
116
|
+
timeout: TimeoutPolicy = NO_TIMEOUT
|
|
117
|
+
idempotency_key: str | None = None
|
|
118
|
+
correlation: Correlation | None = None
|
|
119
|
+
labels: Mapping[str, str] = field(default_factory=dict)
|
|
120
|
+
backend_options: BackendOptions = field(default_factory=BackendOptions)
|
|
121
|
+
|
|
122
|
+
def __post_init__(self) -> None:
|
|
123
|
+
object.__setattr__(self, "labels", MappingProxyType(dict(self.labels)))
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def kind(self) -> ExecutionKind: # pragma: no cover - overridden everywhere
|
|
127
|
+
raise NotImplementedError
|
|
128
|
+
|
|
129
|
+
def required_capabilities(self) -> frozenset[Capability]:
|
|
130
|
+
"""Capabilities a backend must advertise to honour this spec faithfully."""
|
|
131
|
+
required: set[Capability] = {Capability.SUBMIT}
|
|
132
|
+
if self.retry.enabled and self.retry.owner.value == "backend":
|
|
133
|
+
required.add(Capability.RETRY)
|
|
134
|
+
if self.timeout.enabled:
|
|
135
|
+
required.add(Capability.TIMEOUT)
|
|
136
|
+
if self.idempotency_key is not None:
|
|
137
|
+
required.add(Capability.DEDUPLICATION)
|
|
138
|
+
return frozenset(required)
|
|
139
|
+
|
|
140
|
+
def evolve(self, **changes: Any) -> Any:
|
|
141
|
+
"""Return a modified copy. Specs are immutable; this is how you adjust one."""
|
|
142
|
+
return replace(self, **changes)
|
|
143
|
+
|
|
144
|
+
def options_for(self, backend: str) -> JSONObject:
|
|
145
|
+
"""Read this spec's options for one backend (empty when none were set)."""
|
|
146
|
+
return self.backend_options.for_provider(backend)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
150
|
+
class InlineSpec(ExecutionSpec):
|
|
151
|
+
"""Run a callable right now, in this process.
|
|
152
|
+
|
|
153
|
+
The one spec that carries a live Python object: there is no process boundary
|
|
154
|
+
to cross, so there is nothing to serialize and no reason to demand an
|
|
155
|
+
importable name. That is exactly why inline works in a notebook, in a test,
|
|
156
|
+
or on a lambda.
|
|
157
|
+
|
|
158
|
+
Attributes:
|
|
159
|
+
func: The callable to run. Sync or ``async def``.
|
|
160
|
+
args: Positional arguments.
|
|
161
|
+
kwargs: Keyword arguments.
|
|
162
|
+
"""
|
|
163
|
+
|
|
164
|
+
func: Callable[..., Any]
|
|
165
|
+
args: tuple[Any, ...] = ()
|
|
166
|
+
kwargs: Mapping[str, Any] = field(default_factory=dict)
|
|
167
|
+
|
|
168
|
+
def __post_init__(self) -> None:
|
|
169
|
+
# Explicit base call: @dataclass(slots=True) rebuilds the class, which
|
|
170
|
+
# invalidates the zero-argument super() cell in these methods.
|
|
171
|
+
ExecutionSpec.__post_init__(self)
|
|
172
|
+
object.__setattr__(self, "args", tuple(self.args))
|
|
173
|
+
object.__setattr__(self, "kwargs", MappingProxyType(dict(self.kwargs)))
|
|
174
|
+
if not callable(self.func):
|
|
175
|
+
raise TypeError(f"InlineSpec.func must be callable, got {type(self.func).__name__}")
|
|
176
|
+
if not self.name:
|
|
177
|
+
object.__setattr__(self, "name", getattr(self.func, "__qualname__", repr(self.func)))
|
|
178
|
+
|
|
179
|
+
@property
|
|
180
|
+
def kind(self) -> ExecutionKind:
|
|
181
|
+
return ExecutionKind.INLINE
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
185
|
+
class TaskSpec(ExecutionSpec):
|
|
186
|
+
"""Run a named function on a task engine.
|
|
187
|
+
|
|
188
|
+
Attributes:
|
|
189
|
+
task: Portable function reference, ``"package.module:function"``.
|
|
190
|
+
args: Positional arguments. Must be JSON-shaped — they cross a process
|
|
191
|
+
boundary.
|
|
192
|
+
kwargs: Keyword arguments. Must be JSON-shaped.
|
|
193
|
+
priority: Engine priority. Higher runs sooner where supported; the
|
|
194
|
+
absolute scale is engine-specific, only the ordering is portable.
|
|
195
|
+
delay: Wait this long before the task becomes eligible.
|
|
196
|
+
run_at: Absolute time before which the task must not run. Mutually
|
|
197
|
+
exclusive with :attr:`delay`.
|
|
198
|
+
|
|
199
|
+
Arguments are validated as JSON-serializable at construction, so a bad
|
|
200
|
+
payload fails in the caller's stack trace rather than on a worker an hour
|
|
201
|
+
later.
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
task: str
|
|
205
|
+
args: tuple[JSONValue, ...] = ()
|
|
206
|
+
kwargs: Mapping[str, JSONValue] = field(default_factory=dict)
|
|
207
|
+
priority: int = 0
|
|
208
|
+
delay: timedelta | None = None
|
|
209
|
+
run_at: datetime | None = None
|
|
210
|
+
|
|
211
|
+
def __post_init__(self) -> None:
|
|
212
|
+
# Explicit base call: @dataclass(slots=True) rebuilds the class, which
|
|
213
|
+
# invalidates the zero-argument super() cell in these methods.
|
|
214
|
+
ExecutionSpec.__post_init__(self)
|
|
215
|
+
object.__setattr__(self, "args", tuple(self.args))
|
|
216
|
+
object.__setattr__(self, "kwargs", MappingProxyType(dict(self.kwargs)))
|
|
217
|
+
if not self.task:
|
|
218
|
+
raise ValueError("TaskSpec.task is required")
|
|
219
|
+
# Validate the reference eagerly: a malformed path is a programming
|
|
220
|
+
# error, and finding it now beats finding it on a worker.
|
|
221
|
+
FunctionRef.parse(self.task)
|
|
222
|
+
# Same reasoning for the payload — a non-JSON argument must fail in the
|
|
223
|
+
# caller's stack trace, not an hour later inside a worker.
|
|
224
|
+
ensure_json_serializable(list(self.args))
|
|
225
|
+
ensure_json_serializable(dict(self.kwargs))
|
|
226
|
+
if self.delay is not None and self.run_at is not None:
|
|
227
|
+
raise ValueError("set either TaskSpec.delay or TaskSpec.run_at, not both")
|
|
228
|
+
if self.delay is not None and self.delay.total_seconds() < 0:
|
|
229
|
+
raise ValueError("TaskSpec.delay must not be negative")
|
|
230
|
+
if not self.name:
|
|
231
|
+
object.__setattr__(self, "name", self.task)
|
|
232
|
+
|
|
233
|
+
@property
|
|
234
|
+
def kind(self) -> ExecutionKind:
|
|
235
|
+
return ExecutionKind.TASK
|
|
236
|
+
|
|
237
|
+
@property
|
|
238
|
+
def function(self) -> FunctionRef:
|
|
239
|
+
"""The parsed function reference."""
|
|
240
|
+
return FunctionRef.parse(self.task)
|
|
241
|
+
|
|
242
|
+
@property
|
|
243
|
+
def is_deferred(self) -> bool:
|
|
244
|
+
return self.delay is not None or self.run_at is not None
|
|
245
|
+
|
|
246
|
+
def scheduled_for(self, *, now: datetime | None = None) -> datetime | None:
|
|
247
|
+
"""Absolute time this task becomes eligible, or ``None`` if immediate.
|
|
248
|
+
|
|
249
|
+
``now`` is injectable so scheduling logic stays testable without
|
|
250
|
+
freezing the clock globally.
|
|
251
|
+
"""
|
|
252
|
+
if self.run_at is not None:
|
|
253
|
+
return self.run_at
|
|
254
|
+
if self.delay is None:
|
|
255
|
+
return None
|
|
256
|
+
from datetime import UTC
|
|
257
|
+
|
|
258
|
+
base = now if now is not None else datetime.now(UTC)
|
|
259
|
+
return base + self.delay
|
|
260
|
+
|
|
261
|
+
def required_capabilities(self) -> frozenset[Capability]:
|
|
262
|
+
required = set(ExecutionSpec.required_capabilities(self))
|
|
263
|
+
if self.is_deferred:
|
|
264
|
+
required.add(Capability.DELAY)
|
|
265
|
+
if self.priority:
|
|
266
|
+
required.add(Capability.PRIORITY)
|
|
267
|
+
return frozenset(required)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
@dataclass(frozen=True, slots=True, kw_only=True)
|
|
271
|
+
class JobSpec(ExecutionSpec):
|
|
272
|
+
"""Run an isolated workload — a container or a process — to completion.
|
|
273
|
+
|
|
274
|
+
A Job is not a long Task. It has its own image, its own resource envelope and
|
|
275
|
+
its own lifecycle, it does not share a worker with anything else, and it
|
|
276
|
+
returns an exit code rather than a Python value. Collapsing the two would
|
|
277
|
+
force every task engine to grow container semantics it does not have.
|
|
278
|
+
|
|
279
|
+
Attributes:
|
|
280
|
+
job: Job name. Also identifies the pre-declared provider resource for
|
|
281
|
+
runtimes such as Cloud Run Jobs that run an existing Job definition.
|
|
282
|
+
image: Container image. Required by container runtimes, ignored by the
|
|
283
|
+
local subprocess backend.
|
|
284
|
+
command: Full argv. The entrypoint for the local backend; the container
|
|
285
|
+
entrypoint/args override for container runtimes.
|
|
286
|
+
args: Extra arguments appended to :attr:`command`. Kept separate because
|
|
287
|
+
several runtimes model entrypoint and args separately.
|
|
288
|
+
env: Environment variables to inject.
|
|
289
|
+
resources: CPU/memory/GPU request.
|
|
290
|
+
parallelism: Number of task instances per submission (array jobs).
|
|
291
|
+
working_dir: Working directory inside the container/process.
|
|
292
|
+
"""
|
|
293
|
+
|
|
294
|
+
job: str
|
|
295
|
+
image: str | None = None
|
|
296
|
+
command: Sequence[str] = ()
|
|
297
|
+
args: Sequence[str] = ()
|
|
298
|
+
env: Mapping[str, str] = field(default_factory=dict)
|
|
299
|
+
resources: Resources = field(default_factory=Resources)
|
|
300
|
+
parallelism: int = 1
|
|
301
|
+
working_dir: str | None = None
|
|
302
|
+
|
|
303
|
+
def __post_init__(self) -> None:
|
|
304
|
+
# Explicit base call: @dataclass(slots=True) rebuilds the class, which
|
|
305
|
+
# invalidates the zero-argument super() cell in these methods.
|
|
306
|
+
ExecutionSpec.__post_init__(self)
|
|
307
|
+
object.__setattr__(self, "command", tuple(self.command))
|
|
308
|
+
object.__setattr__(self, "args", tuple(self.args))
|
|
309
|
+
object.__setattr__(self, "env", MappingProxyType(dict(self.env)))
|
|
310
|
+
if not self.job:
|
|
311
|
+
raise ValueError("JobSpec.job is required")
|
|
312
|
+
if self.parallelism < 1:
|
|
313
|
+
raise ValueError("JobSpec.parallelism must be >= 1")
|
|
314
|
+
if not self.name:
|
|
315
|
+
object.__setattr__(self, "name", self.job)
|
|
316
|
+
|
|
317
|
+
@property
|
|
318
|
+
def kind(self) -> ExecutionKind:
|
|
319
|
+
return ExecutionKind.JOB
|
|
320
|
+
|
|
321
|
+
@property
|
|
322
|
+
def argv(self) -> tuple[str, ...]:
|
|
323
|
+
"""Full argv: :attr:`command` followed by :attr:`args`."""
|
|
324
|
+
return (*self.command, *self.args)
|
|
325
|
+
|
|
326
|
+
def required_capabilities(self) -> frozenset[Capability]:
|
|
327
|
+
required = set(ExecutionSpec.required_capabilities(self))
|
|
328
|
+
if self.parallelism > 1:
|
|
329
|
+
required.add(Capability.PARALLELISM)
|
|
330
|
+
if self.resources.cpu is not None:
|
|
331
|
+
required.add(Capability.CPU)
|
|
332
|
+
if self.resources.memory is not None:
|
|
333
|
+
required.add(Capability.MEMORY)
|
|
334
|
+
if self.resources.gpu > 0:
|
|
335
|
+
required.add(Capability.GPU)
|
|
336
|
+
if self.env:
|
|
337
|
+
required.add(Capability.ENVIRONMENT)
|
|
338
|
+
return frozenset(required)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
AnySpec = InlineSpec | TaskSpec | JobSpec
|
|
342
|
+
"""Union of every spec kind — what :meth:`taskferry.Taskferry.submit` accepts."""
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
__all__ = [
|
|
346
|
+
"AnySpec",
|
|
347
|
+
"BackendOptions",
|
|
348
|
+
"ExecutionSpec",
|
|
349
|
+
"InlineSpec",
|
|
350
|
+
"JobSpec",
|
|
351
|
+
"Resources",
|
|
352
|
+
"TaskSpec",
|
|
353
|
+
]
|
taskferry/tracking.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Pairing Taskferry ids with engine ids — the bookkeeping every adapter needs.
|
|
2
|
+
|
|
3
|
+
A Taskferry id is Taskferry's. Procrastinate mints an integer, Cloud Run mints a
|
|
4
|
+
resource path, AWS Batch mints a UUID, and none of them will ever store ours. So
|
|
5
|
+
an adapter that wants ``get(execution_id)`` to work has to remember the pairing:
|
|
6
|
+
|
|
7
|
+
```mermaid
|
|
8
|
+
flowchart LR
|
|
9
|
+
SUB["submit()"]
|
|
10
|
+
IDX["ExternalIdIndex<br/>task_9f2c... ↔ 4711"]
|
|
11
|
+
GET["get(task_9f2c...)"]
|
|
12
|
+
ENG["engine.lookup(4711)"]
|
|
13
|
+
|
|
14
|
+
SUB -->|"remember"| IDX
|
|
15
|
+
GET -->|"resolve"| IDX --> ENG
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Five adapters were about to grow the same ``OrderedDict`` plus lock plus eviction
|
|
19
|
+
loop, so it lives here once, with the two properties that matter:
|
|
20
|
+
|
|
21
|
+
**Bounded.** A producer that runs for a month must not accumulate a map for a
|
|
22
|
+
month. Oldest entries are evicted, and an evicted lookup fails loudly.
|
|
23
|
+
|
|
24
|
+
**Honest about its scope.** The index is per-process and in-memory. It cannot
|
|
25
|
+
survive a restart and it is invisible to another replica — which is a real
|
|
26
|
+
limitation of looking an execution up by Taskferry id, not something to paper
|
|
27
|
+
over. Adapters accept the engine's own id as a fallback so an operator holding a
|
|
28
|
+
job id from a dashboard can always look it up, from anywhere.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import threading
|
|
34
|
+
from collections import OrderedDict
|
|
35
|
+
from collections.abc import Callable
|
|
36
|
+
|
|
37
|
+
DEFAULT_CAPACITY = 10_000
|
|
38
|
+
"""Entries kept before the oldest is evicted. Roughly 1 MB at typical id sizes."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ExternalIdIndex:
|
|
42
|
+
"""A bounded, thread-safe, bidirectional map of Taskferry id ↔ engine id.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
capacity: Maximum entries. Zero disables tracking entirely, which is a
|
|
46
|
+
legitimate choice for a fire-and-forget backend that could not look
|
|
47
|
+
an execution up anyway.
|
|
48
|
+
recognises: Predicate deciding whether a string is already an engine id.
|
|
49
|
+
It lets ``resolve`` accept ``"projects/p/locations/eu/executions/x"``
|
|
50
|
+
or ``"4711"`` directly, so an id copied from a console works without
|
|
51
|
+
this process ever having submitted it.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
__slots__ = ("_by_execution", "_capacity", "_lock", "_recognises")
|
|
55
|
+
|
|
56
|
+
def __init__(
|
|
57
|
+
self,
|
|
58
|
+
*,
|
|
59
|
+
capacity: int = DEFAULT_CAPACITY,
|
|
60
|
+
recognises: Callable[[str], bool] | None = None,
|
|
61
|
+
) -> None:
|
|
62
|
+
if capacity < 0:
|
|
63
|
+
raise ValueError("capacity must be >= 0")
|
|
64
|
+
self._capacity = capacity
|
|
65
|
+
self._recognises = recognises
|
|
66
|
+
self._by_execution: OrderedDict[str, str] = OrderedDict()
|
|
67
|
+
self._lock = threading.Lock()
|
|
68
|
+
|
|
69
|
+
def __len__(self) -> int:
|
|
70
|
+
with self._lock:
|
|
71
|
+
return len(self._by_execution)
|
|
72
|
+
|
|
73
|
+
def __contains__(self, execution_id: object) -> bool:
|
|
74
|
+
with self._lock:
|
|
75
|
+
return str(execution_id) in self._by_execution
|
|
76
|
+
|
|
77
|
+
def __repr__(self) -> str:
|
|
78
|
+
return f"ExternalIdIndex(entries={len(self)}, capacity={self._capacity})"
|
|
79
|
+
|
|
80
|
+
def remember(self, execution_id: str, external_id: str | None) -> None:
|
|
81
|
+
"""Record the pairing produced by a submission. ``None`` is ignored."""
|
|
82
|
+
if external_id is None or self._capacity == 0:
|
|
83
|
+
return
|
|
84
|
+
with self._lock:
|
|
85
|
+
self._by_execution[execution_id] = external_id
|
|
86
|
+
self._by_execution.move_to_end(execution_id)
|
|
87
|
+
while len(self._by_execution) > self._capacity:
|
|
88
|
+
self._by_execution.popitem(last=False)
|
|
89
|
+
|
|
90
|
+
def resolve(self, execution_id: str) -> str | None:
|
|
91
|
+
"""The engine id for ``execution_id``, or ``None`` when it is unknown.
|
|
92
|
+
|
|
93
|
+
A remembered pairing wins. Failing that, ``execution_id`` is returned
|
|
94
|
+
unchanged if ``recognises`` says it already *is* an engine id.
|
|
95
|
+
"""
|
|
96
|
+
with self._lock:
|
|
97
|
+
known = self._by_execution.get(execution_id)
|
|
98
|
+
if known is not None:
|
|
99
|
+
self._by_execution.move_to_end(execution_id)
|
|
100
|
+
return known
|
|
101
|
+
if self._recognises is not None and self._recognises(execution_id):
|
|
102
|
+
return execution_id
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
def forget(self, execution_id: str) -> None:
|
|
106
|
+
"""Drop one pairing. Silent when it was not tracked."""
|
|
107
|
+
with self._lock:
|
|
108
|
+
self._by_execution.pop(execution_id, None)
|
|
109
|
+
|
|
110
|
+
def clear(self) -> None:
|
|
111
|
+
with self._lock:
|
|
112
|
+
self._by_execution.clear()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def is_digits(value: str) -> bool:
|
|
116
|
+
"""``recognises`` predicate for engines with numeric ids (Procrastinate)."""
|
|
117
|
+
return value.isdigit()
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def has_prefix(prefix: str) -> Callable[[str], bool]:
|
|
121
|
+
"""``recognises`` predicate for engines with path-shaped ids (Cloud Run, GCP)."""
|
|
122
|
+
|
|
123
|
+
def _check(value: str) -> bool:
|
|
124
|
+
return value.startswith(prefix)
|
|
125
|
+
|
|
126
|
+
return _check
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
__all__ = ["DEFAULT_CAPACITY", "ExternalIdIndex", "has_prefix", "is_digits"]
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: taskferry
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: A portable execution layer for Python — inline, tasks and jobs on the engines you already run.
|
|
5
|
+
Project-URL: Homepage, https://github.com/xiidigital/taskferry
|
|
6
|
+
Project-URL: Documentation, https://taskferry.dev
|
|
7
|
+
Project-URL: Source, https://github.com/xiidigital/taskferry
|
|
8
|
+
Project-URL: Changelog, https://github.com/xiidigital/taskferry/blob/main/packages/taskferry/CHANGELOG.md
|
|
9
|
+
Author: Taskferry authors
|
|
10
|
+
License-Expression: Apache-2.0
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: capabilities,execution,jobs,portability,ports-and-adapters,taskferry,tasks
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.12
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# taskferry
|
|
25
|
+
|
|
26
|
+
**A portable execution layer for Python.**
|
|
27
|
+
|
|
28
|
+
Taskferry models units of work, chooses the right *kind* of execution, and routes
|
|
29
|
+
them to engines that already exist. It is not a task queue, not a worker system,
|
|
30
|
+
not a scheduler and not a workflow engine.
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install taskferry
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
That installs **nothing else**. No Django, no PostgreSQL driver, no Redis client,
|
|
37
|
+
no cloud SDK. The distribution declares zero dependencies and CI proves it in a
|
|
38
|
+
bare virtualenv on every commit.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from taskferry import Taskferry
|
|
42
|
+
|
|
43
|
+
runtime = Taskferry.local()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def add(a: int, b: int) -> int:
|
|
47
|
+
return a + b
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
execution = runtime.inline.submit(add, 20, 22)
|
|
51
|
+
assert execution.result().value == 42
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Three primitives
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
runtime.inline.submit(add, 20, 22) # now, in this process
|
|
58
|
+
runtime.tasks.submit("myapp.tasks:send_email", 42) # later, on an engine
|
|
59
|
+
runtime.jobs.submit("build-cog", image="gdal:latest") # a container, to completion
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
A Job is not "a Task that takes longer": it has its own image, its own resource
|
|
63
|
+
envelope and its own lifecycle, and it returns an exit code rather than a Python
|
|
64
|
+
value.
|
|
65
|
+
|
|
66
|
+
## Built-in backends
|
|
67
|
+
|
|
68
|
+
| Backend | Kind | Use |
|
|
69
|
+
| --- | --- | --- |
|
|
70
|
+
| `inline` | inline | tests, notebooks, CLIs, small tools |
|
|
71
|
+
| `thread` | task | development, single-process applications |
|
|
72
|
+
| `process` | task | rehearsing what a real worker does to your code |
|
|
73
|
+
| `subprocess` | job | local batch workloads |
|
|
74
|
+
|
|
75
|
+
None of them is a queue. They are not durable, not visible across processes, and
|
|
76
|
+
they lose pending work on restart — which is stated plainly in their docstrings
|
|
77
|
+
and in their capability sets. For production, route at a real engine; that is a
|
|
78
|
+
configuration change.
|
|
79
|
+
|
|
80
|
+
## Adapters
|
|
81
|
+
|
|
82
|
+
Install the engine you use. Each registers itself through the `taskferry.backends`
|
|
83
|
+
entry-point group, so naming it in configuration is all it takes.
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pip install taskferry-procrastinate # PostgreSQL-backed tasks
|
|
87
|
+
pip install taskferry-cloudtasks # push-based serverless tasks
|
|
88
|
+
pip install taskferry-cloudrun # serverless container jobs
|
|
89
|
+
pip install taskferry-jobs # AWS Batch, Kubernetes, Azure Container Apps
|
|
90
|
+
pip install taskferry-django # django.tasks integration
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## CLI
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
taskferry backends
|
|
97
|
+
taskferry capabilities pg
|
|
98
|
+
taskferry route --kind job --profile gpu
|
|
99
|
+
taskferry doctor
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## Documentation
|
|
103
|
+
|
|
104
|
+
Full documentation, ADRs and the migration guide live in the
|
|
105
|
+
[repository](https://github.com/xiidigital/taskferry).
|
|
106
|
+
|
|
107
|
+
## License
|
|
108
|
+
|
|
109
|
+
Apache-2.0.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
taskferry/__init__.py,sha256=jlXhwsG9uYObCO9uNw_UBVjLFnbXU7jbmxIBoNEdL4E,5828
|
|
2
|
+
taskferry/aio.py,sha256=K9vqsLg2FQhnT4UOI90HHi8rA6VK-g2YlspFVt7rOg4,16970
|
|
3
|
+
taskferry/capabilities.py,sha256=kDbbbxTBazYKlH9otRn7Qi_63zwN9r7YawqM24cDo88,3140
|
|
4
|
+
taskferry/cli.py,sha256=UDkhgHwqUcqc4Z_0m5FNtnWEhvxGKBv3JEklQKn3IeM,16294
|
|
5
|
+
taskferry/config.py,sha256=4b_y_GzHVRK2Ze7c40D5kpPGEqZZKJeWIKL2s1djARw,14564
|
|
6
|
+
taskferry/envelope.py,sha256=UQdkGJ9lT-UmcX5rZB3xwCGpWGYUht3j0d2dJeyTcT4,7169
|
|
7
|
+
taskferry/errors.py,sha256=r3YrjBGoTmWEB0-vHMZLjEvbmjk3mrnQAmnB-5p6_4Q,4700
|
|
8
|
+
taskferry/execution.py,sha256=rmtg11DjIlE39YlOrDEi6GDU2NR-1m-8SqLyQKupR5U,8466
|
|
9
|
+
taskferry/functions.py,sha256=JC0jAWUALiR7BjtG2FKMbuOuOHr30XvBQqbbj5YkKHk,11774
|
|
10
|
+
taskferry/handle.py,sha256=cBv7oK3WWPYSajN91pxs9Emyptu-VLwwM-0m-hoPsCw,6656
|
|
11
|
+
taskferry/hooks.py,sha256=pKUDyD5pvQF8QlQoK0AlyFZ6jNSmKvZkZEHMmISkYWI,9047
|
|
12
|
+
taskferry/plugins.py,sha256=MNGhZOBju_GmixZGYj0PLEBAXmN5y4ZE_tJ3CSeWVNw,6374
|
|
13
|
+
taskferry/ports.py,sha256=lPO3U3BippQzVZn-AvVgkiG0k4o6qRNHO0F6xARhG_c,13741
|
|
14
|
+
taskferry/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
+
taskferry/retry.py,sha256=qqv2Dl-87WvqIHDoYnwZ8d8wr9AM2UPXO6MDoFV0G7U,6920
|
|
16
|
+
taskferry/router.py,sha256=TP93M0ly3RvALZKBWWWnsJsOlv0arop-NAlaKsSJafc,6252
|
|
17
|
+
taskferry/runtime.py,sha256=LfqDIeYy1XAGriNqnObQrFwV5uc4PlZdGZgVYXBZ8j8,22729
|
|
18
|
+
taskferry/specs.py,sha256=Z1O2FBs7wuASC-1N97Ri-x-egQnHFnmsBh8hl6EYQNM,14169
|
|
19
|
+
taskferry/tracking.py,sha256=kcGMPt2pccWwXMyPgcst4h5tt8BpF9iAhbpM8q6i2QQ,4793
|
|
20
|
+
taskferry/backends/__init__.py,sha256=9odsHuv_OFMRIGn9SvCslBcXaWW-pkFkwyOI2COBUWI,1394
|
|
21
|
+
taskferry/backends/inline.py,sha256=Kxdou00scP_ebXPUbPyyksITk6rnK1jaXgSb_iMJg0g,8215
|
|
22
|
+
taskferry/backends/process.py,sha256=UiJ_Yr-BvmP1m3Eqxd_EaoCcA-CM1g4AWIeYLkEBsqY,10482
|
|
23
|
+
taskferry/backends/subprocess.py,sha256=_f8bg-MEb90cZzdGZs_J15JBltWHPR0jvqIPFmlWhB4,14192
|
|
24
|
+
taskferry/backends/thread.py,sha256=lo9ehxUQWf4JJQTkQdxJmiHeQ3fMV7fw6bl4X2XG9b8,12864
|
|
25
|
+
taskferry/contract/__init__.py,sha256=TJr1qq8IQTjh4moga-8vHQssg4gPOnGv1DkQaC_hZgk,1742
|
|
26
|
+
taskferry/contract/base.py,sha256=NJqGZTmt4lEHJdivx4Pl9EBaqtnRLy5jWOHGRYjnusk,7927
|
|
27
|
+
taskferry/contract/inline.py,sha256=cOaUsBh5IEUky0XSmbCki7VKt8bNYG9T818i-FRSNp8,3143
|
|
28
|
+
taskferry/contract/job.py,sha256=F85ekBdrFS1-67YyAbTFui_7LIVB1E53DV9pZjO06p4,3451
|
|
29
|
+
taskferry/contract/task.py,sha256=6I6Z3q_vkWmO9I-hd0NpjNDVzU0WecVnMFJK01Eif1Y,3352
|
|
30
|
+
taskferry/core/__init__.py,sha256=_xTuiMvnsRRf3yakVRZheHUr-g2-b4uZpovMUHCa1hI,3237
|
|
31
|
+
taskferry/core/capabilities.py,sha256=9Sazy_jVplY_C-T3bHCSPKRjRTbffyIJmJBCUviPYN4,3243
|
|
32
|
+
taskferry/core/config.py,sha256=do2C-mc6ThSkTed_ZF_ktRMEnVsOgHE_WejqH24_FX8,5498
|
|
33
|
+
taskferry/core/correlation.py,sha256=CUsMWhQj_QY9pcEYq5iQ1vDCbypwNj8jdx8Zh-4_jRo,4210
|
|
34
|
+
taskferry/core/delivery.py,sha256=xNBwawpBR7D5lzH5ON94zsaf_gxfJpdADfstLb-D7i8,1073
|
|
35
|
+
taskferry/core/errors.py,sha256=yS48DNSgf-PFobj65OfZdMuLfjsdWOoZrUM1YfiIrOs,1831
|
|
36
|
+
taskferry/core/ids.py,sha256=69bD2Um1FU4eeFWx_HRBA3r2PgflxGjKO9jBQ2MV6QE,1303
|
|
37
|
+
taskferry/core/observability.py,sha256=unXRWzWEsEKKrj1q2ajK0l9fuqzsMJgDDyLDONTb8_w,3881
|
|
38
|
+
taskferry/core/otel.py,sha256=-lZzp8GdwAPmMYJfGoCTcYOR1UVsiaR_YfQoo2a1VCI,2798
|
|
39
|
+
taskferry/core/provider.py,sha256=6aqk468rMnVKl9lEbUAV2MGHhm5eEvFeUU0Q4Kpu_w0,1968
|
|
40
|
+
taskferry/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
|
+
taskferry/core/registry.py,sha256=9pTIXb4HUnRaZWTp_Uh-nlMTDsh7n1vTlFGUsi3kwdM,3425
|
|
42
|
+
taskferry/core/serialization.py,sha256=ROh3w8E2A8c0O1I7SVbkA7HGbOjmv-fZBHG7h-wrtxI,2529
|
|
43
|
+
taskferry/core/typing.py,sha256=IHbJbt5i1ClEDfqiMcAtLMar3t-n8QaRHnCA3tgri6U,649
|
|
44
|
+
taskferry-0.2.0.dist-info/METADATA,sha256=fIBdxHV3m7BRTimEIYeTPzfaQe8NBnggUWJGCETDXHQ,3575
|
|
45
|
+
taskferry-0.2.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
|
|
46
|
+
taskferry-0.2.0.dist-info/entry_points.txt,sha256=bx6DKenzSZRUXdqcoChvz07P3TeLpIteyNl1LWl9f_E,49
|
|
47
|
+
taskferry-0.2.0.dist-info/licenses/LICENSE,sha256=de-gfE0q-xTYImzwC3dj3S7BxVhanf6RmIGjo_7y3aw,11357
|
|
48
|
+
taskferry-0.2.0.dist-info/RECORD,,
|