taskferry 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. taskferry/__init__.py +211 -0
  2. taskferry/aio.py +486 -0
  3. taskferry/backends/__init__.py +38 -0
  4. taskferry/backends/inline.py +235 -0
  5. taskferry/backends/process.py +292 -0
  6. taskferry/backends/subprocess.py +390 -0
  7. taskferry/backends/thread.py +351 -0
  8. taskferry/capabilities.py +90 -0
  9. taskferry/cli.py +445 -0
  10. taskferry/config.py +360 -0
  11. taskferry/contract/__init__.py +56 -0
  12. taskferry/contract/base.py +179 -0
  13. taskferry/contract/inline.py +89 -0
  14. taskferry/contract/job.py +91 -0
  15. taskferry/contract/task.py +91 -0
  16. taskferry/core/__init__.py +130 -0
  17. taskferry/core/capabilities.py +89 -0
  18. taskferry/core/config.py +167 -0
  19. taskferry/core/correlation.py +120 -0
  20. taskferry/core/delivery.py +36 -0
  21. taskferry/core/errors.py +55 -0
  22. taskferry/core/ids.py +37 -0
  23. taskferry/core/observability.py +136 -0
  24. taskferry/core/otel.py +83 -0
  25. taskferry/core/provider.py +50 -0
  26. taskferry/core/py.typed +0 -0
  27. taskferry/core/registry.py +92 -0
  28. taskferry/core/serialization.py +79 -0
  29. taskferry/core/typing.py +16 -0
  30. taskferry/envelope.py +197 -0
  31. taskferry/errors.py +144 -0
  32. taskferry/execution.py +239 -0
  33. taskferry/functions.py +290 -0
  34. taskferry/handle.py +186 -0
  35. taskferry/hooks.py +238 -0
  36. taskferry/plugins.py +183 -0
  37. taskferry/ports.py +356 -0
  38. taskferry/py.typed +0 -0
  39. taskferry/retry.py +205 -0
  40. taskferry/router.py +160 -0
  41. taskferry/runtime.py +609 -0
  42. taskferry/specs.py +353 -0
  43. taskferry/tracking.py +129 -0
  44. taskferry-0.2.0.dist-info/METADATA +109 -0
  45. taskferry-0.2.0.dist-info/RECORD +48 -0
  46. taskferry-0.2.0.dist-info/WHEEL +4 -0
  47. taskferry-0.2.0.dist-info/entry_points.txt +2 -0
  48. taskferry-0.2.0.dist-info/licenses/LICENSE +201 -0
taskferry/specs.py ADDED
@@ -0,0 +1,353 @@
1
+ """What to execute — the portable request objects.
2
+
3
+ ```mermaid
4
+ flowchart TD
5
+ ES["ExecutionSpec<br/>(shared: name · queue · profile · retry · timeout · metadata)"]
6
+ IS["InlineSpec<br/>a callable, right now"]
7
+ TS["TaskSpec<br/>a named function, on a task engine"]
8
+ JS["JobSpec<br/>a container/process, on a batch runtime"]
9
+
10
+ ES --> IS
11
+ ES --> TS
12
+ ES --> JS
13
+ ```
14
+
15
+ Two rules shape every field here.
16
+
17
+ **Only genuinely portable things get a field.** ``queue``, ``priority``,
18
+ ``retry``, ``timeout``, ``image``, ``resources`` mean something on every engine
19
+ that has the corresponding capability. ``procrastinate_lock``,
20
+ ``celery_acks_late``, ``cloud_tasks_dispatch_deadline`` and
21
+ ``kubernetes_node_selector`` do not, and are therefore *not* fields — they go
22
+ into :class:`~taskferry.core.config.ProviderOptions`, namespaced by backend:
23
+
24
+ TaskSpec(
25
+ task="myapp.tasks.refresh_metadata",
26
+ queue="metadata",
27
+ backend_options=BackendOptions({"procrastinate": {"lock": "meta-42"}}),
28
+ )
29
+
30
+ An adapter reads only its own namespace and ignores the rest, so one spec travels
31
+ unchanged from Procrastinate to Cloud Tasks and only the ignored extras differ.
32
+
33
+ **Every spec declares what it needs.** ``required_capabilities()`` turns the
34
+ fields that were actually set into the capabilities a backend must advertise. The
35
+ base backend classes check that before submitting, so asking a thread pool for a
36
+ GPU fails loudly at submit time rather than quietly producing a CPU run.
37
+ """
38
+
39
+ from __future__ import annotations
40
+
41
+ from collections.abc import Callable, Mapping, Sequence
42
+ from dataclasses import dataclass, field, replace
43
+ from datetime import datetime, timedelta
44
+ from types import MappingProxyType
45
+ from typing import Any
46
+
47
+ from taskferry.core.config import ProviderOptions
48
+ from taskferry.core.correlation import Correlation
49
+ from taskferry.core.serialization import ensure_json_serializable
50
+ from taskferry.core.typing import JSONObject, JSONValue
51
+
52
+ from .capabilities import Capability
53
+ from .execution import ExecutionKind
54
+ from .functions import FunctionRef
55
+ from .retry import NO_RETRY, NO_TIMEOUT, RetryPolicy, TimeoutPolicy
56
+
57
+ BackendOptions = ProviderOptions
58
+ """Backend-specific escape hatch, namespaced by backend name.
59
+
60
+ Aliased from :class:`taskferry.core.config.ProviderOptions` — same object, a name
61
+ that reads correctly next to :class:`TaskSpec`."""
62
+
63
+
64
+ @dataclass(frozen=True, slots=True)
65
+ class Resources:
66
+ """Compute a job asks for.
67
+
68
+ ``cpu`` and ``memory`` use Kubernetes quantity syntax (``"1000m"``,
69
+ ``"512Mi"``) because every batch runtime can translate from it and it is
70
+ unambiguous about units. ``gpu`` is a count; ``gpu_type`` is the accelerator
71
+ name when the runtime needs one (``"nvidia-tesla-t4"``).
72
+ """
73
+
74
+ cpu: str | None = None
75
+ memory: str | None = None
76
+ gpu: int = 0
77
+ gpu_type: str | None = None
78
+
79
+ def __post_init__(self) -> None:
80
+ if self.gpu < 0:
81
+ raise ValueError("gpu count must be >= 0")
82
+ if self.gpu_type and not self.gpu:
83
+ raise ValueError("gpu_type was set but gpu count is 0")
84
+
85
+ @property
86
+ def is_empty(self) -> bool:
87
+ return not (self.cpu or self.memory or self.gpu)
88
+
89
+
90
+ @dataclass(frozen=True, slots=True, kw_only=True)
91
+ class ExecutionSpec:
92
+ """Fields common to every kind of execution.
93
+
94
+ Attributes:
95
+ name: Human-readable label. Defaults to something sensible per subclass.
96
+ queue: Logical queue/lane. The primary routing key for tasks.
97
+ profile: Logical resource profile (``"heavy"``, ``"gpu"``). The primary
98
+ routing key for jobs. Never a provider name — that is the router's
99
+ job to decide.
100
+ retry: Portable retry intent. See :mod:`taskferry.retry`.
101
+ timeout: Portable wall-clock timeout intent.
102
+ idempotency_key: Caller-supplied deduplication key. Backends advertising
103
+ ``DEDUPLICATION`` collapse duplicate submissions carrying the same
104
+ key. Taskferry makes **no** exactly-once promise; this is a tool for
105
+ building idempotency, not a guarantee. See ADR-0010.
106
+ correlation: Correlation to propagate. Defaults to the ambient one at
107
+ submit time when left unset.
108
+ labels: Arbitrary metadata forwarded to the engine where supported.
109
+ backend_options: Per-backend escape hatch. See the module docstring.
110
+ """
111
+
112
+ name: str = ""
113
+ queue: str = "default"
114
+ profile: str = "default"
115
+ retry: RetryPolicy = NO_RETRY
116
+ timeout: TimeoutPolicy = NO_TIMEOUT
117
+ idempotency_key: str | None = None
118
+ correlation: Correlation | None = None
119
+ labels: Mapping[str, str] = field(default_factory=dict)
120
+ backend_options: BackendOptions = field(default_factory=BackendOptions)
121
+
122
+ def __post_init__(self) -> None:
123
+ object.__setattr__(self, "labels", MappingProxyType(dict(self.labels)))
124
+
125
+ @property
126
+ def kind(self) -> ExecutionKind: # pragma: no cover - overridden everywhere
127
+ raise NotImplementedError
128
+
129
+ def required_capabilities(self) -> frozenset[Capability]:
130
+ """Capabilities a backend must advertise to honour this spec faithfully."""
131
+ required: set[Capability] = {Capability.SUBMIT}
132
+ if self.retry.enabled and self.retry.owner.value == "backend":
133
+ required.add(Capability.RETRY)
134
+ if self.timeout.enabled:
135
+ required.add(Capability.TIMEOUT)
136
+ if self.idempotency_key is not None:
137
+ required.add(Capability.DEDUPLICATION)
138
+ return frozenset(required)
139
+
140
+ def evolve(self, **changes: Any) -> Any:
141
+ """Return a modified copy. Specs are immutable; this is how you adjust one."""
142
+ return replace(self, **changes)
143
+
144
+ def options_for(self, backend: str) -> JSONObject:
145
+ """Read this spec's options for one backend (empty when none were set)."""
146
+ return self.backend_options.for_provider(backend)
147
+
148
+
149
+ @dataclass(frozen=True, slots=True, kw_only=True)
150
+ class InlineSpec(ExecutionSpec):
151
+ """Run a callable right now, in this process.
152
+
153
+ The one spec that carries a live Python object: there is no process boundary
154
+ to cross, so there is nothing to serialize and no reason to demand an
155
+ importable name. That is exactly why inline works in a notebook, in a test,
156
+ or on a lambda.
157
+
158
+ Attributes:
159
+ func: The callable to run. Sync or ``async def``.
160
+ args: Positional arguments.
161
+ kwargs: Keyword arguments.
162
+ """
163
+
164
+ func: Callable[..., Any]
165
+ args: tuple[Any, ...] = ()
166
+ kwargs: Mapping[str, Any] = field(default_factory=dict)
167
+
168
+ def __post_init__(self) -> None:
169
+ # Explicit base call: @dataclass(slots=True) rebuilds the class, which
170
+ # invalidates the zero-argument super() cell in these methods.
171
+ ExecutionSpec.__post_init__(self)
172
+ object.__setattr__(self, "args", tuple(self.args))
173
+ object.__setattr__(self, "kwargs", MappingProxyType(dict(self.kwargs)))
174
+ if not callable(self.func):
175
+ raise TypeError(f"InlineSpec.func must be callable, got {type(self.func).__name__}")
176
+ if not self.name:
177
+ object.__setattr__(self, "name", getattr(self.func, "__qualname__", repr(self.func)))
178
+
179
+ @property
180
+ def kind(self) -> ExecutionKind:
181
+ return ExecutionKind.INLINE
182
+
183
+
184
+ @dataclass(frozen=True, slots=True, kw_only=True)
185
+ class TaskSpec(ExecutionSpec):
186
+ """Run a named function on a task engine.
187
+
188
+ Attributes:
189
+ task: Portable function reference, ``"package.module:function"``.
190
+ args: Positional arguments. Must be JSON-shaped — they cross a process
191
+ boundary.
192
+ kwargs: Keyword arguments. Must be JSON-shaped.
193
+ priority: Engine priority. Higher runs sooner where supported; the
194
+ absolute scale is engine-specific, only the ordering is portable.
195
+ delay: Wait this long before the task becomes eligible.
196
+ run_at: Absolute time before which the task must not run. Mutually
197
+ exclusive with :attr:`delay`.
198
+
199
+ Arguments are validated as JSON-serializable at construction, so a bad
200
+ payload fails in the caller's stack trace rather than on a worker an hour
201
+ later.
202
+ """
203
+
204
+ task: str
205
+ args: tuple[JSONValue, ...] = ()
206
+ kwargs: Mapping[str, JSONValue] = field(default_factory=dict)
207
+ priority: int = 0
208
+ delay: timedelta | None = None
209
+ run_at: datetime | None = None
210
+
211
+ def __post_init__(self) -> None:
212
+ # Explicit base call: @dataclass(slots=True) rebuilds the class, which
213
+ # invalidates the zero-argument super() cell in these methods.
214
+ ExecutionSpec.__post_init__(self)
215
+ object.__setattr__(self, "args", tuple(self.args))
216
+ object.__setattr__(self, "kwargs", MappingProxyType(dict(self.kwargs)))
217
+ if not self.task:
218
+ raise ValueError("TaskSpec.task is required")
219
+ # Validate the reference eagerly: a malformed path is a programming
220
+ # error, and finding it now beats finding it on a worker.
221
+ FunctionRef.parse(self.task)
222
+ # Same reasoning for the payload — a non-JSON argument must fail in the
223
+ # caller's stack trace, not an hour later inside a worker.
224
+ ensure_json_serializable(list(self.args))
225
+ ensure_json_serializable(dict(self.kwargs))
226
+ if self.delay is not None and self.run_at is not None:
227
+ raise ValueError("set either TaskSpec.delay or TaskSpec.run_at, not both")
228
+ if self.delay is not None and self.delay.total_seconds() < 0:
229
+ raise ValueError("TaskSpec.delay must not be negative")
230
+ if not self.name:
231
+ object.__setattr__(self, "name", self.task)
232
+
233
+ @property
234
+ def kind(self) -> ExecutionKind:
235
+ return ExecutionKind.TASK
236
+
237
+ @property
238
+ def function(self) -> FunctionRef:
239
+ """The parsed function reference."""
240
+ return FunctionRef.parse(self.task)
241
+
242
+ @property
243
+ def is_deferred(self) -> bool:
244
+ return self.delay is not None or self.run_at is not None
245
+
246
+ def scheduled_for(self, *, now: datetime | None = None) -> datetime | None:
247
+ """Absolute time this task becomes eligible, or ``None`` if immediate.
248
+
249
+ ``now`` is injectable so scheduling logic stays testable without
250
+ freezing the clock globally.
251
+ """
252
+ if self.run_at is not None:
253
+ return self.run_at
254
+ if self.delay is None:
255
+ return None
256
+ from datetime import UTC
257
+
258
+ base = now if now is not None else datetime.now(UTC)
259
+ return base + self.delay
260
+
261
+ def required_capabilities(self) -> frozenset[Capability]:
262
+ required = set(ExecutionSpec.required_capabilities(self))
263
+ if self.is_deferred:
264
+ required.add(Capability.DELAY)
265
+ if self.priority:
266
+ required.add(Capability.PRIORITY)
267
+ return frozenset(required)
268
+
269
+
270
+ @dataclass(frozen=True, slots=True, kw_only=True)
271
+ class JobSpec(ExecutionSpec):
272
+ """Run an isolated workload — a container or a process — to completion.
273
+
274
+ A Job is not a long Task. It has its own image, its own resource envelope and
275
+ its own lifecycle, it does not share a worker with anything else, and it
276
+ returns an exit code rather than a Python value. Collapsing the two would
277
+ force every task engine to grow container semantics it does not have.
278
+
279
+ Attributes:
280
+ job: Job name. Also identifies the pre-declared provider resource for
281
+ runtimes such as Cloud Run Jobs that run an existing Job definition.
282
+ image: Container image. Required by container runtimes, ignored by the
283
+ local subprocess backend.
284
+ command: Full argv. The entrypoint for the local backend; the container
285
+ entrypoint/args override for container runtimes.
286
+ args: Extra arguments appended to :attr:`command`. Kept separate because
287
+ several runtimes model entrypoint and args separately.
288
+ env: Environment variables to inject.
289
+ resources: CPU/memory/GPU request.
290
+ parallelism: Number of task instances per submission (array jobs).
291
+ working_dir: Working directory inside the container/process.
292
+ """
293
+
294
+ job: str
295
+ image: str | None = None
296
+ command: Sequence[str] = ()
297
+ args: Sequence[str] = ()
298
+ env: Mapping[str, str] = field(default_factory=dict)
299
+ resources: Resources = field(default_factory=Resources)
300
+ parallelism: int = 1
301
+ working_dir: str | None = None
302
+
303
+ def __post_init__(self) -> None:
304
+ # Explicit base call: @dataclass(slots=True) rebuilds the class, which
305
+ # invalidates the zero-argument super() cell in these methods.
306
+ ExecutionSpec.__post_init__(self)
307
+ object.__setattr__(self, "command", tuple(self.command))
308
+ object.__setattr__(self, "args", tuple(self.args))
309
+ object.__setattr__(self, "env", MappingProxyType(dict(self.env)))
310
+ if not self.job:
311
+ raise ValueError("JobSpec.job is required")
312
+ if self.parallelism < 1:
313
+ raise ValueError("JobSpec.parallelism must be >= 1")
314
+ if not self.name:
315
+ object.__setattr__(self, "name", self.job)
316
+
317
+ @property
318
+ def kind(self) -> ExecutionKind:
319
+ return ExecutionKind.JOB
320
+
321
+ @property
322
+ def argv(self) -> tuple[str, ...]:
323
+ """Full argv: :attr:`command` followed by :attr:`args`."""
324
+ return (*self.command, *self.args)
325
+
326
+ def required_capabilities(self) -> frozenset[Capability]:
327
+ required = set(ExecutionSpec.required_capabilities(self))
328
+ if self.parallelism > 1:
329
+ required.add(Capability.PARALLELISM)
330
+ if self.resources.cpu is not None:
331
+ required.add(Capability.CPU)
332
+ if self.resources.memory is not None:
333
+ required.add(Capability.MEMORY)
334
+ if self.resources.gpu > 0:
335
+ required.add(Capability.GPU)
336
+ if self.env:
337
+ required.add(Capability.ENVIRONMENT)
338
+ return frozenset(required)
339
+
340
+
341
+ AnySpec = InlineSpec | TaskSpec | JobSpec
342
+ """Union of every spec kind — what :meth:`taskferry.Taskferry.submit` accepts."""
343
+
344
+
345
+ __all__ = [
346
+ "AnySpec",
347
+ "BackendOptions",
348
+ "ExecutionSpec",
349
+ "InlineSpec",
350
+ "JobSpec",
351
+ "Resources",
352
+ "TaskSpec",
353
+ ]
taskferry/tracking.py ADDED
@@ -0,0 +1,129 @@
1
+ """Pairing Taskferry ids with engine ids — the bookkeeping every adapter needs.
2
+
3
+ A Taskferry id is Taskferry's. Procrastinate mints an integer, Cloud Run mints a
4
+ resource path, AWS Batch mints a UUID, and none of them will ever store ours. So
5
+ an adapter that wants ``get(execution_id)`` to work has to remember the pairing:
6
+
7
+ ```mermaid
8
+ flowchart LR
9
+ SUB["submit()"]
10
+ IDX["ExternalIdIndex<br/>task_9f2c... ↔ 4711"]
11
+ GET["get(task_9f2c...)"]
12
+ ENG["engine.lookup(4711)"]
13
+
14
+ SUB -->|"remember"| IDX
15
+ GET -->|"resolve"| IDX --> ENG
16
+ ```
17
+
18
+ Five adapters were about to grow the same ``OrderedDict`` plus lock plus eviction
19
+ loop, so it lives here once, with the two properties that matter:
20
+
21
+ **Bounded.** A producer that runs for a month must not accumulate a map for a
22
+ month. Oldest entries are evicted, and an evicted lookup fails loudly.
23
+
24
+ **Honest about its scope.** The index is per-process and in-memory. It cannot
25
+ survive a restart and it is invisible to another replica — which is a real
26
+ limitation of looking an execution up by Taskferry id, not something to paper
27
+ over. Adapters accept the engine's own id as a fallback so an operator holding a
28
+ job id from a dashboard can always look it up, from anywhere.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import threading
34
+ from collections import OrderedDict
35
+ from collections.abc import Callable
36
+
37
+ DEFAULT_CAPACITY = 10_000
38
+ """Entries kept before the oldest is evicted. Roughly 1 MB at typical id sizes."""
39
+
40
+
41
+ class ExternalIdIndex:
42
+ """A bounded, thread-safe, bidirectional map of Taskferry id ↔ engine id.
43
+
44
+ Args:
45
+ capacity: Maximum entries. Zero disables tracking entirely, which is a
46
+ legitimate choice for a fire-and-forget backend that could not look
47
+ an execution up anyway.
48
+ recognises: Predicate deciding whether a string is already an engine id.
49
+ It lets ``resolve`` accept ``"projects/p/locations/eu/executions/x"``
50
+ or ``"4711"`` directly, so an id copied from a console works without
51
+ this process ever having submitted it.
52
+ """
53
+
54
+ __slots__ = ("_by_execution", "_capacity", "_lock", "_recognises")
55
+
56
+ def __init__(
57
+ self,
58
+ *,
59
+ capacity: int = DEFAULT_CAPACITY,
60
+ recognises: Callable[[str], bool] | None = None,
61
+ ) -> None:
62
+ if capacity < 0:
63
+ raise ValueError("capacity must be >= 0")
64
+ self._capacity = capacity
65
+ self._recognises = recognises
66
+ self._by_execution: OrderedDict[str, str] = OrderedDict()
67
+ self._lock = threading.Lock()
68
+
69
+ def __len__(self) -> int:
70
+ with self._lock:
71
+ return len(self._by_execution)
72
+
73
+ def __contains__(self, execution_id: object) -> bool:
74
+ with self._lock:
75
+ return str(execution_id) in self._by_execution
76
+
77
+ def __repr__(self) -> str:
78
+ return f"ExternalIdIndex(entries={len(self)}, capacity={self._capacity})"
79
+
80
+ def remember(self, execution_id: str, external_id: str | None) -> None:
81
+ """Record the pairing produced by a submission. ``None`` is ignored."""
82
+ if external_id is None or self._capacity == 0:
83
+ return
84
+ with self._lock:
85
+ self._by_execution[execution_id] = external_id
86
+ self._by_execution.move_to_end(execution_id)
87
+ while len(self._by_execution) > self._capacity:
88
+ self._by_execution.popitem(last=False)
89
+
90
+ def resolve(self, execution_id: str) -> str | None:
91
+ """The engine id for ``execution_id``, or ``None`` when it is unknown.
92
+
93
+ A remembered pairing wins. Failing that, ``execution_id`` is returned
94
+ unchanged if ``recognises`` says it already *is* an engine id.
95
+ """
96
+ with self._lock:
97
+ known = self._by_execution.get(execution_id)
98
+ if known is not None:
99
+ self._by_execution.move_to_end(execution_id)
100
+ return known
101
+ if self._recognises is not None and self._recognises(execution_id):
102
+ return execution_id
103
+ return None
104
+
105
+ def forget(self, execution_id: str) -> None:
106
+ """Drop one pairing. Silent when it was not tracked."""
107
+ with self._lock:
108
+ self._by_execution.pop(execution_id, None)
109
+
110
+ def clear(self) -> None:
111
+ with self._lock:
112
+ self._by_execution.clear()
113
+
114
+
115
+ def is_digits(value: str) -> bool:
116
+ """``recognises`` predicate for engines with numeric ids (Procrastinate)."""
117
+ return value.isdigit()
118
+
119
+
120
+ def has_prefix(prefix: str) -> Callable[[str], bool]:
121
+ """``recognises`` predicate for engines with path-shaped ids (Cloud Run, GCP)."""
122
+
123
+ def _check(value: str) -> bool:
124
+ return value.startswith(prefix)
125
+
126
+ return _check
127
+
128
+
129
+ __all__ = ["DEFAULT_CAPACITY", "ExternalIdIndex", "has_prefix", "is_digits"]
@@ -0,0 +1,109 @@
1
+ Metadata-Version: 2.5
2
+ Name: taskferry
3
+ Version: 0.2.0
4
+ Summary: A portable execution layer for Python — inline, tasks and jobs on the engines you already run.
5
+ Project-URL: Homepage, https://github.com/xiidigital/taskferry
6
+ Project-URL: Documentation, https://taskferry.dev
7
+ Project-URL: Source, https://github.com/xiidigital/taskferry
8
+ Project-URL: Changelog, https://github.com/xiidigital/taskferry/blob/main/packages/taskferry/CHANGELOG.md
9
+ Author: Taskferry authors
10
+ License-Expression: Apache-2.0
11
+ License-File: LICENSE
12
+ Keywords: capabilities,execution,jobs,portability,ports-and-adapters,taskferry,tasks
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: License :: OSI Approved :: Apache Software License
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: System :: Distributed Computing
20
+ Classifier: Typing :: Typed
21
+ Requires-Python: >=3.12
22
+ Description-Content-Type: text/markdown
23
+
24
+ # taskferry
25
+
26
+ **A portable execution layer for Python.**
27
+
28
+ Taskferry models units of work, chooses the right *kind* of execution, and routes
29
+ them to engines that already exist. It is not a task queue, not a worker system,
30
+ not a scheduler and not a workflow engine.
31
+
32
+ ```bash
33
+ pip install taskferry
34
+ ```
35
+
36
+ That installs **nothing else**. No Django, no PostgreSQL driver, no Redis client,
37
+ no cloud SDK. The distribution declares zero dependencies and CI proves it in a
38
+ bare virtualenv on every commit.
39
+
40
+ ```python
41
+ from taskferry import Taskferry
42
+
43
+ runtime = Taskferry.local()
44
+
45
+
46
+ def add(a: int, b: int) -> int:
47
+ return a + b
48
+
49
+
50
+ execution = runtime.inline.submit(add, 20, 22)
51
+ assert execution.result().value == 42
52
+ ```
53
+
54
+ ## Three primitives
55
+
56
+ ```python
57
+ runtime.inline.submit(add, 20, 22) # now, in this process
58
+ runtime.tasks.submit("myapp.tasks:send_email", 42) # later, on an engine
59
+ runtime.jobs.submit("build-cog", image="gdal:latest") # a container, to completion
60
+ ```
61
+
62
+ A Job is not "a Task that takes longer": it has its own image, its own resource
63
+ envelope and its own lifecycle, and it returns an exit code rather than a Python
64
+ value.
65
+
66
+ ## Built-in backends
67
+
68
+ | Backend | Kind | Use |
69
+ | --- | --- | --- |
70
+ | `inline` | inline | tests, notebooks, CLIs, small tools |
71
+ | `thread` | task | development, single-process applications |
72
+ | `process` | task | rehearsing what a real worker does to your code |
73
+ | `subprocess` | job | local batch workloads |
74
+
75
+ None of them is a queue. They are not durable, not visible across processes, and
76
+ they lose pending work on restart — which is stated plainly in their docstrings
77
+ and in their capability sets. For production, route at a real engine; that is a
78
+ configuration change.
79
+
80
+ ## Adapters
81
+
82
+ Install the engine you use. Each registers itself through the `taskferry.backends`
83
+ entry-point group, so naming it in configuration is all it takes.
84
+
85
+ ```bash
86
+ pip install taskferry-procrastinate # PostgreSQL-backed tasks
87
+ pip install taskferry-cloudtasks # push-based serverless tasks
88
+ pip install taskferry-cloudrun # serverless container jobs
89
+ pip install taskferry-jobs # AWS Batch, Kubernetes, Azure Container Apps
90
+ pip install taskferry-django # django.tasks integration
91
+ ```
92
+
93
+ ## CLI
94
+
95
+ ```bash
96
+ taskferry backends
97
+ taskferry capabilities pg
98
+ taskferry route --kind job --profile gpu
99
+ taskferry doctor
100
+ ```
101
+
102
+ ## Documentation
103
+
104
+ Full documentation, ADRs and the migration guide live in the
105
+ [repository](https://github.com/xiidigital/taskferry).
106
+
107
+ ## License
108
+
109
+ Apache-2.0.
@@ -0,0 +1,48 @@
1
+ taskferry/__init__.py,sha256=jlXhwsG9uYObCO9uNw_UBVjLFnbXU7jbmxIBoNEdL4E,5828
2
+ taskferry/aio.py,sha256=K9vqsLg2FQhnT4UOI90HHi8rA6VK-g2YlspFVt7rOg4,16970
3
+ taskferry/capabilities.py,sha256=kDbbbxTBazYKlH9otRn7Qi_63zwN9r7YawqM24cDo88,3140
4
+ taskferry/cli.py,sha256=UDkhgHwqUcqc4Z_0m5FNtnWEhvxGKBv3JEklQKn3IeM,16294
5
+ taskferry/config.py,sha256=4b_y_GzHVRK2Ze7c40D5kpPGEqZZKJeWIKL2s1djARw,14564
6
+ taskferry/envelope.py,sha256=UQdkGJ9lT-UmcX5rZB3xwCGpWGYUht3j0d2dJeyTcT4,7169
7
+ taskferry/errors.py,sha256=r3YrjBGoTmWEB0-vHMZLjEvbmjk3mrnQAmnB-5p6_4Q,4700
8
+ taskferry/execution.py,sha256=rmtg11DjIlE39YlOrDEi6GDU2NR-1m-8SqLyQKupR5U,8466
9
+ taskferry/functions.py,sha256=JC0jAWUALiR7BjtG2FKMbuOuOHr30XvBQqbbj5YkKHk,11774
10
+ taskferry/handle.py,sha256=cBv7oK3WWPYSajN91pxs9Emyptu-VLwwM-0m-hoPsCw,6656
11
+ taskferry/hooks.py,sha256=pKUDyD5pvQF8QlQoK0AlyFZ6jNSmKvZkZEHMmISkYWI,9047
12
+ taskferry/plugins.py,sha256=MNGhZOBju_GmixZGYj0PLEBAXmN5y4ZE_tJ3CSeWVNw,6374
13
+ taskferry/ports.py,sha256=lPO3U3BippQzVZn-AvVgkiG0k4o6qRNHO0F6xARhG_c,13741
14
+ taskferry/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
15
+ taskferry/retry.py,sha256=qqv2Dl-87WvqIHDoYnwZ8d8wr9AM2UPXO6MDoFV0G7U,6920
16
+ taskferry/router.py,sha256=TP93M0ly3RvALZKBWWWnsJsOlv0arop-NAlaKsSJafc,6252
17
+ taskferry/runtime.py,sha256=LfqDIeYy1XAGriNqnObQrFwV5uc4PlZdGZgVYXBZ8j8,22729
18
+ taskferry/specs.py,sha256=Z1O2FBs7wuASC-1N97Ri-x-egQnHFnmsBh8hl6EYQNM,14169
19
+ taskferry/tracking.py,sha256=kcGMPt2pccWwXMyPgcst4h5tt8BpF9iAhbpM8q6i2QQ,4793
20
+ taskferry/backends/__init__.py,sha256=9odsHuv_OFMRIGn9SvCslBcXaWW-pkFkwyOI2COBUWI,1394
21
+ taskferry/backends/inline.py,sha256=Kxdou00scP_ebXPUbPyyksITk6rnK1jaXgSb_iMJg0g,8215
22
+ taskferry/backends/process.py,sha256=UiJ_Yr-BvmP1m3Eqxd_EaoCcA-CM1g4AWIeYLkEBsqY,10482
23
+ taskferry/backends/subprocess.py,sha256=_f8bg-MEb90cZzdGZs_J15JBltWHPR0jvqIPFmlWhB4,14192
24
+ taskferry/backends/thread.py,sha256=lo9ehxUQWf4JJQTkQdxJmiHeQ3fMV7fw6bl4X2XG9b8,12864
25
+ taskferry/contract/__init__.py,sha256=TJr1qq8IQTjh4moga-8vHQssg4gPOnGv1DkQaC_hZgk,1742
26
+ taskferry/contract/base.py,sha256=NJqGZTmt4lEHJdivx4Pl9EBaqtnRLy5jWOHGRYjnusk,7927
27
+ taskferry/contract/inline.py,sha256=cOaUsBh5IEUky0XSmbCki7VKt8bNYG9T818i-FRSNp8,3143
28
+ taskferry/contract/job.py,sha256=F85ekBdrFS1-67YyAbTFui_7LIVB1E53DV9pZjO06p4,3451
29
+ taskferry/contract/task.py,sha256=6I6Z3q_vkWmO9I-hd0NpjNDVzU0WecVnMFJK01Eif1Y,3352
30
+ taskferry/core/__init__.py,sha256=_xTuiMvnsRRf3yakVRZheHUr-g2-b4uZpovMUHCa1hI,3237
31
+ taskferry/core/capabilities.py,sha256=9Sazy_jVplY_C-T3bHCSPKRjRTbffyIJmJBCUviPYN4,3243
32
+ taskferry/core/config.py,sha256=do2C-mc6ThSkTed_ZF_ktRMEnVsOgHE_WejqH24_FX8,5498
33
+ taskferry/core/correlation.py,sha256=CUsMWhQj_QY9pcEYq5iQ1vDCbypwNj8jdx8Zh-4_jRo,4210
34
+ taskferry/core/delivery.py,sha256=xNBwawpBR7D5lzH5ON94zsaf_gxfJpdADfstLb-D7i8,1073
35
+ taskferry/core/errors.py,sha256=yS48DNSgf-PFobj65OfZdMuLfjsdWOoZrUM1YfiIrOs,1831
36
+ taskferry/core/ids.py,sha256=69bD2Um1FU4eeFWx_HRBA3r2PgflxGjKO9jBQ2MV6QE,1303
37
+ taskferry/core/observability.py,sha256=unXRWzWEsEKKrj1q2ajK0l9fuqzsMJgDDyLDONTb8_w,3881
38
+ taskferry/core/otel.py,sha256=-lZzp8GdwAPmMYJfGoCTcYOR1UVsiaR_YfQoo2a1VCI,2798
39
+ taskferry/core/provider.py,sha256=6aqk468rMnVKl9lEbUAV2MGHhm5eEvFeUU0Q4Kpu_w0,1968
40
+ taskferry/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
41
+ taskferry/core/registry.py,sha256=9pTIXb4HUnRaZWTp_Uh-nlMTDsh7n1vTlFGUsi3kwdM,3425
42
+ taskferry/core/serialization.py,sha256=ROh3w8E2A8c0O1I7SVbkA7HGbOjmv-fZBHG7h-wrtxI,2529
43
+ taskferry/core/typing.py,sha256=IHbJbt5i1ClEDfqiMcAtLMar3t-n8QaRHnCA3tgri6U,649
44
+ taskferry-0.2.0.dist-info/METADATA,sha256=fIBdxHV3m7BRTimEIYeTPzfaQe8NBnggUWJGCETDXHQ,3575
45
+ taskferry-0.2.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
46
+ taskferry-0.2.0.dist-info/entry_points.txt,sha256=bx6DKenzSZRUXdqcoChvz07P3TeLpIteyNl1LWl9f_E,49
47
+ taskferry-0.2.0.dist-info/licenses/LICENSE,sha256=de-gfE0q-xTYImzwC3dj3S7BxVhanf6RmIGjo_7y3aw,11357
48
+ taskferry-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.3
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ taskferry = taskferry.cli:main