offwork 0.1.4__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {offwork-0.1.4 → offwork-0.2.1}/PKG-INFO +8 -6
- {offwork-0.1.4 → offwork-0.2.1}/README.md +5 -5
- {offwork-0.1.4 → offwork-0.2.1}/offwork/__init__.py +8 -1
- offwork-0.2.1/offwork/core/_timeout.py +60 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/graph/tracing.py +115 -118
- offwork-0.2.1/offwork/typing.py +126 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/backends/redis.py +43 -10
- offwork-0.2.1/offwork/worker/backends/ws.py +354 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/remote.py +81 -16
- offwork-0.2.1/offwork/worker/result.py +563 -0
- offwork-0.2.1/offwork/worker/schedule.py +57 -0
- {offwork-0.1.4 → offwork-0.2.1}/pyproject.toml +2 -1
- offwork-0.1.4/offwork/typing.py +0 -50
- offwork-0.1.4/offwork/worker/backends/http.py +0 -243
- offwork-0.1.4/offwork/worker/result.py +0 -312
- offwork-0.1.4/offwork/worker/schedule.py +0 -53
- {offwork-0.1.4 → offwork-0.2.1}/LICENSE +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/__main__.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/_venv.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/__init__.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/clients.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/ed25519.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/envelope.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/errors.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/identity.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/models.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/pairing.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/progress.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/signing.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/task.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/token.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/core/version.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/graph/__init__.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/graph/analyzer.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/graph/decorator.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/graph/graph.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/graph/store.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/py.typed +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/__init__.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/backends/__init__.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/backends/base.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/backends/local.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/backends/rabbitmq.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/deps.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/sandbox/Dockerfile +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/sandbox/__init__.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/sandbox/_protocol.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/sandbox/docker.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/sandbox/guest_agent.py +0 -0
- {offwork-0.1.4 → offwork-0.2.1}/offwork/worker/worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: offwork
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Distributed Python task execution via automatic function serialization
|
|
5
5
|
License: AGPL-3.0-only
|
|
6
6
|
License-File: LICENSE
|
|
@@ -20,8 +20,10 @@ Classifier: Topic :: System :: Distributed Computing
|
|
|
20
20
|
Classifier: Typing :: Typed
|
|
21
21
|
Provides-Extra: rabbitmq
|
|
22
22
|
Provides-Extra: redis
|
|
23
|
+
Provides-Extra: ws
|
|
23
24
|
Requires-Dist: aio-pika (>=9.0) ; extra == "rabbitmq"
|
|
24
25
|
Requires-Dist: redis (>=5.0) ; extra == "redis"
|
|
26
|
+
Requires-Dist: websockets (>=15.0) ; extra == "ws"
|
|
25
27
|
Project-URL: Repository, https://github.com/codeSamuraii/offwork
|
|
26
28
|
Description-Content-Type: text/markdown
|
|
27
29
|
|
|
@@ -76,21 +78,21 @@ pip install offwork[redis]
|
|
|
76
78
|
offwork worker --backend redis://other-machine:6379
|
|
77
79
|
```
|
|
78
80
|
|
|
79
|
-
See [Features](docs/FEATURES.md) for the full API.
|
|
80
|
-
|
|
81
81
|
## Features
|
|
82
82
|
|
|
83
83
|
| | |
|
|
84
84
|
|-|-|
|
|
85
|
-
| **Async-native** | `.run()`, `.
|
|
86
|
-
| **Scheduling** |
|
|
85
|
+
| **Async-native** | `.run()`, `.submit()`, `.map()`, `asyncio.gather` — all coroutines |
|
|
86
|
+
| **Scheduling** | `submit(run_in=delay)`, `submit(run_at=dt)`, `submit(run_every=interval)` with cancellation |
|
|
87
87
|
| **Retry & timeout** | `@offwork.task(timeout=30, retries=3)` with exponential backoff |
|
|
88
88
|
| **Throttling** | `@offwork.task(throttle=timedelta(hours=24)/50)` — rate-limit executions |
|
|
89
|
-
| **Progress & cancellation** | `offwork.progress(3, 10)` inside tasks; `await
|
|
89
|
+
| **Progress & cancellation** | `offwork.progress(3, 10)` inside tasks; `fut.cancel()` / `await fut.cancel()` on client |
|
|
90
90
|
| **Heartbeat & stall detection** | Workers heartbeat every second; clients raise `TaskStalled` on silence |
|
|
91
91
|
| **Package auto-install** | Workers `pip install` missing packages before execution |
|
|
92
92
|
| **Docker sandbox** | Optional container isolation, fully transparent to clients |
|
|
93
93
|
| **Signed execution** | Pre-shared token or PIN pairing + HMAC-SHA256 task authentication |
|
|
94
|
+
| ... | See [Features](docs/FEATURES.md) for more information. |
|
|
95
|
+
|
|
94
96
|
|
|
95
97
|
### Security
|
|
96
98
|
|
|
@@ -49,21 +49,21 @@ pip install offwork[redis]
|
|
|
49
49
|
offwork worker --backend redis://other-machine:6379
|
|
50
50
|
```
|
|
51
51
|
|
|
52
|
-
See [Features](docs/FEATURES.md) for the full API.
|
|
53
|
-
|
|
54
52
|
## Features
|
|
55
53
|
|
|
56
54
|
| | |
|
|
57
55
|
|-|-|
|
|
58
|
-
| **Async-native** | `.run()`, `.
|
|
59
|
-
| **Scheduling** |
|
|
56
|
+
| **Async-native** | `.run()`, `.submit()`, `.map()`, `asyncio.gather` — all coroutines |
|
|
57
|
+
| **Scheduling** | `submit(run_in=delay)`, `submit(run_at=dt)`, `submit(run_every=interval)` with cancellation |
|
|
60
58
|
| **Retry & timeout** | `@offwork.task(timeout=30, retries=3)` with exponential backoff |
|
|
61
59
|
| **Throttling** | `@offwork.task(throttle=timedelta(hours=24)/50)` — rate-limit executions |
|
|
62
|
-
| **Progress & cancellation** | `offwork.progress(3, 10)` inside tasks; `await
|
|
60
|
+
| **Progress & cancellation** | `offwork.progress(3, 10)` inside tasks; `fut.cancel()` / `await fut.cancel()` on client |
|
|
63
61
|
| **Heartbeat & stall detection** | Workers heartbeat every second; clients raise `TaskStalled` on silence |
|
|
64
62
|
| **Package auto-install** | Workers `pip install` missing packages before execution |
|
|
65
63
|
| **Docker sandbox** | Optional container isolation, fully transparent to clients |
|
|
66
64
|
| **Signed execution** | Pre-shared token or PIN pairing + HMAC-SHA256 task authentication |
|
|
65
|
+
| ... | See [Features](docs/FEATURES.md) for more information. |
|
|
66
|
+
|
|
67
67
|
|
|
68
68
|
### Security
|
|
69
69
|
|
|
@@ -62,7 +62,8 @@ from offwork.core.envelope import (
|
|
|
62
62
|
from offwork.core.version import _VERSION
|
|
63
63
|
from offwork.core.progress import ProgressInfo
|
|
64
64
|
from offwork.core.progress import progress as progress
|
|
65
|
-
from offwork.
|
|
65
|
+
from offwork.core._timeout import TimeoutIn, WaitForever, ReturnImmediately
|
|
66
|
+
from offwork.worker.remote import serve, connect, disconnect, _ConnectionContext
|
|
66
67
|
from offwork.worker.result import Result, ResultEnvelope
|
|
67
68
|
from offwork.worker.worker import Worker
|
|
68
69
|
from offwork.worker.worker import execute as execute
|
|
@@ -129,6 +130,12 @@ __all__ = [
|
|
|
129
130
|
"install_package_as",
|
|
130
131
|
"worker_only_import",
|
|
131
132
|
"progress",
|
|
133
|
+
# Timeout types
|
|
134
|
+
"TimeoutIn",
|
|
135
|
+
"WaitForever",
|
|
136
|
+
"ReturnImmediately",
|
|
137
|
+
# Connection handle
|
|
138
|
+
"_ConnectionContext",
|
|
132
139
|
# Serialization
|
|
133
140
|
"serialize",
|
|
134
141
|
"reconstruct",
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Shared timeout type and resolution helper used across the public API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import timedelta
|
|
6
|
+
from typing import Literal
|
|
7
|
+
|
|
8
|
+
# ── Named sentinels ──────────────────────────────────────────────────────────
|
|
9
|
+
|
|
10
|
+
type WaitForever = Literal[False] | Literal[-1]
|
|
11
|
+
"""Sentinel: block until the operation completes, with no deadline."""
|
|
12
|
+
|
|
13
|
+
type ReturnImmediately = Literal[True] | Literal[0]
|
|
14
|
+
"""Sentinel: return as soon as possible (non-blocking / fast-poll)."""
|
|
15
|
+
|
|
16
|
+
# ── Public timeout type ───────────────────────────────────────────────────────
|
|
17
|
+
|
|
18
|
+
type TimeoutIn = float | timedelta | WaitForever | ReturnImmediately
|
|
19
|
+
"""A duration accepted by every wait-style method in the public API.
|
|
20
|
+
|
|
21
|
+
Interpretation
|
|
22
|
+
--------------
|
|
23
|
+
``False`` or ``-1``
|
|
24
|
+
Block indefinitely until the operation completes.
|
|
25
|
+
``True`` or ``0``
|
|
26
|
+
Return as soon as possible (non-blocking or single fast poll).
|
|
27
|
+
``timedelta``
|
|
28
|
+
Wait at most this duration.
|
|
29
|
+
``float`` (positive)
|
|
30
|
+
Wait at most this many seconds.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# ── Internal helper ───────────────────────────────────────────────────────────
|
|
35
|
+
|
|
36
|
+
def resolve_timeout(t: TimeoutIn) -> float | None:
|
|
37
|
+
"""Convert a :data:`TimeoutIn` value to seconds (``float``) or ``None``.
|
|
38
|
+
|
|
39
|
+
Returns
|
|
40
|
+
-------
|
|
41
|
+
None
|
|
42
|
+
Wait forever (corresponds to ``False`` or ``-1``).
|
|
43
|
+
0.0
|
|
44
|
+
Non-blocking / return immediately (corresponds to ``True`` or ``0``).
|
|
45
|
+
positive float
|
|
46
|
+
Maximum seconds to wait.
|
|
47
|
+
"""
|
|
48
|
+
# bool is a subtype of int — check identity before numeric equality
|
|
49
|
+
# so that False is not mistaken for 0 and True is not mistaken for 1.
|
|
50
|
+
if t is False:
|
|
51
|
+
return None
|
|
52
|
+
if t is True:
|
|
53
|
+
return 0.0
|
|
54
|
+
if t == -1:
|
|
55
|
+
return None
|
|
56
|
+
if t == 0:
|
|
57
|
+
return 0.0
|
|
58
|
+
if isinstance(t, timedelta):
|
|
59
|
+
return t.total_seconds()
|
|
60
|
+
return float(t)
|
|
@@ -28,39 +28,141 @@ _R = TypeVar("_R")
|
|
|
28
28
|
_BUILTIN_NAMES = set(dir(builtins))
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
def
|
|
31
|
+
def _make_submit_method(
|
|
32
32
|
wrapper: Callable[..., object], func: Callable[..., object]
|
|
33
33
|
) -> Callable[..., object]:
|
|
34
|
-
"""Create the ``.
|
|
34
|
+
"""Create the ``.submit()`` async method.
|
|
35
|
+
|
|
36
|
+
Submits the function to a remote worker and returns a :class:`Result`
|
|
37
|
+
handle (or a :class:`ScheduleHandle` when *run_every* is set).
|
|
38
|
+
|
|
39
|
+
Scheduling keywords (all optional, at most one of ``run_at`` /
|
|
40
|
+
``run_in`` / ``run_every`` may be given):
|
|
41
|
+
|
|
42
|
+
``run_at``
|
|
43
|
+
:class:`~datetime.datetime` — run once at a specific point in time.
|
|
44
|
+
``run_in``
|
|
45
|
+
:class:`~datetime.timedelta` or ``float`` (seconds) — run once after
|
|
46
|
+
a delay.
|
|
47
|
+
``run_every``
|
|
48
|
+
:class:`~datetime.timedelta` or ``float`` (seconds) — repeat at this
|
|
49
|
+
interval. Returns a :class:`ScheduleHandle` instead of a
|
|
50
|
+
:class:`Result`.
|
|
51
|
+
|
|
52
|
+
Additional keywords for recurring schedules:
|
|
53
|
+
|
|
54
|
+
``_start_at``
|
|
55
|
+
:class:`~datetime.datetime` — first occurrence for *run_every*.
|
|
56
|
+
``run_for``
|
|
57
|
+
:class:`~datetime.timedelta` or ``float`` (seconds) — stop recurring
|
|
58
|
+
after this wall-clock duration.
|
|
59
|
+
``max_runs``
|
|
60
|
+
``int`` — stop recurring after this many executions.
|
|
61
|
+
``backend``
|
|
62
|
+
Override the global backend for this submission.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
async def submit(
|
|
66
|
+
*args: Any,
|
|
67
|
+
run_at: datetime | None = None,
|
|
68
|
+
run_in: timedelta | float | None = None,
|
|
69
|
+
run_every: timedelta | float | None = None,
|
|
70
|
+
_start_at: datetime | None = None,
|
|
71
|
+
run_for: timedelta | float | None = None,
|
|
72
|
+
max_runs: int | None = None,
|
|
73
|
+
backend: str | Backend | None = None,
|
|
74
|
+
**kwargs: Any,
|
|
75
|
+
) -> object:
|
|
76
|
+
if sum(x is not None for x in (run_at, run_in, run_every)) > 1:
|
|
77
|
+
raise ValueError(
|
|
78
|
+
"At most one of run_at, run_in, run_every may be specified"
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
if run_every is not None:
|
|
82
|
+
from offwork.worker.remote import submit_recurring # circular
|
|
83
|
+
|
|
84
|
+
interval = (
|
|
85
|
+
run_every.total_seconds()
|
|
86
|
+
if isinstance(run_every, timedelta)
|
|
87
|
+
else float(run_every)
|
|
88
|
+
)
|
|
89
|
+
start_ts: float | None = None
|
|
90
|
+
if _start_at is not None:
|
|
91
|
+
start_ts = (
|
|
92
|
+
_start_at.timestamp()
|
|
93
|
+
if isinstance(_start_at, datetime)
|
|
94
|
+
else float(_start_at)
|
|
95
|
+
)
|
|
96
|
+
if run_for is None and max_runs is None:
|
|
97
|
+
run_for = timedelta(hours=1)
|
|
98
|
+
run_for_seconds: float | None = None
|
|
99
|
+
if run_for is not None:
|
|
100
|
+
run_for_seconds = (
|
|
101
|
+
run_for.total_seconds()
|
|
102
|
+
if isinstance(run_for, timedelta)
|
|
103
|
+
else float(run_for)
|
|
104
|
+
)
|
|
105
|
+
if run_for_seconds <= 0:
|
|
106
|
+
raise ValueError(f"run_for must be positive, got {run_for}")
|
|
107
|
+
if max_runs is not None and max_runs <= 0:
|
|
108
|
+
raise ValueError(f"max_runs must be positive, got {max_runs}")
|
|
109
|
+
return await submit_recurring(
|
|
110
|
+
func, wrapper, *args,
|
|
111
|
+
_backend=backend, _interval=interval, _start_at=start_ts,
|
|
112
|
+
_run_for=run_for_seconds, _max_runs=max_runs,
|
|
113
|
+
**kwargs,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
if run_at is not None or run_in is not None:
|
|
117
|
+
from offwork.worker.remote import submit_remote_scheduled # circular
|
|
118
|
+
|
|
119
|
+
if run_at is not None:
|
|
120
|
+
scheduled_at = (
|
|
121
|
+
run_at.timestamp()
|
|
122
|
+
if isinstance(run_at, datetime)
|
|
123
|
+
else float(run_at)
|
|
124
|
+
)
|
|
125
|
+
else:
|
|
126
|
+
assert run_in is not None
|
|
127
|
+
delay = (
|
|
128
|
+
run_in.total_seconds()
|
|
129
|
+
if isinstance(run_in, timedelta)
|
|
130
|
+
else float(run_in)
|
|
131
|
+
)
|
|
132
|
+
scheduled_at = _time.time() + delay
|
|
133
|
+
return await submit_remote_scheduled(
|
|
134
|
+
func, wrapper, *args,
|
|
135
|
+
_backend=backend, _scheduled_at=scheduled_at,
|
|
136
|
+
**kwargs,
|
|
137
|
+
)
|
|
35
138
|
|
|
36
|
-
async def start(*args: Any, backend: str | Backend | None = None, **kwargs: Any) -> object:
|
|
37
139
|
from offwork.worker.remote import submit_remote # circular
|
|
38
140
|
|
|
39
141
|
return await submit_remote(func, wrapper, *args, _backend=backend, **kwargs)
|
|
40
142
|
|
|
41
|
-
return
|
|
143
|
+
return submit
|
|
42
144
|
|
|
43
145
|
|
|
44
146
|
def _make_run_method(
|
|
45
|
-
|
|
147
|
+
submit_method: Callable[..., object],
|
|
46
148
|
) -> Callable[..., object]:
|
|
47
149
|
"""Create the ``.run()`` async method that submits and awaits the result."""
|
|
48
150
|
|
|
49
151
|
async def run(*args: object, **kwargs: object) -> object:
|
|
50
|
-
result = await
|
|
152
|
+
result = await submit_method(*args, **kwargs) # type: ignore[misc]
|
|
51
153
|
return await result
|
|
52
154
|
|
|
53
155
|
return run
|
|
54
156
|
|
|
55
157
|
|
|
56
158
|
def _make_map_method(
|
|
57
|
-
|
|
159
|
+
submit_method: Callable[..., object],
|
|
58
160
|
) -> Callable[..., object]:
|
|
59
161
|
"""Create the ``.map()`` async method for batch submission and collection."""
|
|
60
162
|
|
|
61
163
|
async def map(args_list: list[tuple[object, ...]], **kwargs: object) -> list[object]:
|
|
62
164
|
coros: list[Awaitable[object]] = [
|
|
63
|
-
cast(Awaitable[object],
|
|
165
|
+
cast(Awaitable[object], submit_method(*args, **kwargs))
|
|
64
166
|
for args in args_list
|
|
65
167
|
]
|
|
66
168
|
results = await asyncio.gather(*coros)
|
|
@@ -72,120 +174,15 @@ def _make_map_method(
|
|
|
72
174
|
return map
|
|
73
175
|
|
|
74
176
|
|
|
75
|
-
def _make_start_at_method(
|
|
76
|
-
wrapper: Callable[..., object], func: Callable[..., object]
|
|
77
|
-
) -> Callable[..., object]:
|
|
78
|
-
"""Create the ``.start_at()`` method that submits a task scheduled for a specific time."""
|
|
79
|
-
|
|
80
|
-
async def start_at(dt: Any, *args: Any, backend: str | Backend | None = None, **kwargs: Any) -> object:
|
|
81
|
-
from offwork.worker.remote import submit_remote_scheduled # circular
|
|
82
|
-
|
|
83
|
-
ts = dt.timestamp() if isinstance(dt, datetime) else float(dt)
|
|
84
|
-
return await submit_remote_scheduled(
|
|
85
|
-
func, wrapper, *args, _backend=backend, _scheduled_at=ts, **kwargs,
|
|
86
|
-
)
|
|
87
|
-
|
|
88
|
-
return start_at
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def _make_run_at_method(
|
|
92
|
-
start_at_method: Callable[..., object],
|
|
93
|
-
) -> Callable[..., object]:
|
|
94
|
-
"""Create the ``.run_at()`` method that submits at a time and awaits the result."""
|
|
95
|
-
|
|
96
|
-
async def run_at(dt: Any, *args: object, **kwargs: object) -> object:
|
|
97
|
-
result = await start_at_method(dt, *args, **kwargs) # type: ignore[misc]
|
|
98
|
-
return await result
|
|
99
|
-
|
|
100
|
-
return run_at
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
def _make_start_in_method(
|
|
104
|
-
wrapper: Callable[..., object], func: Callable[..., object]
|
|
105
|
-
) -> Callable[..., object]:
|
|
106
|
-
"""Create the ``.start_in()`` method that submits a task after a delay."""
|
|
107
|
-
|
|
108
|
-
async def start_in(delay: Any, *args: Any, backend: str | Backend | None = None, **kwargs: Any) -> object:
|
|
109
|
-
from offwork.worker.remote import submit_remote_scheduled # circular
|
|
110
|
-
|
|
111
|
-
seconds = delay.total_seconds() if isinstance(delay, timedelta) else float(delay)
|
|
112
|
-
return await submit_remote_scheduled(
|
|
113
|
-
func, wrapper, *args, _backend=backend, _scheduled_at=_time.time() + seconds, **kwargs,
|
|
114
|
-
)
|
|
115
|
-
|
|
116
|
-
return start_in
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
def _make_run_in_method(
|
|
120
|
-
start_in_method: Callable[..., object],
|
|
121
|
-
) -> Callable[..., object]:
|
|
122
|
-
"""Create the ``.run_in()`` method that submits after a delay and awaits."""
|
|
123
|
-
|
|
124
|
-
async def run_in(delay: Any, *args: object, **kwargs: object) -> object:
|
|
125
|
-
result = await start_in_method(delay, *args, **kwargs) # type: ignore[misc]
|
|
126
|
-
return await result
|
|
127
|
-
|
|
128
|
-
return run_in
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
def _make_run_every_method(
|
|
132
|
-
wrapper: Callable[..., object], func: Callable[..., object]
|
|
133
|
-
) -> Callable[..., object]:
|
|
134
|
-
"""Create the ``.run_every()`` method for recurring execution."""
|
|
135
|
-
|
|
136
|
-
async def run_every(
|
|
137
|
-
frequency: Any,
|
|
138
|
-
*args: Any,
|
|
139
|
-
_start_at: Any = None,
|
|
140
|
-
run_for: Any = None,
|
|
141
|
-
max_runs: int | None = None,
|
|
142
|
-
backend: str | Backend | None = None,
|
|
143
|
-
**kwargs: Any,
|
|
144
|
-
) -> object:
|
|
145
|
-
from offwork.worker.remote import submit_recurring # circular
|
|
146
|
-
|
|
147
|
-
interval = frequency.total_seconds() if isinstance(frequency, timedelta) else float(frequency)
|
|
148
|
-
start_ts: float | None = None
|
|
149
|
-
if _start_at is not None:
|
|
150
|
-
start_ts = _start_at.timestamp() if isinstance(_start_at, datetime) else float(_start_at)
|
|
151
|
-
if run_for is None and max_runs is None:
|
|
152
|
-
run_for = timedelta(hours=1)
|
|
153
|
-
run_for_seconds: float | None = None
|
|
154
|
-
if run_for is not None:
|
|
155
|
-
run_for_seconds = run_for.total_seconds() if isinstance(run_for, timedelta) else float(run_for)
|
|
156
|
-
if run_for_seconds <= 0:
|
|
157
|
-
raise ValueError(f"run_for must be positive, got {run_for}")
|
|
158
|
-
if max_runs is not None and max_runs <= 0:
|
|
159
|
-
raise ValueError(f"max_runs must be positive, got {max_runs}")
|
|
160
|
-
return await submit_recurring(
|
|
161
|
-
func, wrapper, *args,
|
|
162
|
-
_backend=backend, _interval=interval, _start_at=start_ts,
|
|
163
|
-
_run_for=run_for_seconds, _max_runs=max_runs,
|
|
164
|
-
**kwargs,
|
|
165
|
-
)
|
|
166
|
-
|
|
167
|
-
return run_every
|
|
168
|
-
|
|
169
|
-
|
|
170
177
|
def _attach_traced_attrs(
|
|
171
178
|
wrapper: Callable[..., object], func: Callable[..., object]
|
|
172
179
|
) -> None:
|
|
173
|
-
"""Attach offwork metadata and
|
|
180
|
+
"""Attach offwork metadata and remote-execution methods to a traced wrapper."""
|
|
174
181
|
wrapper.__offwork_traced__ = True # type: ignore[attr-defined]
|
|
175
|
-
|
|
176
|
-
wrapper.
|
|
177
|
-
wrapper.run = _make_run_method(
|
|
178
|
-
wrapper.map = _make_map_method(
|
|
179
|
-
|
|
180
|
-
start_at = _make_start_at_method(wrapper, func)
|
|
181
|
-
wrapper.start_at = start_at # type: ignore[attr-defined]
|
|
182
|
-
wrapper.run_at = _make_run_at_method(start_at) # type: ignore[attr-defined]
|
|
183
|
-
|
|
184
|
-
start_in = _make_start_in_method(wrapper, func)
|
|
185
|
-
wrapper.start_in = start_in # type: ignore[attr-defined]
|
|
186
|
-
wrapper.run_in = _make_run_in_method(start_in) # type: ignore[attr-defined]
|
|
187
|
-
|
|
188
|
-
wrapper.run_every = _make_run_every_method(wrapper, func) # type: ignore[attr-defined]
|
|
182
|
+
submit = _make_submit_method(wrapper, func)
|
|
183
|
+
wrapper.submit = submit # type: ignore[attr-defined]
|
|
184
|
+
wrapper.run = _make_run_method(submit) # type: ignore[attr-defined]
|
|
185
|
+
wrapper.map = _make_map_method(submit) # type: ignore[attr-defined]
|
|
189
186
|
|
|
190
187
|
|
|
191
188
|
def _get_stdlib_dirs() -> list[str]:
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Protocol types for ``@offwork.task``-decorated functions."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime, timedelta
|
|
4
|
+
from typing import Any, TypeVar, Protocol, ParamSpec, overload
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
|
|
7
|
+
from offwork.worker.result import Result
|
|
8
|
+
from offwork.worker.schedule import ScheduleHandle
|
|
9
|
+
|
|
10
|
+
P = ParamSpec("P")
|
|
11
|
+
R = TypeVar("R")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TracedFunction(Protocol[P, R]):
|
|
15
|
+
"""A function decorated with ``@offwork.task``, with remote execution methods.
|
|
16
|
+
|
|
17
|
+
Direct call
|
|
18
|
+
-----------
|
|
19
|
+
Calling the function normally (``func(*args)``) executes it locally,
|
|
20
|
+
exactly as if ``@offwork.task`` were not present.
|
|
21
|
+
|
|
22
|
+
Remote execution
|
|
23
|
+
----------------
|
|
24
|
+
:meth:`submit`
|
|
25
|
+
Submit to a remote worker; return a :class:`~offwork.Result` handle.
|
|
26
|
+
Pass scheduling keywords to run once in the future or on a recurring
|
|
27
|
+
schedule (returns :class:`~offwork.ScheduleHandle` when *run_every*
|
|
28
|
+
is given).
|
|
29
|
+
:meth:`run`
|
|
30
|
+
Submit and ``await`` the result in one call.
|
|
31
|
+
:meth:`map`
|
|
32
|
+
Submit the same function with multiple argument-tuples in parallel.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
__offwork_traced__: bool
|
|
36
|
+
__wrapped__: Callable[P, R]
|
|
37
|
+
|
|
38
|
+
def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R: ...
|
|
39
|
+
|
|
40
|
+
# -- overload: run_every present → ScheduleHandle -------------------------
|
|
41
|
+
|
|
42
|
+
@overload
|
|
43
|
+
async def submit(
|
|
44
|
+
self,
|
|
45
|
+
*args: Any,
|
|
46
|
+
run_every: timedelta | float,
|
|
47
|
+
run_at: None = ...,
|
|
48
|
+
run_in: None = ...,
|
|
49
|
+
_start_at: datetime | None = ...,
|
|
50
|
+
run_for: timedelta | float | None = ...,
|
|
51
|
+
max_runs: int | None = ...,
|
|
52
|
+
backend: Any = ...,
|
|
53
|
+
**kwargs: Any,
|
|
54
|
+
) -> ScheduleHandle: ...
|
|
55
|
+
|
|
56
|
+
# -- overload: no run_every → Result ---------------------------------------
|
|
57
|
+
|
|
58
|
+
@overload
|
|
59
|
+
async def submit(
|
|
60
|
+
self,
|
|
61
|
+
*args: Any,
|
|
62
|
+
run_every: None = ...,
|
|
63
|
+
run_at: datetime | None = ...,
|
|
64
|
+
run_in: timedelta | float | None = ...,
|
|
65
|
+
backend: Any = ...,
|
|
66
|
+
**kwargs: Any,
|
|
67
|
+
) -> Result: ...
|
|
68
|
+
|
|
69
|
+
async def submit(self, *args: Any, **kwargs: Any) -> Result | ScheduleHandle:
|
|
70
|
+
"""Submit the function to a remote worker.
|
|
71
|
+
|
|
72
|
+
Parameters
|
|
73
|
+
----------
|
|
74
|
+
*args
|
|
75
|
+
Positional arguments forwarded to the function.
|
|
76
|
+
run_at
|
|
77
|
+
:class:`~datetime.datetime` — schedule a one-shot run at this
|
|
78
|
+
point in time.
|
|
79
|
+
run_in
|
|
80
|
+
:class:`~datetime.timedelta` or ``float`` (seconds) — schedule
|
|
81
|
+
a one-shot run after this delay.
|
|
82
|
+
run_every
|
|
83
|
+
:class:`~datetime.timedelta` or ``float`` (seconds) — run on a
|
|
84
|
+
recurring schedule at this interval. Returns a
|
|
85
|
+
:class:`~offwork.ScheduleHandle` instead of a :class:`~offwork.Result`.
|
|
86
|
+
_start_at
|
|
87
|
+
First occurrence for *run_every* schedules.
|
|
88
|
+
run_for
|
|
89
|
+
Stop recurring after this wall-clock duration (*run_every* only).
|
|
90
|
+
max_runs
|
|
91
|
+
Stop recurring after this many executions (*run_every* only).
|
|
92
|
+
backend
|
|
93
|
+
Override the global backend for this submission.
|
|
94
|
+
**kwargs
|
|
95
|
+
Keyword arguments forwarded to the function.
|
|
96
|
+
|
|
97
|
+
Returns
|
|
98
|
+
-------
|
|
99
|
+
Result
|
|
100
|
+
When called without *run_every*.
|
|
101
|
+
ScheduleHandle
|
|
102
|
+
When called with *run_every*.
|
|
103
|
+
"""
|
|
104
|
+
...
|
|
105
|
+
|
|
106
|
+
async def run(self, *args: P.args, **kwargs: P.kwargs) -> Any:
|
|
107
|
+
"""Submit and immediately ``await`` the result.
|
|
108
|
+
|
|
109
|
+
Equivalent to ``await (await func.submit(*args, **kwargs))``.
|
|
110
|
+
"""
|
|
111
|
+
...
|
|
112
|
+
|
|
113
|
+
async def map(self, args_list: list[tuple[Any, ...]], **kwargs: Any) -> list[Any]:
|
|
114
|
+
"""Submit the function for each argument-tuple and collect all results.
|
|
115
|
+
|
|
116
|
+
Equivalent to::
|
|
117
|
+
|
|
118
|
+
await asyncio.gather(*(func.run(*a, **kwargs) for a in args_list))
|
|
119
|
+
"""
|
|
120
|
+
...
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class TraceDecorator(Protocol):
|
|
124
|
+
"""The ``@offwork.task`` decorator when called with keyword arguments."""
|
|
125
|
+
|
|
126
|
+
def __call__(self, func: Callable[P, R]) -> TracedFunction[P, R]: ...
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
"""Redis-backed transport using ``RPUSH``/``BLPOP`` for tasks and results."""
|
|
2
2
|
|
|
3
|
+
import math
|
|
3
4
|
import time
|
|
4
5
|
import asyncio
|
|
5
6
|
from typing import Any
|
|
7
|
+
from urllib.parse import parse_qs, urlparse
|
|
6
8
|
from collections.abc import AsyncIterator
|
|
7
9
|
|
|
8
10
|
try:
|
|
9
11
|
import redis.asyncio as _redis
|
|
12
|
+
from redis.exceptions import TimeoutError as RedisTimeoutError
|
|
10
13
|
except ImportError:
|
|
11
14
|
raise ImportError(
|
|
12
15
|
"redis package is required for RedisBackend. "
|
|
@@ -50,7 +53,11 @@ class RedisBackend(Backend):
|
|
|
50
53
|
queue_key: str | None = None,
|
|
51
54
|
result_ttl: int | None = None,
|
|
52
55
|
) -> None:
|
|
53
|
-
|
|
56
|
+
query = parse_qs(urlparse(url).query)
|
|
57
|
+
connect_kwargs: dict[str, Any] = {}
|
|
58
|
+
if "socket_timeout" not in query:
|
|
59
|
+
connect_kwargs["socket_timeout"] = None
|
|
60
|
+
self._redis: Any = _redis.Redis.from_url(url, **connect_kwargs)
|
|
54
61
|
self._queue_key = queue_key or self.DEFAULT_QUEUE_KEY
|
|
55
62
|
self._result_ttl = result_ttl or self.DEFAULT_RESULT_TTL
|
|
56
63
|
|
|
@@ -60,7 +67,13 @@ class RedisBackend(Backend):
|
|
|
60
67
|
async def listen(self) -> AsyncIterator[str]:
|
|
61
68
|
"""Block on ``BLPOP`` and yield task JSON strings as they arrive."""
|
|
62
69
|
while True:
|
|
63
|
-
|
|
70
|
+
try:
|
|
71
|
+
result = await self._redis.blpop(self._queue_key)
|
|
72
|
+
except RedisTimeoutError:
|
|
73
|
+
task = asyncio.current_task()
|
|
74
|
+
if task is not None and task.cancelling():
|
|
75
|
+
raise asyncio.CancelledError() from None
|
|
76
|
+
continue
|
|
64
77
|
if result is None:
|
|
65
78
|
continue
|
|
66
79
|
_, raw = result
|
|
@@ -73,14 +86,34 @@ class RedisBackend(Backend):
|
|
|
73
86
|
|
|
74
87
|
async def get_result(self, task_id: str, timeout: float | None = None) -> str:
|
|
75
88
|
key = f"{self.RESULT_PREFIX}{task_id}"
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
89
|
+
deadline = None if timeout is None else time.monotonic() + max(0.0, timeout)
|
|
90
|
+
while True:
|
|
91
|
+
if deadline is None:
|
|
92
|
+
block_seconds = 0
|
|
93
|
+
else:
|
|
94
|
+
remaining = deadline - time.monotonic()
|
|
95
|
+
if remaining <= 0:
|
|
96
|
+
raise TimeoutError(
|
|
97
|
+
f"Timed out waiting for result of task {task_id}"
|
|
98
|
+
)
|
|
99
|
+
block_seconds = max(1, math.ceil(remaining))
|
|
100
|
+
try:
|
|
101
|
+
result = await self._redis.blpop(key, timeout=block_seconds)
|
|
102
|
+
except RedisTimeoutError:
|
|
103
|
+
task = asyncio.current_task()
|
|
104
|
+
if task is not None and task.cancelling():
|
|
105
|
+
raise asyncio.CancelledError() from None
|
|
106
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
107
|
+
raise TimeoutError(
|
|
108
|
+
f"Timed out waiting for result of task {task_id}"
|
|
109
|
+
) from None
|
|
110
|
+
continue
|
|
111
|
+
if result is None:
|
|
112
|
+
raise TimeoutError(
|
|
113
|
+
f"Timed out waiting for result of task {task_id}"
|
|
114
|
+
)
|
|
115
|
+
_, raw = result
|
|
116
|
+
return raw.decode() if isinstance(raw, bytes) else raw
|
|
84
117
|
|
|
85
118
|
async def try_get_result(self, task_id: str) -> str | None:
|
|
86
119
|
"""Non-blocking ``LPOP``; returns ``None`` if not yet available."""
|