offwork 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {offwork-0.1.0 → offwork-0.1.1}/PKG-INFO +29 -31
- {offwork-0.1.0 → offwork-0.1.1}/README.md +28 -30
- {offwork-0.1.0 → offwork-0.1.1}/offwork/__init__.py +2 -2
- {offwork-0.1.0 → offwork-0.1.1}/offwork/__main__.py +4 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/core/progress.py +1 -1
- {offwork-0.1.0 → offwork-0.1.1}/offwork/core/version.py +1 -1
- offwork-0.1.1/offwork/graph/__init__.py +5 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/graph/analyzer.py +10 -8
- {offwork-0.1.0 → offwork-0.1.1}/offwork/graph/decorator.py +7 -7
- {offwork-0.1.0 → offwork-0.1.1}/offwork/graph/graph.py +1 -1
- {offwork-0.1.0 → offwork-0.1.1}/offwork/graph/store.py +1 -1
- {offwork-0.1.0 → offwork-0.1.1}/offwork/typing.py +3 -3
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/backends/http.py +2 -2
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/backends/rabbitmq.py +29 -24
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/deps.py +4 -4
- {offwork-0.1.0 → offwork-0.1.1}/pyproject.toml +2 -1
- offwork-0.1.0/offwork/graph/__init__.py +0 -5
- {offwork-0.1.0 → offwork-0.1.1}/LICENSE +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/_venv.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/core/__init__.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/core/errors.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/core/models.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/core/pairing.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/core/signing.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/core/task.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/core/token.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/graph/tracing.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/py.typed +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/__init__.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/backends/__init__.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/backends/base.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/backends/local.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/backends/redis.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/remote.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/result.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/sandbox/Dockerfile +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/sandbox/__init__.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/sandbox/_protocol.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/sandbox/docker.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/sandbox/guest_agent.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/schedule.py +0 -0
- {offwork-0.1.0 → offwork-0.1.1}/offwork/worker/worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: offwork
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Distributed Python task execution via automatic function serialization
|
|
5
5
|
License: AGPL-3.0-only
|
|
6
6
|
License-File: LICENSE
|
|
@@ -29,14 +29,13 @@ Description-Content-Type: text/markdown
|
|
|
29
29
|
|
|
30
30
|
**Run any Python function on a remote worker — zero setup, zero deployment.**
|
|
31
31
|
|
|
32
|
-
[](https://pypi.org/project/offwork/)
|
|
33
|
+
[](https://www.python.org/downloads/)
|
|
33
34
|
[](LICENSE)
|
|
34
35
|
[](https://mypy-lang.org/)
|
|
35
36
|
[]()
|
|
36
37
|
|
|
37
|
-
Add `@
|
|
38
|
-
Workers reconstruct and execute everything from scratch — no shared filesystem, no deployment pipeline.
|
|
39
|
-
Missing packages are installed on the fly.
|
|
38
|
+
Add `@offwork.task` to a function. offwork captures its source, all dependencies, and all imports automatically. Workers reconstruct and execute everything from scratch — no shared filesystem, no deployment pipeline. Missing packages are installed on the fly.
|
|
40
39
|
|
|
41
40
|
## Quick start
|
|
42
41
|
|
|
@@ -46,14 +45,14 @@ pip install offwork
|
|
|
46
45
|
|
|
47
46
|
```python
|
|
48
47
|
import asyncio, math, offwork
|
|
49
|
-
|
|
48
|
+
import offwork
|
|
50
49
|
|
|
51
50
|
offwork.connect("local://localhost:9748")
|
|
52
51
|
|
|
53
52
|
def add(a, b):
|
|
54
53
|
return a + b
|
|
55
54
|
|
|
56
|
-
@
|
|
55
|
+
@offwork.task
|
|
57
56
|
def hypotenuse(a: float, b: float) -> float:
|
|
58
57
|
return math.sqrt(add(a**2, b**2))
|
|
59
58
|
|
|
@@ -63,14 +62,32 @@ async def main():
|
|
|
63
62
|
asyncio.run(main())
|
|
64
63
|
```
|
|
65
64
|
|
|
66
|
-
Only the entry point needs `@
|
|
65
|
+
Only the entry point needs `@offwork.task` — everything it calls is captured automatically.
|
|
67
66
|
|
|
68
67
|
```bash
|
|
69
68
|
offwork worker --backend local://localhost:9748 --tmp # start a worker
|
|
70
69
|
python my_script.py # → 5.0
|
|
71
70
|
```
|
|
72
71
|
|
|
73
|
-
For multi-machine, swap `local://` for `redis://` or an `https://` managed broker URL.
|
|
72
|
+
For multi-machine, swap `local://` for `redis://` or an `https://` managed broker URL.
|
|
73
|
+
|
|
74
|
+
## Features
|
|
75
|
+
|
|
76
|
+
| | |
|
|
77
|
+
|-|-|
|
|
78
|
+
| **Auto dependency capture** | Functions, classes, constants, closures — recursive AST analysis |
|
|
79
|
+
| **Package auto-install** | Workers `pip install` missing packages before execution |
|
|
80
|
+
| **Async-native** | `.run()`, `.start()`, `.map()`, `asyncio.gather` |
|
|
81
|
+
| **Retry & timeout** | `@offwork.task(timeout=30, retries=3)` with exponential backoff |
|
|
82
|
+
| **Scheduling** | `.run_in(delay)`, `.run_at(datetime)`, `.run_every(freq)` with cancellation |
|
|
83
|
+
| **Throttling** | `@offwork.task(throttle=timedelta(hours=24)/50)` — rate-limit executions |
|
|
84
|
+
| **Progress & cancellation** | `offwork.progress(3, 10)` inside tasks; `await future.cancel()` on client |
|
|
85
|
+
| **Heartbeat & stall detection** | Workers heartbeat; clients raise `TaskStalled` on silence |
|
|
86
|
+
| **Content-hash caching** | Same code = cache hit, regardless of client |
|
|
87
|
+
| **Pluggable backends** | `local://` (TCP), `redis://`, `amqp://` (RabbitMQ), `https://` (hosted) |
|
|
88
|
+
| **Docker sandbox** | Container isolation, transparent to clients |
|
|
89
|
+
| **Signed execution** | Pre-shared token or PIN pairing + HMAC-SHA256 task authentication |
|
|
90
|
+
| **Graceful shutdown** | Ctrl+C drains in-flight tasks; second Ctrl+C force-quits |
|
|
74
91
|
|
|
75
92
|
## Sandbox
|
|
76
93
|
|
|
@@ -81,7 +98,7 @@ offwork sandbox setup # build image (once)
|
|
|
81
98
|
offwork worker --backend redis://localhost:6379 --sandbox # run with isolation
|
|
82
99
|
```
|
|
83
100
|
|
|
84
|
-
See [Sandbox](docs/SANDBOX.md) for configuration
|
|
101
|
+
See [Sandbox](docs/SANDBOX.md) for configuration.
|
|
85
102
|
|
|
86
103
|
## Signing
|
|
87
104
|
|
|
@@ -89,7 +106,7 @@ Pre-shared token or PIN-based pairing + HMAC-SHA256 — workers reject untrusted
|
|
|
89
106
|
|
|
90
107
|
```bash
|
|
91
108
|
# Token-based (recommended for CI/CD)
|
|
92
|
-
offwork token generate
|
|
109
|
+
offwork token generate
|
|
93
110
|
export OFFWORK_SIGNING_TOKEN=<token> # set on client & worker
|
|
94
111
|
offwork worker --backend redis://localhost:6379 --require-signing
|
|
95
112
|
|
|
@@ -98,25 +115,7 @@ offwork worker --backend redis://localhost:6379 --pair # displays a 6-digi
|
|
|
98
115
|
offwork pair --backend redis://localhost:6379 # on client: enter the PIN
|
|
99
116
|
```
|
|
100
117
|
|
|
101
|
-
After setup, tasks are signed automatically. No client-side code changes. See [Signing & Pairing](docs/SIGNING.md)
|
|
102
|
-
|
|
103
|
-
## Features
|
|
104
|
-
|
|
105
|
-
| | |
|
|
106
|
-
|-|-|
|
|
107
|
-
| **Auto dependency capture** | Functions, classes, constants, closures — recursive AST analysis |
|
|
108
|
-
| **Package auto-install** | Workers `pip install` missing packages before execution |
|
|
109
|
-
| **Async-native** | `.run()`, `.start()`, `.map()`, `asyncio.gather` |
|
|
110
|
-
| **Retry & timeout** | `@trace(timeout=30, retries=3)` with exponential backoff |
|
|
111
|
-
| **Scheduling** | `.run_in(delay)`, `.run_at(datetime)`, `.run_every(freq)` with cancellation |
|
|
112
|
-
| **Throttling** | `@trace(throttle=timedelta(hours=24)/50)` — rate-limit executions |
|
|
113
|
-
| **Progress & cancellation** | `offwork.progress(3, 10)` inside tasks; `await future.cancel()` on client |
|
|
114
|
-
| **Heartbeat & stall detection** | Workers heartbeat; clients raise `TaskStalled` on silence |
|
|
115
|
-
| **Content-hash caching** | Same code = cache hit, regardless of client |
|
|
116
|
-
| **Pluggable backends** | `local://` (same-machine TCP), `redis://`, `amqp://` (RabbitMQ), `http://`/`https://` (hosted broker API) |
|
|
117
|
-
| **Docker sandbox** | Container isolation, transparent to clients |
|
|
118
|
-
| **Signed execution** | Pre-shared token or PIN pairing + HMAC-SHA256 task authentication |
|
|
119
|
-
| **Graceful shutdown** | Ctrl+C drains in-flight tasks; second Ctrl+C force-quits |
|
|
118
|
+
After setup, tasks are signed automatically. No client-side code changes. See [Signing & Pairing](docs/SIGNING.md).
|
|
120
119
|
|
|
121
120
|
## Documentation
|
|
122
121
|
|
|
@@ -126,7 +125,6 @@ After setup, tasks are signed automatically. No client-side code changes. See [S
|
|
|
126
125
|
| **[Technical Overview](docs/TECHNICAL_OVERVIEW.md)** | Architecture, serialization format, internals |
|
|
127
126
|
| **[Signing & Pairing](docs/SIGNING.md)** | Cryptographic task signing protocol |
|
|
128
127
|
| **[Sandbox](docs/SANDBOX.md)** | Docker container isolation |
|
|
129
|
-
| **[Cloud POC](docs/CLOUD_POC.md)** | Local FastAPI + MongoDB + Kubernetes + React prototype for managed hosting |
|
|
130
128
|
|
|
131
129
|
## Examples
|
|
132
130
|
|
|
@@ -2,14 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
**Run any Python function on a remote worker — zero setup, zero deployment.**
|
|
4
4
|
|
|
5
|
-
[](https://pypi.org/project/offwork/)
|
|
6
|
+
[](https://www.python.org/downloads/)
|
|
6
7
|
[](LICENSE)
|
|
7
8
|
[](https://mypy-lang.org/)
|
|
8
9
|
[]()
|
|
9
10
|
|
|
10
|
-
Add `@
|
|
11
|
-
Workers reconstruct and execute everything from scratch — no shared filesystem, no deployment pipeline.
|
|
12
|
-
Missing packages are installed on the fly.
|
|
11
|
+
Add `@offwork.task` to a function. offwork captures its source, all dependencies, and all imports automatically. Workers reconstruct and execute everything from scratch — no shared filesystem, no deployment pipeline. Missing packages are installed on the fly.
|
|
13
12
|
|
|
14
13
|
## Quick start
|
|
15
14
|
|
|
@@ -19,14 +18,14 @@ pip install offwork
|
|
|
19
18
|
|
|
20
19
|
```python
|
|
21
20
|
import asyncio, math, offwork
|
|
22
|
-
|
|
21
|
+
import offwork
|
|
23
22
|
|
|
24
23
|
offwork.connect("local://localhost:9748")
|
|
25
24
|
|
|
26
25
|
def add(a, b):
|
|
27
26
|
return a + b
|
|
28
27
|
|
|
29
|
-
@
|
|
28
|
+
@offwork.task
|
|
30
29
|
def hypotenuse(a: float, b: float) -> float:
|
|
31
30
|
return math.sqrt(add(a**2, b**2))
|
|
32
31
|
|
|
@@ -36,14 +35,32 @@ async def main():
|
|
|
36
35
|
asyncio.run(main())
|
|
37
36
|
```
|
|
38
37
|
|
|
39
|
-
Only the entry point needs `@
|
|
38
|
+
Only the entry point needs `@offwork.task` — everything it calls is captured automatically.
|
|
40
39
|
|
|
41
40
|
```bash
|
|
42
41
|
offwork worker --backend local://localhost:9748 --tmp # start a worker
|
|
43
42
|
python my_script.py # → 5.0
|
|
44
43
|
```
|
|
45
44
|
|
|
46
|
-
For multi-machine, swap `local://` for `redis://` or an `https://` managed broker URL.
|
|
45
|
+
For multi-machine, swap `local://` for `redis://` or an `https://` managed broker URL.
|
|
46
|
+
|
|
47
|
+
## Features
|
|
48
|
+
|
|
49
|
+
| | |
|
|
50
|
+
|-|-|
|
|
51
|
+
| **Auto dependency capture** | Functions, classes, constants, closures — recursive AST analysis |
|
|
52
|
+
| **Package auto-install** | Workers `pip install` missing packages before execution |
|
|
53
|
+
| **Async-native** | `.run()`, `.start()`, `.map()`, `asyncio.gather` |
|
|
54
|
+
| **Retry & timeout** | `@offwork.task(timeout=30, retries=3)` with exponential backoff |
|
|
55
|
+
| **Scheduling** | `.run_in(delay)`, `.run_at(datetime)`, `.run_every(freq)` with cancellation |
|
|
56
|
+
| **Throttling** | `@offwork.task(throttle=timedelta(hours=24)/50)` — rate-limit executions |
|
|
57
|
+
| **Progress & cancellation** | `offwork.progress(3, 10)` inside tasks; `await future.cancel()` on client |
|
|
58
|
+
| **Heartbeat & stall detection** | Workers heartbeat; clients raise `TaskStalled` on silence |
|
|
59
|
+
| **Content-hash caching** | Same code = cache hit, regardless of client |
|
|
60
|
+
| **Pluggable backends** | `local://` (TCP), `redis://`, `amqp://` (RabbitMQ), `https://` (hosted) |
|
|
61
|
+
| **Docker sandbox** | Container isolation, transparent to clients |
|
|
62
|
+
| **Signed execution** | Pre-shared token or PIN pairing + HMAC-SHA256 task authentication |
|
|
63
|
+
| **Graceful shutdown** | Ctrl+C drains in-flight tasks; second Ctrl+C force-quits |
|
|
47
64
|
|
|
48
65
|
## Sandbox
|
|
49
66
|
|
|
@@ -54,7 +71,7 @@ offwork sandbox setup # build image (once)
|
|
|
54
71
|
offwork worker --backend redis://localhost:6379 --sandbox # run with isolation
|
|
55
72
|
```
|
|
56
73
|
|
|
57
|
-
See [Sandbox](docs/SANDBOX.md) for configuration
|
|
74
|
+
See [Sandbox](docs/SANDBOX.md) for configuration.
|
|
58
75
|
|
|
59
76
|
## Signing
|
|
60
77
|
|
|
@@ -62,7 +79,7 @@ Pre-shared token or PIN-based pairing + HMAC-SHA256 — workers reject untrusted
|
|
|
62
79
|
|
|
63
80
|
```bash
|
|
64
81
|
# Token-based (recommended for CI/CD)
|
|
65
|
-
offwork token generate
|
|
82
|
+
offwork token generate
|
|
66
83
|
export OFFWORK_SIGNING_TOKEN=<token> # set on client & worker
|
|
67
84
|
offwork worker --backend redis://localhost:6379 --require-signing
|
|
68
85
|
|
|
@@ -71,25 +88,7 @@ offwork worker --backend redis://localhost:6379 --pair # displays a 6-digi
|
|
|
71
88
|
offwork pair --backend redis://localhost:6379 # on client: enter the PIN
|
|
72
89
|
```
|
|
73
90
|
|
|
74
|
-
After setup, tasks are signed automatically. No client-side code changes. See [Signing & Pairing](docs/SIGNING.md)
|
|
75
|
-
|
|
76
|
-
## Features
|
|
77
|
-
|
|
78
|
-
| | |
|
|
79
|
-
|-|-|
|
|
80
|
-
| **Auto dependency capture** | Functions, classes, constants, closures — recursive AST analysis |
|
|
81
|
-
| **Package auto-install** | Workers `pip install` missing packages before execution |
|
|
82
|
-
| **Async-native** | `.run()`, `.start()`, `.map()`, `asyncio.gather` |
|
|
83
|
-
| **Retry & timeout** | `@trace(timeout=30, retries=3)` with exponential backoff |
|
|
84
|
-
| **Scheduling** | `.run_in(delay)`, `.run_at(datetime)`, `.run_every(freq)` with cancellation |
|
|
85
|
-
| **Throttling** | `@trace(throttle=timedelta(hours=24)/50)` — rate-limit executions |
|
|
86
|
-
| **Progress & cancellation** | `offwork.progress(3, 10)` inside tasks; `await future.cancel()` on client |
|
|
87
|
-
| **Heartbeat & stall detection** | Workers heartbeat; clients raise `TaskStalled` on silence |
|
|
88
|
-
| **Content-hash caching** | Same code = cache hit, regardless of client |
|
|
89
|
-
| **Pluggable backends** | `local://` (same-machine TCP), `redis://`, `amqp://` (RabbitMQ), `http://`/`https://` (hosted broker API) |
|
|
90
|
-
| **Docker sandbox** | Container isolation, transparent to clients |
|
|
91
|
-
| **Signed execution** | Pre-shared token or PIN pairing + HMAC-SHA256 task authentication |
|
|
92
|
-
| **Graceful shutdown** | Ctrl+C drains in-flight tasks; second Ctrl+C force-quits |
|
|
91
|
+
After setup, tasks are signed automatically. No client-side code changes. See [Signing & Pairing](docs/SIGNING.md).
|
|
93
92
|
|
|
94
93
|
## Documentation
|
|
95
94
|
|
|
@@ -99,7 +98,6 @@ After setup, tasks are signed automatically. No client-side code changes. See [S
|
|
|
99
98
|
| **[Technical Overview](docs/TECHNICAL_OVERVIEW.md)** | Architecture, serialization format, internals |
|
|
100
99
|
| **[Signing & Pairing](docs/SIGNING.md)** | Cryptographic task signing protocol |
|
|
101
100
|
| **[Sandbox](docs/SANDBOX.md)** | Docker container isolation |
|
|
102
|
-
| **[Cloud POC](docs/CLOUD_POC.md)** | Local FastAPI + MongoDB + Kubernetes + React prototype for managed hosting |
|
|
103
101
|
|
|
104
102
|
## Examples
|
|
105
103
|
|
|
@@ -53,7 +53,7 @@ from offwork.worker.worker import Worker
|
|
|
53
53
|
from offwork.worker.worker import execute as execute
|
|
54
54
|
from offwork.worker.sandbox import DockerSandbox
|
|
55
55
|
from offwork.worker.schedule import ScheduleHandle
|
|
56
|
-
from offwork.graph.decorator import
|
|
56
|
+
from offwork.graph.decorator import task
|
|
57
57
|
from offwork.worker.backends.base import Backend
|
|
58
58
|
|
|
59
59
|
|
|
@@ -107,7 +107,7 @@ __version__: str = _VERSION
|
|
|
107
107
|
__all__ = [
|
|
108
108
|
"__version__",
|
|
109
109
|
# Primary API
|
|
110
|
-
"
|
|
110
|
+
"task",
|
|
111
111
|
"connect",
|
|
112
112
|
"disconnect",
|
|
113
113
|
"serve",
|
|
@@ -54,6 +54,8 @@ async def _run_in_tmp_venv(args: argparse.Namespace) -> None:
|
|
|
54
54
|
extras: list[str] = []
|
|
55
55
|
if args.backend and args.backend.startswith(("redis://", "rediss://")):
|
|
56
56
|
extras.append("redis")
|
|
57
|
+
if args.backend and args.backend.startswith(("amqp://", "amqps://")):
|
|
58
|
+
extras.append("rabbitmq")
|
|
57
59
|
|
|
58
60
|
async with temp_venv(install_offwork=True, extras=extras) as venv:
|
|
59
61
|
cmd = _build_worker_cmd(str(venv.python), args)
|
|
@@ -315,6 +317,8 @@ def _collect_extras(args: argparse.Namespace, script: Path) -> list[str]:
|
|
|
315
317
|
backend = os.environ.get("OFFWORK_BACKEND", "")
|
|
316
318
|
if backend.startswith(("redis://", "rediss://")) and "redis" not in extras:
|
|
317
319
|
extras.append("redis")
|
|
320
|
+
if backend.startswith(("amqp://", "amqps://")) and "rabbitmq" not in extras:
|
|
321
|
+
extras.append("rabbitmq")
|
|
318
322
|
for extra in _detect_offwork_extras(str(script)):
|
|
319
323
|
if extra not in extras:
|
|
320
324
|
extras.append(extra)
|
|
@@ -67,7 +67,7 @@ def progress(
|
|
|
67
67
|
) -> None:
|
|
68
68
|
"""Report task progress from within a running function.
|
|
69
69
|
|
|
70
|
-
Call this inside a ``@
|
|
70
|
+
Call this inside a ``@offwork.task``-decorated function to report progress
|
|
71
71
|
to the client. When called outside a worker, this is a silent no-op.
|
|
72
72
|
|
|
73
73
|
Accepts either a percentage or a current/total pair::
|
|
@@ -15,20 +15,22 @@ logger = logging.getLogger(__name__)
|
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def _is_trace_decorator(node: ast.expr) -> bool:
|
|
18
|
-
"""Return True if *node* is a ``@
|
|
19
|
-
|
|
18
|
+
"""Return True if *node* is a ``@task``, ``@task(...)``, ``@offwork.task``, or ``@offwork.task(...)`` decorator."""
|
|
19
|
+
# @task or @task(...)
|
|
20
|
+
if isinstance(node, ast.Name) and node.id == "task":
|
|
20
21
|
return True
|
|
21
|
-
if (
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
22
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "task":
|
|
23
|
+
return True
|
|
24
|
+
# @offwork.task or @offwork.task(...)
|
|
25
|
+
if isinstance(node, ast.Attribute) and node.attr == "task" and isinstance(node.value, ast.Name) and node.value.id == "offwork":
|
|
26
|
+
return True
|
|
27
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "task" and isinstance(node.func.value, ast.Name) and node.func.value.id == "offwork":
|
|
26
28
|
return True
|
|
27
29
|
return False
|
|
28
30
|
|
|
29
31
|
|
|
30
32
|
def get_function_source(func: Callable[..., object]) -> str:
|
|
31
|
-
"""Get dedented source of func with @
|
|
33
|
+
"""Get dedented source of func with @task decorator lines stripped."""
|
|
32
34
|
source = textwrap.dedent(inspect.getsource(func))
|
|
33
35
|
tree = ast.parse(source)
|
|
34
36
|
func_def = tree.body[0]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""The ``@
|
|
1
|
+
"""The ``@offwork.task`` decorator for marking functions for remote execution."""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
from datetime import timedelta
|
|
@@ -15,12 +15,12 @@ _P = ParamSpec("_P")
|
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
@overload
|
|
18
|
-
def
|
|
18
|
+
def task(func: Callable[_P, _R]) -> TracedFunction[_P, _R]: ...
|
|
19
19
|
@overload
|
|
20
|
-
def
|
|
20
|
+
def task(*, timeout: float | None = ..., retries: int = ..., retry_delay: float = ..., throttle: timedelta | float | None = ...) -> TraceDecorator: ...
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
def
|
|
23
|
+
def task(
|
|
24
24
|
func: Callable[..., object] | None = None,
|
|
25
25
|
*,
|
|
26
26
|
timeout: float | None = None,
|
|
@@ -35,10 +35,10 @@ def trace(
|
|
|
35
35
|
|
|
36
36
|
Can be used with or without arguments::
|
|
37
37
|
|
|
38
|
-
@
|
|
38
|
+
@offwork.task
|
|
39
39
|
def fast(x): ...
|
|
40
40
|
|
|
41
|
-
@
|
|
41
|
+
@offwork.task(timeout=30, retries=3)
|
|
42
42
|
def flaky(x): ...
|
|
43
43
|
"""
|
|
44
44
|
if func is not None:
|
|
@@ -74,7 +74,7 @@ def _apply_trace(
|
|
|
74
74
|
if throttle_seconds <= 0:
|
|
75
75
|
raise ValueError(f"throttle must be positive, got {throttle}")
|
|
76
76
|
|
|
77
|
-
logger.debug("@
|
|
77
|
+
logger.debug("@offwork.task applied to %s", func.__qualname__)
|
|
78
78
|
graph = Graph.default()
|
|
79
79
|
graph.register(func)
|
|
80
80
|
wrapper = graph.create_wrapper(func)
|
|
@@ -296,7 +296,7 @@ class Graph(TracingMixin):
|
|
|
296
296
|
|
|
297
297
|
@classmethod
|
|
298
298
|
def default(cls) -> "Graph":
|
|
299
|
-
"""Return the singleton default graph used by ``@
|
|
299
|
+
"""Return the singleton default graph used by ``@offwork.task``."""
|
|
300
300
|
if cls._default is None:
|
|
301
301
|
cls._default = Graph()
|
|
302
302
|
return cls._default
|
|
@@ -429,7 +429,7 @@ class Store:
|
|
|
429
429
|
# -- Serialization -------------------------------------------------------
|
|
430
430
|
|
|
431
431
|
def to_dict(self) -> dict[str, Any]:
|
|
432
|
-
"""Export as a dict in v0.
|
|
432
|
+
"""Export as a dict in v0.1.0 format."""
|
|
433
433
|
qname_to_hash = self._refs
|
|
434
434
|
|
|
435
435
|
# Build objects with closure_func_refs converted to hashes
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Protocol types for ``@
|
|
1
|
+
"""Protocol types for ``@offwork.task``-decorated functions."""
|
|
2
2
|
|
|
3
3
|
from datetime import datetime, timedelta
|
|
4
4
|
from typing import Any, TypeVar, Protocol, ParamSpec
|
|
@@ -12,7 +12,7 @@ R = TypeVar("R")
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class TracedFunction(Protocol[P, R]):
|
|
15
|
-
"""A function decorated with ``@
|
|
15
|
+
"""A function decorated with ``@offwork.task``, with remote execution methods."""
|
|
16
16
|
|
|
17
17
|
__offwork_traced__: bool
|
|
18
18
|
__wrapped__: Callable[P, R]
|
|
@@ -43,6 +43,6 @@ class TracedFunction(Protocol[P, R]):
|
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
class TraceDecorator(Protocol):
|
|
46
|
-
"""The ``@
|
|
46
|
+
"""The ``@offwork.task`` decorator when called with keyword arguments."""
|
|
47
47
|
|
|
48
48
|
def __call__(self, func: Callable[P, R]) -> TracedFunction[P, R]: ...
|
|
@@ -22,8 +22,8 @@ class HttpBackend(Backend):
|
|
|
22
22
|
The base URL can point either at the broker root or at the service root;
|
|
23
23
|
when no path is provided, ``/api/v1/broker`` is assumed.
|
|
24
24
|
|
|
25
|
-
Authentication is
|
|
26
|
-
|
|
25
|
+
Authentication is currently supported via an API key, which can be provided by
|
|
26
|
+
including ``?api_key=...`` in the URL and the backend will move it into the
|
|
27
27
|
``X-Offwork-API-Key`` request header.
|
|
28
28
|
"""
|
|
29
29
|
|
|
@@ -67,6 +67,7 @@ class RabbitMQBackend(Backend):
|
|
|
67
67
|
*,
|
|
68
68
|
task_queue: str | None = None,
|
|
69
69
|
result_ttl: int | None = None,
|
|
70
|
+
queue_namespace: str | None = None,
|
|
70
71
|
) -> None:
|
|
71
72
|
self._url = url
|
|
72
73
|
self._task_queue_name = task_queue or self.TASK_QUEUE
|
|
@@ -74,6 +75,14 @@ class RabbitMQBackend(Backend):
|
|
|
74
75
|
self._connection: Any = None
|
|
75
76
|
self._channel: Any = None
|
|
76
77
|
self._lock = asyncio.Lock()
|
|
78
|
+
ns = f"{queue_namespace}." if queue_namespace else ""
|
|
79
|
+
self._result_prefix = f"{ns}{self.RESULT_PREFIX}"
|
|
80
|
+
self._heartbeat_prefix = f"{ns}{self.HEARTBEAT_PREFIX}"
|
|
81
|
+
self._cancel_prefix = f"{ns}{self.CANCEL_PREFIX}"
|
|
82
|
+
self._progress_prefix = f"{ns}{self.PROGRESS_PREFIX}"
|
|
83
|
+
self._schedule_prefix = f"{ns}{self.SCHEDULE_PREFIX}"
|
|
84
|
+
self._throttle_prefix = f"{ns}{self.THROTTLE_PREFIX}"
|
|
85
|
+
self._notify_exchange = f"{ns}{self.NOTIFY_EXCHANGE}"
|
|
77
86
|
|
|
78
87
|
# -- connection management --------------------------------------------------
|
|
79
88
|
|
|
@@ -146,7 +155,7 @@ class RabbitMQBackend(Backend):
|
|
|
146
155
|
redeclares it with the new arguments.
|
|
147
156
|
"""
|
|
148
157
|
try:
|
|
149
|
-
return await channel.declare_queue(name, arguments=arguments)
|
|
158
|
+
return await channel.declare_queue(name, durable=True, arguments=arguments)
|
|
150
159
|
except Exception:
|
|
151
160
|
# Channel is closed by RabbitMQ after PRECONDITION_FAILED.
|
|
152
161
|
# Reopen a fresh channel, purge the stale queue, and retry.
|
|
@@ -157,7 +166,7 @@ class RabbitMQBackend(Backend):
|
|
|
157
166
|
except Exception:
|
|
158
167
|
self._channel = None
|
|
159
168
|
channel = await self._ensure_channel()
|
|
160
|
-
return await channel.declare_queue(name, arguments=arguments)
|
|
169
|
+
return await channel.declare_queue(name, durable=True, arguments=arguments)
|
|
161
170
|
|
|
162
171
|
async def _kv_put(
|
|
163
172
|
self, prefix: str, task_id: str, value: str, ttl_s: int,
|
|
@@ -229,8 +238,8 @@ class RabbitMQBackend(Backend):
|
|
|
229
238
|
async def send_result(self, task_id: str, result_json: str) -> None:
|
|
230
239
|
async with self._lock:
|
|
231
240
|
channel = await self._ensure_channel()
|
|
232
|
-
name = f"{self.
|
|
233
|
-
await channel.declare_queue(name, arguments=self._result_args())
|
|
241
|
+
name = f"{self._result_prefix}{task_id}"
|
|
242
|
+
await channel.declare_queue(name, durable=True, arguments=self._result_args())
|
|
234
243
|
await channel.default_exchange.publish(
|
|
235
244
|
aio_pika.Message(result_json.encode()),
|
|
236
245
|
routing_key=name,
|
|
@@ -239,9 +248,9 @@ class RabbitMQBackend(Backend):
|
|
|
239
248
|
async def get_result(self, task_id: str, timeout: float | None = None) -> str:
|
|
240
249
|
channel = await self._new_channel()
|
|
241
250
|
try:
|
|
242
|
-
name = f"{self.
|
|
251
|
+
name = f"{self._result_prefix}{task_id}"
|
|
243
252
|
queue = await channel.declare_queue(
|
|
244
|
-
name, arguments=self._result_args(),
|
|
253
|
+
name, durable=True, arguments=self._result_args(),
|
|
245
254
|
)
|
|
246
255
|
future: asyncio.Future[str] = asyncio.get_running_loop().create_future()
|
|
247
256
|
|
|
@@ -270,9 +279,9 @@ class RabbitMQBackend(Backend):
|
|
|
270
279
|
async def try_get_result(self, task_id: str) -> str | None:
|
|
271
280
|
async with self._lock:
|
|
272
281
|
channel = await self._ensure_channel()
|
|
273
|
-
name = f"{self.
|
|
282
|
+
name = f"{self._result_prefix}{task_id}"
|
|
274
283
|
queue = await channel.declare_queue(
|
|
275
|
-
name, arguments=self._result_args(),
|
|
284
|
+
name, durable=True, arguments=self._result_args(),
|
|
276
285
|
)
|
|
277
286
|
msg = await queue.get(fail=False)
|
|
278
287
|
if msg is None:
|
|
@@ -285,17 +294,13 @@ class RabbitMQBackend(Backend):
|
|
|
285
294
|
|
|
286
295
|
async def send_heartbeat(self, task_id: str) -> None:
|
|
287
296
|
await self._kv_put(
|
|
288
|
-
self.
|
|
297
|
+
self._heartbeat_prefix, task_id,
|
|
289
298
|
str(time.time()), self.HEARTBEAT_TTL,
|
|
290
299
|
)
|
|
291
300
|
|
|
292
301
|
async def get_heartbeat(self, task_id: str) -> float | None:
|
|
293
|
-
# Consume (ack) the heartbeat rather than peeking. This avoids a
|
|
294
|
-
# RabbitMQ race where nack+requeue bypasses x-max-length=1 and causes
|
|
295
|
-
# the stale heartbeat to be returned on every subsequent poll, making
|
|
296
|
-
# stall detection fire spuriously.
|
|
297
302
|
raw = await self._kv_get(
|
|
298
|
-
self.
|
|
303
|
+
self._heartbeat_prefix, task_id, self.HEARTBEAT_TTL, peek=True,
|
|
299
304
|
)
|
|
300
305
|
return float(raw) if raw is not None else None
|
|
301
306
|
|
|
@@ -303,12 +308,12 @@ class RabbitMQBackend(Backend):
|
|
|
303
308
|
|
|
304
309
|
async def cancel_task(self, task_id: str) -> None:
|
|
305
310
|
await self._kv_put(
|
|
306
|
-
self.
|
|
311
|
+
self._cancel_prefix, task_id, "1", self.CANCEL_TTL,
|
|
307
312
|
)
|
|
308
313
|
|
|
309
314
|
async def is_cancelled(self, task_id: str) -> bool:
|
|
310
315
|
raw = await self._kv_get(
|
|
311
|
-
self.
|
|
316
|
+
self._cancel_prefix, task_id, self.CANCEL_TTL, peek=True,
|
|
312
317
|
)
|
|
313
318
|
return raw is not None
|
|
314
319
|
|
|
@@ -316,24 +321,24 @@ class RabbitMQBackend(Backend):
|
|
|
316
321
|
|
|
317
322
|
async def send_progress(self, task_id: str, progress_json: str) -> None:
|
|
318
323
|
await self._kv_put(
|
|
319
|
-
self.
|
|
324
|
+
self._progress_prefix, task_id, progress_json, self.PROGRESS_TTL,
|
|
320
325
|
)
|
|
321
326
|
|
|
322
327
|
async def get_progress(self, task_id: str) -> str | None:
|
|
323
328
|
return await self._kv_get(
|
|
324
|
-
self.
|
|
329
|
+
self._progress_prefix, task_id, self.PROGRESS_TTL, peek=True,
|
|
325
330
|
)
|
|
326
331
|
|
|
327
332
|
# -- Schedule cancellation -------------------------------------------------
|
|
328
333
|
|
|
329
334
|
async def cancel_schedule(self, schedule_id: str) -> None:
|
|
330
335
|
await self._kv_put(
|
|
331
|
-
self.
|
|
336
|
+
self._schedule_prefix, schedule_id, "1", self.SCHEDULE_TTL,
|
|
332
337
|
)
|
|
333
338
|
|
|
334
339
|
async def is_schedule_cancelled(self, schedule_id: str) -> bool:
|
|
335
340
|
raw = await self._kv_get(
|
|
336
|
-
self.
|
|
341
|
+
self._schedule_prefix, schedule_id, self.SCHEDULE_TTL, peek=True,
|
|
337
342
|
)
|
|
338
343
|
return raw is not None
|
|
339
344
|
|
|
@@ -352,7 +357,7 @@ class RabbitMQBackend(Backend):
|
|
|
352
357
|
# message body. Function names can contain characters rejected by
|
|
353
358
|
# the AMQP queue-name grammar (e.g. ``<locals>``), so we hash them.
|
|
354
359
|
raw = await self._kv_get(
|
|
355
|
-
self.
|
|
360
|
+
self._throttle_prefix, self._safe_suffix(function_name),
|
|
356
361
|
self.THROTTLE_QUEUE_TTL, peek=True,
|
|
357
362
|
)
|
|
358
363
|
if raw is None:
|
|
@@ -364,7 +369,7 @@ class RabbitMQBackend(Backend):
|
|
|
364
369
|
) -> None:
|
|
365
370
|
expiry = time.time() + throttle_seconds
|
|
366
371
|
await self._kv_put(
|
|
367
|
-
self.
|
|
372
|
+
self._throttle_prefix, self._safe_suffix(function_name),
|
|
368
373
|
str(expiry), self.THROTTLE_QUEUE_TTL,
|
|
369
374
|
)
|
|
370
375
|
|
|
@@ -374,7 +379,7 @@ class RabbitMQBackend(Backend):
|
|
|
374
379
|
async with self._lock:
|
|
375
380
|
channel = await self._ensure_channel()
|
|
376
381
|
exchange = await channel.declare_exchange(
|
|
377
|
-
self.
|
|
382
|
+
self._notify_exchange, aio_pika.ExchangeType.FANOUT,
|
|
378
383
|
)
|
|
379
384
|
await exchange.publish(
|
|
380
385
|
aio_pika.Message(task_id.encode()),
|
|
@@ -385,7 +390,7 @@ class RabbitMQBackend(Backend):
|
|
|
385
390
|
channel = await self._new_channel()
|
|
386
391
|
try:
|
|
387
392
|
exchange = await channel.declare_exchange(
|
|
388
|
-
self.
|
|
393
|
+
self._notify_exchange, aio_pika.ExchangeType.FANOUT,
|
|
389
394
|
)
|
|
390
395
|
queue = await channel.declare_queue(exclusive=True)
|
|
391
396
|
await queue.bind(exchange)
|
|
@@ -39,7 +39,7 @@ def install_package_as(package: str) -> Iterator[None]:
|
|
|
39
39
|
"""Declare the pip package name for imports inside this block.
|
|
40
40
|
|
|
41
41
|
At runtime this is a no-op -- the import executes normally. The
|
|
42
|
-
``@
|
|
42
|
+
``@offwork.task`` analyzer detects the ``with`` block in the AST and records
|
|
43
43
|
the *package* name on every :class:`ImportInfo` inside it, so the
|
|
44
44
|
worker knows which pip package to install.
|
|
45
45
|
|
|
@@ -64,7 +64,7 @@ class _WorkerOnlyStub(types.ModuleType):
|
|
|
64
64
|
def _err(self) -> WorkerOnlyError:
|
|
65
65
|
return WorkerOnlyError(
|
|
66
66
|
f"'{self.__name__}' was imported with worker_only_import; "
|
|
67
|
-
"it must only be used inside a @
|
|
67
|
+
"it must only be used inside a @offwork.task function executed on a worker"
|
|
68
68
|
)
|
|
69
69
|
|
|
70
70
|
def __getattr__(self, name: str) -> Any:
|
|
@@ -86,7 +86,7 @@ class _StubAttr:
|
|
|
86
86
|
def _err(self) -> WorkerOnlyError:
|
|
87
87
|
return WorkerOnlyError(
|
|
88
88
|
f"'{self._qualname}' was imported with worker_only_import; "
|
|
89
|
-
"it must only be used inside a @
|
|
89
|
+
"it must only be used inside a @offwork.task function executed on a worker"
|
|
90
90
|
)
|
|
91
91
|
|
|
92
92
|
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
|
@@ -183,7 +183,7 @@ def worker_only_import(package: str | None = None) -> Iterator[None]:
|
|
|
183
183
|
"""Skip installing packages locally; the worker installs them on demand.
|
|
184
184
|
|
|
185
185
|
Imports inside this block resolve to lightweight stubs on the client.
|
|
186
|
-
The ``@
|
|
186
|
+
The ``@offwork.task`` analyzer records the imports as worker-only, and the
|
|
187
187
|
worker installs them via pip before reconstructing the function.
|
|
188
188
|
|
|
189
189
|
The optional *package* argument overrides the pip package name (same
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "offwork"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.1"
|
|
4
4
|
description = "Distributed Python task execution via automatic function serialization"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
@@ -40,6 +40,7 @@ dev = [
|
|
|
40
40
|
"mypy (>=1.19.1,<2.0.0)",
|
|
41
41
|
"ipython (>=9.11.0,<10.0.0)",
|
|
42
42
|
"isort (>=8.0.1,<9.0.0)",
|
|
43
|
+
"twine (>=6.2.0,<7.0.0)",
|
|
43
44
|
]
|
|
44
45
|
|
|
45
46
|
[tool.pytest.ini_options]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|