offwork 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- offwork/__init__.py +167 -0
- offwork/__main__.py +770 -0
- offwork/_venv.py +174 -0
- offwork/core/__init__.py +15 -0
- offwork/core/errors.py +83 -0
- offwork/core/models.py +174 -0
- offwork/core/pairing.py +389 -0
- offwork/core/progress.py +91 -0
- offwork/core/signing.py +91 -0
- offwork/core/task.py +520 -0
- offwork/core/token.py +184 -0
- offwork/core/version.py +10 -0
- offwork/graph/__init__.py +5 -0
- offwork/graph/analyzer.py +637 -0
- offwork/graph/decorator.py +87 -0
- offwork/graph/graph.py +995 -0
- offwork/graph/store.py +500 -0
- offwork/graph/tracing.py +429 -0
- offwork/py.typed +0 -0
- offwork/typing.py +48 -0
- offwork/worker/__init__.py +18 -0
- offwork/worker/backends/__init__.py +3 -0
- offwork/worker/backends/base.py +149 -0
- offwork/worker/backends/http.py +237 -0
- offwork/worker/backends/local.py +452 -0
- offwork/worker/backends/rabbitmq.py +410 -0
- offwork/worker/backends/redis.py +175 -0
- offwork/worker/deps.py +365 -0
- offwork/worker/remote.py +793 -0
- offwork/worker/result.py +276 -0
- offwork/worker/sandbox/Dockerfile +24 -0
- offwork/worker/sandbox/__init__.py +18 -0
- offwork/worker/sandbox/_protocol.py +50 -0
- offwork/worker/sandbox/docker.py +438 -0
- offwork/worker/sandbox/guest_agent.py +622 -0
- offwork/worker/schedule.py +26 -0
- offwork/worker/worker.py +263 -0
- offwork-0.4.0.dist-info/METADATA +143 -0
- offwork-0.4.0.dist-info/RECORD +42 -0
- offwork-0.4.0.dist-info/WHEEL +4 -0
- offwork-0.4.0.dist-info/entry_points.txt +3 -0
- offwork-0.4.0.dist-info/licenses/LICENSE +661 -0
offwork/__main__.py
ADDED
|
@@ -0,0 +1,770 @@
|
|
|
1
|
+
"""CLI entrypoint for offwork.
|
|
2
|
+
|
|
3
|
+
Usage::
|
|
4
|
+
|
|
5
|
+
offwork worker --backend redis://localhost:6379
|
|
6
|
+
offwork worker --backend redis://localhost:6379 --tmp
|
|
7
|
+
offwork worker --backend redis://localhost:6379 --require-signing
|
|
8
|
+
offwork pair --backend redis://localhost:6379 --role worker
|
|
9
|
+
offwork run examples/script.py
|
|
10
|
+
offwork info
|
|
11
|
+
offwork serialize mymodule:csv_to_json
|
|
12
|
+
offwork reconstruct graph.json csv_to_json
|
|
13
|
+
"""
|
|
14
|
+
import os
|
|
15
|
+
import ast
|
|
16
|
+
import sys
|
|
17
|
+
import shutil
|
|
18
|
+
import signal
|
|
19
|
+
import asyncio
|
|
20
|
+
import logging
|
|
21
|
+
import argparse
|
|
22
|
+
import importlib
|
|
23
|
+
import importlib.machinery
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from collections.abc import Callable
|
|
26
|
+
from importlib.metadata import version as pkg_version
|
|
27
|
+
|
|
28
|
+
from offwork import serialize, reconstruct
|
|
29
|
+
from offwork._venv import temp_venv
|
|
30
|
+
from offwork.worker.deps import DEFAULT_IMPORT_TO_PACKAGE
|
|
31
|
+
from offwork.worker.remote import serve
|
|
32
|
+
from offwork.graph.analyzer import _parse_install_package_as, _parse_worker_only_import
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _build_worker_cmd(python: str, args: argparse.Namespace) -> list[str]:
|
|
36
|
+
"""Rebuild the worker CLI command for re-exec, without --tmp."""
|
|
37
|
+
cmd = [python, "-m", "offwork", "worker", "--backend", args.backend]
|
|
38
|
+
cmd.extend(["-c", str(args.concurrency)])
|
|
39
|
+
if args.no_auto_install:
|
|
40
|
+
cmd.append("--no-auto-install")
|
|
41
|
+
if args.verbose:
|
|
42
|
+
cmd.append("--verbose")
|
|
43
|
+
if args.log_level:
|
|
44
|
+
cmd.extend(["--log-level", args.log_level])
|
|
45
|
+
if args.require_signing:
|
|
46
|
+
cmd.append("--require-signing")
|
|
47
|
+
if args.sandbox:
|
|
48
|
+
cmd.append("--sandbox")
|
|
49
|
+
return cmd
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def _run_in_tmp_venv(args: argparse.Namespace) -> None:
|
|
53
|
+
"""Create a temporary venv and re-exec the worker inside it."""
|
|
54
|
+
extras: list[str] = []
|
|
55
|
+
if args.backend and args.backend.startswith(("redis://", "rediss://")):
|
|
56
|
+
extras.append("redis")
|
|
57
|
+
|
|
58
|
+
async with temp_venv(install_offwork=True, extras=extras) as venv:
|
|
59
|
+
cmd = _build_worker_cmd(str(venv.python), args)
|
|
60
|
+
print("Starting worker in temporary venv...", file=sys.stderr)
|
|
61
|
+
await _run_subprocess_async(cmd)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _resolve_log_level(args: argparse.Namespace) -> int:
|
|
65
|
+
"""Determine the log level from CLI arguments."""
|
|
66
|
+
if args.log_level:
|
|
67
|
+
level = getattr(logging, args.log_level.upper(), None)
|
|
68
|
+
if level is None:
|
|
69
|
+
print(f"Error: invalid log level {args.log_level!r}", file=sys.stderr)
|
|
70
|
+
sys.exit(1)
|
|
71
|
+
return level # type: ignore[no-any-return]
|
|
72
|
+
if args.verbose:
|
|
73
|
+
return logging.DEBUG
|
|
74
|
+
return logging.INFO
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _configure_logging(level: int) -> None:
|
|
78
|
+
"""Set up offwork logger with a stderr handler."""
|
|
79
|
+
handler = logging.StreamHandler(sys.stderr)
|
|
80
|
+
handler.setFormatter(logging.Formatter(
|
|
81
|
+
"%(asctime)s %(levelname)-7s %(message)s", datefmt="%H:%M:%S",
|
|
82
|
+
))
|
|
83
|
+
offwork_logger = logging.getLogger("offwork")
|
|
84
|
+
offwork_logger.setLevel(level)
|
|
85
|
+
offwork_logger.addHandler(handler)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _cmd_worker(args: argparse.Namespace) -> None:
|
|
89
|
+
if not args.backend:
|
|
90
|
+
print("Error: --backend is required (or set OFFWORK_BACKEND).", file=sys.stderr)
|
|
91
|
+
sys.exit(1)
|
|
92
|
+
|
|
93
|
+
if args.tmp:
|
|
94
|
+
asyncio.run(_run_in_tmp_venv(args))
|
|
95
|
+
return
|
|
96
|
+
|
|
97
|
+
_configure_logging(_resolve_log_level(args))
|
|
98
|
+
|
|
99
|
+
if args.pair:
|
|
100
|
+
asyncio.run(_pair_then_serve(args))
|
|
101
|
+
return
|
|
102
|
+
|
|
103
|
+
asyncio.run(serve(
|
|
104
|
+
args.backend,
|
|
105
|
+
concurrency=args.concurrency,
|
|
106
|
+
auto_install=not args.no_auto_install,
|
|
107
|
+
sandbox=bool(args.sandbox),
|
|
108
|
+
require_signing=bool(args.require_signing),
|
|
109
|
+
))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
async def _pair_then_serve(args: argparse.Namespace) -> None:
|
|
113
|
+
"""Generate a PIN, pair with a client, then start serving with signing."""
|
|
114
|
+
from offwork.core.pairing import generate_pin, save_shared_key, initiate_pairing
|
|
115
|
+
from offwork.worker.remote import connect, disconnect
|
|
116
|
+
|
|
117
|
+
backend = connect(args.backend)
|
|
118
|
+
|
|
119
|
+
pin = generate_pin()
|
|
120
|
+
print(f"\n Pairing PIN: {pin}\n")
|
|
121
|
+
print(" Enter this PIN on the client with:")
|
|
122
|
+
print(f" offwork pair --backend {args.backend}")
|
|
123
|
+
print("\n Waiting for client...\n")
|
|
124
|
+
|
|
125
|
+
try:
|
|
126
|
+
result = await initiate_pairing(backend, pin, timeout=60.0)
|
|
127
|
+
except Exception as exc:
|
|
128
|
+
print(f" ✗ Pairing failed: {exc}", file=sys.stderr)
|
|
129
|
+
await disconnect()
|
|
130
|
+
sys.exit(1)
|
|
131
|
+
|
|
132
|
+
save_shared_key(result.shared_key, "worker")
|
|
133
|
+
print(f" ✓ Paired successfully. Key saved to ~/.offwork/worker.key")
|
|
134
|
+
print(f" Starting worker with signing enabled...\n")
|
|
135
|
+
await disconnect()
|
|
136
|
+
|
|
137
|
+
await serve(
|
|
138
|
+
args.backend,
|
|
139
|
+
concurrency=args.concurrency,
|
|
140
|
+
auto_install=not args.no_auto_install,
|
|
141
|
+
sandbox=bool(args.sandbox),
|
|
142
|
+
require_signing=True,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _cmd_info(_args: argparse.Namespace) -> None:
|
|
147
|
+
try:
|
|
148
|
+
ver = pkg_version("offwork")
|
|
149
|
+
except Exception:
|
|
150
|
+
ver = "unknown"
|
|
151
|
+
|
|
152
|
+
print(f"offwork {ver}")
|
|
153
|
+
print(f" OFFWORK_BACKEND = {os.environ.get('OFFWORK_BACKEND', '(not set)')}")
|
|
154
|
+
|
|
155
|
+
for dep in ("redis",):
|
|
156
|
+
try:
|
|
157
|
+
dep_ver = pkg_version(dep)
|
|
158
|
+
print(f" {dep}: {dep_ver}")
|
|
159
|
+
except Exception:
|
|
160
|
+
print(f" {dep}: not installed")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _import_target(target: str) -> Callable[..., object]:
|
|
164
|
+
"""Import ``module:function`` and return the callable."""
|
|
165
|
+
if ":" not in target:
|
|
166
|
+
print(
|
|
167
|
+
f"Error: target must be 'module:function', got {target!r}",
|
|
168
|
+
file=sys.stderr,
|
|
169
|
+
)
|
|
170
|
+
sys.exit(1)
|
|
171
|
+
|
|
172
|
+
module_path, func_name = target.rsplit(":", 1)
|
|
173
|
+
mod = importlib.import_module(module_path)
|
|
174
|
+
func: Callable[..., object] | None = getattr(mod, func_name, None)
|
|
175
|
+
if func is None:
|
|
176
|
+
print(
|
|
177
|
+
f"Error: {func_name!r} not found in module {module_path!r}",
|
|
178
|
+
file=sys.stderr,
|
|
179
|
+
)
|
|
180
|
+
sys.exit(1)
|
|
181
|
+
return func
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _cmd_serialize(args: argparse.Namespace) -> None:
|
|
185
|
+
func = _import_target(args.target)
|
|
186
|
+
print(serialize(func))
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _cmd_reconstruct(args: argparse.Namespace) -> None:
|
|
190
|
+
graph_json = Path(args.graph_file).read_text()
|
|
191
|
+
print(reconstruct(graph_json, args.function))
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _parse_script(script: str) -> tuple[str, ast.Module | None]:
|
|
195
|
+
"""Read and parse a script, returning (source, ast_tree_or_None)."""
|
|
196
|
+
source = Path(script).read_text()
|
|
197
|
+
try:
|
|
198
|
+
return source, ast.parse(source)
|
|
199
|
+
except SyntaxError:
|
|
200
|
+
return source, None
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _is_local_package(module_name: str, script_dir: str) -> bool:
|
|
204
|
+
"""Return True if *module_name* is importable from the script's runtime path.
|
|
205
|
+
|
|
206
|
+
The script will be launched with ``cwd`` and its own directory on
|
|
207
|
+
``sys.path`` (see ``_build_script_env``); we ask Python's path-based finder
|
|
208
|
+
whether the name resolves on exactly that list. We do not mutate
|
|
209
|
+
``sys.path`` or ``sys.modules``.
|
|
210
|
+
|
|
211
|
+
If the module isn't found, we leave it alone and let the script fail at
|
|
212
|
+
runtime with the standard ``ModuleNotFoundError`` — that error names the
|
|
213
|
+
missing module and is the clearest signal to the user that their layout or
|
|
214
|
+
invocation directory is wrong.
|
|
215
|
+
"""
|
|
216
|
+
search_path = [script_dir, str(Path.cwd())]
|
|
217
|
+
try:
|
|
218
|
+
spec = importlib.machinery.PathFinder.find_spec(module_name, search_path)
|
|
219
|
+
except (ImportError, ValueError):
|
|
220
|
+
return False
|
|
221
|
+
return spec is not None
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _extract_top_modules(node: ast.AST) -> list[str]:
|
|
225
|
+
"""Extract top-level module names from an Import or ImportFrom node."""
|
|
226
|
+
if isinstance(node, ast.Import):
|
|
227
|
+
return [alias.name.split(".")[0] for alias in node.names]
|
|
228
|
+
if isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
|
|
229
|
+
return [node.module.split(".")[0]]
|
|
230
|
+
return []
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _detect_script_packages(script: str) -> list[str]:
|
|
234
|
+
"""Parse a script file and return pip package names for third-party imports."""
|
|
235
|
+
_source, tree = _parse_script(script)
|
|
236
|
+
if tree is None:
|
|
237
|
+
return []
|
|
238
|
+
|
|
239
|
+
script_dir = str(Path(script).resolve().parent)
|
|
240
|
+
|
|
241
|
+
# module name -> pip package name (None means use default mapping)
|
|
242
|
+
modules: dict[str, str | None] = {}
|
|
243
|
+
skip: set[str] = set()
|
|
244
|
+
for node in ast.iter_child_nodes(tree):
|
|
245
|
+
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
|
246
|
+
for m in _extract_top_modules(node):
|
|
247
|
+
modules.setdefault(m, None)
|
|
248
|
+
elif isinstance(node, ast.With):
|
|
249
|
+
package = _parse_install_package_as(node)
|
|
250
|
+
if package is not None:
|
|
251
|
+
for child in node.body:
|
|
252
|
+
for m in _extract_top_modules(child):
|
|
253
|
+
modules[m] = package
|
|
254
|
+
continue
|
|
255
|
+
if _parse_worker_only_import(node) is not False:
|
|
256
|
+
for child in node.body:
|
|
257
|
+
for m in _extract_top_modules(child):
|
|
258
|
+
skip.add(m)
|
|
259
|
+
|
|
260
|
+
packages: dict[str, None] = {}
|
|
261
|
+
for m, explicit_package in sorted(modules.items()):
|
|
262
|
+
if m in sys.stdlib_module_names or m == "offwork":
|
|
263
|
+
continue
|
|
264
|
+
if m in skip:
|
|
265
|
+
continue
|
|
266
|
+
if _is_local_package(m, script_dir):
|
|
267
|
+
continue
|
|
268
|
+
pkg = explicit_package or DEFAULT_IMPORT_TO_PACKAGE.get(m, m)
|
|
269
|
+
packages.setdefault(pkg, None)
|
|
270
|
+
return list(packages)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _is_connect_or_serve_call(node: ast.Call) -> bool:
|
|
274
|
+
"""Return True if *node* calls ``connect`` or ``serve``."""
|
|
275
|
+
func = node.func
|
|
276
|
+
if isinstance(func, ast.Attribute):
|
|
277
|
+
return func.attr in ("connect", "serve")
|
|
278
|
+
if isinstance(func, ast.Name):
|
|
279
|
+
return func.id in ("connect", "serve")
|
|
280
|
+
return False
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _first_arg_is_redis_url(node: ast.Call) -> bool:
|
|
284
|
+
"""Return True if the first positional arg is a redis:// string literal."""
|
|
285
|
+
if not node.args:
|
|
286
|
+
return False
|
|
287
|
+
first_arg = node.args[0]
|
|
288
|
+
return (
|
|
289
|
+
isinstance(first_arg, ast.Constant)
|
|
290
|
+
and isinstance(first_arg.value, str)
|
|
291
|
+
and first_arg.value.startswith(("redis://", "rediss://"))
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _detect_offwork_extras(script: str) -> list[str]:
|
|
296
|
+
"""Detect offwork extras needed by a script (e.g. redis from connect/serve calls)."""
|
|
297
|
+
_source, tree = _parse_script(script)
|
|
298
|
+
if tree is None:
|
|
299
|
+
return []
|
|
300
|
+
|
|
301
|
+
extras: list[str] = []
|
|
302
|
+
for node in ast.walk(tree):
|
|
303
|
+
if not isinstance(node, ast.Call):
|
|
304
|
+
continue
|
|
305
|
+
if _is_connect_or_serve_call(node) and _first_arg_is_redis_url(node):
|
|
306
|
+
if "redis" not in extras:
|
|
307
|
+
extras.append("redis")
|
|
308
|
+
|
|
309
|
+
return extras
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _collect_extras(args: argparse.Namespace, script: Path) -> list[str]:
|
|
313
|
+
"""Gather offwork extras and third-party packages for a script."""
|
|
314
|
+
extras: list[str] = list(args.extra or [])
|
|
315
|
+
backend = os.environ.get("OFFWORK_BACKEND", "")
|
|
316
|
+
if backend.startswith(("redis://", "rediss://")) and "redis" not in extras:
|
|
317
|
+
extras.append("redis")
|
|
318
|
+
for extra in _detect_offwork_extras(str(script)):
|
|
319
|
+
if extra not in extras:
|
|
320
|
+
extras.append(extra)
|
|
321
|
+
return extras
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _build_script_env() -> dict[str, str]:
|
|
325
|
+
"""Build env dict with cwd prepended to PYTHONPATH."""
|
|
326
|
+
env = os.environ.copy()
|
|
327
|
+
cwd = os.getcwd()
|
|
328
|
+
existing = env.get("PYTHONPATH", "")
|
|
329
|
+
env["PYTHONPATH"] = cwd if not existing else f"{cwd}{os.pathsep}{existing}"
|
|
330
|
+
return env
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
async def _run_subprocess_async(
|
|
334
|
+
cmd: list[str], env: dict[str, str] | None = None
|
|
335
|
+
) -> None:
|
|
336
|
+
"""Run a subprocess, forwarding signals and exiting with its return code."""
|
|
337
|
+
proc = await asyncio.create_subprocess_exec(*cmd, env=env)
|
|
338
|
+
|
|
339
|
+
loop = asyncio.get_running_loop()
|
|
340
|
+
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
341
|
+
try:
|
|
342
|
+
loop.add_signal_handler(sig, proc.send_signal, sig)
|
|
343
|
+
except (NotImplementedError, RuntimeError):
|
|
344
|
+
pass # Windows or no running loop
|
|
345
|
+
|
|
346
|
+
returncode = await proc.wait()
|
|
347
|
+
sys.exit(returncode)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
async def _cmd_run_async(args: argparse.Namespace) -> None:
|
|
351
|
+
script = Path(args.script).resolve()
|
|
352
|
+
if not script.exists():
|
|
353
|
+
print(f"Error: script not found: {script}", file=sys.stderr)
|
|
354
|
+
sys.exit(1)
|
|
355
|
+
|
|
356
|
+
extras = _collect_extras(args, script)
|
|
357
|
+
detected = _detect_script_packages(str(script))
|
|
358
|
+
|
|
359
|
+
async with temp_venv(install_offwork=True, extras=extras) as venv:
|
|
360
|
+
if detected:
|
|
361
|
+
print(f"Installing detected dependencies: {', '.join(detected)}", file=sys.stderr)
|
|
362
|
+
await venv.pip_install(*detected, extra_args=["--quiet"])
|
|
363
|
+
|
|
364
|
+
script_args = list(args.script_args or [])
|
|
365
|
+
if script_args and script_args[0] == "--":
|
|
366
|
+
script_args = script_args[1:]
|
|
367
|
+
|
|
368
|
+
cmd = [str(venv.python), str(script), *script_args]
|
|
369
|
+
await _run_subprocess_async(cmd, env=_build_script_env())
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _cmd_run(args: argparse.Namespace) -> None:
|
|
373
|
+
asyncio.run(_cmd_run_async(args))
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _add_worker_parser(sub: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
377
|
+
p = sub.add_parser("worker", help="Start a offwork worker")
|
|
378
|
+
p.add_argument(
|
|
379
|
+
"--backend",
|
|
380
|
+
default=os.environ.get("OFFWORK_BACKEND"),
|
|
381
|
+
help="Backend URL, e.g. redis://localhost:6379 (default: $OFFWORK_BACKEND)",
|
|
382
|
+
)
|
|
383
|
+
p.add_argument(
|
|
384
|
+
"-c", "--concurrency", type=int, default=1,
|
|
385
|
+
help="Number of concurrent worker tasks (default: 1)",
|
|
386
|
+
)
|
|
387
|
+
p.add_argument("--no-auto-install", action="store_true",
|
|
388
|
+
help="Disable automatic pip dependency installation")
|
|
389
|
+
p.add_argument("--tmp", action="store_true",
|
|
390
|
+
help="Run the worker in a temporary virtual environment (deleted on exit)")
|
|
391
|
+
p.add_argument("--sandbox", action="store_true", default=False,
|
|
392
|
+
help="Run function execution inside an isolated Docker sandbox.")
|
|
393
|
+
p.add_argument("--require-signing", action="store_true", default=False,
|
|
394
|
+
help="Only accept tasks with valid HMAC signatures from paired clients.")
|
|
395
|
+
p.add_argument("--pair", action="store_true", default=False,
|
|
396
|
+
help="Generate a pairing PIN, wait for a client to pair, then start "
|
|
397
|
+
"serving with signing enabled.")
|
|
398
|
+
p.add_argument("-v", "--verbose", action="store_true",
|
|
399
|
+
help="Enable debug logging")
|
|
400
|
+
p.add_argument("--log-level", default=None, metavar="LEVEL",
|
|
401
|
+
help="Set log level (DEBUG, INFO, WARNING, ERROR)")
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _add_run_parser(sub: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
405
|
+
p = sub.add_parser(
|
|
406
|
+
"run", help="Run a Python script in a temporary venv with offwork installed",
|
|
407
|
+
)
|
|
408
|
+
p.add_argument("script", help="Path to the Python script to run")
|
|
409
|
+
p.add_argument("-e", "--extra", action="append", default=[],
|
|
410
|
+
help="Extra pip package to install (repeatable)")
|
|
411
|
+
p.add_argument("script_args", nargs=argparse.REMAINDER,
|
|
412
|
+
help="Arguments to pass to the script")
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _add_serialize_parser(sub: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
416
|
+
p = sub.add_parser("serialize", help="Serialize a function to JSON")
|
|
417
|
+
p.add_argument("target", help="module:function to serialize")
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _add_reconstruct_parser(sub: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
421
|
+
p = sub.add_parser("reconstruct", help="Reconstruct source from graph JSON")
|
|
422
|
+
p.add_argument("graph_file", help="Path to graph JSON file")
|
|
423
|
+
p.add_argument("function", help="Function name to reconstruct")
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _cmd_sandbox(args: argparse.Namespace) -> None:
|
|
427
|
+
"""Handle ``offwork sandbox setup|status|teardown`` subcommands."""
|
|
428
|
+
action = getattr(args, "sandbox_action", None)
|
|
429
|
+
if action is None:
|
|
430
|
+
print("Usage: offwork sandbox {setup|status|teardown}", file=sys.stderr)
|
|
431
|
+
sys.exit(1)
|
|
432
|
+
|
|
433
|
+
if action == "setup":
|
|
434
|
+
asyncio.run(_docker_setup())
|
|
435
|
+
elif action == "status":
|
|
436
|
+
asyncio.run(_sandbox_status())
|
|
437
|
+
elif action == "teardown":
|
|
438
|
+
asyncio.run(_docker_teardown())
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
async def _sandbox_status() -> None:
|
|
442
|
+
"""Print the current Docker sandbox status."""
|
|
443
|
+
if shutil.which("docker") is not None:
|
|
444
|
+
from offwork.worker.sandbox.docker import (
|
|
445
|
+
_image_exists,
|
|
446
|
+
_container_exists,
|
|
447
|
+
_container_running,
|
|
448
|
+
)
|
|
449
|
+
image = os.environ.get("OFFWORK_SANDBOX_DOCKER_IMAGE", "offwork-sandbox")
|
|
450
|
+
container = os.environ.get("OFFWORK_SANDBOX_DOCKER_CONTAINER", "offwork-sandbox")
|
|
451
|
+
img_ok = await _image_exists(image)
|
|
452
|
+
print("Docker:")
|
|
453
|
+
print(f" docker: installed")
|
|
454
|
+
print(f" Image '{image}': {'exists' if img_ok else 'not found'}")
|
|
455
|
+
if await _container_exists(container):
|
|
456
|
+
running = await _container_running(container)
|
|
457
|
+
print(f" Container '{container}': {'running' if running else 'stopped'}")
|
|
458
|
+
else:
|
|
459
|
+
print(f" Container '{container}': not found")
|
|
460
|
+
if not img_ok:
|
|
461
|
+
print(" Hint: run 'offwork sandbox setup' to build the image")
|
|
462
|
+
else:
|
|
463
|
+
print("Docker: not installed")
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
async def _docker_setup() -> None:
|
|
467
|
+
"""Build the Docker sandbox image."""
|
|
468
|
+
if shutil.which("docker") is None:
|
|
469
|
+
print(
|
|
470
|
+
"Error: 'docker' command not found.\n"
|
|
471
|
+
"Install Docker from https://docs.docker.com/get-docker/",
|
|
472
|
+
file=sys.stderr,
|
|
473
|
+
)
|
|
474
|
+
sys.exit(1)
|
|
475
|
+
|
|
476
|
+
from offwork.worker.sandbox.docker import _build_image, _image_exists
|
|
477
|
+
image = os.environ.get("OFFWORK_SANDBOX_DOCKER_IMAGE", "offwork-sandbox")
|
|
478
|
+
if await _image_exists(image):
|
|
479
|
+
print(f"Image '{image}' already exists. Rebuilding...")
|
|
480
|
+
else:
|
|
481
|
+
print(f"Building image '{image}'...")
|
|
482
|
+
await _build_image(image)
|
|
483
|
+
print(f"Done. Start a sandboxed worker with:")
|
|
484
|
+
print(f" offwork worker --backend redis://localhost:6379 --sandbox")
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
async def _docker_teardown() -> None:
|
|
488
|
+
"""Stop and remove the Docker sandbox container and image."""
|
|
489
|
+
if shutil.which("docker") is None:
|
|
490
|
+
print("Docker is not installed, nothing to tear down.", file=sys.stderr)
|
|
491
|
+
return
|
|
492
|
+
|
|
493
|
+
from offwork.worker.sandbox.docker import (
|
|
494
|
+
_docker_wait,
|
|
495
|
+
_image_exists,
|
|
496
|
+
_container_exists,
|
|
497
|
+
_container_running,
|
|
498
|
+
)
|
|
499
|
+
container = os.environ.get("OFFWORK_SANDBOX_DOCKER_CONTAINER", "offwork-sandbox")
|
|
500
|
+
image = os.environ.get("OFFWORK_SANDBOX_DOCKER_IMAGE", "offwork-sandbox")
|
|
501
|
+
|
|
502
|
+
if await _container_exists(container):
|
|
503
|
+
if await _container_running(container):
|
|
504
|
+
print(f"Stopping container '{container}'...")
|
|
505
|
+
await _docker_wait("stop", container)
|
|
506
|
+
print(f"Removing container '{container}'...")
|
|
507
|
+
await _docker_wait("rm", container)
|
|
508
|
+
|
|
509
|
+
if await _image_exists(image):
|
|
510
|
+
print(f"Removing image '{image}'...")
|
|
511
|
+
await _docker_wait("rmi", image)
|
|
512
|
+
|
|
513
|
+
print("Done.")
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _add_sandbox_parser(sub: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
517
|
+
p = sub.add_parser("sandbox", help="Manage the Docker sandbox")
|
|
518
|
+
sandbox_sub = p.add_subparsers(dest="sandbox_action")
|
|
519
|
+
sandbox_sub.add_parser("setup", help="Build the Docker sandbox image")
|
|
520
|
+
sandbox_sub.add_parser("status", help="Show Docker sandbox status")
|
|
521
|
+
sandbox_sub.add_parser("teardown", help="Stop and remove the Docker sandbox")
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
# -- Pairing -----------------------------------------------------------------
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _cmd_pair(args: argparse.Namespace) -> None:
|
|
528
|
+
"""Handle ``offwork pair`` — client-side PIN-based key exchange."""
|
|
529
|
+
from offwork.core.pairing import load_shared_key
|
|
530
|
+
|
|
531
|
+
if not args.backend:
|
|
532
|
+
print("Error: --backend is required (or set OFFWORK_BACKEND).", file=sys.stderr)
|
|
533
|
+
sys.exit(1)
|
|
534
|
+
|
|
535
|
+
role = args.role
|
|
536
|
+
|
|
537
|
+
# Check for existing key
|
|
538
|
+
existing = load_shared_key(role)
|
|
539
|
+
if existing is not None and not args.force:
|
|
540
|
+
print(
|
|
541
|
+
f"A shared key already exists for role '{role}'.\n"
|
|
542
|
+
"Use --force to overwrite it, or 'offwork pair --clear' to remove it.",
|
|
543
|
+
file=sys.stderr,
|
|
544
|
+
)
|
|
545
|
+
sys.exit(1)
|
|
546
|
+
|
|
547
|
+
_configure_logging(logging.INFO if not args.verbose else logging.DEBUG)
|
|
548
|
+
|
|
549
|
+
asyncio.run(_pair_async(args, role))
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _cmd_pair_clear(args: argparse.Namespace) -> None:
|
|
553
|
+
"""Handle ``offwork pair --clear``."""
|
|
554
|
+
from offwork.core.pairing import clear_shared_key
|
|
555
|
+
|
|
556
|
+
role = args.role
|
|
557
|
+
|
|
558
|
+
if clear_shared_key(role):
|
|
559
|
+
print(f"Shared key for '{role}' removed.")
|
|
560
|
+
else:
|
|
561
|
+
print(f"No shared key found for '{role}'.")
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
async def _pair_async(args: argparse.Namespace, role: str) -> None:
|
|
565
|
+
"""Run the pairing protocol asynchronously."""
|
|
566
|
+
from offwork.core.pairing import (
|
|
567
|
+
generate_pin,
|
|
568
|
+
save_shared_key,
|
|
569
|
+
initiate_pairing,
|
|
570
|
+
respond_to_pairing,
|
|
571
|
+
)
|
|
572
|
+
from offwork.worker.remote import connect, disconnect
|
|
573
|
+
|
|
574
|
+
backend = connect(args.backend)
|
|
575
|
+
|
|
576
|
+
try:
|
|
577
|
+
if role == "worker":
|
|
578
|
+
# Worker is the initiator: generate PIN and show it
|
|
579
|
+
pin = args.pin or generate_pin()
|
|
580
|
+
print(f"\n Pairing PIN: {pin}\n")
|
|
581
|
+
print(" Enter this PIN on the client side.")
|
|
582
|
+
print(" Waiting for client...\n")
|
|
583
|
+
result = await initiate_pairing(backend, pin, timeout=args.timeout)
|
|
584
|
+
else:
|
|
585
|
+
# Client is the responder: ask for PIN
|
|
586
|
+
pin = args.pin
|
|
587
|
+
if not pin:
|
|
588
|
+
pin = input(" Enter pairing PIN: ").strip()
|
|
589
|
+
print(" Waiting for worker...\n")
|
|
590
|
+
result = await respond_to_pairing(backend, pin, timeout=args.timeout)
|
|
591
|
+
|
|
592
|
+
save_shared_key(result.shared_key, role)
|
|
593
|
+
print(f" \u2713 Paired successfully as '{role}'.")
|
|
594
|
+
print(f" Peer role: {result.peer_role}")
|
|
595
|
+
print(f" Key saved to ~/.offwork/{role}.key\n")
|
|
596
|
+
finally:
|
|
597
|
+
await disconnect()
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _add_pair_parser(sub: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
601
|
+
p = sub.add_parser(
|
|
602
|
+
"pair",
|
|
603
|
+
help="Pair this machine with a worker using a PIN code (client-side)",
|
|
604
|
+
)
|
|
605
|
+
p.add_argument(
|
|
606
|
+
"--backend",
|
|
607
|
+
default=os.environ.get("OFFWORK_BACKEND"),
|
|
608
|
+
help="Backend URL for the pairing channel (default: $OFFWORK_BACKEND)",
|
|
609
|
+
)
|
|
610
|
+
p.add_argument(
|
|
611
|
+
"--role", default="client", choices=("client", "worker"),
|
|
612
|
+
help="Role of this machine in the pairing (default: client). "
|
|
613
|
+
"Use 'offwork worker --pair' instead of '--role worker'.",
|
|
614
|
+
)
|
|
615
|
+
p.add_argument(
|
|
616
|
+
"--pin", default=None,
|
|
617
|
+
help="PIN code (prompted interactively if omitted)",
|
|
618
|
+
)
|
|
619
|
+
p.add_argument(
|
|
620
|
+
"--timeout", type=float, default=60.0,
|
|
621
|
+
help="Seconds to wait for the peer (default: 60)",
|
|
622
|
+
)
|
|
623
|
+
p.add_argument(
|
|
624
|
+
"--force", action="store_true",
|
|
625
|
+
help="Overwrite existing shared key",
|
|
626
|
+
)
|
|
627
|
+
p.add_argument(
|
|
628
|
+
"--clear", action="store_true",
|
|
629
|
+
help="Remove the shared key for this role and exit (skips pairing)",
|
|
630
|
+
)
|
|
631
|
+
p.add_argument("-v", "--verbose", action="store_true",
|
|
632
|
+
help="Enable debug logging")
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
636
|
+
parser = argparse.ArgumentParser(
|
|
637
|
+
prog="offwork", description="offwork - distributed task execution",
|
|
638
|
+
)
|
|
639
|
+
sub = parser.add_subparsers(dest="command")
|
|
640
|
+
_add_worker_parser(sub)
|
|
641
|
+
_add_run_parser(sub)
|
|
642
|
+
sub.add_parser("info", help="Show offwork configuration")
|
|
643
|
+
_add_serialize_parser(sub)
|
|
644
|
+
_add_reconstruct_parser(sub)
|
|
645
|
+
_add_sandbox_parser(sub)
|
|
646
|
+
_add_pair_parser(sub)
|
|
647
|
+
_add_token_parser(sub)
|
|
648
|
+
return parser
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _dispatch_pair(args: argparse.Namespace) -> None:
|
|
652
|
+
"""Route ``offwork pair`` to clear or pairing handler."""
|
|
653
|
+
if args.clear:
|
|
654
|
+
_cmd_pair_clear(args)
|
|
655
|
+
else:
|
|
656
|
+
_cmd_pair(args)
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
# -- Token -------------------------------------------------------------------
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def _cmd_token(args: argparse.Namespace) -> None:
|
|
663
|
+
"""Handle ``offwork token`` subcommands."""
|
|
664
|
+
action = getattr(args, "token_action", None)
|
|
665
|
+
if action is None:
|
|
666
|
+
print("Usage: offwork token {generate|show|clear}", file=sys.stderr)
|
|
667
|
+
sys.exit(1)
|
|
668
|
+
|
|
669
|
+
if action == "generate":
|
|
670
|
+
_cmd_token_generate(args)
|
|
671
|
+
elif action == "show":
|
|
672
|
+
_cmd_token_show(args)
|
|
673
|
+
elif action == "clear":
|
|
674
|
+
_cmd_token_clear(args)
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _cmd_token_generate(args: argparse.Namespace) -> None:
|
|
678
|
+
"""Generate a new signing token."""
|
|
679
|
+
from offwork.core.token import generate_token, save_token, load_token
|
|
680
|
+
|
|
681
|
+
existing = load_token()
|
|
682
|
+
if existing is not None and not args.force:
|
|
683
|
+
print(
|
|
684
|
+
"A signing token already exists.\n"
|
|
685
|
+
"Use --force to overwrite it, or 'offwork token clear' to remove it.",
|
|
686
|
+
file=sys.stderr,
|
|
687
|
+
)
|
|
688
|
+
sys.exit(1)
|
|
689
|
+
|
|
690
|
+
token_hex = generate_token()
|
|
691
|
+
save_token(token_hex)
|
|
692
|
+
|
|
693
|
+
print(f"\n Token generated and saved to ~/.offwork/token\n")
|
|
694
|
+
print(f" Token: {token_hex}\n")
|
|
695
|
+
print(" Set this on both client and worker:")
|
|
696
|
+
print(f" export OFFWORK_SIGNING_TOKEN={token_hex}\n")
|
|
697
|
+
print(" Or start the worker with signing enabled:")
|
|
698
|
+
print(" offwork worker --backend redis://localhost:6379 --require-signing\n")
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def _cmd_token_show(_args: argparse.Namespace) -> None:
|
|
702
|
+
"""Show the current signing token status."""
|
|
703
|
+
from offwork.core.token import load_token, _TOKEN_ENV_VAR
|
|
704
|
+
|
|
705
|
+
env_val = os.environ.get(_TOKEN_ENV_VAR)
|
|
706
|
+
if env_val is not None:
|
|
707
|
+
print(f" Source: {_TOKEN_ENV_VAR} environment variable")
|
|
708
|
+
print(f" Token: {env_val.strip()[:16]}... (truncated)")
|
|
709
|
+
return
|
|
710
|
+
|
|
711
|
+
token = load_token()
|
|
712
|
+
if token is not None:
|
|
713
|
+
print(f" Source: ~/.offwork/token")
|
|
714
|
+
print(f" Token: {token[:16]}... (truncated)")
|
|
715
|
+
else:
|
|
716
|
+
print(" No signing token configured.")
|
|
717
|
+
print(" Generate one with: offwork token generate")
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def _cmd_token_clear(_args: argparse.Namespace) -> None:
|
|
721
|
+
"""Remove the saved signing token."""
|
|
722
|
+
from offwork.core.token import clear_token
|
|
723
|
+
|
|
724
|
+
if clear_token():
|
|
725
|
+
print("Signing token removed.")
|
|
726
|
+
else:
|
|
727
|
+
print("No signing token found.")
|
|
728
|
+
|
|
729
|
+
|
|
730
|
+
def _add_token_parser(sub: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
731
|
+
p = sub.add_parser(
|
|
732
|
+
"token",
|
|
733
|
+
help="Manage signing tokens for automated deployments",
|
|
734
|
+
)
|
|
735
|
+
token_sub = p.add_subparsers(dest="token_action")
|
|
736
|
+
gen = token_sub.add_parser(
|
|
737
|
+
"generate",
|
|
738
|
+
help="Generate a new signing token",
|
|
739
|
+
)
|
|
740
|
+
gen.add_argument(
|
|
741
|
+
"--force", action="store_true",
|
|
742
|
+
help="Overwrite an existing token",
|
|
743
|
+
)
|
|
744
|
+
token_sub.add_parser("show", help="Show current token status")
|
|
745
|
+
token_sub.add_parser("clear", help="Remove the saved token")
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
_COMMAND_HANDLERS: dict[str, Callable[[argparse.Namespace], None]] = {
|
|
749
|
+
"worker": _cmd_worker,
|
|
750
|
+
"run": _cmd_run,
|
|
751
|
+
"info": _cmd_info,
|
|
752
|
+
"serialize": _cmd_serialize,
|
|
753
|
+
"reconstruct": _cmd_reconstruct,
|
|
754
|
+
"sandbox": _cmd_sandbox,
|
|
755
|
+
"pair": _dispatch_pair,
|
|
756
|
+
"token": _cmd_token,
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def main() -> None:
|
|
761
|
+
parser = _build_parser()
|
|
762
|
+
args = parser.parse_args()
|
|
763
|
+
if args.command is None:
|
|
764
|
+
parser.print_help()
|
|
765
|
+
sys.exit(1)
|
|
766
|
+
_COMMAND_HANDLERS[args.command](args)
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
if __name__ == "__main__":
|
|
770
|
+
main()
|