offwork 0.4.0__py3-none-any.whl

offwork/worker/worker.py

@@ -0,0 +1,263 @@
+"""Worker: reconstruct functions from serialized stores, cache, and execute."""
+
+import asyncio
+import hashlib
+import inspect
+import logging
+import functools
+import contextvars
+from typing import Any
+from dataclasses import field, dataclass
+from collections.abc import Callable
+
+from offwork.core.task import Task, resolve_args
+from offwork.core.errors import WorkerError
+from offwork.core.models import FunctionNode
+from offwork.graph.store import Store
+from offwork.worker.deps import ensure_dependencies
+from offwork.worker.sandbox import DockerSandbox
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class BuildInfo:
+    """Metadata about how a function was resolved for execution."""
+
+    cache_hit: bool
+    installed_packages: list[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class _CachedFunction:
+    """A cached compiled function with its namespace and metadata."""
+
+    namespace: dict[str, Any]
+    func: Callable[..., Any]
+    subgraph_key: str
+    source: str
+
+
+def _compute_subgraph_key(store: Store, function_name: str) -> str:
+    """Compute a cache key from all content hashes in the function's subgraph."""
+    root_hash = store._resolve_function_hash(function_name)
+    all_hashes = sorted(store.walk(root_hash))
+    return hashlib.sha256(":".join(all_hashes).encode()).hexdigest()[:16]
+
+
+def _extract_target_callable(
+    namespace: dict[str, Any],
+    store: Store,
+    function_name: str,
+) -> Any:
+    """Extract the target callable from an exec'd namespace."""
+    target_qname, nodes = store.collect(function_name)
+    target_node = nodes[target_qname]
+
+    if target_node.owner_class:
+        return _extract_method(namespace, target_node)
+    return _extract_function(namespace, target_node)
+
+
+def _extract_method(namespace: dict[str, Any], node: FunctionNode) -> Any:
+    """Look up a method on a class in the reconstructed namespace."""
+    assert node.owner_class is not None
+    class_name = node.owner_class.rsplit(".", 1)[-1]
+    cls = namespace.get(class_name)
+    if cls is None:
+        raise WorkerError(f"Class '{class_name}' not found in reconstructed namespace")
+    func = getattr(cls, node.name, None)
+    if func is None:
+        raise WorkerError(f"Method '{node.name}' not found on class '{class_name}'")
+    return func
+
+
+def _extract_function(namespace: dict[str, Any], node: FunctionNode) -> Any:
+    """Look up a standalone function in the reconstructed namespace."""
+    func = namespace.get(node.name)
+    if func is None:
+        raise WorkerError(f"Function '{node.name}' not found in reconstructed namespace")
+    return func
+
+
+class Worker:
+    """Execute functions from serialized offwork graphs with caching.
+
+    Parameters
+    ----------
+    auto_install
+        Automatically install missing third-party dependencies via pip.
+    import_to_package
+        Extra import-name -> pip-package-name mappings (merged with defaults).
+    sandbox
+        Optional :class:`~offwork.worker.sandbox.DockerSandbox` or
+        ``True`` to create one with default settings.  When provided,
+        function execution is delegated to the sandbox instead of
+        running ``exec`` in the host process.
+    """
+
+    def __init__(
+        self,
+        auto_install: bool = True,
+        import_to_package: dict[str, str] | None = None,
+        sandbox: DockerSandbox | bool | None = None,
+    ) -> None:
+        self._import_to_package = import_to_package
+        self._auto_install = auto_install
+        self._cache: dict[str, _CachedFunction] = {}
+        self._last_build_info: BuildInfo | None = None
+
+        if sandbox is True:
+            self._sandbox: DockerSandbox | None = DockerSandbox()
+        elif isinstance(sandbox, DockerSandbox):
+            self._sandbox = sandbox
+        else:
+            self._sandbox = None
+
+    async def _get_cached(self, json_str: str, function_name: str) -> _CachedFunction:
+        """Return the cached (or freshly built) function for *function_name*."""
+        store = Store.from_json(json_str)
+        key = _compute_subgraph_key(store, function_name)
+        if key not in self._cache:
+            self._cache[key] = await self._build(store, function_name, key)
+        else:
+            self._last_build_info = BuildInfo(cache_hit=True)
+        return self._cache[key]
+
+    @property
+    def sandboxed(self) -> bool:
+        """Whether execution is delegated to a Docker sandbox."""
+        return self._sandbox is not None
+
+    async def run(self, task: Task) -> Any:
+        """Execute a :class:`Task`, resolving serialized object arguments.
+
+        When a sandbox is configured, the full source + args are sent to
+        the sandbox executor.  Otherwise async functions are awaited
+        directly and sync functions run in a thread executor.
+        """
+        cached = await self._get_cached(task.graph_json, task.function_name)
+        logger.debug("Executing %s (cache key: %s)", task.function_name, cached.subgraph_key)
+
+        if self.sandboxed:
+            assert self._sandbox is not None
+            # Determine owner_class for method dispatch inside the sandbox
+            store = Store.from_json(task.graph_json)
+            target_qname, nodes = store.collect(task.function_name)
+            target_node = nodes[target_qname]
+            return await self._sandbox.execute(
+                cached.source,
+                target_node.name,
+                task.args,
+                task.kwargs,
+                owner_class=target_node.owner_class,
+            )
+
+        args, kwargs = resolve_args(task.args, task.kwargs, cached.namespace)
+
+        if inspect.iscoroutinefunction(cached.func):
+            return await cached.func(*args, **kwargs)
+
+        loop = asyncio.get_running_loop()
+        ctx = contextvars.copy_context()
+        return await loop.run_in_executor(
+            None, ctx.run, functools.partial(cached.func, *args, **kwargs),
+        )
+
+    async def run_with_policy(self, task: Task) -> Any:
+        """Execute a :class:`Task` with retry and timeout enforcement.
+
+        Reads ``task.retries``, ``task.timeout``, and ``task.retry_delay``
+        to apply exponential-backoff retries and per-attempt timeouts.
+        """
+        last_exc: Exception | None = None
+        for attempt in range(1 + task.retries):
+            try:
+                if task.timeout is not None:
+                    return await asyncio.wait_for(
+                        self.run(task), timeout=task.timeout,
+                    )
+                return await self.run(task)
+            except Exception as exc:
+                last_exc = exc
+                if attempt < task.retries:
+                    delay = task.retry_delay * (2 ** attempt)
+                    logger.warning(
+                        "Task %s attempt %d/%d failed, retrying in %.1fs: %s",
+                        task.task_id, attempt + 1, task.retries, delay, exc,
+                    )
+                    await asyncio.sleep(delay)
+        raise last_exc  # type: ignore[misc]
+
+    def cache_info(self) -> dict[str, Any]:
+        """Return cache statistics."""
+        return {"size": len(self._cache), "keys": list(self._cache.keys())}
+
+    def clear_cache(self) -> None:
+        """Drop all cached functions."""
+        self._cache.clear()
+
+    # -- internals -------------------------------------------------------------
+
+    def last_build_info(self) -> BuildInfo | None:
+        """Return metadata about the most recent execution's build phase."""
+        return self._last_build_info
+
+    async def _build(
+        self, store: Store, function_name: str, key: str
+    ) -> _CachedFunction:
+        installed_packages: list[str] = []
+        if self._auto_install:
+            install_result = await ensure_dependencies(
+                store, function_name, self._import_to_package
+            )
+            installed_packages = install_result.installed
+
+        self._last_build_info = BuildInfo(
+            cache_hit=False,
+            installed_packages=installed_packages,
+        )
+
+        source = store.reconstruct(function_name)
+        logger.debug("Reconstructed source for %s:\n%s", function_name, source)
+
+        code = compile(source, f"<offwork:{function_name}>", "exec")
+        namespace: dict[str, Any] = {}
+        exec(code, namespace)  # noqa: S102
+
+        func = _extract_target_callable(namespace, store, function_name)
+        return _CachedFunction(
+            namespace=namespace, func=func, subgraph_key=key, source=source
+        )
+
+
+async def execute(
+    json_str_or_task: str | Task,
+    function_name: str | None = None,
+    *args: Any,
+    **kwargs: Any,
+) -> Any:
+    """One-shot execution of a function from a serialized offwork graph.
+
+    Accepts either a JSON string + function name, or a :class:`Task`::
+
+        await execute(json_str, "my_func", arg1, arg2)
+        await execute(task)
+    """
+    worker = Worker()
+    if isinstance(json_str_or_task, Task):
+        return await worker.run(json_str_or_task)
+    if function_name is None:
+        raise TypeError("function_name is required when passing a JSON string")
+
+    cached = await worker._get_cached(json_str_or_task, function_name)
+    resolved_args, resolved_kwargs = resolve_args(args, {}, cached.namespace)
+
+    if inspect.iscoroutinefunction(cached.func):
+        return await cached.func(*resolved_args, **resolved_kwargs)
+
+    loop = asyncio.get_running_loop()
+    ctx = contextvars.copy_context()
+    return await loop.run_in_executor(
+        None, ctx.run, functools.partial(cached.func, *resolved_args, **resolved_kwargs),
+    )

offwork-0.4.0.dist-info/METADATA

@@ -0,0 +1,143 @@
+Metadata-Version: 2.4
+Name: offwork
+Version: 0.4.0
+Summary: Distributed Python task execution via automatic function serialization
+License: AGPL-3.0-only
+License-File: LICENSE
+Keywords: distributed,remote-execution,serialization,task-queue
+Author: Rémi Héneault
+Author-email: remi@heneault.fr
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Typing :: Typed
+Provides-Extra: rabbitmq
+Provides-Extra: redis
+Requires-Dist: aio-pika (>=9.0) ; extra == "rabbitmq"
+Requires-Dist: redis (>=5.0) ; extra == "redis"
+Project-URL: Repository, https://github.com/codeSamuraii/offwork
+Description-Content-Type: text/markdown
+
+# offwork
+
+**Run any Python function on a remote worker — zero setup, zero deployment.**
+
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/downloads/)
+[![License: AGPL-3.0](https://img.shields.io/badge/license-AGPL--3.0-green)](LICENSE)
+[![Typed](https://img.shields.io/badge/typing-strict%20mypy-blue)](https://mypy-lang.org/)
+[![Zero Dependencies](https://img.shields.io/badge/dependencies-0-brightgreen)]()
+
+Add `@trace` to a function. offwork captures its source, dependencies, and imports automatically.
+Workers reconstruct and execute everything from scratch — no shared filesystem, no deployment pipeline.
+Missing packages are installed on the fly.
+
+## Quick start
+
+```bash
+pip install offwork
+```
+
+```python
+import asyncio, math, offwork
+from offwork import trace
+
+offwork.connect("local://localhost:9748")
+
+def add(a, b):
+    return a + b
+
+@trace
+def hypotenuse(a: float, b: float) -> float:
+    return math.sqrt(add(a**2, b**2))
+
+async def main():
+    print(await hypotenuse.run(3.0, 4.0))  # 5.0
+
+asyncio.run(main())
+```
+
+Only the entry point needs `@trace` — everything it calls is captured automatically.
+
+```bash
+offwork worker --backend local://localhost:9748 --tmp   # start a worker
+python my_script.py                                    # → 5.0
+```
+
+For multi-machine, swap `local://` for `redis://` or an `https://` managed broker URL. That's it.
+
+## Sandbox
+
+Run tasks inside Docker containers for isolation — transparent to clients:
+
+```bash
+offwork sandbox setup                                      # build image (once)
+offwork worker --backend redis://localhost:6379 --sandbox  # run with isolation
+```
+
+See [Sandbox](docs/SANDBOX.md) for configuration and management.
+
+## Signing
+
+Pre-shared token or PIN-based pairing + HMAC-SHA256 — workers reject untrusted or tampered tasks:
+
+```bash
+# Token-based (recommended for CI/CD)
+offwork token generate                                       # generate once
+export OFFWORK_SIGNING_TOKEN=<token>                         # set on client & worker
+offwork worker --backend redis://localhost:6379 --require-signing
+
+# PIN-based pairing (interactive)
+offwork worker --backend redis://localhost:6379 --pair       # displays a 6-digit PIN
+offwork pair --backend redis://localhost:6379                # on client: enter the PIN
+```
+
+After setup, tasks are signed automatically. No client-side code changes. See [Signing & Pairing](docs/SIGNING.md) for details.
+
+## Features
+
+| | |
+|-|-|
+| **Auto dependency capture** | Functions, classes, constants, closures — recursive AST analysis |
+| **Package auto-install** | Workers `pip install` missing packages before execution |
+| **Async-native** | `.run()`, `.start()`, `.map()`, `asyncio.gather` |
+| **Retry & timeout** | `@trace(timeout=30, retries=3)` with exponential backoff |
+| **Scheduling** | `.run_in(delay)`, `.run_at(datetime)`, `.run_every(freq)` with cancellation |
+| **Throttling** | `@trace(throttle=timedelta(hours=24)/50)` — rate-limit executions |
+| **Progress & cancellation** | `offwork.progress(3, 10)` inside tasks; `await future.cancel()` on client |
+| **Heartbeat & stall detection** | Workers heartbeat; clients raise `TaskStalled` on silence |
+| **Content-hash caching** | Same code = cache hit, regardless of client |
+| **Pluggable backends** | `local://` (same-machine TCP), `redis://`, `amqp://` (RabbitMQ), `http://`/`https://` (hosted broker API) |
+| **Docker sandbox** | Container isolation, transparent to clients |
+| **Signed execution** | Pre-shared token or PIN pairing + HMAC-SHA256 task authentication |
+| **Graceful shutdown** | Ctrl+C drains in-flight tasks; second Ctrl+C force-quits |
+
+## Documentation
+
+| | |
+|-|-|
+| **[Quick Start](docs/QUICK_START.md)** | Tutorial and API walkthrough |
+| **[Technical Overview](docs/TECHNICAL_OVERVIEW.md)** | Architecture, serialization format, internals |
+| **[Signing & Pairing](docs/SIGNING.md)** | Cryptographic task signing protocol |
+| **[Sandbox](docs/SANDBOX.md)** | Docker container isolation |
+| **[Cloud POC](docs/CLOUD_POC.md)** | Local FastAPI + MongoDB + Kubernetes + React prototype for managed hosting |
+
+## Examples
+
+```bash
+offwork worker --backend local://localhost:9748 --tmp
+offwork run examples/remote_execution.py
+```
+
+[`remote_execution.py`](examples/remote_execution.py) · [`async_execution.py`](examples/async_execution.py) · [`package_installation.py`](examples/package_installation.py) · [`progress_reporting.py`](examples/progress_reporting.py) · [`cancellation.py`](examples/cancellation.py) · [`scheduling.py`](examples/scheduling.py) · [`throttling_and_retry.py`](examples/throttling_and_retry.py) · [`large_module.py`](examples/large_module.py)
+
+## License
+
+[AGPL-3.0](LICENSE)
+

offwork-0.4.0.dist-info/RECORD

@@ -0,0 +1,42 @@
+offwork/__init__.py,sha256=AfZmUAbdo0BQVBEyL-vnd7oNaenjTMoxtKuJ9PblTmM,4092
+offwork/__main__.py,sha256=z9AGC3gTLg1p56vYDRSm8Ufzp0XTv3WI-OjwyzBr2qQ,27001
+offwork/_venv.py,sha256=V5KLob55JgfexY0L8lwEbi2wgHUSYISaJZkpOyHZIhY,5552
+offwork/core/__init__.py,sha256=z1ODdcKmBgRWishd_u1QSPxITiOfn2QKiA6sxcg4PyE,388
+offwork/core/errors.py,sha256=7zbLr7dx3csz9HKvkf1pRklD-UDlWoz55c8TyPACkWA,2279
+offwork/core/models.py,sha256=C9LsXUgMHKkuIHaH5Pl8S7SuGcrbLbLWKb22WDsCrkE,6717
+offwork/core/pairing.py,sha256=ChWm18_2qHQCWBv6MXjIuESRXmft0651Agk3xhqGv4A,11747
+offwork/core/progress.py,sha256=VGBWVZLdi4vyWJ2xFYd273zcuK217paE_oPlhvJ0gEM,2687
+offwork/core/signing.py,sha256=GZMjs49RHUroYB4TkpOlHMw_BTGu4moihtqSJKRxRNk,2980
+offwork/core/task.py,sha256=qh-znMt3Aj_am1iZYbtVDFDeyx1Ji8cA4zyoZ44bxAw,19039
+offwork/core/token.py,sha256=0BoLjOW0WQPbrNjxTjrmZiyHe9VnkRXbjTMkDaj_cNE,5162
+offwork/core/version.py,sha256=hQN8MYNj67BuO2wPFVnnbRkXz89wbL-oRQYqMkIji0w,318
+offwork/graph/__init__.py,sha256=sAqTD-tzaAPQo5R5WvhbaXf3g4ErCu1jg0tx26hSFYc,185
+offwork/graph/analyzer.py,sha256=MqT9KebuELqkBmXHrLy6mNrLJ1D01FT-oE-YCtlwMdk,22829
+offwork/graph/decorator.py,sha256=d_MU9MyPqVBg_K8TXIrcFT-wa1f1T--QD3klOB5eDK4,2751
+offwork/graph/graph.py,sha256=16EudFrYoBo6haJoDpTO5JqhURfq4Mqe9Nqg1TNZ76g,39624
+offwork/graph/store.py,sha256=R9F_34DXwbI5OYriMQTZC_VwIGCxznf1uU3xOTnV27Y,17539
+offwork/graph/tracing.py,sha256=AlOsYaj2Bo_FT_Q5Ta2_NyVzOOsbdK_sx2yssbxqlrA,15241
+offwork/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+offwork/typing.py,sha256=WEBuqsWHLPhRpwgFsH63WUBACZwLhkcA4M_tSg49mP8,1571
+offwork/worker/__init__.py,sha256=sXesG9Aawd-LQBPBo_YBoX9bPGRUBhkmPRx3RJj_jBQ,501
+offwork/worker/backends/__init__.py,sha256=NMibQgmgkdi3b2VGG2llAJ57j6ZLdWG5_rZEDHLXO84,72
+offwork/worker/backends/base.py,sha256=qSSMcdFh8486_i4Dxa8UhqbZeN-hRb6qL_JlxN3p-iI,5430
+offwork/worker/backends/http.py,sha256=YHrGHyIp5T8mVFQw51BwItCpP2duItxgXX3QOyVkUqc,8871
+offwork/worker/backends/local.py,sha256=-fA7er5ubf3fCTP5CKQAZLOLj9I8apsiYXaYYYSAvyo,16588
+offwork/worker/backends/rabbitmq.py,sha256=4jjguMi51uzkRRUsae34pFDsSpehmCtPcucXOJ2aTTs,16153
+offwork/worker/backends/redis.py,sha256=Hkj6YVlE0wKK77WDEghi2sACxVKIVIYN-0n0zzNIeNo,6503
+offwork/worker/deps.py,sha256=f3RxKPPdnllYK9TTvBLm1PzydS0w-OzaAb1QizBzw0U,12205
+offwork/worker/remote.py,sha256=eQw7x0t9gSX3TzDppCEU84TGobtPXpz6Jgky3QZkpME,27079
+offwork/worker/result.py,sha256=ZV3gyviXD6jqrf2Hgrsh9WQIW_YcFwuxN-KVXBAo5qM,9901
+offwork/worker/sandbox/Dockerfile,sha256=cp_aqrLMyu2LDUldDiGWfpc6WIL5EoRGbIIsIRaCSxQ,607
+offwork/worker/sandbox/__init__.py,sha256=LZXATgdqas9IUZz8CdQtxDnOHBVzgZEcFwRBUAQ2RBg,573
+offwork/worker/sandbox/_protocol.py,sha256=5e2xxF3pz3LD0HL2QCxGbqSNePzd446jIoBndCMi0Cc,1575
+offwork/worker/sandbox/docker.py,sha256=fTwZyUo9aYE4AqZoGRaORJ_2aDJqBkmQILJ3m0GsTSI,15893
+offwork/worker/sandbox/guest_agent.py,sha256=c5KQG8i2LGX3QHRC2tQQhNtSRReyGvrfSnoQ5EfVdns,21132
+offwork/worker/schedule.py,sha256=t9V13Cw-CwUsz2IlURBAowQ8G7SZHhXwj4VqWIW4Vsk,771
+offwork/worker/worker.py,sha256=Lun0MJYW7IBIEpRK9VajenzQESEhk9OfimEeQonJnbQ,9583
+offwork-0.4.0.dist-info/METADATA,sha256=-4PJFxDRkxwT70jaNWqI2vRfS3XtbNcqf1NKSaKSZvQ,5844
+offwork-0.4.0.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88
+offwork-0.4.0.dist-info/entry_points.txt,sha256=gWtWiCZ5OVBoXKMDmieFpHl8IDwBPvcTjCr4rI1a-oM,49
+offwork-0.4.0.dist-info/licenses/LICENSE,sha256=Hbi0KIkhodUCF0-cV2J6BDtbxiJHek97TguGflNvBHM,34526
+offwork-0.4.0.dist-info/RECORD,,

offwork-0.4.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.4.0
+Root-Is-Purelib: true
+Tag: py3-none-any

offwork-0.4.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+offwork=offwork.__main__:main
+