run-yonder 0.1.0__py3-none-any.whl

run_yonder-0.1.0.dist-info/METADATA

@@ -0,0 +1,271 @@
+Metadata-Version: 2.4
+Name: run-yonder
+Version: 0.1.0
+Summary: Run Python functions inside containers via a decorator.
+Project-URL: Source, https://gitlab.com/berge472/yonder
+Author-email: Jason Berger <berge472@gmail.com>
+License: MIT
+Keywords: cloudpickle,container,decorator,docker,remote-execution
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.9
+Requires-Dist: cloudpickle>=3.0
+Provides-Extra: dev
+Requires-Dist: docker>=7.0; extra == 'dev'
+Requires-Dist: kubernetes>=28.0; extra == 'dev'
+Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Provides-Extra: docker
+Requires-Dist: docker>=7.0; extra == 'docker'
+Provides-Extra: k8s
+Requires-Dist: kubernetes>=28.0; extra == 'k8s'
+Description-Content-Type: text/x-rst
+
+.. image:: doc/assets/images/logo.svg
+   :alt: yonder
+   :align: center
+   :width: 380px
+
+|
+
+yonder
+======
+
+Run Python functions on container-backed runners via a decorator.
+Functions and their arguments are serialized with
+`cloudpickle <https://github.com/cloudpipe/cloudpickle>`_, shipped to the
+runner, executed there, and the result is shipped back.
+
+Install
+-------
+
+.. code:: bash
+
+    # core package
+    pip install -e .
+
+    # with the bundled Docker runner
+    pip install -e ".[docker]"
+
+    # with the bundled Kubernetes runner
+    pip install -e ".[k8s]"
+
+Quick start
+-----------
+
+.. code:: python
+
+    import yonder
+    from yonder import DockerRunner
+
+    # Existing image. If the image doesn't already have cloudpickle, set
+    # inject_cloudpickle=True to derive an image that does (cached across
+    # runs, keyed by the base image). `when=` is an optional zero-arg
+    # predicate that makes the runner a no-op (calls run locally) when it
+    # returns False — useful for "use the container only if the library
+    # isn't already installed on the host".
+    runnerA = DockerRunner(
+        image="url/to/image",
+        env={"ENV_VARA": "value"},
+        mounts={"/host/path": "/in/container"},
+        inject_cloudpickle=True,
+        when=lambda: liba_not_present(),
+    )
+
+    # Build from a Dockerfile (context defaults to the Dockerfile's directory).
+    runnerB = DockerRunner(
+        dockerfile="./docker/myimg.Dockerfile",
+        build_args={"VERSION": "1.2.3"},
+    )
+
+    # On-demand: fresh container per call instead of one long-lived container.
+    ephemeral = DockerRunner(image="url/to/image", on_demand=True)
+
+    # Attach to a container you manage outside yonder. yonder will start it
+    # if it's stopped, but never removes it on close.
+    external = DockerRunner(container="my-dev-container")
+
+    # Register them by name.
+    yonder.register({
+        "runnerA": runnerA,
+        "runnerB": runnerB,
+        "ephemeral": ephemeral,
+        "external": external,
+    })
+
+    # Decorate. Dispatch policy lives on the runner (via the runner's
+    # `when=`); the decorator just names the target.
+    @yonder.yonder(runner="runnerA")
+    def functionA(a, b):
+        return a + b
+
+    @yonder.yonder(runner="runnerB")
+    def functionB(a, b):
+        return a * b
+
+Lazy start by default
+~~~~~~~~~~~~~~~~~~~~~
+
+A registered runner is **not** built/started until it's actually needed.
+The first decorated call that targets a runner triggers its ``start()``
+(image build/pull, persistent container creation). Runners that are never
+called incur no cost.
+
+You can pre-warm explicitly when you'd rather pay startup cost up front:
+
+.. code:: python
+
+    runnerA.start()       # pre-warm a specific runner
+    yonder.wait()         # pre-warm every registered runner (parallel)
+
+Both are opt-in. On-demand runners always behave the same way — they spin
+up a fresh container per call regardless of pre-warming.
+
+Tear down
+~~~~~~~~~
+
+Tear down when done:
+
+.. code:: python
+
+    yonder.closeAll()
+    # or, per-runner:
+    runnerA.close()
+
+``DockerRunner`` is also a context manager:
+
+.. code:: python
+
+    with DockerRunner(image="python:3.11-slim") as r:
+        yonder.register({"tmp": r})
+        yonder.wait()
+        ...
+    # container removed on exit
+
+DockerRunner options
+--------------------
+
+.. code:: python
+
+    DockerRunner(
+        # Exactly one of these three is required:
+        image=None,          # use/pull an existing image; yonder creates the container
+        dockerfile=None,     # build an image locally first; yonder creates the container
+        container=None,      # attach to an existing container (managed outside yonder)
+
+        build_context=None,  # override the build context directory (dockerfile mode)
+        build_args=None,     # dict of Docker ARGs (dockerfile mode)
+
+        env=None,            # environment variables (image/dockerfile mode only)
+        mounts=None,         # {host_path: container_path} (image/dockerfile mode only)
+        on_demand=False,     # one-shot container per call (image/dockerfile mode only)
+        name=None,           # container/tag name (auto-generated if omitted)
+        inject_cloudpickle=False,  # derive `FROM image + RUN pip install cloudpickle`
+                                   # (image mode only; tagged by base-image hash so the
+                                   # docker layer cache makes re-runs cheap)
+        client=None,         # pre-built docker.DockerClient (else docker.from_env())
+    )
+
+K8sRunner
+---------
+
+Attach to an existing Kubernetes pod and exec into one of its containers.
+Pods are treated like externally-managed containers — yonder never creates
+or removes them.
+
+.. code:: python
+
+    from yonder import K8sRunner
+
+    # By pod name.
+    by_name = K8sRunner(
+        pod="my-pod-abc123",
+        namespace="default",
+        container="app",  # optional; defaults to the pod's first container
+    )
+
+    # By label selector — first Running pod that matches is used.
+    by_selector = K8sRunner(
+        selector="app=my-app,env=dev",
+        namespace="default",
+    )
+
+.. code:: python
+
+    K8sRunner(
+        # Exactly one of these two is required:
+        pod=None,            # attach by pod name
+        selector=None,       # attach by label selector
+
+        namespace="default", # pod namespace
+        container=None,      # container within the pod (default: first)
+        kubeconfig=None,     # kubeconfig path (else KUBECONFIG/~/.kube/config,
+                             # falling back to in-cluster config)
+        context=None,        # kubeconfig context
+        name=None,           # runner name (auto-derived if omitted)
+        api_client=None,     # pre-built kubernetes.client.ApiClient
+    )
+
+The target container must have ``python`` on ``PATH`` and ``cloudpickle``
+installed (same Python minor version as the host is strongly recommended).
+
+Architecture
+------------
+
+- **``Runner``** (``yonder.Runner``) — abstract base. Implementations hold
+  their own config and expose ``run(payload) -> bytes``, plus optional
+  ``start()`` / ``close()`` lifecycle hooks.
+- **``DockerRunner``** — default implementation; uses the
+  `Docker SDK for Python <https://docker-py.readthedocs.io/>`_. Supports
+  existing images, Dockerfile builds, and attaching to externally-managed
+  containers; persistent and on-demand modes.
+- **``K8sRunner``** — Kubernetes implementation; uses the
+  `Kubernetes Python client <https://github.com/kubernetes-client/python>`_
+  to exec into a pod selected by name or label selector.
+- **``YonderSession``** — process-wide name→runner registry. Populate with
+  ``yonder.register({...})``, bring everything online with
+  ``yonder.wait()``, tear everything down with ``yonder.closeAll()``.
+- **Decorator** — serializes ``(func, args, kwargs)`` with cloudpickle,
+  calls the named runner's ``run(payload)``, deserializes the envelope, and
+  either returns the value or re-raises the exception.
+
+Writing your own Runner
+-----------------------
+
+.. code:: python
+
+    from yonder import Runner, register
+
+    class MyRunner(Runner):
+        def __init__(self, **config):
+            ...
+
+        def start(self):
+            # eager init (build/pull image, warm container, etc.)
+            ...
+
+        def run(self, payload: bytes) -> bytes:
+            # send payload to your execution environment,
+            # return the cloudpickled envelope
+            ...
+
+        def close(self):
+            ...
+
+    register({"mine": MyRunner(...)})
+
+Container requirements
+----------------------
+
+Images used with ``DockerRunner`` must have:
+
+- ``python`` on ``PATH``
+- ``cloudpickle`` installed (``pip install cloudpickle``). If your base
+  image doesn't have it, pass ``inject_cloudpickle=True`` and yonder will
+  build a derived image that does (one-time, cached).
+- Ideally the same Python minor version as the host (cloudpickle pickles
+  function bytecode, which is not portable across CPython minor versions).

run_yonder-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+yonder/__init__.py,sha256=inhtrjsHPEg76En7NrFcut_Gs_aZdl5lxMNkmR-GDcg,595
+yonder/decorator.py,sha256=S49_AGY1siJSygZ-s2lp8EnMCm4tUOlnUqF_K9ScG2g,4472
+yonder/paths.py,sha256=446OXCfZEjUIBzjxepFjfvpgD7ezPUZKOJGs7MTItF0,2314
+yonder/session.py,sha256=1KT4kME2cRfqYmJAJgTMO7o3a9IrR_klvPIIobuC4O4,5067
+yonder/runners/__init__.py,sha256=NJj2bCqYMocXnm1UaG2JL9wFx3GUiKHchX8LAYinLX4,136
+yonder/runners/base.py,sha256=fa4F76FsWcYv29vWc05iVNDbrIvN6VGX3JOTOEagwT8,2735
+yonder/runners/docker.py,sha256=MNJGByghoiVltG6IrSFrvd_-0w32o2jr1Hu8EhPUPLc,26265
+yonder/runners/k8s.py,sha256=HI7rOWohV6rfFdZ68cXXPCHSQSfyeHuQh82PpicbxIM,9478
+run_yonder-0.1.0.dist-info/METADATA,sha256=FrMXaQvCpSvcDDase1dk9Dn-sZXCCjRSOrTXf9Wnzd0,9026
+run_yonder-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+run_yonder-0.1.0.dist-info/RECORD,,

run_yonder-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

yonder/__init__.py

@@ -0,0 +1,33 @@
+"""yonder — run Python functions on registered container runners."""
+
+from .decorator import yonder
+from .paths import HostPath
+from .runners.base import Runner
+from .runners.docker import DockerRunner
+from .runners.k8s import K8sRunner
+from .session import (
+    YonderSession,
+    closeAll,
+    get,
+    getSession,
+    register,
+    setLogLevel,
+    wait,
+)
+
+__all__ = [
+    "yonder",
+    "register",
+    "get",
+    "wait",
+    "closeAll",
+    "setLogLevel",
+    "getSession",
+    "YonderSession",
+    "Runner",
+    "DockerRunner",
+    "K8sRunner",
+    "HostPath",
+]
+
+__version__ = "0.1.0"

yonder/decorator.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import functools
+import pickle
+
+import cloudpickle
+
+from .paths import translate as _translate_host_paths
+from .session import get as _get_runner
+
+_DOCS_BASE = "https://jason-berger.gitlab.io/yonder"
+_REF_MODE_DOCS = f"{_DOCS_BASE}/pages/runners.html#runner-modes"
+_REF_MODE_EXAMPLE = f"{_DOCS_BASE}/pages/examples.html#rdkit-example"
+
+
+def yonder(runner: str):
+    """Decorator: run the wrapped function on a registered runner.
+
+    ``runner`` is the name previously registered via :func:`yonder.register`.
+
+    Whether a call dispatches to the runner or runs locally is controlled
+    by the runner's own ``when`` predicate (see :class:`~yonder.Runner`).
+    If the runner has no ``when``, every call dispatches.
+
+    Transport is selected by the runner via its ``mode`` attribute. With
+    ``mode="cloudpickle"`` (the default), the function is shipped as
+    bytecode via cloudpickle and host/container Python minor versions must
+    match. With ``mode="reference"``, the function is shipped as
+    ``(module, qualname, args, kwargs)`` and the container imports the
+    module under its own Python; the function must be defined at module
+    scope in a non-``__main__`` module.
+    """
+
+    def decorate(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            runner_obj = _get_runner(runner)
+            runner_when = getattr(runner_obj, "when", None)
+            if runner_when is not None and not runner_when():
+                # Local dispatch: HostPath instances pass through unchanged
+                # (they're str subclasses, so open(p)/Path(p)/etc. all work).
+                return func(*args, **kwargs)
+
+            # Remote dispatch: rewrite any HostPath in args/kwargs to its
+            # in-runner equivalent before serializing.
+            args = tuple(
+                _translate_host_paths(a, runner_obj.translate_host_path)
+                for a in args
+            )
+            kwargs = {
+                k: _translate_host_paths(v, runner_obj.translate_host_path)
+                for k, v in kwargs.items()
+            }
+
+            mode = getattr(runner_obj, "mode", "cloudpickle")
+            if mode == "reference":
+                _validate_reference(func)
+                payload = pickle.dumps(
+                    {
+                        "module": func.__module__,
+                        "qualname": func.__qualname__,
+                        "args": args,
+                        "kwargs": kwargs,
+                    }
+                )
+                result_bytes = runner_obj.run(payload)
+                envelope = pickle.loads(result_bytes)
+            else:
+                payload = cloudpickle.dumps(
+                    {"func": func, "args": args, "kwargs": kwargs}
+                )
+                result_bytes = runner_obj.run(payload)
+                envelope = cloudpickle.loads(result_bytes)
+
+            if envelope["status"] == "ok":
+                return envelope["value"]
+            raise envelope["exception"]
+
+        wrapper.__yonder__ = True
+        wrapper.__yonder_runner__ = runner
+        # Reference-mode bootstrap reads this to skip the wrapper and call
+        # the original function inside the container.
+        wrapper.__yonder_inner__ = func
+        return wrapper
+
+    return decorate
+
+
+def _validate_reference(func) -> None:
+    if func.__qualname__ != func.__name__:
+        raise TypeError(
+            f"reference-mode runners require module-scope functions, but "
+            f"`{func.__qualname__}` is defined inside another scope. Move "
+            f"the function to the top of a module file (e.g. tasks.py) and "
+            f"import it where you need it.\n"
+            f"  docs:    {_REF_MODE_DOCS}\n"
+            f"  example: {_REF_MODE_EXAMPLE}"
+        )
+    if func.__module__ == "__main__":
+        raise TypeError(
+            f"reference-mode runners cannot ship `{func.__name__}` because "
+            f"it is defined in the same script you're running "
+            f"(module `__main__`). The container would have to re-execute "
+            f"your script to import it, which loops back through this code "
+            f"and recurses. Move `{func.__name__}` into a separate "
+            f"importable module (e.g. tasks.py) and import it from your "
+            f"script — see the rdkit example below.\n"
+            f"  docs:    {_REF_MODE_DOCS}\n"
+            f"  example: {_REF_MODE_EXAMPLE}"
+        )

yonder/paths.py

@@ -0,0 +1,62 @@
+"""Path translation between host and runner filesystems.
+
+Users wrap a host-side filesystem path in :class:`HostPath` to tell the
+decorator "translate this against the runner's mount table when this
+call dispatches remotely; pass it through unchanged when it runs
+locally." ``HostPath`` is a ``str`` subclass, so a ``HostPath`` always
+works anywhere a path string works — ``open(p)``, ``pathlib.Path(p)``,
+``os.path.join(p, ...)`` — without unwrapping.
+"""
+from __future__ import annotations
+
+import os
+from typing import Any, Callable
+
+
+class HostPath(str):
+    """A host-filesystem path that yonder should translate to its
+    in-runner equivalent when a call dispatches remotely.
+
+    Inherits from :class:`str`, so it can be used anywhere a path string
+    can::
+
+        runner = DockerRunner(image="...", mounts={"/home/jason/data": "/data"})
+
+        @yonder(runner="r")
+        def process(path: str) -> str:
+            return open(path).read()
+
+        process(HostPath("/home/jason/data/file.txt"))
+        # remote dispatch: decorator rewrites to "/data/file.txt"
+        # local dispatch (runner.when() returned False): passes through as
+        # "/home/jason/data/file.txt"
+
+    If no mount on the resolved runner covers the path, translation
+    raises :class:`ValueError` before any payload is shipped.
+    """
+
+    def __new__(cls, value):
+        return super().__new__(cls, os.fspath(value))
+
+    def __repr__(self) -> str:  # pragma: no cover
+        return f"HostPath({str.__repr__(self)})"
+
+
+def translate(value: Any, translate_fn: Callable[[str], str]) -> Any:
+    """Walk ``value`` and apply ``translate_fn`` to every :class:`HostPath`
+    found inside it.
+
+    Recurses into ``list``, ``tuple``, and ``dict`` so wrapping a path
+    inside an ordinary container still gets translated. Other types pass
+    through unchanged — this is opt-in via ``HostPath``, not a blanket
+    string-rewriter.
+    """
+    if isinstance(value, HostPath):
+        return translate_fn(value)
+    if isinstance(value, list):
+        return [translate(v, translate_fn) for v in value]
+    if isinstance(value, tuple):
+        return tuple(translate(v, translate_fn) for v in value)
+    if isinstance(value, dict):
+        return {k: translate(v, translate_fn) for k, v in value.items()}
+    return value

yonder/runners/__init__.py

@@ -0,0 +1,5 @@
+from .base import Runner
+from .docker import DockerRunner
+from .k8s import K8sRunner
+
+__all__ = ["Runner", "DockerRunner", "K8sRunner"]

yonder/runners/base.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Callable, Optional
+
+
+class Runner(ABC):
+    """A self-contained execution target for a yonder-decorated function.
+
+    Concrete runners hold their own configuration (image, env, mounts, etc.)
+    and expose a single :meth:`run` method that takes a cloudpickled payload
+    of the form ``{"func": ..., "args": (...), "kwargs": {...}}`` and returns
+    the cloudpickled envelope produced by the in-container bootstrap.
+
+    :meth:`start` and :meth:`close` are optional lifecycle hooks; the default
+    implementations are no-ops, which is correct for stateless / on-demand
+    runners. Long-lived runners override them to manage their container.
+
+    ``when`` is an optional zero-arg predicate stored on the runner itself.
+    If set and it returns ``False``, the decorator skips this runner and
+    runs the function locally — independent of any ``when=`` passed to the
+    ``@yonder`` decorator. It composes with the decorator's predicate as
+    AND: the runner is used only when both return ``True``.
+
+    ``mode`` selects the transport. ``"cloudpickle"`` (default) ships the
+    function as bytecode via cloudpickle; the host and container Pythons
+    must match minor versions. ``"reference"`` ships the function as
+    ``(module, qualname, args, kwargs)`` and lets the container import
+    its own copy — bytecode never crosses the wire and host/container
+    Python versions can differ. Reference mode requires the function to
+    be defined at module scope in a non-``__main__`` module.
+    """
+
+    when: Optional[Callable[[], bool]] = None
+    mode: str = "cloudpickle"
+
+    @abstractmethod
+    def run(self, payload: bytes) -> bytes: ...
+
+    def translate_host_path(self, path: str) -> str:
+        """Translate a host-side path to its equivalent inside the runner.
+
+        Called by the decorator on every :class:`yonder.HostPath` it finds
+        in a call's args/kwargs before dispatching. Default implementation
+        raises — subclasses that mount host paths into the runner (e.g.
+        :class:`yonder.DockerRunner`) should override.
+
+        Should raise :class:`ValueError` if no mount on this runner covers
+        ``path``.
+        """
+        raise NotImplementedError(
+            f"{type(self).__name__} does not support HostPath translation. "
+            f"Pass the in-runner path directly instead."
+        )
+
+    def start(self) -> None:
+        """Optional eager initialization. Default no-op."""
+
+    def close(self) -> None:
+        """Optional cleanup. Default no-op."""
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        self.close()