horus-runtime 0.2.2__tar.gz → 0.3.0__tar.gz

{horus_runtime-0.2.2 → horus_runtime-0.3.0}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: horus-runtime
-Version: 0.2.2
+Version: 0.3.0
 Summary: The Horus workflow manager runtime. Execute advanced workflows with multi-environment support, HPC integration, automatic artifact management, and more.
 License-File: LICENSE
 Classifier: License :: OSI Approved :: GNU Affero General Public License v3
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.13
 Requires-Dist: click>=8.3.3
+Requires-Dist: cloudpickle>=3.0
 Requires-Dist: loguru~=0.7
 Requires-Dist: pydantic-settings~=2.0
 Requires-Dist: pydantic~=2.0

{horus_runtime-0.2.2 → horus_runtime-0.3.0}/pyproject.toml

@@ -11,6 +11,7 @@ classifiers = [
 ]
 dependencies = [
   "click>=8.3.3",
+  "cloudpickle>=3.0",
   "loguru~=0.7",
   "pydantic~=2.0",
   "pydantic_settings~=2.0",
@@ -34,6 +35,7 @@ pickle = "horus_builtin.artifact.pickle"
 shell = "horus_builtin.executor.shell"
 python_exec = "horus_builtin.executor.python_exec"
 python_fn = "horus_builtin.executor.python_fn"
+python_fn_external = "horus_builtin.executor.python_fn_external"
 
 [project.entry-points."horus.interaction"]
 common = "horus_builtin.interaction.common"
@@ -110,6 +112,10 @@ strict = true
 exclude = ["build", "dist"]
 plugins = ["pydantic.mypy"]
 
+[[tool.mypy.overrides]]
+module = ["cloudpickle.*"]
+ignore_missing_imports = true
+
 [tool.pydantic-mypy]
 init_typed = true
 init_forbid_extra = true

{horus_runtime-0.2.2 → horus_runtime-0.3.0}/src/horus_builtin/event/tui_subscriber.py

@@ -29,7 +29,7 @@ import sys
 import time
 from collections import deque
 from collections.abc import Iterator
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from typing import TYPE_CHECKING, ClassVar, NamedTuple
 
 from pydantic import PrivateAttr
@@ -258,6 +258,8 @@ class WorkflowTUISubscriber(BaseEventSubscriber):
     _last_transfer: tuple[str, float] | None = PrivateAttr(default=None)
     _error: tuple[str, str] | None = PrivateAttr(default=None)
     _paused: bool = PrivateAttr(default=False)
+    _finished: bool = PrivateAttr(default=False)
+    _finished_at: float | None = PrivateAttr(default=None)
 
     def setup(self) -> None:
         """No startup work needed."""
@@ -328,6 +330,16 @@ class WorkflowTUISubscriber(BaseEventSubscriber):
                     self._capture_error(exc)
                     raise
                 finally:
+                    # Leave the DAG/table/log view on screen instead of
+                    # tearing it down the instant the run returns, so the
+                    # user can still see final task state. Only wait on an
+                    # interactive TTY, non-interactive runs (CI, pipes)
+                    # would otherwise hang forever on this.
+                    self._finished = True
+                    self._finished_at = time.monotonic()
+                    if sys.stdin.isatty():
+                        with suppress(EOFError, KeyboardInterrupt):
+                            input()
                     self._live = None
         finally:
             horus_logger.restore_terminal()
@@ -459,6 +471,10 @@ class WorkflowTUISubscriber(BaseEventSubscriber):
         ]
         if self._error is not None:
             sections.append(self._render_error())
+        if self._finished:
+            sections.append(
+                Text(_("Workflow finished, press Enter to close"), style="dim")
+            )
         return Group(*sections)
 
     def _render_header(self, workflow: BaseWorkflow) -> RenderableType:
@@ -467,7 +483,7 @@ class WorkflowTUISubscriber(BaseEventSubscriber):
         elapsed = (
             None
             if self._started_at is None
-            else time.monotonic() - self._started_at
+            else (self._finished_at or time.monotonic()) - self._started_at
         )
         line = Text.assemble(
             (workflow.name, "bold"),

horus_runtime-0.3.0/src/horus_builtin/executor/_remote_function_call.py

@@ -0,0 +1,117 @@
+#
+# horus-runtime
+# Copyright (C) 2026 Temple Compute
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+"""
+Launcher shipped to the target by
+:class:`~horus_builtin.executor.python_fn_external.
+ExternalPythonFunctionExecutor`.
+
+It is copied to the task working directory and run there by the target's
+interpreter, so it is deliberately a *script*: it must import nothing from
+horus-runtime at module level (the payload may pull horus classes in while
+unpickling, but the launcher itself must start on any interpreter that has
+cloudpickle).
+
+Protocol, both halves of which are hard-coded here and in the executor:
+
+- ``argv[1]``: cloudpickled ``(func, kwargs)`` payload to read.
+- ``argv[2]``: where to write the cloudpickled outcome.
+- outcome: ``{"ok": bool, "value": ..., "traceback": str | None,
+  "exception": BaseException | None}``.
+
+The outcome file, not the exit code, is what the executor believes: an exit
+code can only say *that* something failed, and "subprocess exited 1" is a
+useless thing to hand a scientist whose function raised on line 40.
+"""
+
+import asyncio
+import inspect
+import sys
+import traceback
+from collections.abc import Awaitable
+from typing import Any
+
+import cloudpickle
+
+
+async def _resolve(awaitable: Awaitable[Any]) -> Any:
+    """Await *awaitable*, the entry point ``asyncio.run`` needs."""
+    return await awaitable
+
+
+def _write_outcome(result_path: str, outcome: dict[str, Any]) -> None:
+    """
+    Write *outcome* to *result_path*, degrading to text if it will not pickle.
+
+    A return value or an exception can be unpicklable (an open file handle
+    captured in ``__init__``, say). Losing the object is survivable; losing
+    the traceback that explains the run is not, so the fallback keeps the
+    text.
+    """
+    try:
+        data = cloudpickle.dumps(outcome)
+    except Exception:
+        data = cloudpickle.dumps(
+            {
+                "ok": False,
+                "value": None,
+                "exception": None,
+                "traceback": (
+                    f"{outcome.get('traceback') or ''}\n"
+                    f"Could not send the result back to the orchestrator:\n"
+                    f"{traceback.format_exc()}"
+                ).strip(),
+            }
+        )
+    with open(result_path, "wb") as fh:
+        fh.write(data)
+
+
+def main(payload_path: str, result_path: str) -> int:
+    """
+    Call the pickled function and report the outcome. Returns an exit code.
+    """
+    with open(payload_path, "rb") as fh:
+        func, kwargs = cloudpickle.load(fh)
+
+    outcome: dict[str, Any]
+    try:
+        value = func(**kwargs)
+        # The in-process executor awaits coroutine functions, so an async task
+        # body must keep working here.
+        if inspect.isawaitable(value):
+            value = asyncio.run(_resolve(value))
+        outcome = {
+            "ok": True,
+            "value": value,
+            "exception": None,
+            "traceback": None,
+        }
+    except Exception as exc:
+        outcome = {
+            "ok": False,
+            "value": None,
+            "exception": exc,
+            "traceback": traceback.format_exc(),
+        }
+
+    _write_outcome(result_path, outcome)
+    return 0 if outcome["ok"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1], sys.argv[2]))

{horus_runtime-0.2.2 → horus_runtime-0.3.0}/src/horus_builtin/executor/python_exec.py

@@ -20,12 +20,15 @@ Defines the PythonExecExecutor class, which represents an executor that runs a
 a Python code task in-process in the Horus runtime.
 """
 
+import os
 from typing import TYPE_CHECKING, ClassVar
 
 from horus_builtin.executor._cwd_lock import chdir_locked
 from horus_builtin.runtime.python_string import PythonCodeStringRuntime
 from horus_runtime.context import HorusContext
 from horus_runtime.core.executor.base import BaseExecutor, RuntimeFilterType
+from horus_runtime.core.resources import InProcessScope, ResourceScope
+from horus_runtime.core.target.channel import ChannelProcess
 from horus_runtime.i18n import tr as _
 from horus_runtime.settings import runtime_settings
 
@@ -46,6 +49,15 @@ class PythonExecExecutor(BaseExecutor):
 
     runtimes: ClassVar[RuntimeFilterType] = (PythonCodeStringRuntime,)
 
+    async def resource_scope(
+        self, task: "BaseTask", process: ChannelProcess | None = None
+    ) -> ResourceScope:
+        """
+        Declare that the work runs inside the orchestrator process.
+        """
+        del task, process
+        return InProcessScope(pid=os.getpid())
+
     async def _execute(self, task: "BaseTask") -> None:
         """
         Runs the task in-process by executing the Python code specified in the

{horus_runtime-0.2.2 → horus_runtime-0.3.0}/src/horus_builtin/executor/python_fn.py

@@ -20,6 +20,7 @@ Python executor for in-memory workflows in horus-runtime. (Function
 executor).
 """
 
+import os
 from inspect import isawaitable
 from typing import ClassVar
 
@@ -30,11 +31,49 @@ from horus_builtin.runtime.python import (
 )
 from horus_runtime.core.artifact.base import BaseArtifact
 from horus_runtime.core.executor.base import BaseExecutor, RuntimeFilterType
+from horus_runtime.core.resources import InProcessScope, ResourceScope
+from horus_runtime.core.target.channel import ChannelProcess
 from horus_runtime.core.task.base import BaseTask
 from horus_runtime.i18n import tr as _
 from horus_runtime.logging import horus_logger
 
 
+async def parse_result_artifacts(
+    task: BaseTask, result: PythonFunctionReturnType
+) -> None:
+    """
+    Parse the result of a Python function execution to extract any declared
+    side-product artifacts.
+
+    Module-level so the out-of-process executor applies the exact same rules
+    to a result that came back over a pipe as the in-process one does to a
+    live return value.
+    """
+    if isawaitable(result):
+        result = await result
+
+    if result is None:
+        return
+    if isinstance(result, BaseArtifact):
+        task.side_artifacts = [result]
+        return
+    if isinstance(result, list) and all(
+        isinstance(r, BaseArtifact) for r in result
+    ):
+        task.side_artifacts = result
+        return
+
+    horus_logger.log.warning(
+        _(
+            "Task %(task_id)s returned an unexpected value from its "
+            "Python function runtime. Expected BaseArtifact, "
+            "list[BaseArtifact], or None; got: %(result)s. Skipping side "
+            "artifact handling."
+        )
+        % {"task_id": task.id, "result": result}
+    )
+
+
 class PythonFunctionExecutor(BaseExecutor):
     """
     Executor for running Python functions in-memory.
@@ -48,6 +87,15 @@ class PythonFunctionExecutor(BaseExecutor):
 
     runtimes: ClassVar[RuntimeFilterType] = (PythonFunctionRuntime,)
 
+    async def resource_scope(
+        self, task: "BaseTask", process: ChannelProcess | None = None
+    ) -> ResourceScope:
+        """
+        Declare that the work runs inside the orchestrator process.
+        """
+        del task, process
+        return InProcessScope(pid=os.getpid())
+
     async def _execute(self, task: "BaseTask") -> None:
         """
         Executes the Python function specified in the task's runtime.
@@ -78,26 +126,4 @@ class PythonFunctionExecutor(BaseExecutor):
         Parse the result of a Python function execution to extract any declared
         side-product artifacts.
         """
-        if isawaitable(result):
-            result = await result
-
-        if result is None:
-            return
-        if isinstance(result, BaseArtifact):
-            task.side_artifacts = [result]
-            return
-        if isinstance(result, list) and all(
-            isinstance(r, BaseArtifact) for r in result
-        ):
-            task.side_artifacts = result
-            return
-
-        horus_logger.log.warning(
-            _(
-                "Task %(task_id)s returned an unexpected value from its "
-                "Python function runtime. Expected BaseArtifact, "
-                "list[BaseArtifact], or None; got: %(result)s. Skipping side "
-                "artifact handling."
-            )
-            % {"task_id": task.id, "result": result}
-        )
+        await parse_result_artifacts(task, result)

horus_runtime-0.3.0/src/horus_builtin/executor/python_fn_external.py

@@ -0,0 +1,233 @@
+#
+# horus-runtime
+# Copyright (C) 2026 Temple Compute
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+"""
+Run a ``python_function`` runtime in a subprocess **on the task's target**
+instead of inside the orchestrator.
+
+Why this exists, given ``python_fn`` already runs functions:
+
+- ``PythonFunctionExecutor`` ignores ``task.target``. A function task with an
+  SSH target runs on the orchestrator, which is a correctness bug and not
+  merely a metrics gap. This executor honours the target like every other
+  executor does, by going through ``target.run_command``.
+- Because the work is a real process, it inherits the default
+  :meth:`BaseExecutor.resource_scope` (a ``ProcessTreeScope`` with a live
+  pid) and becomes measurable with no extra code here.
+
+**Limitation: live DAG mutation does not work here.** A task body may call
+``workflow.add_task()`` / ``add_edge()`` on the running workflow (see
+``core/workflow/base.py``, "Safe to call from inside a running task"). That
+depends on sharing memory with the orchestrator and cannot cross a process
+boundary, so tasks that mutate the DAG must keep using the in-process
+``python_function`` executor. The same reasoning applies to any function that
+takes the live ``task``; rather than hand it a lookalike copy whose mutations
+would silently go nowhere, this executor refuses such a function up front.
+"""
+
+import asyncio
+import shlex
+import sys
+from contextlib import aclosing
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar
+
+import cloudpickle
+
+from horus_builtin.executor.python_fn import parse_result_artifacts
+from horus_builtin.runtime.python import PythonFunctionRuntime
+from horus_runtime.core.executor.base import BaseExecutor, RuntimeFilterType
+from horus_runtime.core.task.exceptions import TaskExecutionError
+from horus_runtime.i18n import tr as _
+from horus_runtime.logging import horus_logger
+from horus_runtime.settings import runtime_settings
+
+if TYPE_CHECKING:
+    from horus_runtime.core.task.base import BaseTask
+
+_LAUNCHER_SOURCE = Path(__file__).with_name("_remote_function_call.py")
+_LAUNCHER_NAME = ".horus_function_call.py"
+_PAYLOAD_NAME = ".horus_function_payload.pkl"
+_RESULT_NAME = ".horus_function_result.pkl"
+
+
+class ExternalPythonFunctionExecutor(BaseExecutor):
+    """
+    Execute a Python function in a subprocess on the task's target.
+    """
+
+    kind: str = "python_function_external"
+    kind_name: ClassVar[str] = "External Python Function Executor"
+    kind_description: ClassVar[str] = _(
+        "Executes a Python function in a subprocess on the task's target."
+    )
+
+    runtimes: ClassVar[RuntimeFilterType] = (PythonFunctionRuntime,)
+
+    python: str | None = None
+    """
+    Interpreter used on the target. It needs ``cloudpickle`` importable, plus
+    whatever the function itself imports, including ``horus_runtime`` if the
+    function takes or returns artifacts.
+
+    ``None`` (the default) means: the orchestrator's own interpreter when the
+    target is co-located with it, otherwise plain ``python``. The first is the
+    one interpreter that provably satisfies those requirements, since it is
+    running this code; the second is the only thing that can be assumed about
+    a machine we have never seen.
+
+    Environment management is deliberately *not* reimplemented here. The
+    ``horus-environments`` executors already build conda/uv/virtualenv
+    environments on the target and put the interpreter at a known path, so
+    composing with them is a matter of pointing this field at one, e.g.
+    ``python: .horus_python_environment/bin/python``. Relative paths resolve
+    against the task working directory, which is the command's cwd.
+    """
+
+    def _interpreter(self, task: "BaseTask") -> str:
+        """Return the interpreter command to invoke on the target."""
+        if self.python is not None:
+            return self.python
+        if task.target.is_orchestrator_local:
+            return sys.executable
+        return "python"
+
+    async def _execute(self, task: "BaseTask") -> None:
+        """
+        Ship the function to the target, run it there, bring the result back.
+        """
+        assert isinstance(task.runtime, PythonFunctionRuntime)
+        func, kwargs = await task.runtime.setup_runtime(task)
+
+        if "task" in kwargs:
+            # PythonFunctionRuntime injects the live task whenever the
+            # signature asks for it. A copy of it on the far side of a pipe
+            # would look right and do nothing (mutations, DAG edits and
+            # interactions all die with the subprocess), so fail loudly here
+            # instead of subtly there.
+            raise TaskExecutionError(
+                _(
+                    "Task %(task_id)s uses the '%(kind)s' executor but its "
+                    "function accepts a 'task' argument. The live task object "
+                    "cannot cross a process boundary; use the in-process "
+                    "'python_function' executor, or drop the 'task' parameter "
+                    "(and any **kwargs that would receive it)."
+                )
+                % {"task_id": task.id, "kind": self.kind}
+            )
+
+        await task.target.mkdir(task.working_dir)
+
+        # cloudpickle rather than pickle: closing over local state is a
+        # supported way to build a workflow programmatically, and plain pickle
+        # cannot represent a closure. It also already prefers a by-reference
+        # pickle for anything importable, so the cheap payload is the default
+        # and the expensive one only appears when it is the only option.
+        payload = f"{task.working_dir}/{_PAYLOAD_NAME}"
+        result_path = f"{task.working_dir}/{_RESULT_NAME}"
+        launcher = f"{task.working_dir}/{_LAUNCHER_NAME}"
+        await task.target.put_file(cloudpickle.dumps((func, kwargs)), payload)
+        await task.target.put_file(_LAUNCHER_SOURCE, launcher)
+
+        command = (
+            f"{self._interpreter(task)} {shlex.quote(launcher)} "
+            f"{shlex.quote(payload)} {shlex.quote(result_path)}"
+        )
+        horus_logger.log.debug(
+            _("Executing function for task %(task_id)s: %(command)s")
+            % {"task_id": task.id, "command": command}
+        )
+
+        proc = await task.target.run_command(
+            command,
+            cwd=task.working_dir,
+            env={
+                runtime_settings.SIDE_ARTIFACTS_DIR_ENV: str(
+                    task.side_artifacts_dir
+                ),
+            },
+        )
+        try:
+            async with aclosing(proc.stream()) as stream:
+                async for stream_name, line in stream:
+                    text = line.decode("utf-8", errors="replace").rstrip()
+                    if stream_name == "stdout":
+                        horus_logger.log.info(text)
+                    else:
+                        horus_logger.log.warning(text)
+        except asyncio.CancelledError:
+            proc.kill()
+            await proc.wait()
+            raise
+        await proc.wait()
+
+        outcome = await self._read_outcome(task, result_path, proc.returncode)
+        await parse_result_artifacts(task, outcome["value"])
+
+    async def _read_outcome(
+        self, task: "BaseTask", result_path: str, returncode: int | None
+    ) -> dict[str, Any]:
+        """
+        Read the launcher's outcome file and re-raise a remote failure here.
+
+        The exit code is only a fallback: what the caller needs is the
+        exception the function raised, with its own traceback. When it made it
+        back it is re-raised as itself, so ``except MyError`` around a workflow
+        run still catches what it always caught, with the target-side frames
+        attached as a note.
+        """
+        try:
+            raw = await task.target.get_file(result_path)
+            outcome: dict[str, Any] = cloudpickle.loads(raw)
+        except Exception as exc:
+            # No outcome file at all: the interpreter never got far enough to
+            # write one (missing python, missing cloudpickle, killed job). The
+            # streamed stderr above is the real diagnosis; point at it.
+            raise TaskExecutionError(
+                _(
+                    "Function subprocess for task %(task_id)s produced no "
+                    "result (exit code %(code)s); see the task log for the "
+                    "interpreter's own output. Cause: %(err)s"
+                )
+                % {
+                    "task_id": task.id,
+                    "code": returncode,
+                    "err": exc,
+                }
+            ) from exc
+
+        if outcome["ok"]:
+            return outcome
+
+        remote_traceback = outcome["traceback"] or ""
+        exception = outcome["exception"]
+        if isinstance(exception, BaseException):
+            exception.add_note(
+                _("Raised on target %(target)s:\n%(traceback)s")
+                % {
+                    "target": task.target.location_id,
+                    "traceback": remote_traceback,
+                }
+            )
+            raise exception
+        raise TaskExecutionError(
+            _("Function failed on target %(target)s:\n%(traceback)s")
+            % {
+                "target": task.target.location_id,
+                "traceback": remote_traceback,
+            }
+        )

{horus_runtime-0.2.2 → horus_runtime-0.3.0}/src/horus_builtin/runtime/command.py

@@ -21,6 +21,8 @@ Command implementation for the runtime.
 
 from typing import TYPE_CHECKING, ClassVar
 
+from pydantic import Field
+
 from horus_builtin.runtime.substitution import substitute
 from horus_runtime.context import HorusContext
 from horus_runtime.core.runtime.base import BaseRuntime
@@ -51,7 +53,7 @@ class CommandRuntime(BaseRuntime[str]):
     through untouched.
     """
 
-    formatted_command: str = ""
+    formatted_command: str = Field(default="", exclude=True)
     """
     The formatted command after processing any placeholders.
     """