interloper-docker 0.2.0__tar.gz → 0.3.0__tar.gz

{interloper_docker-0.2.0 → interloper_docker-0.3.0}/PKG-INFO

@@ -1,14 +1,12 @@
 Metadata-Version: 2.3
 Name: interloper-docker
-Version: 0.2.0
+Version: 0.3.0
 Summary: Interloper Docker integration
 Author: Guillaume Onfroy
 Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
 Requires-Dist: docker>=7.1.0
 Requires-Dist: interloper-core
+Requires-Dist: interloper-scheduler
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 
-# interloper-docker
-
-Docker execution support for Interloper.

{interloper_docker-0.2.0 → interloper_docker-0.3.0}/pyproject.toml

@@ -3,24 +3,20 @@
 # ###############
 [project]
 name = "interloper-docker"
-version = "0.2.0"
+version = "0.3.0"
 description = "Interloper Docker integration"
 readme = "README.md"
 authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
 requires-python = ">=3.10"
-dependencies = [
-    "docker>=7.1.0",
-    "interloper-core",
-]
-
-[project.optional-dependencies]
+dependencies = ["docker>=7.1.0", "interloper-core", "interloper-scheduler"]
 
 [build-system]
-requires = ["uv_build>=0.9.5,<0.10.0"]
+requires = ["uv_build>=0.11.5,<0.12"]
 build-backend = "uv_build"
 
 [tool.uv.sources]
 interloper-core = { workspace = true }
+interloper-scheduler = { workspace = true }
 
 # ###############
 # RUFF
@@ -33,7 +29,6 @@ extend-select = ["E", "I", "UP", "ANN001", "ANN201", "ANN202"]
 
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["F401", "F403"]
-"**/schemas/**" = ["E501"]
 "tests/**" = ["ANN", "F811"]
 
 # ###############
@@ -43,4 +38,4 @@ extend-select = ["E", "I", "UP", "ANN001", "ANN201", "ANN202"]
 include = ["src"]
 typeCheckingMode = "basic"
 reportMissingParameterType = true
-ignore = ["libs/**", "tests/**", "scripts/**"]
+ignore = ["tests/**"]

{interloper_docker-0.2.0 → interloper_docker-0.3.0}/src/interloper_docker/__init__.py

@@ -1,9 +1,9 @@
 """Interloper Docker integration for container-based asset execution."""
 
-from interloper_docker.backfiller import DockerBackfiller
+from interloper_docker.launcher import DockerLauncher
 from interloper_docker.runner import DockerRunner
 
 __all__ = [
-    "DockerBackfiller",
+    "DockerLauncher",
     "DockerRunner",
 ]

interloper_docker-0.3.0/src/interloper_docker/launcher.py

@@ -0,0 +1,165 @@
+"""Docker launcher: runs each job in its own container."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Any
+from uuid import UUID
+
+import docker
+from interloper.catalog.base import Catalog
+from interloper_scheduler.launcher import Launcher, RunState, RunStatus
+
+logger = logging.getLogger(__name__)
+
+# TODO: Implement a grace period to check if the container is running before returning in order to avoid
+# stale run statuses due to container startup errors?
+
+# TODO: `launch` command should supoort --catalog option to pass the catalog as a serialized string
+# Then use this instead of INTERLOPER_CATALOG
+
+
+class DockerLauncher(Launcher):
+    """Launches each run in its own Docker container.
+
+    The container executes the ``interloper launch <run_id>`` CLI command,
+    which hydrates the DAG from the database and runs it to completion.
+
+    Postgres connection parameters are passed as plain values. The caller
+    (``_build_launcher``) injects the app-level defaults; any overrides
+    from the launcher YAML config take precedence.
+    """
+
+    def __init__(
+        self,
+        catalog: Catalog,
+        postgres_host: str,
+        postgres_port: int,
+        postgres_user: str,
+        postgres_password: str,
+        postgres_database: str,
+        image: str = "interloper:latest-scheduler",
+        runner_type: str = "multi_thread",
+        runner_config: dict[str, Any] | None = None,
+        volumes: dict[str, dict[str, str]] | None = None,
+    ) -> None:
+        """Initialize the Docker launcher.
+
+        Args:
+            catalog: Catalog to inject into the container so it builds an identical catalog.
+            postgres_host: Postgres host to inject into the container.
+            postgres_port: Postgres port to inject into the container.
+            postgres_user: Postgres user to inject into the container.
+            postgres_password: Postgres password to inject into the container.
+            postgres_database: Postgres database to inject into the container.
+            image: Docker image to use.
+            runner_type: Runner type name forwarded to the container.
+            runner_config: Runner-specific kwargs forwarded to the container.
+            volumes: Volume mounts for the container.  When
+                ``runner_type`` is ``"docker"``, the Docker socket is
+                mounted automatically if not already included.
+        """
+        super().__init__(runner_type=runner_type, runner_config=runner_config)
+        self._client = docker.from_env()
+        self._catalog = catalog
+        self._image = image
+        self._postgres_host = postgres_host
+        self._postgres_port = postgres_port
+        self._postgres_user = postgres_user
+        self._postgres_password = postgres_password
+        self._postgres_database = postgres_database
+        self._volumes = dict(volumes or {})
+        if runner_type == "docker" and "/var/run/docker.sock" not in self._volumes:
+            self._volumes["/var/run/docker.sock"] = {"bind": "/var/run/docker.sock", "mode": "rw"}
+
+    def launch(self, run_id: UUID) -> None:
+        """Start a container that executes a single run.
+
+        Args:
+            run_id: The run UUID to execute.
+        """
+        environment = self._build_environment()
+        container_name = f"interloper_run_{str(run_id)[:8]}"
+
+        try:
+            container = self._client.containers.run(
+                image=self._image,
+                name=container_name,
+                command=["interloper", "launch", str(run_id)],
+                environment=environment,
+                volumes=self._volumes if self._volumes else None,
+                user="root" if self._runner_type == "docker" else None,
+                detach=True,
+                auto_remove=False,
+                labels={"interloper.run_id": str(run_id)},
+            )
+            logger.info("Started container %s for run %s", container.short_id, run_id)
+        except Exception:
+            logger.exception("Failed to start container for run %s", run_id)
+            raise
+
+    def describe_run(self, run_id: UUID) -> RunState:
+        """Return the authoritative state of a run's container.
+
+        Used by the reaper to catch failed runs as soon as the
+        container terminates, without waiting for the fallback timeout.
+
+        Args:
+            run_id: The run UUID to describe.
+
+        Returns:
+            A :class:`RunState` indicating whether the container is
+            still running, has succeeded, has failed, or is gone.
+        """
+        container_name = f"interloper_run_{str(run_id)[:8]}"
+        try:
+            container = self._client.containers.get(container_name)
+        except Exception:
+            return RunState(status=RunStatus.NOT_FOUND)
+
+        try:
+            container.reload()
+        except Exception:
+            return RunState(status=RunStatus.NOT_FOUND)
+
+        state = container.attrs.get("State", {}) if container.attrs else {}
+        docker_status = (state.get("Status") or container.status or "").lower()
+
+        # "running", "created", "restarting", "paused" — still alive
+        if docker_status in ("running", "created", "restarting", "paused"):
+            return RunState(status=RunStatus.RUNNING)
+
+        # Terminal states: "exited", "dead", "removing"
+        exit_code = state.get("ExitCode")
+        if exit_code == 0:
+            return RunState(status=RunStatus.SUCCEEDED)
+
+        # Anything else is a failure
+        parts = [f"Container {container.short_id} status={docker_status}"]
+        if exit_code is not None:
+            parts.append(f"exit_code={exit_code}")
+        if state.get("OOMKilled"):
+            parts.append("OOMKilled")
+        error = state.get("Error") or ""
+        if error:
+            parts.append(f"error={error}")
+        return RunState(status=RunStatus.FAILED, error=" ".join(parts))
+
+    def _build_environment(self) -> dict[str, str]:
+        """Build environment variables for the container."""
+        environment: dict[str, str] = {
+            "INTERLOPER_POSTGRES_HOST": self._postgres_host,
+            "INTERLOPER_POSTGRES_PORT": str(self._postgres_port),
+            "INTERLOPER_POSTGRES_USER": self._postgres_user,
+            "INTERLOPER_POSTGRES_PASSWORD": self._postgres_password,
+            "INTERLOPER_POSTGRES_DATABASE": self._postgres_database,
+            "INTERLOPER_CATALOG": json.dumps(self._catalog.to_paths()),
+            "INTERLOPER_RUNNER_TYPE": self._runner_type,
+            "INTERLOPER_RUNNER_CONFIG": json.dumps(self._runner_config),
+        }
+        encryption_key = os.environ.get("SECRETS_ENCRYPTION_KEY")
+        if encryption_key:
+            environment["SECRETS_ENCRYPTION_KEY"] = encryption_key
+        return environment

interloper_docker-0.3.0/src/interloper_docker/runner.py

@@ -0,0 +1,343 @@
+"""Docker-based runner that executes each asset in its own container.
+
+Each submitted asset runs inside a fresh container. A mini-DAG comprising
+the target asset and all its upstream ancestors is sent to the container
+via inline JSON so the asset can resolve its upstream dependencies from IO
+without recomputing them.
+
+**Real-time events** are streamed via **stderr** using the ``@EVENT:``
+prefix (see :class:`~interloper.events.StderrEventHandler`).  Events for
+the target asset are forwarded to the host EventBus; events for
+non-materializable parent assets and container-internal ``RUN_*`` events
+are dropped.  The host updates internal state with ``emit=False`` to
+avoid duplicate events.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+from concurrent.futures import Future, ThreadPoolExecutor
+from typing import Any
+
+import docker
+from docker.client import DockerClient
+from docker.models.containers import Container
+from interloper.asset.base import Asset
+from interloper.errors import RunnerError
+from interloper.events import EventBus, EventType
+from interloper.events.event import parse_event_from_log_line
+from interloper.partitioning.base import Partition, PartitionWindow
+from interloper.partitioning.time import TimePartition, TimePartitionWindow
+from interloper.runner.sync_runner import SyncRunner
+from pydantic import Field, PrivateAttr
+
+logger = logging.getLogger(__name__)
+
+# Events emitted by the container's inner run — not forwarded to the host
+# because the host manages its own run lifecycle.
+_RUN_EVENTS = frozenset(
+    {
+        EventType.RUN_STARTED,
+        EventType.RUN_COMPLETED,
+        EventType.RUN_FAILED,
+    }
+)
+
+
+class DockerRunner(SyncRunner):
+    """Execute assets as individual Docker containers.
+
+    For each asset, constructs a mini-DAG comprising the asset and all its
+    upstream ancestors. The mini-DAG is sent to the container via inline JSON.
+    Inside the container, all non-target assets are marked as
+    ``materializable=False`` to avoid recomputation while still enabling
+    IO-based dependency resolution.
+
+    Events are emitted by the container process and streamed to the host
+    via stderr.  The host forwards them to its EventBus (for persistence
+    by the scheduler) and updates internal state silently (``emit=False``)
+    to avoid duplicate events.
+
+    Fully synchronous::
+
+        with DockerRunner(image="my-image", on_event=log_event) as runner:
+            result = runner.run(dag)
+    """
+
+    image: str = "interloper:latest-worker"
+    max_containers: int = 4
+    env_vars: dict[str, str] = Field(default_factory=dict)
+    volumes: dict[str, dict[str, str]] | list[str] = Field(default_factory=dict)
+    fail_fast: bool = False
+    reraise: bool = False
+    auto_remove: bool = True
+
+    _docker: DockerClient = PrivateAttr()
+    _poll_pool: ThreadPoolExecutor | None = PrivateAttr(default=None)
+    _log_threads: dict[str, threading.Thread] = PrivateAttr(default_factory=dict)
+    _stop_log_streaming: threading.Event = PrivateAttr(default_factory=threading.Event)
+    _container_map: dict[Future[Any], Container] = PrivateAttr(default_factory=dict)
+
+    def model_post_init(self, context: Any) -> None:
+        """Initialize Docker client after model initialization."""
+        super().model_post_init(context)
+        self._docker = docker.from_env()
+
+    # ------------------------------------------------------------------
+    # Scheduling primitives
+    # ------------------------------------------------------------------
+
+    @property
+    def _capacity(self) -> int:
+        """Maximum number of concurrent containers."""
+        return self.max_containers
+
+    def _on_start(self) -> None:
+        """Create the polling thread pool."""
+        self._stop_log_streaming.clear()
+        self._poll_pool = ThreadPoolExecutor(max_workers=self.max_containers)
+
+    def _on_end(self) -> None:
+        """Shut down log streaming and the polling pool."""
+        self._stop_log_streaming.set()
+        for thread in self._log_threads.values():
+            thread.join(timeout=2.0)
+        self._log_threads.clear()
+        if self._poll_pool is not None:
+            self._poll_pool.shutdown(wait=True, cancel_futures=False)
+            self._poll_pool = None
+        self._container_map.clear()
+
+    def _submit_asset(
+        self,
+        asset: Asset,
+        partition_or_window: Partition | PartitionWindow | None,
+    ) -> Future[Any]:
+        """Launch an asset in a Docker container and return a polling Future.
+
+        Returns:
+            A Future that raises :class:`RunnerError` on container failure.
+        """
+        if self._poll_pool is None:
+            raise RunnerError("Poll pool not initialized")
+
+        mini_dag = self.state.dag.mini_dag(asset.id)
+        dag_spec = mini_dag.to_spec().model_dump(mode="json")
+
+        cmd = self._build_command(dag_spec, partition_or_window, self.state.run_id)
+        name = self._build_name(asset)
+        env = self._build_env()
+        volumes = self._build_volumes()
+
+        # emit=False: the container emits ASSET_STARTED via stderr
+        self.state.mark_asset_running(asset, emit=False)
+
+        try:
+            container = self._docker.containers.run(
+                image=self.image,
+                name=name,
+                command=cmd,
+                environment=env,
+                volumes=volumes if volumes else None,
+                labels={"interloper.asset_id": asset.id},
+                remove=False,
+                detach=True,
+                stdout=True,
+                stderr=True,
+            )
+        except Exception as e:
+            # Container never started — emit from the host
+            self.state.mark_asset_failed(asset, str(e))
+            done: Future[None] = Future()
+            done.set_result(None)
+            return done
+
+        self._start_log_streaming(container, target_asset_id=asset.id)
+
+        future = self._poll_pool.submit(self._poll_container, container)
+        self._container_map[future] = container
+        return future
+
+    def _handle_completed(self, future: Future[Any], asset: Asset) -> None:
+        """Process a completed container future and clean up.
+
+        Updates internal state silently (``emit=False``) because the
+        container already emitted the real events.  Assets already in a
+        terminal state (e.g. marked failed during ``_submit_asset``) are
+        skipped.
+        """
+        container = self._container_map.pop(future, None)
+        if container is not None:
+            self._stop_container_log_streaming(container)
+
+        info = self.state.asset_executions.get(asset.id)
+        if not (info and info.is_terminal):
+            try:
+                future.result()
+            except Exception as e:
+                self.state.mark_asset_failed(asset, str(e), emit=False)
+                if self.fail_fast or self.reraise:
+                    raise
+            else:
+                self.state.mark_asset_completed(asset, emit=False)
+
+        if container is not None and self.auto_remove:
+            try:
+                container.remove()
+            except Exception:  # noqa: BLE001, S110
+                pass
+
+    def _handle_flushed_future(self, future: Future[Any], asset: Asset) -> None:
+        """Clean up container after flush."""
+        container = self._container_map.pop(future, None)
+        if container is not None:
+            self._stop_container_log_streaming(container)
+
+        info = self.state.asset_executions.get(asset.id)
+        if not (info and info.is_terminal):
+            try:
+                future.result()
+            except Exception as e:  # noqa: BLE001
+                self.state.mark_asset_failed(asset, str(e), emit=False)
+            else:
+                self.state.mark_asset_completed(asset, emit=False)
+
+        if container is not None and self.auto_remove:
+            try:
+                container.remove()
+            except Exception:  # noqa: BLE001, S110
+                pass
+
+    # ------------------------------------------------------------------
+    # Container polling
+    # ------------------------------------------------------------------
+
+    def _poll_container(self, container: Container) -> None:
+        """Block until the container exits; raise on failure.
+
+        Raises:
+            RunnerError: If the container exits with a non-zero code.
+        """
+        result = container.wait()
+        status_code = result.get("StatusCode", 1)
+        if status_code != 0:
+            cid = (container.id or "unknown")[:12]
+            raise RunnerError(f"Container {cid} exited with code {status_code}")
+
+    # ------------------------------------------------------------------
+    # Command and environment builders
+    # ------------------------------------------------------------------
+
+    def _build_command(
+        self,
+        dag_spec: dict[str, Any],
+        partition_or_window: Partition | PartitionWindow | None,
+        run_id: str,
+    ) -> list[str]:
+        """Build the CLI command for asset execution in a container."""
+        cmd = [
+            "interloper",
+            "run",
+            "--format",
+            "inline",
+            f"--run-id={run_id}",
+            json.dumps(dag_spec),
+        ]
+
+        if isinstance(partition_or_window, TimePartition):
+            cmd.extend(["--date", partition_or_window.value.strftime("%Y-%m-%d")])
+        elif isinstance(partition_or_window, TimePartitionWindow):
+            cmd.extend(
+                [
+                    "--start-date",
+                    partition_or_window.start.strftime("%Y-%m-%d"),
+                    "--end-date",
+                    partition_or_window.end.strftime("%Y-%m-%d"),
+                ]
+            )
+
+        return cmd
+
+    def _build_env(self) -> dict[str, str]:
+        """Build the environment variables for the container."""
+        env = dict(self.env_vars)
+        env["INTERLOPER_EVENTS_TO_STDERR"] = "true"
+        return env
+
+    def _build_volumes(self) -> dict[str, dict[str, str]]:
+        """Build the volume mounts for the container."""
+        volumes: dict[str, dict[str, str]] = {}
+        if isinstance(self.volumes, dict):
+            volumes.update(self.volumes)
+        elif isinstance(self.volumes, list):
+            for volume in self.volumes:
+                parts = volume.split(":")
+                volumes[parts[0]] = {"bind": parts[1], "mode": "rw"}
+        return volumes
+
+    def _build_name(self, asset: Asset) -> str:
+        """Build the name for the container."""
+        return f"interloper_run_{self.state.run_id[:8]}_{asset.id[:8]}"
+
+    # ------------------------------------------------------------------
+    # Real-time event streaming (stderr)
+    # ------------------------------------------------------------------
+
+    def _start_log_streaming(self, container: Container, *, target_asset_id: str) -> None:
+        """Stream events from the container's stderr to the host EventBus.
+
+        Only events belonging to the **target asset** are forwarded.
+        Events for non-materializable parent assets in the mini-DAG and
+        container-internal ``RUN_*`` events are dropped.
+
+        Args:
+            container: The Docker container to stream from.
+            target_asset_id: Only forward events with this ``asset_id``.
+        """
+        cid = (container.id or "???")[:12]
+
+        def stream_logs() -> None:
+            buf = ""
+            try:
+                for chunk in container.logs(stream=True, follow=True, stdout=False, stderr=True):
+                    if self._stop_log_streaming.is_set():
+                        break
+                    buf += chunk.decode("utf-8", errors="ignore")
+                    while "\n" in buf:
+                        line, buf = buf.split("\n", 1)
+                        line = line.rstrip()
+                        if not line:
+                            continue
+                        try:
+                            event = parse_event_from_log_line(line)
+                            if event is not None:
+                                if event.type in _RUN_EVENTS:
+                                    continue
+                                event_asset_id = event.metadata.get("asset_id")
+                                if event_asset_id and event_asset_id != target_asset_id:
+                                    continue
+                                EventBus.emit(event.type, metadata=event.metadata)
+                                continue
+                        except Exception:  # noqa: BLE001, S110
+                            pass
+                        logger.debug("[container %s] %s", cid, line)
+            except Exception:  # noqa: BLE001, S110
+                pass
+            buf = buf.rstrip()
+            if buf:
+                logger.debug("[container %s] %s", cid, buf)
+
+        thread = threading.Thread(target=stream_logs, daemon=True)
+        thread.start()
+        if container.id is not None:
+            self._log_threads[container.id] = thread
+
+    def _stop_container_log_streaming(self, container: Container) -> None:
+        """Stop and clean up the log streaming thread for a container."""
+        if container.id is None:
+            return
+        thread = self._log_threads.pop(container.id, None)
+        if thread is not None:
+            thread.join(timeout=1.0)

interloper_docker-0.2.0/README.md

@@ -1,3 +0,0 @@
-# interloper-docker
-
-Docker execution support for Interloper.

interloper_docker-0.2.0/src/interloper_docker/backfiller.py

@@ -1,336 +0,0 @@
-"""Docker Backfiller implementation for Interloper.
-
-This backfiller starts a Docker container and invokes the Interloper CLI inside it
-using an inline JSON config. It runs the entire DAG in the container, delegating
-asset scheduling to the configured backfiller in the inline config (typically in_process).
-"""
-
-from __future__ import annotations
-
-import threading
-from collections.abc import Callable
-from time import sleep
-
-import docker
-from docker.errors import NotFound
-from docker.models.containers import Container
-from interloper.backfillers.base import Backfiller
-from interloper.cli.config import Config
-from interloper.dag.base import DAG
-from interloper.errors import PartitionError
-from interloper.events.base import Event, EventBus, parse_event_from_log_line
-from interloper.partitioning.base import Partition, PartitionWindow
-from interloper.partitioning.time import TimePartition, TimePartitionWindow
-from interloper.runners.base import Runner
-from interloper.runners.results import ExecutionStatus, RunResult
-from interloper.serialization.backfiller import BackfillerSpec
-
-
-class DockerBackfiller(Backfiller[Container]):
-    """Run an Interloper DAG inside a Docker container via the Interloper CLI.
-
-    The image must contain the `interloper` package (CLI available on PATH).
-    """
-
-    def __init__(
-        self,
-        image: str,
-        env_vars: dict[str, str] | None = None,
-        max_containers: int = 1,
-        runner: Runner | None = None,
-        volumes: dict[str, dict[str, str]] | list[str] | None = None,
-        dind: bool = False,
-        on_event: Callable[[Event], None] | None = None,
-    ) -> None:
-        """Initialize the DockerBackfiller.
-
-        Args:
-            image: Docker image to use
-            env_vars: Environment variables to pass to the container
-            max_containers: Maximum number of concurrent containers (default 1)
-            runner: Runner to use for running assets
-            volumes: Volume mounts for the container
-            dind: If True, mount the Docker socket to enable Docker-in-Docker
-            on_event: Optional event handler for lifecycle events
-        """
-        super().__init__(runner=runner, on_event=on_event)
-
-        # Force the runner to re-raise exceptions to make sure the container's exit code is propagated.
-        self.runner._reraise = True
-
-        self._image = image
-        self._env_vars = env_vars or {}
-        self._max_containers = max_containers
-        self._volumes = volumes or {}
-        self._dind = dind
-        self._docker = docker.from_env()
-
-        # Track log streaming threads for cleanup
-        self._log_threads: dict[str, threading.Thread] = {}
-        self._stop_log_streaming = threading.Event()
-
-    @property
-    def _capacity(self) -> int:
-        """Maximum number of concurrent containers."""
-        return self._max_containers
-
-    def _on_start(self) -> None:
-        self._stop_log_streaming.clear()
-
-    def _on_end(self) -> None:
-        # Signal all log streaming threads to stop
-        self._stop_log_streaming.set()
-
-        # Wait for threads to finish
-        for thread in self._log_threads.values():
-            thread.join(timeout=2.0)
-        self._log_threads.clear()
-
-    def _build_command(
-        self,
-        dag: DAG,
-        partition_or_window: Partition | PartitionWindow | None,
-        backfill_id: str,
-    ) -> list[str]:
-        """Build the CLI command for a partition.
-
-        Args:
-            dag: The DAG to execute
-            partition_or_window: The partition or window
-            backfill_id: The backfill ID
-
-        Returns:
-            Command list for the container
-        """
-        config = Config(dag=dag, runner=self.runner)
-
-        cmd = [
-            "interloper",
-            "run",
-            "--format=inline",
-            f"--backfill-id={backfill_id}",
-            config.to_json(),
-        ]
-
-        if partition_or_window is None:
-            return cmd
-
-        if isinstance(partition_or_window, TimePartition):
-            cmd.extend(["--date", partition_or_window.value.strftime("%Y-%m-%d")])
-        elif isinstance(partition_or_window, TimePartitionWindow):
-            cmd.extend(
-                [
-                    "--start-date",
-                    partition_or_window.start.strftime("%Y-%m-%d"),
-                    "--end-date",
-                    partition_or_window.end.strftime("%Y-%m-%d"),
-                ]
-            )
-        else:
-            raise PartitionError("Unsupported partition or window type")
-        return cmd
-
-    def _build_env(self) -> dict[str, str]:
-        """Build the environment variables for the container."""
-        env = dict(self._env_vars)
-        # Enable log-based event streaming
-        env["INTERLOPER_EVENTS_TO_STDERR"] = "true"
-        return env
-
-    def _build_volumes(self) -> dict[str, dict[str, str]]:
-        """Build the volume mounts for the container."""
-        volumes = {}
-        if isinstance(self._volumes, dict):
-            volumes.update(self._volumes)
-        elif isinstance(self._volumes, list):
-            for volume in self._volumes:
-                volumes[volume.split(":")[0]] = {"bind": volume.split(":")[1], "mode": "rw"}
-        if self._dind:
-            volumes["/var/run/docker.sock"] = {"bind": "/var/run/docker.sock", "mode": "rw"}
-        return volumes
-
-    def _build_name(self, partition_or_window: Partition | PartitionWindow | None) -> str:
-        """Build the name for the container."""
-        name = f"interloper_backfill_{self.state.backfill_id[:8]}"
-        if partition_or_window is not None:
-            name += f"-{partition_or_window}"
-        return name.replace(":", "-").replace("_", "-").lower()
-
-    def _start_log_streaming(self, container: Container) -> None:
-        """Start a background thread to stream logs and parse events from a container.
-
-        Args:
-            container: The Docker container to stream logs from
-        """
-
-        def stream_logs() -> None:
-            try:
-                # Stream logs from the container (both stdout and stderr)
-                for log_line in container.logs(stream=True, follow=True, stdout=True, stderr=True):
-                    if self._stop_log_streaming.is_set():
-                        break
-
-                    try:
-                        line = log_line.decode("utf-8", errors="ignore")
-                        event = parse_event_from_log_line(line)
-                        if event is not None:
-                            EventBus.get_instance().emit(event)
-                    except Exception:
-                        # Ignore parsing errors, continue streaming
-                        pass
-            except Exception:
-                # Container may have been removed or stopped
-                pass
-
-        thread = threading.Thread(target=stream_logs, daemon=True)
-        thread.start()
-        if container.id is not None:
-            self._log_threads[container.id] = thread
-
-    def _stop_container_log_streaming(self, container: Container) -> None:
-        """Stop and clean up the log streaming thread for a container.
-
-        Args:
-            container: The Docker container to stop streaming for
-        """
-        if container.id is None:
-            return
-        thread = self._log_threads.pop(container.id, None)
-        if thread is not None:
-            # Thread will stop on next iteration due to container exit
-            thread.join(timeout=1.0)
-
-    def _submit_run(
-        self,
-        dag: DAG,
-        partition_or_window: Partition | PartitionWindow | None,
-    ) -> Container:
-        """Submit execution of a run in a Docker container.
-
-        Args:
-            dag: The DAG to execute
-            partition_or_window: Either a Partition or PartitionWindow object
-
-        Returns:
-            The container as the handle
-        """
-        cmd = self._build_command(dag, partition_or_window, self.state.backfill_id)
-        env = self._build_env()
-        volumes = self._build_volumes()
-        name = self._build_name(partition_or_window)
-
-        self.state.mark_run_running(partition_or_window)
-
-        container = self._docker.containers.run(
-            image=self._image,
-            name=name,
-            command=cmd,
-            environment=env,
-            volumes=volumes if volumes else None,
-            remove=False,
-            detach=True,
-            stdout=True,
-            stderr=True,
-        )
-        # Store partition in container object for _wait_any
-        setattr(container, "_interloper_partition", partition_or_window)
-
-        # Start log streaming for event collection
-        self._start_log_streaming(container)
-
-        return container
-
-    def _wait_any(self, handles: list[Container]) -> Container:
-        """Wait for any container to complete by polling.
-
-        Args:
-            handles: List of container objects to wait for
-
-        Returns:
-            The container that completed
-        """
-        while True:
-            for container in handles:
-                container.reload()
-
-                if container.status in ("exited", "dead"):
-                    # Stop log streaming for this container
-                    self._stop_container_log_streaming(container)
-
-                    result = container.wait()
-                    status_code = result.get("StatusCode", 1)
-
-                    # Get partition from container object
-                    partition = getattr(container, "_interloper_partition", None)
-
-                    if status_code == 0:
-                        # TODO: This is not the true RunResult, we need to get it from the container?
-                        #       Missing the asset_executions.
-                        result = RunResult(partition, ExecutionStatus.COMPLETED)
-                        self.state.mark_run_completed(partition, result)
-                    else:
-                        self.state.mark_run_failed(partition, f"Container exited with code {status_code}")
-
-                        try:
-                            logs = container.logs(stdout=True, stderr=True)
-                            if logs:
-                                print("=============== START OF RUN CONTAINER LOGS ==================")
-                                print(logs.decode("utf-8", errors="ignore"))
-                                print("================ END OF RUN CONTAINER LOGS ===================")
-                        except Exception:
-                            pass
-
-                    # Remove the container after processing
-                    try:
-                        container.remove()
-                    except Exception as e:
-                        print(f"Error removing container {container.id}: {e}")
-                        pass
-
-                    return container
-
-            sleep(1.0)
-
-    def _cancel_all(self, handles: list[Container]) -> None:
-        """Best-effort cancellation of outstanding containers.
-
-        Args:
-            handles: List of container objects to cancel
-        """
-        for container in handles:
-            partition = getattr(container, "_interloper_partition", None)
-
-            # Stop log streaming for this container
-            self._stop_container_log_streaming(container)
-
-            try:
-                container.stop(timeout=5)
-            except NotFound:
-                # Container already removed, mark as cancelled
-                if partition is not None:
-                    self.state.mark_run_cancelled(partition)
-            except Exception:
-                try:
-                    container.kill()
-                except NotFound:
-                    # Container already removed
-                    self.state.mark_run_cancelled(partition)
-                except Exception:
-                    pass
-            else:
-                # Only mark as cancelled if we successfully stopped/killed
-                if partition is not None:
-                    self.state.mark_run_cancelled(partition)
-
-    def to_spec(self) -> BackfillerSpec:
-        """Convert to serializable spec."""
-        return BackfillerSpec(
-            path=self.path,
-            init=dict(
-                image=self._image,
-                env_vars=self._env_vars,
-                volumes=self._volumes,
-                max_containers=self._max_containers,
-                dind=self._dind,
-            ),
-        )

interloper_docker-0.2.0/src/interloper_docker/runner.py

@@ -1,253 +0,0 @@
-"""Docker-based runner that runs each asset in its own container.
-
-Each submitted asset is executed inside a fresh container. To allow an asset
-to resolve its upstream dependencies from IO without recomputing them, we pass
-to the container a mini-DAG consisting of the target asset plus all its
-upstream ancestors. The container runs the Interloper CLI with an inline
-config, similar to the `DockerBackfiller`.
-"""
-
-from __future__ import annotations
-
-from collections.abc import Callable
-
-import docker
-from docker.models.containers import Container
-from interloper.assets.base import Asset
-from interloper.cli.config import Config
-from interloper.dag.base import DAG
-from interloper.errors import PartitionError, RunnerError
-from interloper.events.base import Event
-from interloper.partitioning.base import Partition, PartitionWindow
-from interloper.partitioning.time import TimePartition, TimePartitionWindow
-from interloper.runners.base import Runner
-from interloper.serialization.runner import RunnerSpec
-
-
-class DockerRunner(Runner[Container]):
-    """Execute assets as individual Docker containers.
-
-    For each asset, constructs a mini-DAG comprising the asset and all its
-    upstream ancestors. The mini-DAG is sent to the container via inline JSON.
-    Inside the container, all non-target assets are marked as
-    `materializable=False` prior to execution to avoid recomputation while
-    still enabling IO-based dependency resolution.
-    """
-
-    def __init__(
-        self,
-        image: str,
-        max_containers: int = 4,
-        env_vars: dict[str, str] | None = None,
-        volumes: dict[str, dict[str, str]] | list[str] | None = None,
-        fail_fast: bool = False,
-        reraise: bool = False,
-        on_event: Callable[[Event], None] | None = None,
-    ) -> None:
-        """Initialize the DockerRunner.
-
-        Args:
-            image: Docker image to use for container execution.
-            max_containers: Maximum number of concurrent containers.
-            env_vars: Environment variables to pass to the container.
-            volumes: Volume mounts for the container.
-            fail_fast: Stop execution on first failure.
-            reraise: Re-raise exceptions.
-            on_event: Optional event handler for lifecycle events.
-        """
-        super().__init__(fail_fast=fail_fast, reraise=reraise, on_event=on_event)
-        self._image = image
-        self._max_containers = max_containers
-        self._env_vars = env_vars or {}
-        self._volumes = volumes or {}
-        self._docker = docker.from_env()
-
-    @property
-    def _capacity(self) -> int:
-        return self._max_containers
-
-    def _build_command(
-        self,
-        dag: DAG,
-        partition_or_window: Partition | PartitionWindow | None,
-        run_id: str,
-    ) -> list[str]:
-        """Build the CLI command for asset execution in a container.
-
-        Args:
-            dag: The DAG to execute.
-            partition_or_window: The partition or window.
-            run_id: The run ID.
-
-        Returns:
-            Command list for the container.
-        """
-        config = Config(dag=dag)
-
-        cmd = [
-            "interloper",
-            "run",
-            "--format",
-            "inline",
-            f"--run-id={run_id}",
-            config.to_json(),
-        ]
-
-        if isinstance(partition_or_window, TimePartition):
-            cmd.extend(["--date", partition_or_window.value.strftime("%Y-%m-%d")])
-        elif isinstance(partition_or_window, TimePartitionWindow):
-            cmd.extend(
-                [
-                    "--start-date",
-                    partition_or_window.start.strftime("%Y-%m-%d"),
-                    "--end-date",
-                    partition_or_window.end.strftime("%Y-%m-%d"),
-                ]
-            )
-        else:
-            raise PartitionError("Unsupported partition or window type")
-        return cmd
-
-    def _build_env(self) -> dict[str, str]:
-        """Build the environment variables for the container."""
-        return dict(self._env_vars)
-
-    def _build_volumes(self) -> dict[str, dict[str, str]]:
-        """Build the volume mounts for the container."""
-        volumes = {}
-        if isinstance(self._volumes, dict):
-            volumes.update(self._volumes)
-        elif isinstance(self._volumes, list):
-            for volume in self._volumes:
-                volumes[volume.split(":")[0]] = {"bind": volume.split(":")[1], "mode": "rw"}
-        return volumes
-
-    def _build_name(self, asset: Asset) -> str:
-        """Build the name for the container."""
-        name = f"interloper_run_{self.state.run_id[:8]}-{asset.instance_key}"
-        return name.replace(":", "-").replace("_", "-").lower()
-
-    def _submit_asset(
-        self,
-        asset: Asset,
-        partition_or_window: Partition | PartitionWindow | None,
-    ) -> Container:
-        """Submit execution of an asset and return the container object for completion tracking.
-
-        IMPORTANT: this method is not calling the `_execute_asset` method of the base class.
-        Therefore, the state has to be updated manually here and in `_wait_any` below.
-
-        Args:
-            asset: The asset to execute
-            partition_or_window: Either a Partition or PartitionWindow object
-
-        Returns:
-            The container object for the asset execution
-        """
-        # Build a mini-DAG: target asset + its parents (non-materializable)
-        mini_dag = self.state.dag.mini_dag(asset.instance_key)
-
-        cmd = self._build_command(mini_dag, partition_or_window, self.state.run_id)
-        name = self._build_name(asset)
-        env = self._build_env()
-        volumes = self._build_volumes()
-
-        self.state.mark_asset_running(asset)
-
-        container = self._docker.containers.run(
-            image=self._image,
-            name=name,
-            command=cmd,
-            environment=env,
-            volumes=volumes if volumes else None,
-            labels={"interloper.asset_key": asset.instance_key},
-            remove=False,
-            detach=True,
-            stdout=True,
-            stderr=True,
-        )
-
-        return container
-
-    def _wait_any(self, handles: list[Container]) -> Container:
-        """Wait for any container to finish by polling.
-
-        IMPORTANT: the `_execute_asset` method of the base class is not called by `_submit_asset`.
-        Therefore, the state has to be updated manually here and in `_submit_asset` above.
-
-        Args:
-            handles: List of container objects to wait for
-
-        Returns:
-            The container object that finished
-        """
-
-        while True:
-            for container in handles:
-                container.reload()
-
-                if container.status in ("exited", "dead"):
-                    result = container.wait()
-                    status_code = result.get("StatusCode", 1)
-
-                    # Map back to asset
-                    asset: Asset | None = None
-                    asset_key = container.labels.get("interloper.asset_key")
-                    if asset_key and asset_key in self.state.dag.asset_map:
-                        asset = self.state.dag.asset_map[asset_key]
-                    if asset is None:
-                        raise RunnerError("Failed to map container to asset")
-
-                    if status_code == 0:
-                        self.state.mark_asset_completed(asset)
-                    else:
-                        self.state.mark_asset_failed(asset, f"Container {container.id} exited with code {status_code}")
-
-                        try:
-                            logs = container.logs(stdout=True, stderr=True)
-                            if logs:
-                                print("=============== START OF ASSET CONTAINER LOGS ================")
-                                print(logs.decode("utf-8", errors="ignore"))
-                                print("================ END OF ASSET CONTAINER LOGS =================")
-
-                        except Exception:
-                            pass
-
-                        if self._reraise or self._fail_fast:
-                            raise RunnerError(f"Container {container.id} exited with code {status_code}")
-
-                    # Remove the container after processing
-                    try:
-                        container.remove()
-                    except Exception as e:
-                        print(f"Error removing container {container.id}: {e}")
-                        pass
-
-                    return container
-
-    def _cancel_all(self, handles: list[Container]) -> None:
-        for container in handles:
-            try:
-                container.stop(timeout=2)
-            except Exception:
-                try:
-                    container.kill()
-                except Exception:
-                    pass
-            finally:
-                asset_key = container.labels.get("interloper.asset_key")
-                asset = self.state.dag.asset_map[asset_key]
-                self.state.mark_asset_cancelled(asset)
-
-    def to_spec(self) -> RunnerSpec:
-        return RunnerSpec(
-            path=self.path,
-            init=dict(
-                image=self._image,
-                max_containers=self._max_containers,
-                env_vars=self._env_vars,
-                volumes=self._volumes,
-                fail_fast=self._fail_fast,
-                reraise=self._reraise,
-            ),
-        )