dagster-docker 0.17.19__py3-none-any.whl → 0.28.5__py3-none-any.whl

dagster_docker/__init__.py

@@ -1,11 +1,15 @@
-from dagster._core.utils import check_dagster_package_version
+from dagster_shared.libraries import DagsterLibraryRegistry
 
-from .docker_executor import docker_executor as docker_executor
-from .docker_run_launcher import DockerRunLauncher as DockerRunLauncher
-from .ops import (
+from dagster_docker.docker_executor import docker_executor as docker_executor
+from dagster_docker.docker_run_launcher import DockerRunLauncher as DockerRunLauncher
+from dagster_docker.ops import (
     docker_container_op as docker_container_op,
     execute_docker_container as execute_docker_container,
 )
-from .version import __version__
+from dagster_docker.pipes import (
+    PipesDockerClient as PipesDockerClient,
+    PipesDockerLogsMessageReader as PipesDockerLogsMessageReader,
+)
+from dagster_docker.version import __version__
 
-check_dagster_package_version("dagster-docker", __version__)
+DagsterLibraryRegistry.register("dagster-docker", __version__)

dagster_docker/container_context.py

@@ -1,4 +1,5 @@
-from typing import TYPE_CHECKING, Any, Mapping, NamedTuple, Optional, Sequence, cast
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any, NamedTuple, Optional, cast
 
 from dagster import (
     Array,
@@ -10,10 +11,10 @@ from dagster import (
 from dagster._config import process_config
 from dagster._core.container_context import process_shared_container_context_config
 from dagster._core.errors import DagsterInvalidConfigError
-from dagster._core.storage.pipeline_run import DagsterRun
+from dagster._core.storage.dagster_run import DagsterRun
 
 if TYPE_CHECKING:
-    from . import DockerRunLauncher
+    from dagster_docker import DockerRunLauncher
 
 DOCKER_CONTAINER_CONTEXT_SCHEMA = {
     "registry": Field(
@@ -79,7 +80,7 @@ class DockerContainerContext(
         networks: Optional[Sequence[str]] = None,
         container_kwargs: Optional[Mapping[str, Any]] = None,
     ):
-        return super(DockerContainerContext, cls).__new__(
+        return super().__new__(
             cls,
             registry=check.opt_nullable_mapping_param(registry, "registry"),
             env_vars=check.opt_sequence_param(env_vars, "env_vars", of_type=str),
@@ -102,7 +103,7 @@ class DockerContainerContext(
         )
 
     @staticmethod
-    def create_for_run(pipeline_run: DagsterRun, run_launcher: Optional["DockerRunLauncher"]):
+    def create_for_run(dagster_run: DagsterRun, run_launcher: Optional["DockerRunLauncher"]):
         context = DockerContainerContext()
 
         # First apply the instance / run_launcher-level context
@@ -117,8 +118,8 @@ class DockerContainerContext(
             )
 
         run_container_context = (
-            pipeline_run.pipeline_code_origin.repository_origin.container_context
-            if pipeline_run.pipeline_code_origin
+            dagster_run.job_code_origin.repository_origin.container_context
+            if dagster_run.job_code_origin
             else None
         )
 
@@ -154,7 +155,7 @@ class DockerContainerContext(
                 run_docker_container_context,
             )
 
-        processed_context_value = cast(Mapping[str, Any], processed_container_context.value)
+        processed_context_value = cast("Mapping[str, Any]", processed_container_context.value)
 
         return shared_container_context.merge(
             DockerContainerContext(

dagster_docker/docker_executor.py

@@ -1,13 +1,18 @@
-from typing import Iterator, Optional, cast
+from collections.abc import Iterator
+from typing import TYPE_CHECKING, Optional, cast
 
 import dagster._check as check
 import docker
 import docker.errors
 from dagster import Field, IntSource, executor
-from dagster._annotations import experimental
+from dagster._annotations import beta
 from dagster._core.definitions.executor_definition import multiple_process_executor_requirements
-from dagster._core.events import DagsterEvent, EngineEventData, MetadataEntry
+from dagster._core.events import DagsterEvent, EngineEventData
 from dagster._core.execution.retries import RetryMode, get_retries_config
+from dagster._core.execution.step_dependency_config import (
+    StepDependencyConfig,
+    get_step_dependency_config_field,
+)
 from dagster._core.execution.tags import get_tag_concurrency_limits_config
 from dagster._core.executor.base import Executor
 from dagster._core.executor.init import InitExecutorContext
@@ -17,15 +22,15 @@ from dagster._core.executor.step_delegating.step_handler.base import (
     StepHandler,
     StepHandlerContext,
 )
-from dagster._core.origin import PipelinePythonOrigin
 from dagster._core.utils import parse_env_var
-from dagster._grpc.types import ExecuteStepArgs
-from dagster._serdes.utils import hash_str
 from dagster._utils.merger import merge_dicts
+from dagster_shared.serdes.utils import hash_str
 
+from dagster_docker.container_context import DockerContainerContext
 from dagster_docker.utils import DOCKER_CONFIG_SCHEMA, validate_docker_config, validate_docker_image
 
-from .container_context import DockerContainerContext
+if TYPE_CHECKING:
+    from dagster._core.origin import JobPythonOrigin
 
 
 @executor(
@@ -43,14 +48,14 @@ from .container_context import DockerContainerContext
                 ),
             ),
             "tag_concurrency_limits": get_tag_concurrency_limits_config(),
+            "step_dependency_config": get_step_dependency_config_field(),
         },
     ),
     requirements=multiple_process_executor_requirements(),
 )
-@experimental
+@beta
 def docker_executor(init_context: InitExecutorContext) -> Executor:
-    """
-    Executor which launches steps as Docker containers.
+    """Executor which launches steps as Docker containers.
 
     To use the `docker_executor`, set it as the `executor_def` when defining a job:
 
@@ -101,6 +106,9 @@ def docker_executor(init_context: InitExecutorContext) -> Executor:
         retries=check.not_none(RetryMode.from_config(retries)),
         max_concurrent=max_concurrent,
         tag_concurrency_limits=tag_concurrency_limits,
+        step_dependency_config=StepDependencyConfig.from_config(
+            config.get("step_dependency_config")  # type: ignore
+        ),
     )
 
 
@@ -118,10 +126,10 @@ class DockerStepHandler(StepHandler):
         )
 
     def _get_image(self, step_handler_context: StepHandlerContext):
-        from . import DockerRunLauncher
+        from dagster_docker import DockerRunLauncher
 
         image = cast(
-            PipelinePythonOrigin, step_handler_context.pipeline_run.pipeline_code_origin
+            "JobPythonOrigin", step_handler_context.dagster_run.job_code_origin
         ).repository_origin.container_image
         if not image:
             image = self._image
@@ -138,13 +146,13 @@ class DockerStepHandler(StepHandler):
 
     def _get_docker_container_context(self, step_handler_context: StepHandlerContext):
         # This doesn't vary per step: would be good to have a hook where it can be set once
-        # for the whole StepHandler but we need access to the PipelineRun for that
+        # for the whole StepHandler but we need access to the DagsterRun for that
 
-        from .docker_run_launcher import DockerRunLauncher
+        from dagster_docker.docker_run_launcher import DockerRunLauncher
 
         run_launcher = step_handler_context.instance.run_launcher
         run_target = DockerContainerContext.create_for_run(
-            step_handler_context.pipeline_run,
+            step_handler_context.dagster_run,
             run_launcher if isinstance(run_launcher, DockerRunLauncher) else None,
         )
 
@@ -172,11 +180,17 @@ class DockerStepHandler(StepHandler):
             )
         return client
 
-    def _get_container_name(self, execute_step_args: ExecuteStepArgs):
-        run_id = execute_step_args.pipeline_run_id
-        step_keys_to_execute = check.not_none(execute_step_args.step_keys_to_execute)
+    def _get_step_key(self, step_handler_context: StepHandlerContext) -> str:
+        step_keys_to_execute = cast(
+            "list[str]", step_handler_context.execute_step_args.step_keys_to_execute
+        )
         assert len(step_keys_to_execute) == 1, "Launching multiple steps is not currently supported"
-        step_key = step_keys_to_execute[0]
+        return step_keys_to_execute[0]
+
+    def _get_container_name(self, step_handler_context: StepHandlerContext):
+        execute_step_args = step_handler_context.execute_step_args
+        run_id = execute_step_args.run_id
+        step_key = self._get_step_key(step_handler_context)
 
         step_name = f"dagster-step-{hash_str(run_id + step_key)}"
 
@@ -200,17 +214,20 @@ class DockerStepHandler(StepHandler):
         assert len(step_keys_to_execute) == 1, "Launching multiple steps is not currently supported"
         step_key = step_keys_to_execute[0]
 
+        container_kwargs = {**container_context.container_kwargs}
+        container_kwargs.pop("stop_timeout", None)
+
         env_vars = dict([parse_env_var(env_var) for env_var in container_context.env_vars])
-        env_vars["DAGSTER_RUN_JOB_NAME"] = step_handler_context.pipeline_run.job_name
+        env_vars["DAGSTER_RUN_JOB_NAME"] = step_handler_context.dagster_run.job_name
         env_vars["DAGSTER_RUN_STEP_KEY"] = step_key
         return client.containers.create(
             step_image,
-            name=self._get_container_name(execute_step_args),
+            name=self._get_container_name(step_handler_context),
             detach=True,
             network=container_context.networks[0] if len(container_context.networks) else None,
             command=execute_step_args.get_command_args(),
             environment=env_vars,
-            **container_context.container_kwargs,
+            **container_kwargs,
         )
 
     def launch_step(self, step_handler_context: StepHandlerContext) -> Iterator[DagsterEvent]:
@@ -245,9 +262,9 @@ class DockerStepHandler(StepHandler):
         yield DagsterEvent.step_worker_starting(
             step_handler_context.get_step_context(step_key),
             message="Launching step in Docker container.",
-            metadata_entries=[
-                MetadataEntry("Docker container id", value=step_container.id),
-            ],
+            metadata={
+                "Docker container id": step_container.id,
+            },
         )
         step_container.start()
 
@@ -256,9 +273,15 @@ class DockerStepHandler(StepHandler):
 
         client = self._get_client(container_context)
 
-        container_name = self._get_container_name(step_handler_context.execute_step_args)
+        container_name = self._get_container_name(step_handler_context)
+        step_key = self._get_step_key(step_handler_context)
 
-        container = client.containers.get(container_name)
+        try:
+            container = client.containers.get(container_name)
+        except docker.errors.NotFound:
+            return CheckStepHealthResult.unhealthy(
+                reason=f"Docker container {container_name} for step {step_key} could not be found."
+            )
 
         if container.status == "running":
             return CheckStepHealthResult.healthy()
@@ -276,7 +299,7 @@ class DockerStepHandler(StepHandler):
             return CheckStepHealthResult.healthy()
 
         return CheckStepHealthResult.unhealthy(
-            reason=f"Container status is {container.status}. Return code is {str(ret_code)}."
+            reason=f"Container status is {container.status}. Return code is {ret_code}."
         )
 
     def terminate_step(self, step_handler_context: StepHandlerContext) -> Iterator[DagsterEvent]:
@@ -285,12 +308,12 @@ class DockerStepHandler(StepHandler):
         step_keys_to_execute = check.not_none(
             step_handler_context.execute_step_args.step_keys_to_execute
         )
-        assert (
-            len(step_keys_to_execute) == 1
-        ), "Terminating multiple steps is not currently supported"
+        assert len(step_keys_to_execute) == 1, (
+            "Terminating multiple steps is not currently supported"
+        )
         step_key = step_keys_to_execute[0]
 
-        container_name = self._get_container_name(step_handler_context.execute_step_args)
+        container_name = self._get_container_name(step_handler_context)
 
         yield DagsterEvent.engine_event(
             step_handler_context.get_step_context(step_key),
@@ -302,4 +325,6 @@ class DockerStepHandler(StepHandler):
 
         container = client.containers.get(container_name)
 
-        container.stop()
+        stop_timeout = container_context.container_kwargs.get("stop_timeout")
+
+        container.stop(timeout=stop_timeout)

dagster_docker/docker_run_launcher.py

@@ -1,3 +1,7 @@
+import json
+from collections.abc import Mapping
+from typing import Any, Optional
+
 import dagster._check as check
 import docker
 from dagster._core.launcher.base import (
@@ -7,16 +11,17 @@ from dagster._core.launcher.base import (
     RunLauncher,
     WorkerStatus,
 )
-from dagster._core.storage.pipeline_run import DagsterRun
+from dagster._core.storage.dagster_run import DagsterRun
 from dagster._core.storage.tags import DOCKER_IMAGE_TAG
 from dagster._core.utils import parse_env_var
 from dagster._grpc.types import ExecuteRunArgs, ResumeRunArgs
 from dagster._serdes import ConfigurableClass
+from dagster._serdes.config_class import ConfigurableClassData
+from typing_extensions import Self
 
+from dagster_docker.container_context import DockerContainerContext
 from dagster_docker.utils import DOCKER_CONFIG_SCHEMA, validate_docker_config, validate_docker_image
 
-from .container_context import DockerContainerContext
-
 DOCKER_CONTAINER_ID_TAG = "docker/container_id"
 
 
@@ -25,7 +30,7 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
 
     def __init__(
         self,
-        inst_data=None,
+        inst_data: Optional[ConfigurableClassData] = None,
         image=None,
         registry=None,
         env_vars=None,
@@ -61,12 +66,14 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
     def config_type(cls):
         return DOCKER_CONFIG_SCHEMA
 
-    @staticmethod
-    def from_config_value(inst_data, config_value):
-        return DockerRunLauncher(inst_data=inst_data, **config_value)
+    @classmethod
+    def from_config_value(
+        cls, inst_data: ConfigurableClassData, config_value: Mapping[str, Any]
+    ) -> Self:
+        return cls(inst_data=inst_data, **config_value)
 
-    def get_container_context(self, pipeline_run: DagsterRun) -> DockerContainerContext:
-        return DockerContainerContext.create_for_run(pipeline_run, self)
+    def get_container_context(self, dagster_run: DagsterRun) -> DockerContainerContext:
+        return DockerContainerContext.create_for_run(dagster_run, self)
 
     def _get_client(self, container_context: DockerContainerContext):
         client = docker.client.from_env()
@@ -78,8 +85,8 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
             )
         return client
 
-    def _get_docker_image(self, pipeline_code_origin):
-        docker_image = pipeline_code_origin.repository_origin.container_image
+    def _get_docker_image(self, job_code_origin):
+        docker_image = job_code_origin.repository_origin.container_image
 
         if not docker_image:
             docker_image = self.image
@@ -97,6 +104,17 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
 
         client = self._get_client(container_context)
 
+        container_kwargs = {**container_context.container_kwargs}
+        labels = container_kwargs.pop("labels", {})
+
+        container_kwargs.pop("stop_timeout", None)
+
+        if isinstance(labels, list):
+            labels = {key: "" for key in labels}
+
+        labels["dagster/run_id"] = run.run_id
+        labels["dagster/job_name"] = run.job_name
+
         try:
             container = client.containers.create(
                 image=docker_image,
@@ -104,10 +122,11 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
                 detach=True,
                 environment=docker_env,
                 network=container_context.networks[0] if len(container_context.networks) else None,
-                **container_context.container_kwargs,
+                labels=labels,
+                **container_kwargs,
             )
 
-        except docker.errors.ImageNotFound:
+        except docker.errors.ImageNotFound:  # pyright: ignore[reportAttributeAccessIssue]
             client.images.pull(docker_image)
             container = client.containers.create(
                 image=docker_image,
@@ -115,7 +134,8 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
                 detach=True,
                 environment=docker_env,
                 network=container_context.networks[0] if len(container_context.networks) else None,
-                **container_context.container_kwargs,
+                labels=labels,
+                **container_kwargs,
             )
 
         if len(container_context.networks) > 1:
@@ -124,31 +144,26 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
                 network.connect(container)
 
         self._instance.report_engine_event(
-            message=(
-                "Launching run in a new container {container_id} with image {docker_image}".format(
-                    container_id=container.id,
-                    docker_image=docker_image,
-                )
-            ),
-            pipeline_run=run,
+            message=f"Launching run in a new container {container.id} with image {docker_image}",
+            dagster_run=run,
             cls=self.__class__,
         )
 
         self._instance.add_run_tags(
             run.run_id,
-            {DOCKER_CONTAINER_ID_TAG: container.id, DOCKER_IMAGE_TAG: docker_image},
+            {DOCKER_CONTAINER_ID_TAG: container.id, DOCKER_IMAGE_TAG: docker_image},  # pyright: ignore[reportArgumentType]
         )
 
         container.start()
 
     def launch_run(self, context: LaunchRunContext) -> None:
-        run = context.pipeline_run
-        pipeline_code_origin = check.not_none(context.pipeline_code_origin)
-        docker_image = self._get_docker_image(pipeline_code_origin)
+        run = context.dagster_run
+        job_code_origin = check.not_none(context.job_code_origin)
+        docker_image = self._get_docker_image(job_code_origin)
 
         command = ExecuteRunArgs(
-            pipeline_origin=pipeline_code_origin,
-            pipeline_run_id=run.run_id,
+            job_origin=job_code_origin,
+            run_id=run.run_id,
             instance_ref=self._instance.get_ref(),
         ).get_command_args()
 
@@ -159,20 +174,20 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
         return True
 
     def resume_run(self, context: ResumeRunContext) -> None:
-        run = context.pipeline_run
-        pipeline_code_origin = check.not_none(context.pipeline_code_origin)
-        docker_image = self._get_docker_image(pipeline_code_origin)
+        run = context.dagster_run
+        job_code_origin = check.not_none(context.job_code_origin)
+        docker_image = self._get_docker_image(job_code_origin)
 
         command = ResumeRunArgs(
-            pipeline_origin=pipeline_code_origin,
-            pipeline_run_id=run.run_id,
+            job_origin=job_code_origin,
+            run_id=run.run_id,
             instance_ref=self._instance.get_ref(),
         ).get_command_args()
 
         self._launch_container_with_command(run, docker_image, command)
 
     def _get_container(self, run):
-        if not run or run.is_finished:
+        if not run:
             return None
 
         container_id = run.tags.get(DOCKER_CONTAINER_ID_TAG)
@@ -184,24 +199,31 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
 
         try:
             return self._get_client(container_context).containers.get(container_id)
-        except Exception:
+        except docker.errors.NotFound:  # pyright: ignore[reportAttributeAccessIssue]
             return None
 
     def terminate(self, run_id):
         run = self._instance.get_run_by_id(run_id)
+
+        if not run or run.is_finished:
+            return False
+
+        self._instance.report_run_canceling(run)
+
         container = self._get_container(run)
 
+        container_context = self.get_container_context(run)
+        stop_timeout = container_context.container_kwargs.get("stop_timeout")
+
         if not container:
             self._instance.report_engine_event(
                 message="Unable to get docker container to send termination request to.",
-                pipeline_run=run,
+                dagster_run=run,
                 cls=self.__class__,
             )
             return False
 
-        self._instance.report_run_canceling(run)
-
-        container.stop()
+        container.stop(timeout=stop_timeout)
 
         return True
 
@@ -210,11 +232,22 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
         return True
 
     def check_run_worker_health(self, run: DagsterRun):
+        container_id = run.tags.get(DOCKER_CONTAINER_ID_TAG)
+
+        if not container_id:
+            return CheckRunHealthResult(WorkerStatus.NOT_FOUND, msg="No container ID tag for run.")
+
         container = self._get_container(run)
         if container is None:
-            return CheckRunHealthResult(WorkerStatus.NOT_FOUND)
+            return CheckRunHealthResult(
+                WorkerStatus.NOT_FOUND, msg=f"Could not find container with ID {container_id}."
+            )
         if container.status == "running":
             return CheckRunHealthResult(WorkerStatus.RUNNING)
-        return CheckRunHealthResult(
-            WorkerStatus.FAILED, msg=f"Container status is {container.status}"
+
+        container_state = container.attrs.get("State")
+        failure_string = f"Container status is {container.status}." + (
+            f" Container state: {json.dumps(container_state)}" if container_state else ""
         )
+
+        return CheckRunHealthResult(WorkerStatus.FAILED, msg=failure_string)

dagster_docker/ops/__init__.py

@@ -1,4 +1,4 @@
-from .docker_container_op import (
+from dagster_docker.ops.docker_container_op import (
     docker_container_op as docker_container_op,
     execute_docker_container as execute_docker_container,
 )

dagster_docker/ops/docker_container_op.py

@@ -1,14 +1,16 @@
-from typing import Any, Mapping, Optional, Sequence
+from collections.abc import Mapping, Sequence
+from typing import Any, Optional
 
 import docker
+import docker.errors
 from dagster import Field, In, Nothing, OpExecutionContext, StringSource, op
-from dagster._annotations import experimental
+from dagster._annotations import beta
 from dagster._core.utils import parse_env_var
-from dagster._serdes.utils import hash_str
+from dagster_shared.serdes.utils import hash_str
 
-from ..container_context import DockerContainerContext
-from ..docker_run_launcher import DockerRunLauncher
-from ..utils import DOCKER_CONFIG_SCHEMA, validate_docker_image
+from dagster_docker.container_context import DockerContainerContext
+from dagster_docker.docker_run_launcher import DockerRunLauncher
+from dagster_docker.utils import DOCKER_CONFIG_SCHEMA, validate_docker_image
 
 DOCKER_CONTAINER_OP_CONFIG = {
     **DOCKER_CONFIG_SCHEMA,
@@ -44,7 +46,6 @@ def _get_client(docker_container_context: DockerContainerContext):
 def _get_container_name(run_id, op_name, retry_number):
     container_name = hash_str(run_id + op_name)
 
-    retry_number = retry_number
     if retry_number > 0:
         container_name = f"{container_name}-{retry_number}"
 
@@ -72,7 +73,7 @@ def _create_container(
     )
 
 
-@experimental
+@beta
 def execute_docker_container(
     context: OpExecutionContext,
     image: str,
@@ -83,8 +84,7 @@ def execute_docker_container(
     env_vars: Optional[Sequence[str]] = None,
     container_kwargs: Optional[Mapping[str, Any]] = None,
 ):
-    """
-    This function is a utility for executing a Docker container from within a Dagster op.
+    """This function is a utility for executing a Docker container from within a Dagster op.
 
     Args:
         image (str): The image to use for the launched Docker container.
@@ -105,10 +105,12 @@ def execute_docker_container(
             of available options.
     """
     run_container_context = DockerContainerContext.create_for_run(
-        context.pipeline_run,
-        context.instance.run_launcher
-        if isinstance(context.instance.run_launcher, DockerRunLauncher)
-        else None,
+        context.dagster_run,
+        (
+            context.instance.run_launcher
+            if isinstance(context.instance.run_launcher, DockerRunLauncher)
+            else None
+        ),
     )
 
     validate_docker_image(image)
@@ -139,7 +141,7 @@ def execute_docker_container(
     container.start()
 
     for line in container.logs(stdout=True, stderr=True, stream=True, follow=True):
-        print(line)
+        print(line)  # noqa: T201
 
     exit_status = container.wait()["StatusCode"]
 
@@ -148,10 +150,9 @@ def execute_docker_container(
 
 
 @op(ins={"start_after": In(Nothing)}, config_schema=DOCKER_CONTAINER_OP_CONFIG)
-@experimental
+@beta
 def docker_container_op(context):
-    """
-    An op that runs a Docker container using the docker Python API.
+    """An op that runs a Docker container using the docker Python API.
 
     Contrast with the `docker_executor`, which runs each Dagster op in a Dagster job in its
     own Docker container.

dagster_docker/pipes.py

@@ -0,0 +1,208 @@
+from collections.abc import Iterator, Mapping, Sequence
+from contextlib import contextmanager
+from typing import Any, Optional, Union
+
+import docker
+import docker.errors
+from dagster import (
+    OpExecutionContext,
+    _check as check,
+)
+from dagster._core.definitions.resource_annotation import TreatAsResourceParam
+from dagster._core.execution.context.asset_execution_context import AssetExecutionContext
+from dagster._core.pipes.client import (
+    PipesClient,
+    PipesClientCompletedInvocation,
+    PipesContextInjector,
+    PipesMessageReader,
+)
+from dagster._core.pipes.context import PipesMessageHandler
+from dagster._core.pipes.utils import (
+    PipesEnvContextInjector,
+    extract_message_or_forward_to_stdout,
+    open_pipes_session,
+)
+from dagster_pipes import DagsterPipesError, PipesDefaultMessageWriter, PipesExtras, PipesParams
+
+
+class PipesDockerLogsMessageReader(PipesMessageReader):
+    @contextmanager
+    def read_messages(
+        self,
+        handler: PipesMessageHandler,
+    ) -> Iterator[PipesParams]:
+        self._handler = handler
+        try:
+            yield {PipesDefaultMessageWriter.STDIO_KEY: PipesDefaultMessageWriter.STDERR}
+        finally:
+            self._handler = None
+
+    def consume_docker_logs(self, container) -> None:
+        handler = check.not_none(
+            self._handler, "Can only consume logs within context manager scope."
+        )
+        for log_line in container.logs(stdout=True, stderr=True, stream=True, follow=True):
+            if isinstance(log_line, bytes):
+                log_entry = log_line.decode("utf-8")
+            elif isinstance(log_line, str):
+                log_entry = log_line
+            else:
+                continue
+
+            extract_message_or_forward_to_stdout(handler, log_entry)
+
+    def no_messages_debug_text(self) -> str:
+        return "Attempted to read messages by extracting them from docker logs directly."
+
+
+class PipesDockerClient(PipesClient, TreatAsResourceParam):
+    """A pipes client that runs external processes in docker containers.
+
+    By default context is injected via environment variables and messages are parsed out of the
+    log stream, with other logs forwarded to stdout of the orchestration process.
+
+    Args:
+        env (Optional[Mapping[str, str]]): An optional dict of environment variables to pass to the
+            container.
+        register (Optional[Mapping[str, str]]): An optional dict of registry credentials to login to
+            the docker client.
+        context_injector (Optional[PipesContextInjector]): A context injector to use to inject
+            context into the docker container process. Defaults to :py:class:`PipesEnvContextInjector`.
+        message_reader (Optional[PipesMessageReader]): A message reader to use to read messages
+            from the docker container process. Defaults to :py:class:`DockerLogsMessageReader`.
+    """
+
+    def __init__(
+        self,
+        env: Optional[Mapping[str, str]] = None,
+        registry: Optional[Mapping[str, str]] = None,
+        context_injector: Optional[PipesContextInjector] = None,
+        message_reader: Optional[PipesMessageReader] = None,
+    ):
+        self.env = check.opt_mapping_param(env, "env", key_type=str, value_type=str)
+        self.registry = check.opt_mapping_param(registry, "registry", key_type=str, value_type=str)
+        self.context_injector = (
+            check.opt_inst_param(
+                context_injector,
+                "context_injector",
+                PipesContextInjector,
+            )
+            or PipesEnvContextInjector()
+        )
+
+        self.message_reader = (
+            check.opt_inst_param(message_reader, "message_reader", PipesMessageReader)
+            or PipesDockerLogsMessageReader()
+        )
+
+    @classmethod
+    def _is_dagster_maintained(cls) -> bool:
+        return True
+
+    def run(  # pyright: ignore[reportIncompatibleMethodOverride]
+        self,
+        *,
+        context: Union[OpExecutionContext, AssetExecutionContext],
+        image: str,
+        extras: Optional[PipesExtras] = None,
+        command: Optional[Union[str, Sequence[str]]] = None,
+        env: Optional[Mapping[str, str]] = None,
+        registry: Optional[Mapping[str, str]] = None,
+        container_kwargs: Optional[Mapping[str, Any]] = None,
+    ) -> PipesClientCompletedInvocation:
+        """Create a docker container and run it to completion, enriched with the pipes protocol.
+
+        Args:
+            image (str):
+                The image for the container to use.
+            command (Optional[Union[str, Sequence[str]]]):
+                The command for the container use.
+            env (Optional[Mapping[str,str]]):
+                A mapping of environment variable names to values to set on the first
+                container in the pod spec, on top of those configured on resource.
+            registry (Optional[Mapping[str, str]]:
+                A mapping containing url, username, and password to be used
+                with docker client login.
+            container_kwargs (Optional[Mapping[str, Any]]:
+                Arguments to be forwarded to docker client containers.create.
+            extras (Optional[PipesExtras]):
+                Extra values to pass along as part of the ext protocol.
+            context_injector (Optional[PipesContextInjector]):
+                Override the default ext protocol context injection.
+            message_reader (Optional[PipesMessageReader]):
+                Override the default ext protocol message reader.
+
+        Returns:
+            PipesClientCompletedInvocation: Wrapper containing results reported by the external
+                process.
+        """
+        with open_pipes_session(
+            context=context,
+            context_injector=self.context_injector,
+            message_reader=self.message_reader,
+            extras=extras,
+        ) as pipes_session:
+            client = docker.client.from_env()
+            registry = registry or self.registry
+            if registry:
+                client.login(
+                    registry=registry["url"],
+                    username=registry["username"],
+                    password=registry["password"],
+                )
+
+            try:
+                container = self._create_container(
+                    client=client,
+                    image=image,
+                    command=command,
+                    env=env,
+                    open_pipes_session_env=pipes_session.get_bootstrap_env_vars(),
+                    container_kwargs=container_kwargs,
+                )
+            except docker.errors.ImageNotFound:
+                client.images.pull(image)
+                container = self._create_container(
+                    client=client,
+                    image=image,
+                    command=command,
+                    env=env,
+                    open_pipes_session_env=pipes_session.get_bootstrap_env_vars(),
+                    container_kwargs=container_kwargs,
+                )
+
+            result = container.start()
+            try:
+                if isinstance(self.message_reader, PipesDockerLogsMessageReader):
+                    self.message_reader.consume_docker_logs(container)
+
+                result = container.wait()
+                if result["StatusCode"] != 0:
+                    raise DagsterPipesError(f"Container exited with non-zero status code: {result}")
+            finally:
+                container.stop()
+        return PipesClientCompletedInvocation(pipes_session)
+
+    def _create_container(
+        self,
+        client,
+        image: str,
+        command: Optional[Union[str, Sequence[str]]],
+        env: Optional[Mapping[str, str]],
+        container_kwargs: Optional[Mapping[str, Any]],
+        open_pipes_session_env: Mapping[str, str],
+    ):
+        kwargs = dict(container_kwargs or {})
+        kwargs_env = kwargs.pop("environment", {})
+        return client.containers.create(
+            image=image,
+            command=command,
+            detach=True,
+            environment={
+                **open_pipes_session_env,
+                **(self.env or {}),
+                **(env or {}),
+                **kwargs_env,
+            },
+            **kwargs,
+        )

dagster_docker/py.typed

@@ -0,0 +1 @@
+partial

dagster_docker/utils.py

@@ -6,7 +6,7 @@ from dagster import (
 from dagster._utils.merger import merge_dicts
 from docker_image import reference
 
-from .container_context import DOCKER_CONTAINER_CONTEXT_SCHEMA
+from dagster_docker.container_context import DOCKER_CONTAINER_CONTEXT_SCHEMA
 
 DOCKER_CONFIG_SCHEMA = merge_dicts(
     {
@@ -55,8 +55,4 @@ def validate_docker_image(docker_image):
         # validate that the docker image name is valid
         reference.Reference.parse(docker_image)
     except Exception as e:
-        raise Exception(
-            "Docker image name {docker_image} is not correctly formatted".format(
-                docker_image=docker_image
-            )
-        ) from e
+        raise Exception(f"Docker image name {docker_image} is not correctly formatted") from e

dagster_docker/version.py

@@ -1 +1 @@
-__version__ = "0.17.19"
+__version__ = "0.28.5"

dagster_docker-0.28.5.dist-info/METADATA

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: dagster-docker
+Version: 0.28.5
+Summary: A Dagster integration for docker
+Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-docker
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10,<3.14
+License-File: LICENSE
+Requires-Dist: dagster==1.12.5
+Requires-Dist: docker
+Requires-Dist: docker-image-py
+Provides-Extra: test
+Requires-Dist: flaky; extra == "test"
+Requires-Dist: botocore>=1.21.49; extra == "test"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagster_docker-0.28.5.dist-info/RECORD

@@ -0,0 +1,15 @@
+dagster_docker/__init__.py,sha256=rnJ8jarKzaUQl3kIXVFR9h2kNinX0Enjx_mIROBgHdY,624
+dagster_docker/container_context.py,sha256=pbM6btkjXppI5mXaU-2YhRSUTkCQfhzvPHz72WzM3Ec,6485
+dagster_docker/docker_executor.py,sha256=i9wszCPICeduvseI4OPfYOob8NShlBnG0ZLlD0p8EKY,12826
+dagster_docker/docker_run_launcher.py,sha256=3rGXZ1wp9gbPyn1iP5XLiTxF-JzazCVpt50qvvfTSm8,8458
+dagster_docker/pipes.py,sha256=Ibp5ZqYmISxHZBaOwVwed2phCZaRfimmDSG87lVrRj0,8242
+dagster_docker/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
+dagster_docker/utils.py,sha256=oo3JziSJL0CrHMtvQL9KA3OUEWTBEHA1PdY4sqotZy0,1821
+dagster_docker/version.py,sha256=SdGElACDuKJX1c_iqPcfu_quaKOShee-DePM8qfbbJU,23
+dagster_docker/ops/__init__.py,sha256=-86lzmRMHY6zwn33K7hP1_CI0Z14c4-8urBKl7tlIA4,161
+dagster_docker/ops/docker_container_op.py,sha256=zzhdaKlAhBWn1bAHWpy-9ym6evxK1eZniSfACuR7E_E,6434
+dagster_docker-0.28.5.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_docker-0.28.5.dist-info/METADATA,sha256=ZhrMexywE8Dz17ILRMTY-fzxDhrlr1MRYm-5yfqBA8c,1043
+dagster_docker-0.28.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_docker-0.28.5.dist-info/top_level.txt,sha256=FvMtaf9uYIb-jIVJgoiNi1tJ31JZmvh55GxzszLahKs,15
+dagster_docker-0.28.5.dist-info/RECORD,,

{dagster_docker-0.17.19.dist-info → dagster_docker-0.28.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.33.6)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{dagster_docker-0.17.19.dist-info → dagster_docker-0.28.5.dist-info/licenses}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 Elementl, Inc.
+   Copyright 2025 Dagster Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

dagster_docker-0.17.19.dist-info/METADATA

@@ -1,22 +0,0 @@
-Metadata-Version: 2.1
-Name: dagster-docker
-Version: 0.17.19
-Summary: A Dagster integration for docker
-Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-docker
-Author: Elementl
-Author-email: hello@elementl.com
-License: Apache-2.0
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Operating System :: OS Independent
-License-File: LICENSE
-Requires-Dist: dagster (==1.1.19)
-Requires-Dist: docker
-Requires-Dist: docker-image-py
-
-UNKNOWN
-

dagster_docker-0.17.19.dist-info/RECORD

@@ -1,13 +0,0 @@
-dagster_docker/__init__.py,sha256=FVXqcMHKY8GXFaqIl5aTpX5fHHTsRu4dGJ6EYkusfdM,421
-dagster_docker/container_context.py,sha256=ytE70Wj2emHOopviwZ6aE7bkc3sywCvJTo4T9WVJVmo,6484
-dagster_docker/docker_executor.py,sha256=gMbZLVOGJvMuWIED7Lmhf4iT5JivW2fxtRX79B24Eqw,11815
-dagster_docker/docker_run_launcher.py,sha256=S5_EFHOOo4q82BipDN1mRHX0GJa1H3wucLz44wqnrdo,7220
-dagster_docker/utils.py,sha256=pviR2koL8w_UQq14UY2wquT1zcy5VDEg9sOteybURsY,1892
-dagster_docker/version.py,sha256=LKtJnPhS5GRuhOh9obHGnDlncpF9oAEamdbIKdiL3Tw,24
-dagster_docker/ops/__init__.py,sha256=ZGKgOE1FC-vGfR4bk2kg4FXSg4LVsV2PzIK_XyAUqOk,143
-dagster_docker/ops/docker_container_op.py,sha256=jJFwgFnbNlCyq2g6AiUI41Rd_0WSMhhtyBQeez9Ay1k,6362
-dagster_docker-0.17.19.dist-info/LICENSE,sha256=-gtoVIAZYUHYmNHISZg982FI4Oh19mV1nxgTVW8eCB8,11344
-dagster_docker-0.17.19.dist-info/METADATA,sha256=FHEWIB02zH3sDvYolvv0XWAM3YdHwhuWjY6f-ipc7fk,721
-dagster_docker-0.17.19.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
-dagster_docker-0.17.19.dist-info/top_level.txt,sha256=FvMtaf9uYIb-jIVJgoiNi1tJ31JZmvh55GxzszLahKs,15
-dagster_docker-0.17.19.dist-info/RECORD,,