dagster-docker 0.23.10__tar.gz → 0.28.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/LICENSE +1 -1
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/MANIFEST.in +1 -0
- {dagster-docker-0.23.10/dagster_docker.egg-info → dagster_docker-0.28.1}/PKG-INFO +20 -4
- dagster_docker-0.28.1/README.md +4 -0
- dagster_docker-0.28.1/dagster_docker/__init__.py +15 -0
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker/container_context.py +5 -4
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker/docker_executor.py +49 -23
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker/docker_run_launcher.py +42 -14
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker/ops/__init__.py +1 -1
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker/ops/docker_container_op.py +10 -8
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker/pipes.py +6 -6
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker/utils.py +1 -1
- dagster_docker-0.28.1/dagster_docker/version.py +1 -0
- {dagster-docker-0.23.10 → dagster_docker-0.28.1/dagster_docker.egg-info}/PKG-INFO +20 -4
- dagster_docker-0.28.1/dagster_docker.egg-info/requires.txt +7 -0
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/setup.py +15 -5
- dagster-docker-0.23.10/README.md +0 -4
- dagster-docker-0.23.10/dagster_docker/__init__.py +0 -15
- dagster-docker-0.23.10/dagster_docker/version.py +0 -1
- dagster-docker-0.23.10/dagster_docker.egg-info/requires.txt +0 -3
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker/py.typed +0 -0
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker.egg-info/SOURCES.txt +0 -0
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker.egg-info/dependency_links.txt +0 -0
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker.egg-info/not-zip-safe +0 -0
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker.egg-info/top_level.txt +0 -0
- {dagster-docker-0.23.10 → dagster_docker-0.28.1}/setup.cfg +0 -0
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright
|
|
189
|
+
Copyright 2025 Dagster Labs, Inc.
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
|
@@ -1,17 +1,33 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-docker
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.28.1
|
|
4
4
|
Summary: A Dagster integration for docker
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-docker
|
|
6
6
|
Author: Dagster Labs
|
|
7
7
|
Author-email: hello@dagsterlabs.com
|
|
8
8
|
License: Apache-2.0
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
10
9
|
Classifier: Programming Language :: Python :: 3.9
|
|
11
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
14
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
|
-
Requires-Python: >=3.
|
|
16
|
+
Requires-Python: >=3.9,<3.14
|
|
17
17
|
License-File: LICENSE
|
|
18
|
+
Requires-Dist: dagster==1.12.1
|
|
19
|
+
Requires-Dist: docker
|
|
20
|
+
Requires-Dist: docker-image-py
|
|
21
|
+
Provides-Extra: test
|
|
22
|
+
Requires-Dist: flaky; extra == "test"
|
|
23
|
+
Requires-Dist: botocore>=1.21.49; extra == "test"
|
|
24
|
+
Dynamic: author
|
|
25
|
+
Dynamic: author-email
|
|
26
|
+
Dynamic: classifier
|
|
27
|
+
Dynamic: home-page
|
|
28
|
+
Dynamic: license
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
Dynamic: provides-extra
|
|
31
|
+
Dynamic: requires-dist
|
|
32
|
+
Dynamic: requires-python
|
|
33
|
+
Dynamic: summary
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from dagster_shared.libraries import DagsterLibraryRegistry
|
|
2
|
+
|
|
3
|
+
from dagster_docker.docker_executor import docker_executor as docker_executor
|
|
4
|
+
from dagster_docker.docker_run_launcher import DockerRunLauncher as DockerRunLauncher
|
|
5
|
+
from dagster_docker.ops import (
|
|
6
|
+
docker_container_op as docker_container_op,
|
|
7
|
+
execute_docker_container as execute_docker_container,
|
|
8
|
+
)
|
|
9
|
+
from dagster_docker.pipes import (
|
|
10
|
+
PipesDockerClient as PipesDockerClient,
|
|
11
|
+
PipesDockerLogsMessageReader as PipesDockerLogsMessageReader,
|
|
12
|
+
)
|
|
13
|
+
from dagster_docker.version import __version__
|
|
14
|
+
|
|
15
|
+
DagsterLibraryRegistry.register("dagster-docker", __version__)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
from
|
|
1
|
+
from collections.abc import Mapping, Sequence
|
|
2
|
+
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, cast
|
|
2
3
|
|
|
3
4
|
from dagster import (
|
|
4
5
|
Array,
|
|
@@ -13,7 +14,7 @@ from dagster._core.errors import DagsterInvalidConfigError
|
|
|
13
14
|
from dagster._core.storage.dagster_run import DagsterRun
|
|
14
15
|
|
|
15
16
|
if TYPE_CHECKING:
|
|
16
|
-
from
|
|
17
|
+
from dagster_docker import DockerRunLauncher
|
|
17
18
|
|
|
18
19
|
DOCKER_CONTAINER_CONTEXT_SCHEMA = {
|
|
19
20
|
"registry": Field(
|
|
@@ -79,7 +80,7 @@ class DockerContainerContext(
|
|
|
79
80
|
networks: Optional[Sequence[str]] = None,
|
|
80
81
|
container_kwargs: Optional[Mapping[str, Any]] = None,
|
|
81
82
|
):
|
|
82
|
-
return super(
|
|
83
|
+
return super().__new__(
|
|
83
84
|
cls,
|
|
84
85
|
registry=check.opt_nullable_mapping_param(registry, "registry"),
|
|
85
86
|
env_vars=check.opt_sequence_param(env_vars, "env_vars", of_type=str),
|
|
@@ -154,7 +155,7 @@ class DockerContainerContext(
|
|
|
154
155
|
run_docker_container_context,
|
|
155
156
|
)
|
|
156
157
|
|
|
157
|
-
processed_context_value = cast(Mapping[str, Any], processed_container_context.value)
|
|
158
|
+
processed_context_value = cast("Mapping[str, Any]", processed_container_context.value)
|
|
158
159
|
|
|
159
160
|
return shared_container_context.merge(
|
|
160
161
|
DockerContainerContext(
|
|
@@ -1,13 +1,18 @@
|
|
|
1
|
-
from
|
|
1
|
+
from collections.abc import Iterator
|
|
2
|
+
from typing import TYPE_CHECKING, Optional, cast
|
|
2
3
|
|
|
3
4
|
import dagster._check as check
|
|
4
5
|
import docker
|
|
5
6
|
import docker.errors
|
|
6
7
|
from dagster import Field, IntSource, executor
|
|
7
|
-
from dagster._annotations import
|
|
8
|
+
from dagster._annotations import beta
|
|
8
9
|
from dagster._core.definitions.executor_definition import multiple_process_executor_requirements
|
|
9
10
|
from dagster._core.events import DagsterEvent, EngineEventData
|
|
10
11
|
from dagster._core.execution.retries import RetryMode, get_retries_config
|
|
12
|
+
from dagster._core.execution.step_dependency_config import (
|
|
13
|
+
StepDependencyConfig,
|
|
14
|
+
get_step_dependency_config_field,
|
|
15
|
+
)
|
|
11
16
|
from dagster._core.execution.tags import get_tag_concurrency_limits_config
|
|
12
17
|
from dagster._core.executor.base import Executor
|
|
13
18
|
from dagster._core.executor.init import InitExecutorContext
|
|
@@ -17,15 +22,15 @@ from dagster._core.executor.step_delegating.step_handler.base import (
|
|
|
17
22
|
StepHandler,
|
|
18
23
|
StepHandlerContext,
|
|
19
24
|
)
|
|
20
|
-
from dagster._core.origin import JobPythonOrigin
|
|
21
25
|
from dagster._core.utils import parse_env_var
|
|
22
|
-
from dagster._grpc.types import ExecuteStepArgs
|
|
23
|
-
from dagster._serdes.utils import hash_str
|
|
24
26
|
from dagster._utils.merger import merge_dicts
|
|
27
|
+
from dagster_shared.serdes.utils import hash_str
|
|
25
28
|
|
|
29
|
+
from dagster_docker.container_context import DockerContainerContext
|
|
26
30
|
from dagster_docker.utils import DOCKER_CONFIG_SCHEMA, validate_docker_config, validate_docker_image
|
|
27
31
|
|
|
28
|
-
|
|
32
|
+
if TYPE_CHECKING:
|
|
33
|
+
from dagster._core.origin import JobPythonOrigin
|
|
29
34
|
|
|
30
35
|
|
|
31
36
|
@executor(
|
|
@@ -43,11 +48,12 @@ from .container_context import DockerContainerContext
|
|
|
43
48
|
),
|
|
44
49
|
),
|
|
45
50
|
"tag_concurrency_limits": get_tag_concurrency_limits_config(),
|
|
51
|
+
"step_dependency_config": get_step_dependency_config_field(),
|
|
46
52
|
},
|
|
47
53
|
),
|
|
48
54
|
requirements=multiple_process_executor_requirements(),
|
|
49
55
|
)
|
|
50
|
-
@
|
|
56
|
+
@beta
|
|
51
57
|
def docker_executor(init_context: InitExecutorContext) -> Executor:
|
|
52
58
|
"""Executor which launches steps as Docker containers.
|
|
53
59
|
|
|
@@ -100,6 +106,9 @@ def docker_executor(init_context: InitExecutorContext) -> Executor:
|
|
|
100
106
|
retries=check.not_none(RetryMode.from_config(retries)),
|
|
101
107
|
max_concurrent=max_concurrent,
|
|
102
108
|
tag_concurrency_limits=tag_concurrency_limits,
|
|
109
|
+
step_dependency_config=StepDependencyConfig.from_config(
|
|
110
|
+
config.get("step_dependency_config") # type: ignore
|
|
111
|
+
),
|
|
103
112
|
)
|
|
104
113
|
|
|
105
114
|
|
|
@@ -117,10 +126,10 @@ class DockerStepHandler(StepHandler):
|
|
|
117
126
|
)
|
|
118
127
|
|
|
119
128
|
def _get_image(self, step_handler_context: StepHandlerContext):
|
|
120
|
-
from
|
|
129
|
+
from dagster_docker import DockerRunLauncher
|
|
121
130
|
|
|
122
131
|
image = cast(
|
|
123
|
-
JobPythonOrigin, step_handler_context.dagster_run.job_code_origin
|
|
132
|
+
"JobPythonOrigin", step_handler_context.dagster_run.job_code_origin
|
|
124
133
|
).repository_origin.container_image
|
|
125
134
|
if not image:
|
|
126
135
|
image = self._image
|
|
@@ -139,7 +148,7 @@ class DockerStepHandler(StepHandler):
|
|
|
139
148
|
# This doesn't vary per step: would be good to have a hook where it can be set once
|
|
140
149
|
# for the whole StepHandler but we need access to the DagsterRun for that
|
|
141
150
|
|
|
142
|
-
from .docker_run_launcher import DockerRunLauncher
|
|
151
|
+
from dagster_docker.docker_run_launcher import DockerRunLauncher
|
|
143
152
|
|
|
144
153
|
run_launcher = step_handler_context.instance.run_launcher
|
|
145
154
|
run_target = DockerContainerContext.create_for_run(
|
|
@@ -171,11 +180,17 @@ class DockerStepHandler(StepHandler):
|
|
|
171
180
|
)
|
|
172
181
|
return client
|
|
173
182
|
|
|
174
|
-
def
|
|
175
|
-
|
|
176
|
-
|
|
183
|
+
def _get_step_key(self, step_handler_context: StepHandlerContext) -> str:
|
|
184
|
+
step_keys_to_execute = cast(
|
|
185
|
+
"list[str]", step_handler_context.execute_step_args.step_keys_to_execute
|
|
186
|
+
)
|
|
177
187
|
assert len(step_keys_to_execute) == 1, "Launching multiple steps is not currently supported"
|
|
178
|
-
|
|
188
|
+
return step_keys_to_execute[0]
|
|
189
|
+
|
|
190
|
+
def _get_container_name(self, step_handler_context: StepHandlerContext):
|
|
191
|
+
execute_step_args = step_handler_context.execute_step_args
|
|
192
|
+
run_id = execute_step_args.run_id
|
|
193
|
+
step_key = self._get_step_key(step_handler_context)
|
|
179
194
|
|
|
180
195
|
step_name = f"dagster-step-{hash_str(run_id + step_key)}"
|
|
181
196
|
|
|
@@ -199,17 +214,20 @@ class DockerStepHandler(StepHandler):
|
|
|
199
214
|
assert len(step_keys_to_execute) == 1, "Launching multiple steps is not currently supported"
|
|
200
215
|
step_key = step_keys_to_execute[0]
|
|
201
216
|
|
|
217
|
+
container_kwargs = {**container_context.container_kwargs}
|
|
218
|
+
container_kwargs.pop("stop_timeout", None)
|
|
219
|
+
|
|
202
220
|
env_vars = dict([parse_env_var(env_var) for env_var in container_context.env_vars])
|
|
203
221
|
env_vars["DAGSTER_RUN_JOB_NAME"] = step_handler_context.dagster_run.job_name
|
|
204
222
|
env_vars["DAGSTER_RUN_STEP_KEY"] = step_key
|
|
205
223
|
return client.containers.create(
|
|
206
224
|
step_image,
|
|
207
|
-
name=self._get_container_name(
|
|
225
|
+
name=self._get_container_name(step_handler_context),
|
|
208
226
|
detach=True,
|
|
209
227
|
network=container_context.networks[0] if len(container_context.networks) else None,
|
|
210
228
|
command=execute_step_args.get_command_args(),
|
|
211
229
|
environment=env_vars,
|
|
212
|
-
**
|
|
230
|
+
**container_kwargs,
|
|
213
231
|
)
|
|
214
232
|
|
|
215
233
|
def launch_step(self, step_handler_context: StepHandlerContext) -> Iterator[DagsterEvent]:
|
|
@@ -255,9 +273,15 @@ class DockerStepHandler(StepHandler):
|
|
|
255
273
|
|
|
256
274
|
client = self._get_client(container_context)
|
|
257
275
|
|
|
258
|
-
container_name = self._get_container_name(step_handler_context
|
|
276
|
+
container_name = self._get_container_name(step_handler_context)
|
|
277
|
+
step_key = self._get_step_key(step_handler_context)
|
|
259
278
|
|
|
260
|
-
|
|
279
|
+
try:
|
|
280
|
+
container = client.containers.get(container_name)
|
|
281
|
+
except docker.errors.NotFound:
|
|
282
|
+
return CheckStepHealthResult.unhealthy(
|
|
283
|
+
reason=f"Docker container {container_name} for step {step_key} could not be found."
|
|
284
|
+
)
|
|
261
285
|
|
|
262
286
|
if container.status == "running":
|
|
263
287
|
return CheckStepHealthResult.healthy()
|
|
@@ -284,12 +308,12 @@ class DockerStepHandler(StepHandler):
|
|
|
284
308
|
step_keys_to_execute = check.not_none(
|
|
285
309
|
step_handler_context.execute_step_args.step_keys_to_execute
|
|
286
310
|
)
|
|
287
|
-
assert (
|
|
288
|
-
|
|
289
|
-
)
|
|
311
|
+
assert len(step_keys_to_execute) == 1, (
|
|
312
|
+
"Terminating multiple steps is not currently supported"
|
|
313
|
+
)
|
|
290
314
|
step_key = step_keys_to_execute[0]
|
|
291
315
|
|
|
292
|
-
container_name = self._get_container_name(step_handler_context
|
|
316
|
+
container_name = self._get_container_name(step_handler_context)
|
|
293
317
|
|
|
294
318
|
yield DagsterEvent.engine_event(
|
|
295
319
|
step_handler_context.get_step_context(step_key),
|
|
@@ -301,4 +325,6 @@ class DockerStepHandler(StepHandler):
|
|
|
301
325
|
|
|
302
326
|
container = client.containers.get(container_name)
|
|
303
327
|
|
|
304
|
-
|
|
328
|
+
stop_timeout = container_context.container_kwargs.get("stop_timeout")
|
|
329
|
+
|
|
330
|
+
container.stop(timeout=stop_timeout)
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
|
|
1
|
+
import json
|
|
2
|
+
from collections.abc import Mapping
|
|
3
|
+
from typing import Any, Optional
|
|
2
4
|
|
|
3
5
|
import dagster._check as check
|
|
4
6
|
import docker
|
|
@@ -17,10 +19,9 @@ from dagster._serdes import ConfigurableClass
|
|
|
17
19
|
from dagster._serdes.config_class import ConfigurableClassData
|
|
18
20
|
from typing_extensions import Self
|
|
19
21
|
|
|
22
|
+
from dagster_docker.container_context import DockerContainerContext
|
|
20
23
|
from dagster_docker.utils import DOCKER_CONFIG_SCHEMA, validate_docker_config, validate_docker_image
|
|
21
24
|
|
|
22
|
-
from .container_context import DockerContainerContext
|
|
23
|
-
|
|
24
25
|
DOCKER_CONTAINER_ID_TAG = "docker/container_id"
|
|
25
26
|
|
|
26
27
|
|
|
@@ -103,6 +104,17 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
|
|
|
103
104
|
|
|
104
105
|
client = self._get_client(container_context)
|
|
105
106
|
|
|
107
|
+
container_kwargs = {**container_context.container_kwargs}
|
|
108
|
+
labels = container_kwargs.pop("labels", {})
|
|
109
|
+
|
|
110
|
+
container_kwargs.pop("stop_timeout", None)
|
|
111
|
+
|
|
112
|
+
if isinstance(labels, list):
|
|
113
|
+
labels = {key: "" for key in labels}
|
|
114
|
+
|
|
115
|
+
labels["dagster/run_id"] = run.run_id
|
|
116
|
+
labels["dagster/job_name"] = run.job_name
|
|
117
|
+
|
|
106
118
|
try:
|
|
107
119
|
container = client.containers.create(
|
|
108
120
|
image=docker_image,
|
|
@@ -110,10 +122,11 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
|
|
|
110
122
|
detach=True,
|
|
111
123
|
environment=docker_env,
|
|
112
124
|
network=container_context.networks[0] if len(container_context.networks) else None,
|
|
113
|
-
|
|
125
|
+
labels=labels,
|
|
126
|
+
**container_kwargs,
|
|
114
127
|
)
|
|
115
128
|
|
|
116
|
-
except docker.errors.ImageNotFound:
|
|
129
|
+
except docker.errors.ImageNotFound: # pyright: ignore[reportAttributeAccessIssue]
|
|
117
130
|
client.images.pull(docker_image)
|
|
118
131
|
container = client.containers.create(
|
|
119
132
|
image=docker_image,
|
|
@@ -121,7 +134,8 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
|
|
|
121
134
|
detach=True,
|
|
122
135
|
environment=docker_env,
|
|
123
136
|
network=container_context.networks[0] if len(container_context.networks) else None,
|
|
124
|
-
|
|
137
|
+
labels=labels,
|
|
138
|
+
**container_kwargs,
|
|
125
139
|
)
|
|
126
140
|
|
|
127
141
|
if len(container_context.networks) > 1:
|
|
@@ -137,7 +151,7 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
|
|
|
137
151
|
|
|
138
152
|
self._instance.add_run_tags(
|
|
139
153
|
run.run_id,
|
|
140
|
-
{DOCKER_CONTAINER_ID_TAG: container.id, DOCKER_IMAGE_TAG: docker_image},
|
|
154
|
+
{DOCKER_CONTAINER_ID_TAG: container.id, DOCKER_IMAGE_TAG: docker_image}, # pyright: ignore[reportArgumentType]
|
|
141
155
|
)
|
|
142
156
|
|
|
143
157
|
container.start()
|
|
@@ -173,7 +187,7 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
|
|
|
173
187
|
self._launch_container_with_command(run, docker_image, command)
|
|
174
188
|
|
|
175
189
|
def _get_container(self, run):
|
|
176
|
-
if not run
|
|
190
|
+
if not run:
|
|
177
191
|
return None
|
|
178
192
|
|
|
179
193
|
container_id = run.tags.get(DOCKER_CONTAINER_ID_TAG)
|
|
@@ -185,19 +199,22 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
|
|
|
185
199
|
|
|
186
200
|
try:
|
|
187
201
|
return self._get_client(container_context).containers.get(container_id)
|
|
188
|
-
except
|
|
202
|
+
except docker.errors.NotFound: # pyright: ignore[reportAttributeAccessIssue]
|
|
189
203
|
return None
|
|
190
204
|
|
|
191
205
|
def terminate(self, run_id):
|
|
192
206
|
run = self._instance.get_run_by_id(run_id)
|
|
193
207
|
|
|
194
|
-
if not run:
|
|
208
|
+
if not run or run.is_finished:
|
|
195
209
|
return False
|
|
196
210
|
|
|
197
211
|
self._instance.report_run_canceling(run)
|
|
198
212
|
|
|
199
213
|
container = self._get_container(run)
|
|
200
214
|
|
|
215
|
+
container_context = self.get_container_context(run)
|
|
216
|
+
stop_timeout = container_context.container_kwargs.get("stop_timeout")
|
|
217
|
+
|
|
201
218
|
if not container:
|
|
202
219
|
self._instance.report_engine_event(
|
|
203
220
|
message="Unable to get docker container to send termination request to.",
|
|
@@ -206,7 +223,7 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
|
|
|
206
223
|
)
|
|
207
224
|
return False
|
|
208
225
|
|
|
209
|
-
container.stop()
|
|
226
|
+
container.stop(timeout=stop_timeout)
|
|
210
227
|
|
|
211
228
|
return True
|
|
212
229
|
|
|
@@ -215,11 +232,22 @@ class DockerRunLauncher(RunLauncher, ConfigurableClass):
|
|
|
215
232
|
return True
|
|
216
233
|
|
|
217
234
|
def check_run_worker_health(self, run: DagsterRun):
|
|
235
|
+
container_id = run.tags.get(DOCKER_CONTAINER_ID_TAG)
|
|
236
|
+
|
|
237
|
+
if not container_id:
|
|
238
|
+
return CheckRunHealthResult(WorkerStatus.NOT_FOUND, msg="No container ID tag for run.")
|
|
239
|
+
|
|
218
240
|
container = self._get_container(run)
|
|
219
241
|
if container is None:
|
|
220
|
-
return CheckRunHealthResult(
|
|
242
|
+
return CheckRunHealthResult(
|
|
243
|
+
WorkerStatus.NOT_FOUND, msg=f"Could not find container with ID {container_id}."
|
|
244
|
+
)
|
|
221
245
|
if container.status == "running":
|
|
222
246
|
return CheckRunHealthResult(WorkerStatus.RUNNING)
|
|
223
|
-
|
|
224
|
-
|
|
247
|
+
|
|
248
|
+
container_state = container.attrs.get("State")
|
|
249
|
+
failure_string = f"Container status is {container.status}." + (
|
|
250
|
+
f" Container state: {json.dumps(container_state)}" if container_state else ""
|
|
225
251
|
)
|
|
252
|
+
|
|
253
|
+
return CheckRunHealthResult(WorkerStatus.FAILED, msg=failure_string)
|
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
from
|
|
1
|
+
from collections.abc import Mapping, Sequence
|
|
2
|
+
from typing import Any, Optional
|
|
2
3
|
|
|
3
4
|
import docker
|
|
5
|
+
import docker.errors
|
|
4
6
|
from dagster import Field, In, Nothing, OpExecutionContext, StringSource, op
|
|
5
|
-
from dagster._annotations import
|
|
7
|
+
from dagster._annotations import beta
|
|
6
8
|
from dagster._core.utils import parse_env_var
|
|
7
|
-
from
|
|
9
|
+
from dagster_shared.serdes.utils import hash_str
|
|
8
10
|
|
|
9
|
-
from
|
|
10
|
-
from
|
|
11
|
-
from
|
|
11
|
+
from dagster_docker.container_context import DockerContainerContext
|
|
12
|
+
from dagster_docker.docker_run_launcher import DockerRunLauncher
|
|
13
|
+
from dagster_docker.utils import DOCKER_CONFIG_SCHEMA, validate_docker_image
|
|
12
14
|
|
|
13
15
|
DOCKER_CONTAINER_OP_CONFIG = {
|
|
14
16
|
**DOCKER_CONFIG_SCHEMA,
|
|
@@ -71,7 +73,7 @@ def _create_container(
|
|
|
71
73
|
)
|
|
72
74
|
|
|
73
75
|
|
|
74
|
-
@
|
|
76
|
+
@beta
|
|
75
77
|
def execute_docker_container(
|
|
76
78
|
context: OpExecutionContext,
|
|
77
79
|
image: str,
|
|
@@ -148,7 +150,7 @@ def execute_docker_container(
|
|
|
148
150
|
|
|
149
151
|
|
|
150
152
|
@op(ins={"start_after": In(Nothing)}, config_schema=DOCKER_CONTAINER_OP_CONFIG)
|
|
151
|
-
@
|
|
153
|
+
@beta
|
|
152
154
|
def docker_container_op(context):
|
|
153
155
|
"""An op that runs a Docker container using the docker Python API.
|
|
154
156
|
|
|
@@ -1,13 +1,15 @@
|
|
|
1
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
1
2
|
from contextlib import contextmanager
|
|
2
|
-
from typing import Any,
|
|
3
|
+
from typing import Any, Optional, Union
|
|
3
4
|
|
|
4
5
|
import docker
|
|
6
|
+
import docker.errors
|
|
5
7
|
from dagster import (
|
|
6
8
|
OpExecutionContext,
|
|
7
9
|
_check as check,
|
|
8
10
|
)
|
|
9
|
-
from dagster._annotations import experimental
|
|
10
11
|
from dagster._core.definitions.resource_annotation import TreatAsResourceParam
|
|
12
|
+
from dagster._core.execution.context.asset_execution_context import AssetExecutionContext
|
|
11
13
|
from dagster._core.pipes.client import (
|
|
12
14
|
PipesClient,
|
|
13
15
|
PipesClientCompletedInvocation,
|
|
@@ -23,7 +25,6 @@ from dagster._core.pipes.utils import (
|
|
|
23
25
|
from dagster_pipes import DagsterPipesError, PipesDefaultMessageWriter, PipesExtras, PipesParams
|
|
24
26
|
|
|
25
27
|
|
|
26
|
-
@experimental
|
|
27
28
|
class PipesDockerLogsMessageReader(PipesMessageReader):
|
|
28
29
|
@contextmanager
|
|
29
30
|
def read_messages(
|
|
@@ -54,7 +55,6 @@ class PipesDockerLogsMessageReader(PipesMessageReader):
|
|
|
54
55
|
return "Attempted to read messages by extracting them from docker logs directly."
|
|
55
56
|
|
|
56
57
|
|
|
57
|
-
@experimental
|
|
58
58
|
class PipesDockerClient(PipesClient, TreatAsResourceParam):
|
|
59
59
|
"""A pipes client that runs external processes in docker containers.
|
|
60
60
|
|
|
@@ -99,10 +99,10 @@ class PipesDockerClient(PipesClient, TreatAsResourceParam):
|
|
|
99
99
|
def _is_dagster_maintained(cls) -> bool:
|
|
100
100
|
return True
|
|
101
101
|
|
|
102
|
-
def run(
|
|
102
|
+
def run( # pyright: ignore[reportIncompatibleMethodOverride]
|
|
103
103
|
self,
|
|
104
104
|
*,
|
|
105
|
-
context: OpExecutionContext,
|
|
105
|
+
context: Union[OpExecutionContext, AssetExecutionContext],
|
|
106
106
|
image: str,
|
|
107
107
|
extras: Optional[PipesExtras] = None,
|
|
108
108
|
command: Optional[Union[str, Sequence[str]]] = None,
|
|
@@ -6,7 +6,7 @@ from dagster import (
|
|
|
6
6
|
from dagster._utils.merger import merge_dicts
|
|
7
7
|
from docker_image import reference
|
|
8
8
|
|
|
9
|
-
from .container_context import DOCKER_CONTAINER_CONTEXT_SCHEMA
|
|
9
|
+
from dagster_docker.container_context import DOCKER_CONTAINER_CONTEXT_SCHEMA
|
|
10
10
|
|
|
11
11
|
DOCKER_CONFIG_SCHEMA = merge_dicts(
|
|
12
12
|
{
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.28.1"
|
|
@@ -1,17 +1,33 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-docker
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.28.1
|
|
4
4
|
Summary: A Dagster integration for docker
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-docker
|
|
6
6
|
Author: Dagster Labs
|
|
7
7
|
Author-email: hello@dagsterlabs.com
|
|
8
8
|
License: Apache-2.0
|
|
9
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
10
9
|
Classifier: Programming Language :: Python :: 3.9
|
|
11
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
14
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
|
-
Requires-Python: >=3.
|
|
16
|
+
Requires-Python: >=3.9,<3.14
|
|
17
17
|
License-File: LICENSE
|
|
18
|
+
Requires-Dist: dagster==1.12.1
|
|
19
|
+
Requires-Dist: docker
|
|
20
|
+
Requires-Dist: docker-image-py
|
|
21
|
+
Provides-Extra: test
|
|
22
|
+
Requires-Dist: flaky; extra == "test"
|
|
23
|
+
Requires-Dist: botocore>=1.21.49; extra == "test"
|
|
24
|
+
Dynamic: author
|
|
25
|
+
Dynamic: author-email
|
|
26
|
+
Dynamic: classifier
|
|
27
|
+
Dynamic: home-page
|
|
28
|
+
Dynamic: license
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
Dynamic: provides-extra
|
|
31
|
+
Dynamic: requires-dist
|
|
32
|
+
Dynamic: requires-python
|
|
33
|
+
Dynamic: summary
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
-
from typing import Dict
|
|
3
2
|
|
|
4
3
|
from setuptools import find_packages, setup
|
|
5
4
|
|
|
6
5
|
|
|
7
6
|
def get_version() -> str:
|
|
8
|
-
version:
|
|
7
|
+
version: dict[str, str] = {}
|
|
9
8
|
with open(Path(__file__).parent / "dagster_docker/version.py", encoding="utf8") as fp:
|
|
10
9
|
exec(fp.read(), version)
|
|
11
10
|
|
|
@@ -24,16 +23,27 @@ setup(
|
|
|
24
23
|
description="A Dagster integration for docker",
|
|
25
24
|
url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-docker",
|
|
26
25
|
classifiers=[
|
|
27
|
-
"Programming Language :: Python :: 3.8",
|
|
28
26
|
"Programming Language :: Python :: 3.9",
|
|
29
27
|
"Programming Language :: Python :: 3.10",
|
|
30
28
|
"Programming Language :: Python :: 3.11",
|
|
31
29
|
"Programming Language :: Python :: 3.12",
|
|
30
|
+
"Programming Language :: Python :: 3.13",
|
|
32
31
|
"License :: OSI Approved :: Apache Software License",
|
|
33
32
|
"Operating System :: OS Independent",
|
|
34
33
|
],
|
|
35
34
|
packages=find_packages(exclude=["dagster_docker_tests*"]),
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
include_package_data=True,
|
|
36
|
+
python_requires=">=3.9,<3.14",
|
|
37
|
+
install_requires=[
|
|
38
|
+
"dagster==1.12.1",
|
|
39
|
+
"docker",
|
|
40
|
+
"docker-image-py",
|
|
41
|
+
],
|
|
42
|
+
extras_require={
|
|
43
|
+
"test": [
|
|
44
|
+
"flaky",
|
|
45
|
+
"botocore>=1.21.49", # first botocore version that works on python 3.9+
|
|
46
|
+
],
|
|
47
|
+
},
|
|
38
48
|
zip_safe=False,
|
|
39
49
|
)
|
dagster-docker-0.23.10/README.md
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
from dagster._core.libraries import DagsterLibraryRegistry
|
|
2
|
-
|
|
3
|
-
from .docker_executor import docker_executor as docker_executor
|
|
4
|
-
from .docker_run_launcher import DockerRunLauncher as DockerRunLauncher
|
|
5
|
-
from .ops import (
|
|
6
|
-
docker_container_op as docker_container_op,
|
|
7
|
-
execute_docker_container as execute_docker_container,
|
|
8
|
-
)
|
|
9
|
-
from .pipes import (
|
|
10
|
-
PipesDockerClient as PipesDockerClient,
|
|
11
|
-
PipesDockerLogsMessageReader as PipesDockerLogsMessageReader,
|
|
12
|
-
)
|
|
13
|
-
from .version import __version__
|
|
14
|
-
|
|
15
|
-
DagsterLibraryRegistry.register("dagster-docker", __version__)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.23.10"
|
|
File without changes
|
|
File without changes
|
{dagster-docker-0.23.10 → dagster_docker-0.28.1}/dagster_docker.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|