prefect-client 2.20.4__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +74 -110
- prefect/_internal/compatibility/deprecated.py +6 -115
- prefect/_internal/compatibility/experimental.py +4 -79
- prefect/_internal/compatibility/migration.py +166 -0
- prefect/_internal/concurrency/__init__.py +2 -2
- prefect/_internal/concurrency/api.py +1 -35
- prefect/_internal/concurrency/calls.py +0 -6
- prefect/_internal/concurrency/cancellation.py +0 -3
- prefect/_internal/concurrency/event_loop.py +0 -20
- prefect/_internal/concurrency/inspection.py +3 -3
- prefect/_internal/concurrency/primitives.py +1 -0
- prefect/_internal/concurrency/services.py +23 -0
- prefect/_internal/concurrency/threads.py +35 -0
- prefect/_internal/concurrency/waiters.py +0 -28
- prefect/_internal/integrations.py +7 -0
- prefect/_internal/pydantic/__init__.py +0 -45
- prefect/_internal/pydantic/annotations/pendulum.py +2 -2
- prefect/_internal/pydantic/v1_schema.py +21 -22
- prefect/_internal/pydantic/v2_schema.py +0 -2
- prefect/_internal/pydantic/v2_validated_func.py +18 -23
- prefect/_internal/pytz.py +1 -1
- prefect/_internal/retries.py +61 -0
- prefect/_internal/schemas/bases.py +45 -177
- prefect/_internal/schemas/fields.py +1 -43
- prefect/_internal/schemas/validators.py +47 -233
- prefect/agent.py +3 -695
- prefect/artifacts.py +173 -14
- prefect/automations.py +39 -4
- prefect/blocks/abstract.py +1 -1
- prefect/blocks/core.py +405 -153
- prefect/blocks/fields.py +2 -57
- prefect/blocks/notifications.py +43 -28
- prefect/blocks/redis.py +168 -0
- prefect/blocks/system.py +67 -20
- prefect/blocks/webhook.py +2 -9
- prefect/cache_policies.py +239 -0
- prefect/client/__init__.py +4 -0
- prefect/client/base.py +33 -27
- prefect/client/cloud.py +65 -20
- prefect/client/collections.py +1 -1
- prefect/client/orchestration.py +650 -442
- prefect/client/schemas/actions.py +115 -100
- prefect/client/schemas/filters.py +46 -52
- prefect/client/schemas/objects.py +228 -178
- prefect/client/schemas/responses.py +18 -36
- prefect/client/schemas/schedules.py +55 -36
- prefect/client/schemas/sorting.py +2 -0
- prefect/client/subscriptions.py +8 -7
- prefect/client/types/flexible_schedule_list.py +11 -0
- prefect/client/utilities.py +9 -6
- prefect/concurrency/asyncio.py +60 -11
- prefect/concurrency/context.py +24 -0
- prefect/concurrency/events.py +2 -2
- prefect/concurrency/services.py +46 -16
- prefect/concurrency/sync.py +51 -7
- prefect/concurrency/v1/asyncio.py +143 -0
- prefect/concurrency/v1/context.py +27 -0
- prefect/concurrency/v1/events.py +61 -0
- prefect/concurrency/v1/services.py +116 -0
- prefect/concurrency/v1/sync.py +92 -0
- prefect/context.py +246 -149
- prefect/deployments/__init__.py +33 -18
- prefect/deployments/base.py +10 -15
- prefect/deployments/deployments.py +2 -1048
- prefect/deployments/flow_runs.py +178 -0
- prefect/deployments/runner.py +72 -173
- prefect/deployments/schedules.py +31 -25
- prefect/deployments/steps/__init__.py +0 -1
- prefect/deployments/steps/core.py +7 -0
- prefect/deployments/steps/pull.py +15 -21
- prefect/deployments/steps/utility.py +2 -1
- prefect/docker/__init__.py +20 -0
- prefect/docker/docker_image.py +82 -0
- prefect/engine.py +15 -2475
- prefect/events/actions.py +17 -23
- prefect/events/cli/automations.py +20 -7
- prefect/events/clients.py +142 -80
- prefect/events/filters.py +14 -18
- prefect/events/related.py +74 -75
- prefect/events/schemas/__init__.py +0 -5
- prefect/events/schemas/automations.py +55 -46
- prefect/events/schemas/deployment_triggers.py +7 -197
- prefect/events/schemas/events.py +46 -65
- prefect/events/schemas/labelling.py +10 -14
- prefect/events/utilities.py +4 -5
- prefect/events/worker.py +23 -8
- prefect/exceptions.py +15 -0
- prefect/filesystems.py +30 -529
- prefect/flow_engine.py +827 -0
- prefect/flow_runs.py +379 -7
- prefect/flows.py +470 -360
- prefect/futures.py +382 -331
- prefect/infrastructure/__init__.py +5 -26
- prefect/infrastructure/base.py +3 -320
- prefect/infrastructure/provisioners/__init__.py +5 -3
- prefect/infrastructure/provisioners/cloud_run.py +13 -8
- prefect/infrastructure/provisioners/container_instance.py +14 -9
- prefect/infrastructure/provisioners/ecs.py +10 -8
- prefect/infrastructure/provisioners/modal.py +8 -5
- prefect/input/__init__.py +4 -0
- prefect/input/actions.py +2 -4
- prefect/input/run_input.py +9 -9
- prefect/logging/formatters.py +2 -4
- prefect/logging/handlers.py +9 -14
- prefect/logging/loggers.py +5 -5
- prefect/main.py +72 -0
- prefect/plugins.py +2 -64
- prefect/profiles.toml +16 -2
- prefect/records/__init__.py +1 -0
- prefect/records/base.py +223 -0
- prefect/records/filesystem.py +207 -0
- prefect/records/memory.py +178 -0
- prefect/records/result_store.py +64 -0
- prefect/results.py +577 -504
- prefect/runner/runner.py +117 -47
- prefect/runner/server.py +32 -34
- prefect/runner/storage.py +3 -12
- prefect/runner/submit.py +2 -10
- prefect/runner/utils.py +2 -2
- prefect/runtime/__init__.py +1 -0
- prefect/runtime/deployment.py +1 -0
- prefect/runtime/flow_run.py +40 -5
- prefect/runtime/task_run.py +1 -0
- prefect/serializers.py +28 -39
- prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
- prefect/settings.py +209 -332
- prefect/states.py +160 -63
- prefect/task_engine.py +1478 -57
- prefect/task_runners.py +383 -287
- prefect/task_runs.py +240 -0
- prefect/task_worker.py +463 -0
- prefect/tasks.py +684 -374
- prefect/transactions.py +410 -0
- prefect/types/__init__.py +72 -86
- prefect/types/entrypoint.py +13 -0
- prefect/utilities/annotations.py +4 -3
- prefect/utilities/asyncutils.py +227 -148
- prefect/utilities/callables.py +137 -45
- prefect/utilities/collections.py +134 -86
- prefect/utilities/dispatch.py +27 -14
- prefect/utilities/dockerutils.py +11 -4
- prefect/utilities/engine.py +186 -32
- prefect/utilities/filesystem.py +4 -5
- prefect/utilities/importtools.py +26 -27
- prefect/utilities/pydantic.py +128 -38
- prefect/utilities/schema_tools/hydration.py +18 -1
- prefect/utilities/schema_tools/validation.py +30 -0
- prefect/utilities/services.py +35 -9
- prefect/utilities/templating.py +12 -2
- prefect/utilities/timeout.py +20 -5
- prefect/utilities/urls.py +195 -0
- prefect/utilities/visualization.py +1 -0
- prefect/variables.py +78 -59
- prefect/workers/__init__.py +0 -1
- prefect/workers/base.py +237 -244
- prefect/workers/block.py +5 -226
- prefect/workers/cloud.py +6 -0
- prefect/workers/process.py +265 -12
- prefect/workers/server.py +29 -11
- {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/METADATA +28 -24
- prefect_client-3.0.0.dist-info/RECORD +201 -0
- {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/WHEEL +1 -1
- prefect/_internal/pydantic/_base_model.py +0 -51
- prefect/_internal/pydantic/_compat.py +0 -82
- prefect/_internal/pydantic/_flags.py +0 -20
- prefect/_internal/pydantic/_types.py +0 -8
- prefect/_internal/pydantic/utilities/config_dict.py +0 -72
- prefect/_internal/pydantic/utilities/field_validator.py +0 -150
- prefect/_internal/pydantic/utilities/model_construct.py +0 -56
- prefect/_internal/pydantic/utilities/model_copy.py +0 -55
- prefect/_internal/pydantic/utilities/model_dump.py +0 -136
- prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
- prefect/_internal/pydantic/utilities/model_fields.py +0 -50
- prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
- prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
- prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
- prefect/_internal/pydantic/utilities/model_validate.py +0 -75
- prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
- prefect/_internal/pydantic/utilities/model_validator.py +0 -87
- prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
- prefect/_vendor/fastapi/__init__.py +0 -25
- prefect/_vendor/fastapi/applications.py +0 -946
- prefect/_vendor/fastapi/background.py +0 -3
- prefect/_vendor/fastapi/concurrency.py +0 -44
- prefect/_vendor/fastapi/datastructures.py +0 -58
- prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
- prefect/_vendor/fastapi/dependencies/models.py +0 -64
- prefect/_vendor/fastapi/dependencies/utils.py +0 -877
- prefect/_vendor/fastapi/encoders.py +0 -177
- prefect/_vendor/fastapi/exception_handlers.py +0 -40
- prefect/_vendor/fastapi/exceptions.py +0 -46
- prefect/_vendor/fastapi/logger.py +0 -3
- prefect/_vendor/fastapi/middleware/__init__.py +0 -1
- prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
- prefect/_vendor/fastapi/middleware/cors.py +0 -3
- prefect/_vendor/fastapi/middleware/gzip.py +0 -3
- prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
- prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
- prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
- prefect/_vendor/fastapi/openapi/__init__.py +0 -0
- prefect/_vendor/fastapi/openapi/constants.py +0 -2
- prefect/_vendor/fastapi/openapi/docs.py +0 -203
- prefect/_vendor/fastapi/openapi/models.py +0 -480
- prefect/_vendor/fastapi/openapi/utils.py +0 -485
- prefect/_vendor/fastapi/param_functions.py +0 -340
- prefect/_vendor/fastapi/params.py +0 -453
- prefect/_vendor/fastapi/py.typed +0 -0
- prefect/_vendor/fastapi/requests.py +0 -4
- prefect/_vendor/fastapi/responses.py +0 -40
- prefect/_vendor/fastapi/routing.py +0 -1331
- prefect/_vendor/fastapi/security/__init__.py +0 -15
- prefect/_vendor/fastapi/security/api_key.py +0 -98
- prefect/_vendor/fastapi/security/base.py +0 -6
- prefect/_vendor/fastapi/security/http.py +0 -172
- prefect/_vendor/fastapi/security/oauth2.py +0 -227
- prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
- prefect/_vendor/fastapi/security/utils.py +0 -10
- prefect/_vendor/fastapi/staticfiles.py +0 -1
- prefect/_vendor/fastapi/templating.py +0 -3
- prefect/_vendor/fastapi/testclient.py +0 -1
- prefect/_vendor/fastapi/types.py +0 -3
- prefect/_vendor/fastapi/utils.py +0 -235
- prefect/_vendor/fastapi/websockets.py +0 -7
- prefect/_vendor/starlette/__init__.py +0 -1
- prefect/_vendor/starlette/_compat.py +0 -28
- prefect/_vendor/starlette/_exception_handler.py +0 -80
- prefect/_vendor/starlette/_utils.py +0 -88
- prefect/_vendor/starlette/applications.py +0 -261
- prefect/_vendor/starlette/authentication.py +0 -159
- prefect/_vendor/starlette/background.py +0 -43
- prefect/_vendor/starlette/concurrency.py +0 -59
- prefect/_vendor/starlette/config.py +0 -151
- prefect/_vendor/starlette/convertors.py +0 -87
- prefect/_vendor/starlette/datastructures.py +0 -707
- prefect/_vendor/starlette/endpoints.py +0 -130
- prefect/_vendor/starlette/exceptions.py +0 -60
- prefect/_vendor/starlette/formparsers.py +0 -276
- prefect/_vendor/starlette/middleware/__init__.py +0 -17
- prefect/_vendor/starlette/middleware/authentication.py +0 -52
- prefect/_vendor/starlette/middleware/base.py +0 -220
- prefect/_vendor/starlette/middleware/cors.py +0 -176
- prefect/_vendor/starlette/middleware/errors.py +0 -265
- prefect/_vendor/starlette/middleware/exceptions.py +0 -74
- prefect/_vendor/starlette/middleware/gzip.py +0 -113
- prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
- prefect/_vendor/starlette/middleware/sessions.py +0 -82
- prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
- prefect/_vendor/starlette/middleware/wsgi.py +0 -147
- prefect/_vendor/starlette/py.typed +0 -0
- prefect/_vendor/starlette/requests.py +0 -328
- prefect/_vendor/starlette/responses.py +0 -347
- prefect/_vendor/starlette/routing.py +0 -933
- prefect/_vendor/starlette/schemas.py +0 -154
- prefect/_vendor/starlette/staticfiles.py +0 -248
- prefect/_vendor/starlette/status.py +0 -199
- prefect/_vendor/starlette/templating.py +0 -231
- prefect/_vendor/starlette/testclient.py +0 -804
- prefect/_vendor/starlette/types.py +0 -30
- prefect/_vendor/starlette/websockets.py +0 -193
- prefect/blocks/kubernetes.py +0 -119
- prefect/deprecated/__init__.py +0 -0
- prefect/deprecated/data_documents.py +0 -350
- prefect/deprecated/packaging/__init__.py +0 -12
- prefect/deprecated/packaging/base.py +0 -96
- prefect/deprecated/packaging/docker.py +0 -146
- prefect/deprecated/packaging/file.py +0 -92
- prefect/deprecated/packaging/orion.py +0 -80
- prefect/deprecated/packaging/serializers.py +0 -171
- prefect/events/instrument.py +0 -135
- prefect/infrastructure/container.py +0 -824
- prefect/infrastructure/kubernetes.py +0 -920
- prefect/infrastructure/process.py +0 -289
- prefect/manifests.py +0 -20
- prefect/new_flow_engine.py +0 -449
- prefect/new_task_engine.py +0 -423
- prefect/pydantic/__init__.py +0 -76
- prefect/pydantic/main.py +0 -39
- prefect/software/__init__.py +0 -2
- prefect/software/base.py +0 -50
- prefect/software/conda.py +0 -199
- prefect/software/pip.py +0 -122
- prefect/software/python.py +0 -52
- prefect/task_server.py +0 -322
- prefect_client-2.20.4.dist-info/RECORD +0 -294
- /prefect/{_internal/pydantic/utilities → client/types}/__init__.py +0 -0
- /prefect/{_vendor → concurrency/v1}/__init__.py +0 -0
- {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/LICENSE +0 -0
- {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/top_level.txt +0 -0
@@ -1,824 +0,0 @@
|
|
1
|
-
"""
|
2
|
-
DEPRECATION WARNING:
|
3
|
-
|
4
|
-
This module is deprecated as of March 2024 and will not be available after September 2024.
|
5
|
-
It has been replaced by the Docker worker from the prefect-docker package, which offers enhanced functionality and better performance.
|
6
|
-
|
7
|
-
For upgrade instructions, see https://docs.prefect.io/latest/guides/upgrade-guide-agents-to-workers/.
|
8
|
-
"""
|
9
|
-
import json
|
10
|
-
import re
|
11
|
-
import shlex
|
12
|
-
import sys
|
13
|
-
import urllib.parse
|
14
|
-
import warnings
|
15
|
-
from abc import ABC, abstractmethod
|
16
|
-
from typing import TYPE_CHECKING, Dict, Generator, List, Optional, Tuple, Union
|
17
|
-
|
18
|
-
import anyio.abc
|
19
|
-
import packaging.version
|
20
|
-
|
21
|
-
from prefect._internal.compatibility.deprecated import deprecated_class
|
22
|
-
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
23
|
-
|
24
|
-
if HAS_PYDANTIC_V2:
|
25
|
-
from pydantic.v1 import Field, validator
|
26
|
-
else:
|
27
|
-
from pydantic import Field, validator
|
28
|
-
|
29
|
-
from typing_extensions import Literal
|
30
|
-
|
31
|
-
import prefect
|
32
|
-
from prefect.blocks.core import Block, SecretStr
|
33
|
-
from prefect.exceptions import InfrastructureNotAvailable, InfrastructureNotFound
|
34
|
-
from prefect.infrastructure.base import Infrastructure, InfrastructureResult
|
35
|
-
from prefect.settings import PREFECT_API_URL
|
36
|
-
from prefect.utilities.asyncutils import run_sync_in_worker_thread, sync_compatible
|
37
|
-
from prefect.utilities.collections import AutoEnum
|
38
|
-
from prefect.utilities.dockerutils import (
|
39
|
-
format_outlier_version_name,
|
40
|
-
get_prefect_image_name,
|
41
|
-
parse_image_tag,
|
42
|
-
)
|
43
|
-
from prefect.utilities.importtools import lazy_import
|
44
|
-
from prefect.utilities.slugify import slugify
|
45
|
-
|
46
|
-
if TYPE_CHECKING:
|
47
|
-
import docker
|
48
|
-
from docker import DockerClient
|
49
|
-
from docker.models.containers import Container
|
50
|
-
else:
|
51
|
-
docker = lazy_import("docker")
|
52
|
-
|
53
|
-
|
54
|
-
# Labels to apply to all containers started by Prefect
|
55
|
-
CONTAINER_LABELS = {
|
56
|
-
"io.prefect.version": prefect.__version__,
|
57
|
-
}
|
58
|
-
|
59
|
-
|
60
|
-
class ImagePullPolicy(AutoEnum):
|
61
|
-
IF_NOT_PRESENT = AutoEnum.auto()
|
62
|
-
ALWAYS = AutoEnum.auto()
|
63
|
-
NEVER = AutoEnum.auto()
|
64
|
-
|
65
|
-
|
66
|
-
@deprecated_class(
|
67
|
-
start_date="Mar 2024",
|
68
|
-
help="Use the `DockerRegistryCredentials` class from prefect-docker instead.",
|
69
|
-
)
|
70
|
-
class BaseDockerLogin(Block, ABC):
|
71
|
-
_logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/14a315b79990200db7341e42553e23650b34bb96-250x250.png"
|
72
|
-
_block_schema_capabilities = ["docker-login"]
|
73
|
-
|
74
|
-
@abstractmethod
|
75
|
-
async def login(self) -> "DockerClient":
|
76
|
-
"""
|
77
|
-
Log in and return an authenticated `DockerClient`.
|
78
|
-
(DEPRECATED) Use `get_docker_client` instead of `login`.
|
79
|
-
"""
|
80
|
-
|
81
|
-
@abstractmethod
|
82
|
-
async def get_docker_client(self) -> "DockerClient":
|
83
|
-
"""
|
84
|
-
Log in and return an authenticated `DockerClient`.
|
85
|
-
"""
|
86
|
-
|
87
|
-
def _login(self, username, password, registry_url, reauth) -> "DockerClient":
|
88
|
-
client = self._get_docker_client()
|
89
|
-
|
90
|
-
client.login(
|
91
|
-
username=username,
|
92
|
-
password=password,
|
93
|
-
registry=registry_url,
|
94
|
-
# See https://github.com/docker/docker-py/issues/2256 for information on
|
95
|
-
# the default value for reauth.
|
96
|
-
reauth=reauth,
|
97
|
-
)
|
98
|
-
|
99
|
-
return client
|
100
|
-
|
101
|
-
@staticmethod
|
102
|
-
def _get_docker_client():
|
103
|
-
try:
|
104
|
-
with warnings.catch_warnings():
|
105
|
-
# Silence warnings due to use of deprecated methods within dockerpy
|
106
|
-
# See https://github.com/docker/docker-py/pull/2931
|
107
|
-
warnings.filterwarnings(
|
108
|
-
"ignore",
|
109
|
-
message="distutils Version classes are deprecated.*",
|
110
|
-
category=DeprecationWarning,
|
111
|
-
)
|
112
|
-
|
113
|
-
docker_client = docker.from_env()
|
114
|
-
|
115
|
-
except docker.errors.DockerException as exc:
|
116
|
-
raise RuntimeError("Could not connect to Docker.") from exc
|
117
|
-
|
118
|
-
return docker_client
|
119
|
-
|
120
|
-
|
121
|
-
@deprecated_class(
|
122
|
-
start_date="Mar 2024",
|
123
|
-
help="Use the `DockerRegistryCredentials` class from prefect-docker instead.",
|
124
|
-
)
|
125
|
-
class DockerRegistry(BaseDockerLogin):
|
126
|
-
"""
|
127
|
-
DEPRECATION WARNING:
|
128
|
-
|
129
|
-
This class is deprecated as of March 2024 and will not be available after September 2024.
|
130
|
-
It has been replaced by `DockerRegistryCredentials` from the `prefect-docker` package, which
|
131
|
-
offers enhanced functionality and better a better user experience.
|
132
|
-
|
133
|
-
Connects to a Docker registry.
|
134
|
-
|
135
|
-
Requires a Docker Engine to be connectable.
|
136
|
-
|
137
|
-
Attributes:
|
138
|
-
username: The username to log into the registry with.
|
139
|
-
password: The password to log into the registry with.
|
140
|
-
registry_url: The URL to the registry such as registry.hub.docker.com. Generally, "http" or "https" can be
|
141
|
-
omitted.
|
142
|
-
reauth: If already logged into the registry, should login be performed again?
|
143
|
-
This setting defaults to `True` to support common token authentication
|
144
|
-
patterns such as ECR.
|
145
|
-
"""
|
146
|
-
|
147
|
-
_block_type_name = "Docker Registry"
|
148
|
-
_documentation_url = "https://docs.prefect.io/api-ref/prefect/infrastructure/#prefect.infrastructure.docker.DockerRegistry"
|
149
|
-
|
150
|
-
username: str = Field(
|
151
|
-
default=..., description="The username to log into the registry with."
|
152
|
-
)
|
153
|
-
password: SecretStr = Field(
|
154
|
-
default=..., description="The password to log into the registry with."
|
155
|
-
)
|
156
|
-
registry_url: str = Field(
|
157
|
-
default=...,
|
158
|
-
description=(
|
159
|
-
'The URL to the registry. Generally, "http" or "https" can be omitted.'
|
160
|
-
),
|
161
|
-
)
|
162
|
-
reauth: bool = Field(
|
163
|
-
default=True,
|
164
|
-
description="Whether or not to reauthenticate on each interaction.",
|
165
|
-
)
|
166
|
-
|
167
|
-
@sync_compatible
|
168
|
-
async def login(self) -> "DockerClient":
|
169
|
-
warnings.warn(
|
170
|
-
(
|
171
|
-
"`login` is deprecated. Instead, use `get_docker_client` to obtain an"
|
172
|
-
" authenticated `DockerClient`."
|
173
|
-
),
|
174
|
-
category=DeprecationWarning,
|
175
|
-
stacklevel=3,
|
176
|
-
)
|
177
|
-
return await self.get_docker_client()
|
178
|
-
|
179
|
-
@sync_compatible
|
180
|
-
async def get_docker_client(self) -> "DockerClient":
|
181
|
-
client = await run_sync_in_worker_thread(
|
182
|
-
self._login,
|
183
|
-
self.username,
|
184
|
-
self.password.get_secret_value(),
|
185
|
-
self.registry_url,
|
186
|
-
self.reauth,
|
187
|
-
)
|
188
|
-
|
189
|
-
return client
|
190
|
-
|
191
|
-
|
192
|
-
class DockerContainerResult(InfrastructureResult):
|
193
|
-
"""Contains information about a completed Docker container"""
|
194
|
-
|
195
|
-
|
196
|
-
@deprecated_class(
|
197
|
-
start_date="Mar 2024",
|
198
|
-
help="Use the Docker worker from prefect-docker instead."
|
199
|
-
" Refer to the upgrade guide for more information:"
|
200
|
-
" https://docs.prefect.io/latest/guides/upgrade-guide-agents-to-workers/.",
|
201
|
-
)
|
202
|
-
class DockerContainer(Infrastructure):
|
203
|
-
"""
|
204
|
-
Runs a command in a container.
|
205
|
-
|
206
|
-
Requires a Docker Engine to be connectable. Docker settings will be retrieved from
|
207
|
-
the environment.
|
208
|
-
|
209
|
-
Click [here](https://docs.prefect.io/guides/deployment/docker) to see a tutorial.
|
210
|
-
|
211
|
-
Attributes:
|
212
|
-
auto_remove: If set, the container will be removed on completion. Otherwise,
|
213
|
-
the container will remain after exit for inspection.
|
214
|
-
command: A list of strings specifying the command to run in the container to
|
215
|
-
start the flow run. In most cases you should not override this.
|
216
|
-
env: Environment variables to set for the container.
|
217
|
-
image: An optional string specifying the tag of a Docker image to use.
|
218
|
-
Defaults to the Prefect image.
|
219
|
-
image_pull_policy: Specifies if the image should be pulled. One of 'ALWAYS',
|
220
|
-
'NEVER', 'IF_NOT_PRESENT'.
|
221
|
-
image_registry: A `DockerRegistry` block containing credentials to use if `image` is stored in a private
|
222
|
-
image registry.
|
223
|
-
labels: An optional dictionary of labels, mapping name to value.
|
224
|
-
name: An optional name for the container.
|
225
|
-
network_mode: Set the network mode for the created container. Defaults to 'host'
|
226
|
-
if a local API url is detected, otherwise the Docker default of 'bridge' is
|
227
|
-
used. If 'networks' is set, this cannot be set.
|
228
|
-
networks: An optional list of strings specifying Docker networks to connect the
|
229
|
-
container to.
|
230
|
-
stream_output: If set, stream output from the container to local standard output.
|
231
|
-
volumes: An optional list of volume mount strings in the format of
|
232
|
-
"local_path:container_path".
|
233
|
-
memswap_limit: Total memory (memory + swap), -1 to disable swap. Should only be
|
234
|
-
set if `mem_limit` is also set. If `mem_limit` is set, this defaults to
|
235
|
-
allowing the container to use as much swap as memory. For example, if
|
236
|
-
`mem_limit` is 300m and `memswap_limit` is not set, the container can use
|
237
|
-
600m in total of memory and swap.
|
238
|
-
mem_limit: Memory limit of the created container. Accepts float values to enforce
|
239
|
-
a limit in bytes or a string with a unit e.g. 100000b, 1000k, 128m, 1g.
|
240
|
-
If a string is given without a unit, bytes are assumed.
|
241
|
-
privileged: Give extended privileges to this container.
|
242
|
-
|
243
|
-
## Connecting to a locally hosted Prefect API
|
244
|
-
|
245
|
-
If using a local API URL on Linux, we will update the network mode default to 'host'
|
246
|
-
to enable connectivity. If using another OS or an alternative network mode is used,
|
247
|
-
we will replace 'localhost' in the API URL with 'host.docker.internal'. Generally,
|
248
|
-
this will enable connectivity, but the API URL can be provided as an environment
|
249
|
-
variable to override inference in more complex use-cases.
|
250
|
-
|
251
|
-
Note, if using 'host.docker.internal' in the API URL on Linux, the API must be bound
|
252
|
-
to 0.0.0.0 or the Docker IP address to allow connectivity. On macOS, this is not
|
253
|
-
necessary and the API is connectable while bound to localhost.
|
254
|
-
"""
|
255
|
-
|
256
|
-
type: Literal["docker-container"] = Field(
|
257
|
-
default="docker-container", description="The type of infrastructure."
|
258
|
-
)
|
259
|
-
image: str = Field(
|
260
|
-
default_factory=get_prefect_image_name,
|
261
|
-
description="Tag of a Docker image to use. Defaults to the Prefect image.",
|
262
|
-
)
|
263
|
-
image_pull_policy: Optional[ImagePullPolicy] = Field(
|
264
|
-
default=None, description="Specifies if the image should be pulled."
|
265
|
-
)
|
266
|
-
image_registry: Optional[DockerRegistry] = None
|
267
|
-
networks: List[str] = Field(
|
268
|
-
default_factory=list,
|
269
|
-
description=(
|
270
|
-
"A list of strings specifying Docker networks to connect the container to."
|
271
|
-
),
|
272
|
-
)
|
273
|
-
network_mode: Optional[str] = Field(
|
274
|
-
default=None,
|
275
|
-
description=(
|
276
|
-
"The network mode for the created container (e.g. host, bridge). If"
|
277
|
-
" 'networks' is set, this cannot be set."
|
278
|
-
),
|
279
|
-
)
|
280
|
-
auto_remove: bool = Field(
|
281
|
-
default=False,
|
282
|
-
description="If set, the container will be removed on completion.",
|
283
|
-
)
|
284
|
-
volumes: List[str] = Field(
|
285
|
-
default_factory=list,
|
286
|
-
description=(
|
287
|
-
"A list of volume mount strings in the format of"
|
288
|
-
' "local_path:container_path".'
|
289
|
-
),
|
290
|
-
)
|
291
|
-
stream_output: bool = Field(
|
292
|
-
default=True,
|
293
|
-
description=(
|
294
|
-
"If set, the output will be streamed from the container to local standard"
|
295
|
-
" output."
|
296
|
-
),
|
297
|
-
)
|
298
|
-
memswap_limit: Union[int, str] = Field(
|
299
|
-
default=None,
|
300
|
-
description=(
|
301
|
-
"Total memory (memory + swap), -1 to disable swap. Should only be "
|
302
|
-
"set if `mem_limit` is also set. If `mem_limit` is set, this defaults to"
|
303
|
-
"allowing the container to use as much swap as memory. For example, if "
|
304
|
-
"`mem_limit` is 300m and `memswap_limit` is not set, the container can use "
|
305
|
-
"600m in total of memory and swap."
|
306
|
-
),
|
307
|
-
)
|
308
|
-
mem_limit: Union[float, str] = Field(
|
309
|
-
default=None,
|
310
|
-
description=(
|
311
|
-
"Memory limit of the created container. Accepts float values to enforce "
|
312
|
-
"a limit in bytes or a string with a unit e.g. 100000b, 1000k, 128m, 1g. "
|
313
|
-
"If a string is given without a unit, bytes are assumed."
|
314
|
-
),
|
315
|
-
)
|
316
|
-
privileged: bool = Field(
|
317
|
-
default=False,
|
318
|
-
description="Give extended privileges to this container.",
|
319
|
-
)
|
320
|
-
|
321
|
-
_block_type_name = "Docker Container"
|
322
|
-
_logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/14a315b79990200db7341e42553e23650b34bb96-250x250.png"
|
323
|
-
_documentation_url = "https://docs.prefect.io/api-ref/prefect/infrastructure/#prefect.infrastructure.DockerContainer"
|
324
|
-
|
325
|
-
@validator("labels")
|
326
|
-
def convert_labels_to_docker_format(cls, labels: Dict[str, str]):
|
327
|
-
labels = labels or {}
|
328
|
-
new_labels = {}
|
329
|
-
for name, value in labels.items():
|
330
|
-
if "/" in name:
|
331
|
-
namespace, key = name.split("/", maxsplit=1)
|
332
|
-
new_namespace = ".".join(reversed(namespace.split(".")))
|
333
|
-
new_labels[f"{new_namespace}.{key}"] = value
|
334
|
-
else:
|
335
|
-
new_labels[name] = value
|
336
|
-
return new_labels
|
337
|
-
|
338
|
-
@validator("volumes")
|
339
|
-
def check_volume_format(cls, volumes):
|
340
|
-
for volume in volumes:
|
341
|
-
if ":" not in volume:
|
342
|
-
raise ValueError(
|
343
|
-
"Invalid volume specification. "
|
344
|
-
f"Expected format 'path:container_path', but got {volume!r}"
|
345
|
-
)
|
346
|
-
|
347
|
-
return volumes
|
348
|
-
|
349
|
-
@sync_compatible
|
350
|
-
async def run(
|
351
|
-
self,
|
352
|
-
task_status: Optional[anyio.abc.TaskStatus] = None,
|
353
|
-
) -> Optional[bool]:
|
354
|
-
if not self.command:
|
355
|
-
raise ValueError("Docker container cannot be run with empty command.")
|
356
|
-
|
357
|
-
# The `docker` library uses requests instead of an async http library so it must
|
358
|
-
# be run in a thread to avoid blocking the event loop.
|
359
|
-
container = await run_sync_in_worker_thread(self._create_and_start_container)
|
360
|
-
container_pid = self._get_infrastructure_pid(container_id=container.id)
|
361
|
-
|
362
|
-
# Mark as started and return the infrastructure id
|
363
|
-
if task_status:
|
364
|
-
task_status.started(container_pid)
|
365
|
-
|
366
|
-
# Monitor the container
|
367
|
-
container = await run_sync_in_worker_thread(
|
368
|
-
self._watch_container_safe, container
|
369
|
-
)
|
370
|
-
|
371
|
-
exit_code = container.attrs["State"].get("ExitCode")
|
372
|
-
return DockerContainerResult(
|
373
|
-
status_code=exit_code if exit_code is not None else -1,
|
374
|
-
identifier=container_pid,
|
375
|
-
)
|
376
|
-
|
377
|
-
async def kill(self, infrastructure_pid: str, grace_seconds: int = 30):
|
378
|
-
docker_client = self._get_client()
|
379
|
-
base_url, container_id = self._parse_infrastructure_pid(infrastructure_pid)
|
380
|
-
|
381
|
-
if docker_client.api.base_url != base_url:
|
382
|
-
raise InfrastructureNotAvailable(
|
383
|
-
"".join(
|
384
|
-
[
|
385
|
-
(
|
386
|
-
f"Unable to stop container {container_id!r}: the current"
|
387
|
-
" Docker API "
|
388
|
-
),
|
389
|
-
(
|
390
|
-
f"URL {docker_client.api.base_url!r} does not match the"
|
391
|
-
" expected "
|
392
|
-
),
|
393
|
-
f"API base URL {base_url}.",
|
394
|
-
]
|
395
|
-
)
|
396
|
-
)
|
397
|
-
try:
|
398
|
-
container = docker_client.containers.get(container_id=container_id)
|
399
|
-
except docker.errors.NotFound:
|
400
|
-
raise InfrastructureNotFound(
|
401
|
-
f"Unable to stop container {container_id!r}: The container was not"
|
402
|
-
" found."
|
403
|
-
)
|
404
|
-
|
405
|
-
try:
|
406
|
-
container.stop(timeout=grace_seconds)
|
407
|
-
except Exception:
|
408
|
-
raise
|
409
|
-
|
410
|
-
def preview(self):
|
411
|
-
# TODO: build and document a more sophisticated preview
|
412
|
-
docker_client = self._get_client()
|
413
|
-
try:
|
414
|
-
return json.dumps(self._build_container_settings(docker_client))
|
415
|
-
finally:
|
416
|
-
docker_client.close()
|
417
|
-
|
418
|
-
async def generate_work_pool_base_job_template(self):
|
419
|
-
from prefect.workers.utilities import (
|
420
|
-
get_default_base_job_template_for_infrastructure_type,
|
421
|
-
)
|
422
|
-
|
423
|
-
base_job_template = await get_default_base_job_template_for_infrastructure_type(
|
424
|
-
self.get_corresponding_worker_type()
|
425
|
-
)
|
426
|
-
if base_job_template is None:
|
427
|
-
return await super().generate_work_pool_base_job_template()
|
428
|
-
for key, value in self.dict(exclude_unset=True, exclude_defaults=True).items():
|
429
|
-
if key == "command":
|
430
|
-
base_job_template["variables"]["properties"]["command"][
|
431
|
-
"default"
|
432
|
-
] = shlex.join(value)
|
433
|
-
elif key == "image_registry":
|
434
|
-
self.logger.warning(
|
435
|
-
"Image registry blocks are not supported by Docker"
|
436
|
-
" work pools. Please authenticate to your registry using"
|
437
|
-
" the `docker login` command on your worker instances."
|
438
|
-
)
|
439
|
-
elif key in [
|
440
|
-
"type",
|
441
|
-
"block_type_slug",
|
442
|
-
"_block_document_id",
|
443
|
-
"_block_document_name",
|
444
|
-
"_is_anonymous",
|
445
|
-
]:
|
446
|
-
continue
|
447
|
-
elif key == "image_pull_policy":
|
448
|
-
new_value = None
|
449
|
-
if value == ImagePullPolicy.ALWAYS:
|
450
|
-
new_value = "Always"
|
451
|
-
elif value == ImagePullPolicy.NEVER:
|
452
|
-
new_value = "Never"
|
453
|
-
elif value == ImagePullPolicy.IF_NOT_PRESENT:
|
454
|
-
new_value = "IfNotPresent"
|
455
|
-
|
456
|
-
base_job_template["variables"]["properties"][key]["default"] = new_value
|
457
|
-
elif key in base_job_template["variables"]["properties"]:
|
458
|
-
base_job_template["variables"]["properties"][key]["default"] = value
|
459
|
-
else:
|
460
|
-
self.logger.warning(
|
461
|
-
f"Variable {key!r} is not supported by Docker work pools. Skipping."
|
462
|
-
)
|
463
|
-
|
464
|
-
return base_job_template
|
465
|
-
|
466
|
-
def get_corresponding_worker_type(self):
|
467
|
-
return "docker"
|
468
|
-
|
469
|
-
def _get_infrastructure_pid(self, container_id: str) -> str:
|
470
|
-
"""Generates a Docker infrastructure_pid string in the form of
|
471
|
-
`<docker_host_base_url>:<container_id>`.
|
472
|
-
"""
|
473
|
-
docker_client = self._get_client()
|
474
|
-
base_url = docker_client.api.base_url
|
475
|
-
docker_client.close()
|
476
|
-
return f"{base_url}:{container_id}"
|
477
|
-
|
478
|
-
def _parse_infrastructure_pid(self, infrastructure_pid: str) -> Tuple[str, str]:
|
479
|
-
"""Splits a Docker infrastructure_pid into its component parts"""
|
480
|
-
|
481
|
-
# base_url can contain `:` so we only want the last item of the split
|
482
|
-
base_url, container_id = infrastructure_pid.rsplit(":", 1)
|
483
|
-
return base_url, str(container_id)
|
484
|
-
|
485
|
-
def _build_container_settings(
|
486
|
-
self,
|
487
|
-
docker_client: "DockerClient",
|
488
|
-
) -> Dict:
|
489
|
-
network_mode = self._get_network_mode()
|
490
|
-
return dict(
|
491
|
-
image=self.image,
|
492
|
-
network=self.networks[0] if self.networks else None,
|
493
|
-
network_mode=network_mode,
|
494
|
-
command=self.command,
|
495
|
-
environment=self._get_environment_variables(network_mode),
|
496
|
-
auto_remove=self.auto_remove,
|
497
|
-
labels={**CONTAINER_LABELS, **self.labels},
|
498
|
-
extra_hosts=self._get_extra_hosts(docker_client),
|
499
|
-
name=self._get_container_name(),
|
500
|
-
volumes=self.volumes,
|
501
|
-
mem_limit=self.mem_limit,
|
502
|
-
memswap_limit=self.memswap_limit,
|
503
|
-
privileged=self.privileged,
|
504
|
-
)
|
505
|
-
|
506
|
-
def _create_and_start_container(self) -> "Container":
|
507
|
-
if self.image_registry:
|
508
|
-
# If an image registry block was supplied, load an authenticated Docker
|
509
|
-
# client from the block. Otherwise, use an unauthenticated client to
|
510
|
-
# pull images from public registries.
|
511
|
-
docker_client = self.image_registry.get_docker_client()
|
512
|
-
else:
|
513
|
-
docker_client = self._get_client()
|
514
|
-
container_settings = self._build_container_settings(docker_client)
|
515
|
-
|
516
|
-
if self._should_pull_image(docker_client):
|
517
|
-
self.logger.info(f"Pulling image {self.image!r}...")
|
518
|
-
self._pull_image(docker_client)
|
519
|
-
|
520
|
-
container = self._create_container(docker_client, **container_settings)
|
521
|
-
|
522
|
-
# Add additional networks after the container is created; only one network can
|
523
|
-
# be attached at creation time
|
524
|
-
if len(self.networks) > 1:
|
525
|
-
for network_name in self.networks[1:]:
|
526
|
-
network = docker_client.networks.get(network_name)
|
527
|
-
network.connect(container)
|
528
|
-
|
529
|
-
# Start the container
|
530
|
-
container.start()
|
531
|
-
|
532
|
-
docker_client.close()
|
533
|
-
|
534
|
-
return container
|
535
|
-
|
536
|
-
def _get_image_and_tag(self) -> Tuple[str, Optional[str]]:
|
537
|
-
return parse_image_tag(self.image)
|
538
|
-
|
539
|
-
def _determine_image_pull_policy(self) -> ImagePullPolicy:
|
540
|
-
"""
|
541
|
-
Determine the appropriate image pull policy.
|
542
|
-
|
543
|
-
1. If they specified an image pull policy, use that.
|
544
|
-
|
545
|
-
2. If they did not specify an image pull policy and gave us
|
546
|
-
the "latest" tag, use ImagePullPolicy.always.
|
547
|
-
|
548
|
-
3. If they did not specify an image pull policy and did not
|
549
|
-
specify a tag, use ImagePullPolicy.always.
|
550
|
-
|
551
|
-
4. If they did not specify an image pull policy and gave us
|
552
|
-
a tag other than "latest", use ImagePullPolicy.if_not_present.
|
553
|
-
|
554
|
-
This logic matches the behavior of Kubernetes.
|
555
|
-
See:https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting
|
556
|
-
"""
|
557
|
-
if not self.image_pull_policy:
|
558
|
-
_, tag = self._get_image_and_tag()
|
559
|
-
if tag == "latest" or not tag:
|
560
|
-
return ImagePullPolicy.ALWAYS
|
561
|
-
return ImagePullPolicy.IF_NOT_PRESENT
|
562
|
-
return self.image_pull_policy
|
563
|
-
|
564
|
-
def _get_network_mode(self) -> Optional[str]:
|
565
|
-
# User's value takes precedence; this may collide with the incompatible options
|
566
|
-
# mentioned below.
|
567
|
-
if self.network_mode:
|
568
|
-
if sys.platform != "linux" and self.network_mode == "host":
|
569
|
-
warnings.warn(
|
570
|
-
f"{self.network_mode!r} network mode is not supported on platform "
|
571
|
-
f"{sys.platform!r} and may not work as intended."
|
572
|
-
)
|
573
|
-
return self.network_mode
|
574
|
-
|
575
|
-
# Network mode is not compatible with networks or ports (we do not support ports
|
576
|
-
# yet though)
|
577
|
-
if self.networks:
|
578
|
-
return None
|
579
|
-
|
580
|
-
# Check for a local API connection
|
581
|
-
api_url = self.env.get("PREFECT_API_URL", PREFECT_API_URL.value())
|
582
|
-
|
583
|
-
if api_url:
|
584
|
-
try:
|
585
|
-
_, netloc, _, _, _, _ = urllib.parse.urlparse(api_url)
|
586
|
-
except Exception as exc:
|
587
|
-
warnings.warn(
|
588
|
-
f"Failed to parse host from API URL {api_url!r} with exception: "
|
589
|
-
f"{exc}\nThe network mode will not be inferred."
|
590
|
-
)
|
591
|
-
return None
|
592
|
-
|
593
|
-
host = netloc.split(":")[0]
|
594
|
-
|
595
|
-
# If using a locally hosted API, use a host network on linux
|
596
|
-
if sys.platform == "linux" and (host == "127.0.0.1" or host == "localhost"):
|
597
|
-
return "host"
|
598
|
-
|
599
|
-
# Default to unset
|
600
|
-
return None
|
601
|
-
|
602
|
-
def _should_pull_image(self, docker_client: "DockerClient") -> bool:
|
603
|
-
"""
|
604
|
-
Decide whether we need to pull the Docker image.
|
605
|
-
"""
|
606
|
-
image_pull_policy = self._determine_image_pull_policy()
|
607
|
-
|
608
|
-
if image_pull_policy is ImagePullPolicy.ALWAYS:
|
609
|
-
return True
|
610
|
-
elif image_pull_policy is ImagePullPolicy.NEVER:
|
611
|
-
return False
|
612
|
-
elif image_pull_policy is ImagePullPolicy.IF_NOT_PRESENT:
|
613
|
-
try:
|
614
|
-
# NOTE: images.get() wants the tag included with the image
|
615
|
-
# name, while images.pull() wants them split.
|
616
|
-
docker_client.images.get(self.image)
|
617
|
-
except docker.errors.ImageNotFound:
|
618
|
-
self.logger.debug(f"Could not find Docker image locally: {self.image}")
|
619
|
-
return True
|
620
|
-
return False
|
621
|
-
|
622
|
-
def _pull_image(self, docker_client: "DockerClient"):
|
623
|
-
"""
|
624
|
-
Pull the image we're going to use to create the container.
|
625
|
-
"""
|
626
|
-
image, tag = self._get_image_and_tag()
|
627
|
-
|
628
|
-
return docker_client.images.pull(image, tag)
|
629
|
-
|
630
|
-
def _create_container(self, docker_client: "DockerClient", **kwargs) -> "Container":
|
631
|
-
"""
|
632
|
-
Create a docker container with retries on name conflicts.
|
633
|
-
|
634
|
-
If the container already exists with the given name, an incremented index is
|
635
|
-
added.
|
636
|
-
"""
|
637
|
-
# Create the container with retries on name conflicts (with an incremented idx)
|
638
|
-
index = 0
|
639
|
-
container = None
|
640
|
-
name = original_name = kwargs.pop("name")
|
641
|
-
|
642
|
-
while not container:
|
643
|
-
from docker.errors import APIError
|
644
|
-
|
645
|
-
try:
|
646
|
-
display_name = repr(name) if name else "with auto-generated name"
|
647
|
-
self.logger.info(f"Creating Docker container {display_name}...")
|
648
|
-
container = docker_client.containers.create(name=name, **kwargs)
|
649
|
-
except APIError as exc:
|
650
|
-
if "Conflict" in str(exc) and "container name" in str(exc):
|
651
|
-
self.logger.info(
|
652
|
-
f"Docker container name {display_name} already exists; "
|
653
|
-
"retrying..."
|
654
|
-
)
|
655
|
-
index += 1
|
656
|
-
name = f"{original_name}-{index}"
|
657
|
-
else:
|
658
|
-
raise
|
659
|
-
|
660
|
-
self.logger.info(
|
661
|
-
f"Docker container {container.name!r} has status {container.status!r}"
|
662
|
-
)
|
663
|
-
return container
|
664
|
-
|
665
|
-
def _watch_container_safe(self, container: "Container") -> "Container":
|
666
|
-
# Monitor the container capturing the latest snapshot while capturing
|
667
|
-
# not found errors
|
668
|
-
docker_client = self._get_client()
|
669
|
-
|
670
|
-
try:
|
671
|
-
for latest_container in self._watch_container(docker_client, container.id):
|
672
|
-
container = latest_container
|
673
|
-
except docker.errors.NotFound:
|
674
|
-
# The container was removed during watching
|
675
|
-
self.logger.warning(
|
676
|
-
f"Docker container {container.name} was removed before we could wait "
|
677
|
-
"for its completion."
|
678
|
-
)
|
679
|
-
finally:
|
680
|
-
docker_client.close()
|
681
|
-
|
682
|
-
return container
|
683
|
-
|
684
|
-
def _watch_container(
|
685
|
-
self, docker_client: "DockerClient", container_id: str
|
686
|
-
) -> Generator[None, None, "Container"]:
|
687
|
-
container: "Container" = docker_client.containers.get(container_id)
|
688
|
-
|
689
|
-
status = container.status
|
690
|
-
self.logger.info(
|
691
|
-
f"Docker container {container.name!r} has status {container.status!r}"
|
692
|
-
)
|
693
|
-
yield container
|
694
|
-
|
695
|
-
if self.stream_output:
|
696
|
-
try:
|
697
|
-
for log in container.logs(stream=True):
|
698
|
-
log: bytes
|
699
|
-
print(log.decode().rstrip())
|
700
|
-
except docker.errors.APIError as exc:
|
701
|
-
if "marked for removal" in str(exc):
|
702
|
-
self.logger.warning(
|
703
|
-
f"Docker container {container.name} was marked for removal"
|
704
|
-
" before logs could be retrieved. Output will not be"
|
705
|
-
" streamed. "
|
706
|
-
)
|
707
|
-
else:
|
708
|
-
self.logger.exception(
|
709
|
-
"An unexpected Docker API error occurred while streaming"
|
710
|
-
f" output from container {container.name}."
|
711
|
-
)
|
712
|
-
|
713
|
-
container.reload()
|
714
|
-
if container.status != status:
|
715
|
-
self.logger.info(
|
716
|
-
f"Docker container {container.name!r} has status"
|
717
|
-
f" {container.status!r}"
|
718
|
-
)
|
719
|
-
yield container
|
720
|
-
|
721
|
-
container.wait()
|
722
|
-
self.logger.info(
|
723
|
-
f"Docker container {container.name!r} has status {container.status!r}"
|
724
|
-
)
|
725
|
-
yield container
|
726
|
-
|
727
|
-
def _get_client(self):
|
728
|
-
try:
|
729
|
-
with warnings.catch_warnings():
|
730
|
-
# Silence warnings due to use of deprecated methods within dockerpy
|
731
|
-
# See https://github.com/docker/docker-py/pull/2931
|
732
|
-
warnings.filterwarnings(
|
733
|
-
"ignore",
|
734
|
-
message="distutils Version classes are deprecated.*",
|
735
|
-
category=DeprecationWarning,
|
736
|
-
)
|
737
|
-
|
738
|
-
docker_client = docker.from_env()
|
739
|
-
|
740
|
-
except docker.errors.DockerException as exc:
|
741
|
-
raise RuntimeError("Could not connect to Docker.") from exc
|
742
|
-
|
743
|
-
return docker_client
|
744
|
-
|
745
|
-
def _get_container_name(self) -> Optional[str]:
|
746
|
-
"""
|
747
|
-
Generates a container name to match the configured name, ensuring it is Docker
|
748
|
-
compatible.
|
749
|
-
"""
|
750
|
-
# Must match `/?[a-zA-Z0-9][a-zA-Z0-9_.-]+` in the end
|
751
|
-
if not self.name:
|
752
|
-
return None
|
753
|
-
|
754
|
-
return (
|
755
|
-
slugify(
|
756
|
-
self.name,
|
757
|
-
lowercase=False,
|
758
|
-
# Docker does not limit length but URL limits apply eventually so
|
759
|
-
# limit the length for safety
|
760
|
-
max_length=250,
|
761
|
-
# Docker allows these characters for container names
|
762
|
-
regex_pattern=r"[^a-zA-Z0-9_.-]+",
|
763
|
-
).lstrip(
|
764
|
-
# Docker does not allow leading underscore, dash, or period
|
765
|
-
"_-."
|
766
|
-
)
|
767
|
-
# Docker does not allow 0 character names so cast to null if the name is
|
768
|
-
# empty after slufification
|
769
|
-
or None
|
770
|
-
)
|
771
|
-
|
772
|
-
def _get_extra_hosts(self, docker_client) -> Dict[str, str]:
|
773
|
-
"""
|
774
|
-
A host.docker.internal -> host-gateway mapping is necessary for communicating
|
775
|
-
with the API on Linux machines. Docker Desktop on macOS will automatically
|
776
|
-
already have this mapping.
|
777
|
-
"""
|
778
|
-
if sys.platform == "linux" and (
|
779
|
-
# Do not warn if the user has specified a host manually that does not use
|
780
|
-
# a local address
|
781
|
-
"PREFECT_API_URL" not in self.env
|
782
|
-
or re.search(
|
783
|
-
".*(localhost)|(127.0.0.1)|(host.docker.internal).*",
|
784
|
-
self.env["PREFECT_API_URL"],
|
785
|
-
)
|
786
|
-
):
|
787
|
-
user_version = packaging.version.parse(
|
788
|
-
format_outlier_version_name(docker_client.version()["Version"])
|
789
|
-
)
|
790
|
-
required_version = packaging.version.parse("20.10.0")
|
791
|
-
|
792
|
-
if user_version < required_version:
|
793
|
-
warnings.warn(
|
794
|
-
"`host.docker.internal` could not be automatically resolved to"
|
795
|
-
" your local ip address. This feature is not supported on Docker"
|
796
|
-
f" Engine v{user_version}, upgrade to v{required_version}+ if you"
|
797
|
-
" encounter issues."
|
798
|
-
)
|
799
|
-
return {}
|
800
|
-
else:
|
801
|
-
# Compatibility for linux -- https://github.com/docker/cli/issues/2290
|
802
|
-
# Only supported by Docker v20.10.0+ which is our minimum recommend version
|
803
|
-
return {"host.docker.internal": "host-gateway"}
|
804
|
-
|
805
|
-
def _get_environment_variables(self, network_mode):
|
806
|
-
# If the API URL has been set by the base environment rather than the by the
|
807
|
-
# user, update the value to ensure connectivity when using a bridge network by
|
808
|
-
# updating local connections to use the docker internal host unless the
|
809
|
-
# network mode is "host" where localhost is available already.
|
810
|
-
env = {**self._base_environment(), **self.env}
|
811
|
-
|
812
|
-
if (
|
813
|
-
"PREFECT_API_URL" in env
|
814
|
-
and "PREFECT_API_URL" not in self.env
|
815
|
-
and network_mode != "host"
|
816
|
-
):
|
817
|
-
env["PREFECT_API_URL"] = (
|
818
|
-
env["PREFECT_API_URL"]
|
819
|
-
.replace("localhost", "host.docker.internal")
|
820
|
-
.replace("127.0.0.1", "host.docker.internal")
|
821
|
-
)
|
822
|
-
|
823
|
-
# Drop null values allowing users to "unset" variables
|
824
|
-
return {key: value for key, value in env.items() if value is not None}
|