prefect-client 2.19.2__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. prefect/__init__.py +8 -56
  2. prefect/_internal/compatibility/deprecated.py +6 -115
  3. prefect/_internal/compatibility/experimental.py +4 -79
  4. prefect/_internal/concurrency/api.py +0 -34
  5. prefect/_internal/concurrency/calls.py +0 -6
  6. prefect/_internal/concurrency/cancellation.py +0 -3
  7. prefect/_internal/concurrency/event_loop.py +0 -20
  8. prefect/_internal/concurrency/inspection.py +3 -3
  9. prefect/_internal/concurrency/threads.py +35 -0
  10. prefect/_internal/concurrency/waiters.py +0 -28
  11. prefect/_internal/pydantic/__init__.py +0 -45
  12. prefect/_internal/pydantic/v1_schema.py +21 -22
  13. prefect/_internal/pydantic/v2_schema.py +0 -2
  14. prefect/_internal/pydantic/v2_validated_func.py +18 -23
  15. prefect/_internal/schemas/bases.py +44 -177
  16. prefect/_internal/schemas/fields.py +1 -43
  17. prefect/_internal/schemas/validators.py +60 -158
  18. prefect/artifacts.py +161 -14
  19. prefect/automations.py +39 -4
  20. prefect/blocks/abstract.py +1 -1
  21. prefect/blocks/core.py +268 -148
  22. prefect/blocks/fields.py +2 -57
  23. prefect/blocks/kubernetes.py +8 -12
  24. prefect/blocks/notifications.py +40 -20
  25. prefect/blocks/system.py +22 -11
  26. prefect/blocks/webhook.py +2 -9
  27. prefect/client/base.py +4 -4
  28. prefect/client/cloud.py +8 -13
  29. prefect/client/orchestration.py +347 -341
  30. prefect/client/schemas/actions.py +92 -86
  31. prefect/client/schemas/filters.py +20 -40
  32. prefect/client/schemas/objects.py +151 -145
  33. prefect/client/schemas/responses.py +16 -24
  34. prefect/client/schemas/schedules.py +47 -35
  35. prefect/client/subscriptions.py +2 -2
  36. prefect/client/utilities.py +5 -2
  37. prefect/concurrency/asyncio.py +3 -1
  38. prefect/concurrency/events.py +1 -1
  39. prefect/concurrency/services.py +6 -3
  40. prefect/context.py +195 -27
  41. prefect/deployments/__init__.py +5 -6
  42. prefect/deployments/base.py +7 -5
  43. prefect/deployments/flow_runs.py +185 -0
  44. prefect/deployments/runner.py +50 -45
  45. prefect/deployments/schedules.py +28 -23
  46. prefect/deployments/steps/__init__.py +0 -1
  47. prefect/deployments/steps/core.py +1 -0
  48. prefect/deployments/steps/pull.py +7 -21
  49. prefect/engine.py +12 -2422
  50. prefect/events/actions.py +17 -23
  51. prefect/events/cli/automations.py +19 -6
  52. prefect/events/clients.py +14 -37
  53. prefect/events/filters.py +14 -18
  54. prefect/events/related.py +2 -2
  55. prefect/events/schemas/__init__.py +0 -5
  56. prefect/events/schemas/automations.py +55 -46
  57. prefect/events/schemas/deployment_triggers.py +7 -197
  58. prefect/events/schemas/events.py +34 -65
  59. prefect/events/schemas/labelling.py +10 -14
  60. prefect/events/utilities.py +2 -3
  61. prefect/events/worker.py +2 -3
  62. prefect/filesystems.py +6 -517
  63. prefect/{new_flow_engine.py → flow_engine.py} +313 -72
  64. prefect/flow_runs.py +377 -5
  65. prefect/flows.py +307 -166
  66. prefect/futures.py +186 -345
  67. prefect/infrastructure/__init__.py +0 -27
  68. prefect/infrastructure/provisioners/__init__.py +5 -3
  69. prefect/infrastructure/provisioners/cloud_run.py +11 -6
  70. prefect/infrastructure/provisioners/container_instance.py +11 -7
  71. prefect/infrastructure/provisioners/ecs.py +6 -4
  72. prefect/infrastructure/provisioners/modal.py +8 -5
  73. prefect/input/actions.py +2 -4
  74. prefect/input/run_input.py +5 -7
  75. prefect/logging/formatters.py +0 -2
  76. prefect/logging/handlers.py +3 -11
  77. prefect/logging/loggers.py +2 -2
  78. prefect/manifests.py +2 -1
  79. prefect/records/__init__.py +1 -0
  80. prefect/records/result_store.py +42 -0
  81. prefect/records/store.py +9 -0
  82. prefect/results.py +43 -39
  83. prefect/runner/runner.py +19 -15
  84. prefect/runner/server.py +6 -10
  85. prefect/runner/storage.py +3 -8
  86. prefect/runner/submit.py +2 -2
  87. prefect/runner/utils.py +2 -2
  88. prefect/serializers.py +24 -35
  89. prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
  90. prefect/settings.py +70 -133
  91. prefect/states.py +17 -47
  92. prefect/task_engine.py +697 -58
  93. prefect/task_runners.py +269 -301
  94. prefect/task_server.py +53 -34
  95. prefect/tasks.py +327 -337
  96. prefect/transactions.py +220 -0
  97. prefect/types/__init__.py +61 -82
  98. prefect/utilities/asyncutils.py +195 -136
  99. prefect/utilities/callables.py +311 -43
  100. prefect/utilities/collections.py +23 -38
  101. prefect/utilities/dispatch.py +11 -3
  102. prefect/utilities/dockerutils.py +4 -0
  103. prefect/utilities/engine.py +140 -20
  104. prefect/utilities/importtools.py +97 -27
  105. prefect/utilities/pydantic.py +128 -38
  106. prefect/utilities/schema_tools/hydration.py +5 -1
  107. prefect/utilities/templating.py +12 -2
  108. prefect/variables.py +78 -61
  109. prefect/workers/__init__.py +0 -1
  110. prefect/workers/base.py +15 -17
  111. prefect/workers/process.py +3 -8
  112. prefect/workers/server.py +2 -2
  113. {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/METADATA +22 -21
  114. prefect_client-3.0.0rc1.dist-info/RECORD +176 -0
  115. prefect/_internal/pydantic/_base_model.py +0 -51
  116. prefect/_internal/pydantic/_compat.py +0 -82
  117. prefect/_internal/pydantic/_flags.py +0 -20
  118. prefect/_internal/pydantic/_types.py +0 -8
  119. prefect/_internal/pydantic/utilities/__init__.py +0 -0
  120. prefect/_internal/pydantic/utilities/config_dict.py +0 -72
  121. prefect/_internal/pydantic/utilities/field_validator.py +0 -150
  122. prefect/_internal/pydantic/utilities/model_construct.py +0 -56
  123. prefect/_internal/pydantic/utilities/model_copy.py +0 -55
  124. prefect/_internal/pydantic/utilities/model_dump.py +0 -136
  125. prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
  126. prefect/_internal/pydantic/utilities/model_fields.py +0 -50
  127. prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
  128. prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
  129. prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
  130. prefect/_internal/pydantic/utilities/model_validate.py +0 -75
  131. prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
  132. prefect/_internal/pydantic/utilities/model_validator.py +0 -87
  133. prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
  134. prefect/_vendor/__init__.py +0 -0
  135. prefect/_vendor/fastapi/__init__.py +0 -25
  136. prefect/_vendor/fastapi/applications.py +0 -946
  137. prefect/_vendor/fastapi/background.py +0 -3
  138. prefect/_vendor/fastapi/concurrency.py +0 -44
  139. prefect/_vendor/fastapi/datastructures.py +0 -58
  140. prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
  141. prefect/_vendor/fastapi/dependencies/models.py +0 -64
  142. prefect/_vendor/fastapi/dependencies/utils.py +0 -877
  143. prefect/_vendor/fastapi/encoders.py +0 -177
  144. prefect/_vendor/fastapi/exception_handlers.py +0 -40
  145. prefect/_vendor/fastapi/exceptions.py +0 -46
  146. prefect/_vendor/fastapi/logger.py +0 -3
  147. prefect/_vendor/fastapi/middleware/__init__.py +0 -1
  148. prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
  149. prefect/_vendor/fastapi/middleware/cors.py +0 -3
  150. prefect/_vendor/fastapi/middleware/gzip.py +0 -3
  151. prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
  152. prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
  153. prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
  154. prefect/_vendor/fastapi/openapi/__init__.py +0 -0
  155. prefect/_vendor/fastapi/openapi/constants.py +0 -2
  156. prefect/_vendor/fastapi/openapi/docs.py +0 -203
  157. prefect/_vendor/fastapi/openapi/models.py +0 -480
  158. prefect/_vendor/fastapi/openapi/utils.py +0 -485
  159. prefect/_vendor/fastapi/param_functions.py +0 -340
  160. prefect/_vendor/fastapi/params.py +0 -453
  161. prefect/_vendor/fastapi/requests.py +0 -4
  162. prefect/_vendor/fastapi/responses.py +0 -40
  163. prefect/_vendor/fastapi/routing.py +0 -1331
  164. prefect/_vendor/fastapi/security/__init__.py +0 -15
  165. prefect/_vendor/fastapi/security/api_key.py +0 -98
  166. prefect/_vendor/fastapi/security/base.py +0 -6
  167. prefect/_vendor/fastapi/security/http.py +0 -172
  168. prefect/_vendor/fastapi/security/oauth2.py +0 -227
  169. prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
  170. prefect/_vendor/fastapi/security/utils.py +0 -10
  171. prefect/_vendor/fastapi/staticfiles.py +0 -1
  172. prefect/_vendor/fastapi/templating.py +0 -3
  173. prefect/_vendor/fastapi/testclient.py +0 -1
  174. prefect/_vendor/fastapi/types.py +0 -3
  175. prefect/_vendor/fastapi/utils.py +0 -235
  176. prefect/_vendor/fastapi/websockets.py +0 -7
  177. prefect/_vendor/starlette/__init__.py +0 -1
  178. prefect/_vendor/starlette/_compat.py +0 -28
  179. prefect/_vendor/starlette/_exception_handler.py +0 -80
  180. prefect/_vendor/starlette/_utils.py +0 -88
  181. prefect/_vendor/starlette/applications.py +0 -261
  182. prefect/_vendor/starlette/authentication.py +0 -159
  183. prefect/_vendor/starlette/background.py +0 -43
  184. prefect/_vendor/starlette/concurrency.py +0 -59
  185. prefect/_vendor/starlette/config.py +0 -151
  186. prefect/_vendor/starlette/convertors.py +0 -87
  187. prefect/_vendor/starlette/datastructures.py +0 -707
  188. prefect/_vendor/starlette/endpoints.py +0 -130
  189. prefect/_vendor/starlette/exceptions.py +0 -60
  190. prefect/_vendor/starlette/formparsers.py +0 -276
  191. prefect/_vendor/starlette/middleware/__init__.py +0 -17
  192. prefect/_vendor/starlette/middleware/authentication.py +0 -52
  193. prefect/_vendor/starlette/middleware/base.py +0 -220
  194. prefect/_vendor/starlette/middleware/cors.py +0 -176
  195. prefect/_vendor/starlette/middleware/errors.py +0 -265
  196. prefect/_vendor/starlette/middleware/exceptions.py +0 -74
  197. prefect/_vendor/starlette/middleware/gzip.py +0 -113
  198. prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
  199. prefect/_vendor/starlette/middleware/sessions.py +0 -82
  200. prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
  201. prefect/_vendor/starlette/middleware/wsgi.py +0 -147
  202. prefect/_vendor/starlette/requests.py +0 -328
  203. prefect/_vendor/starlette/responses.py +0 -347
  204. prefect/_vendor/starlette/routing.py +0 -933
  205. prefect/_vendor/starlette/schemas.py +0 -154
  206. prefect/_vendor/starlette/staticfiles.py +0 -248
  207. prefect/_vendor/starlette/status.py +0 -199
  208. prefect/_vendor/starlette/templating.py +0 -231
  209. prefect/_vendor/starlette/testclient.py +0 -804
  210. prefect/_vendor/starlette/types.py +0 -30
  211. prefect/_vendor/starlette/websockets.py +0 -193
  212. prefect/agent.py +0 -698
  213. prefect/deployments/deployments.py +0 -1042
  214. prefect/deprecated/__init__.py +0 -0
  215. prefect/deprecated/data_documents.py +0 -350
  216. prefect/deprecated/packaging/__init__.py +0 -12
  217. prefect/deprecated/packaging/base.py +0 -96
  218. prefect/deprecated/packaging/docker.py +0 -146
  219. prefect/deprecated/packaging/file.py +0 -92
  220. prefect/deprecated/packaging/orion.py +0 -80
  221. prefect/deprecated/packaging/serializers.py +0 -171
  222. prefect/events/instrument.py +0 -135
  223. prefect/infrastructure/base.py +0 -323
  224. prefect/infrastructure/container.py +0 -818
  225. prefect/infrastructure/kubernetes.py +0 -920
  226. prefect/infrastructure/process.py +0 -289
  227. prefect/new_task_engine.py +0 -423
  228. prefect/pydantic/__init__.py +0 -76
  229. prefect/pydantic/main.py +0 -39
  230. prefect/software/__init__.py +0 -2
  231. prefect/software/base.py +0 -50
  232. prefect/software/conda.py +0 -199
  233. prefect/software/pip.py +0 -122
  234. prefect/software/python.py +0 -52
  235. prefect/workers/block.py +0 -218
  236. prefect_client-2.19.2.dist-info/RECORD +0 -292
  237. {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/LICENSE +0 -0
  238. {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/WHEEL +0 -0
  239. {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/top_level.txt +0 -0
@@ -1,818 +0,0 @@
1
- """
2
- DEPRECATION WARNING:
3
-
4
- This module is deprecated as of March 2024 and will not be available after September 2024.
5
- It has been replaced by the Docker worker from the prefect-docker package, which offers enhanced functionality and better performance.
6
-
7
- For upgrade instructions, see https://docs.prefect.io/latest/guides/upgrade-guide-agents-to-workers/.
8
- """
9
- import json
10
- import re
11
- import shlex
12
- import sys
13
- import urllib.parse
14
- import warnings
15
- from abc import ABC, abstractmethod
16
- from typing import TYPE_CHECKING, Dict, Generator, List, Optional, Tuple, Union
17
-
18
- import anyio.abc
19
- import packaging.version
20
-
21
- from prefect._internal.compatibility.deprecated import deprecated_class
22
- from prefect._internal.pydantic import HAS_PYDANTIC_V2
23
-
24
- if HAS_PYDANTIC_V2:
25
- from pydantic.v1 import Field, validator
26
- else:
27
- from pydantic import Field, validator
28
-
29
- from typing_extensions import Literal
30
-
31
- import prefect
32
- from prefect.blocks.core import Block, SecretStr
33
- from prefect.exceptions import InfrastructureNotAvailable, InfrastructureNotFound
34
- from prefect.infrastructure.base import Infrastructure, InfrastructureResult
35
- from prefect.settings import PREFECT_API_URL
36
- from prefect.utilities.asyncutils import run_sync_in_worker_thread, sync_compatible
37
- from prefect.utilities.collections import AutoEnum
38
- from prefect.utilities.dockerutils import (
39
- format_outlier_version_name,
40
- get_prefect_image_name,
41
- parse_image_tag,
42
- )
43
- from prefect.utilities.importtools import lazy_import
44
- from prefect.utilities.slugify import slugify
45
-
46
- if TYPE_CHECKING:
47
- import docker
48
- from docker import DockerClient
49
- from docker.models.containers import Container
50
- else:
51
- docker = lazy_import("docker")
52
-
53
-
54
- # Labels to apply to all containers started by Prefect
55
- CONTAINER_LABELS = {
56
- "io.prefect.version": prefect.__version__,
57
- }
58
-
59
-
60
- class ImagePullPolicy(AutoEnum):
61
- IF_NOT_PRESENT = AutoEnum.auto()
62
- ALWAYS = AutoEnum.auto()
63
- NEVER = AutoEnum.auto()
64
-
65
-
66
- @deprecated_class(
67
- start_date="Mar 2024",
68
- help="Use the `DockerRegistryCredentials` class from prefect-docker instead.",
69
- )
70
- class BaseDockerLogin(Block, ABC):
71
- _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/14a315b79990200db7341e42553e23650b34bb96-250x250.png"
72
- _block_schema_capabilities = ["docker-login"]
73
-
74
- @abstractmethod
75
- async def login(self) -> "DockerClient":
76
- """
77
- Log in and return an authenticated `DockerClient`.
78
- (DEPRECATED) Use `get_docker_client` instead of `login`.
79
- """
80
-
81
- @abstractmethod
82
- async def get_docker_client(self) -> "DockerClient":
83
- """
84
- Log in and return an authenticated `DockerClient`.
85
- """
86
-
87
- def _login(self, username, password, registry_url, reauth) -> "DockerClient":
88
- client = self._get_docker_client()
89
-
90
- client.login(
91
- username=username,
92
- password=password,
93
- registry=registry_url,
94
- # See https://github.com/docker/docker-py/issues/2256 for information on
95
- # the default value for reauth.
96
- reauth=reauth,
97
- )
98
-
99
- return client
100
-
101
- @staticmethod
102
- def _get_docker_client():
103
- try:
104
- with warnings.catch_warnings():
105
- # Silence warnings due to use of deprecated methods within dockerpy
106
- # See https://github.com/docker/docker-py/pull/2931
107
- warnings.filterwarnings(
108
- "ignore",
109
- message="distutils Version classes are deprecated.*",
110
- category=DeprecationWarning,
111
- )
112
-
113
- docker_client = docker.from_env()
114
-
115
- except docker.errors.DockerException as exc:
116
- raise RuntimeError("Could not connect to Docker.") from exc
117
-
118
- return docker_client
119
-
120
-
121
- @deprecated_class(
122
- start_date="Mar 2024",
123
- help="Use the `DockerRegistryCredentials` class from prefect-docker instead.",
124
- )
125
- class DockerRegistry(BaseDockerLogin):
126
- """
127
- Connects to a Docker registry.
128
-
129
- Requires a Docker Engine to be connectable.
130
-
131
- Attributes:
132
- username: The username to log into the registry with.
133
- password: The password to log into the registry with.
134
- registry_url: The URL to the registry such as registry.hub.docker.com. Generally, "http" or "https" can be
135
- omitted.
136
- reauth: If already logged into the registry, should login be performed again?
137
- This setting defaults to `True` to support common token authentication
138
- patterns such as ECR.
139
- """
140
-
141
- _block_type_name = "Docker Registry"
142
- _documentation_url = "https://docs.prefect.io/api-ref/prefect/infrastructure/#prefect.infrastructure.docker.DockerRegistry"
143
-
144
- username: str = Field(
145
- default=..., description="The username to log into the registry with."
146
- )
147
- password: SecretStr = Field(
148
- default=..., description="The password to log into the registry with."
149
- )
150
- registry_url: str = Field(
151
- default=...,
152
- description=(
153
- 'The URL to the registry. Generally, "http" or "https" can be omitted.'
154
- ),
155
- )
156
- reauth: bool = Field(
157
- default=True,
158
- description="Whether or not to reauthenticate on each interaction.",
159
- )
160
-
161
- @sync_compatible
162
- async def login(self) -> "DockerClient":
163
- warnings.warn(
164
- (
165
- "`login` is deprecated. Instead, use `get_docker_client` to obtain an"
166
- " authenticated `DockerClient`."
167
- ),
168
- category=DeprecationWarning,
169
- stacklevel=3,
170
- )
171
- return await self.get_docker_client()
172
-
173
- @sync_compatible
174
- async def get_docker_client(self) -> "DockerClient":
175
- client = await run_sync_in_worker_thread(
176
- self._login,
177
- self.username,
178
- self.password.get_secret_value(),
179
- self.registry_url,
180
- self.reauth,
181
- )
182
-
183
- return client
184
-
185
-
186
- class DockerContainerResult(InfrastructureResult):
187
- """Contains information about a completed Docker container"""
188
-
189
-
190
- @deprecated_class(
191
- start_date="Mar 2024",
192
- help="Use the Docker worker from prefect-docker instead."
193
- " Refer to the upgrade guide for more information:"
194
- " https://docs.prefect.io/latest/guides/upgrade-guide-agents-to-workers/.",
195
- )
196
- class DockerContainer(Infrastructure):
197
- """
198
- Runs a command in a container.
199
-
200
- Requires a Docker Engine to be connectable. Docker settings will be retrieved from
201
- the environment.
202
-
203
- Click [here](https://docs.prefect.io/guides/deployment/docker) to see a tutorial.
204
-
205
- Attributes:
206
- auto_remove: If set, the container will be removed on completion. Otherwise,
207
- the container will remain after exit for inspection.
208
- command: A list of strings specifying the command to run in the container to
209
- start the flow run. In most cases you should not override this.
210
- env: Environment variables to set for the container.
211
- image: An optional string specifying the tag of a Docker image to use.
212
- Defaults to the Prefect image.
213
- image_pull_policy: Specifies if the image should be pulled. One of 'ALWAYS',
214
- 'NEVER', 'IF_NOT_PRESENT'.
215
- image_registry: A `DockerRegistry` block containing credentials to use if `image` is stored in a private
216
- image registry.
217
- labels: An optional dictionary of labels, mapping name to value.
218
- name: An optional name for the container.
219
- network_mode: Set the network mode for the created container. Defaults to 'host'
220
- if a local API url is detected, otherwise the Docker default of 'bridge' is
221
- used. If 'networks' is set, this cannot be set.
222
- networks: An optional list of strings specifying Docker networks to connect the
223
- container to.
224
- stream_output: If set, stream output from the container to local standard output.
225
- volumes: An optional list of volume mount strings in the format of
226
- "local_path:container_path".
227
- memswap_limit: Total memory (memory + swap), -1 to disable swap. Should only be
228
- set if `mem_limit` is also set. If `mem_limit` is set, this defaults to
229
- allowing the container to use as much swap as memory. For example, if
230
- `mem_limit` is 300m and `memswap_limit` is not set, the container can use
231
- 600m in total of memory and swap.
232
- mem_limit: Memory limit of the created container. Accepts float values to enforce
233
- a limit in bytes or a string with a unit e.g. 100000b, 1000k, 128m, 1g.
234
- If a string is given without a unit, bytes are assumed.
235
- privileged: Give extended privileges to this container.
236
-
237
- ## Connecting to a locally hosted Prefect API
238
-
239
- If using a local API URL on Linux, we will update the network mode default to 'host'
240
- to enable connectivity. If using another OS or an alternative network mode is used,
241
- we will replace 'localhost' in the API URL with 'host.docker.internal'. Generally,
242
- this will enable connectivity, but the API URL can be provided as an environment
243
- variable to override inference in more complex use-cases.
244
-
245
- Note, if using 'host.docker.internal' in the API URL on Linux, the API must be bound
246
- to 0.0.0.0 or the Docker IP address to allow connectivity. On macOS, this is not
247
- necessary and the API is connectable while bound to localhost.
248
- """
249
-
250
- type: Literal["docker-container"] = Field(
251
- default="docker-container", description="The type of infrastructure."
252
- )
253
- image: str = Field(
254
- default_factory=get_prefect_image_name,
255
- description="Tag of a Docker image to use. Defaults to the Prefect image.",
256
- )
257
- image_pull_policy: Optional[ImagePullPolicy] = Field(
258
- default=None, description="Specifies if the image should be pulled."
259
- )
260
- image_registry: Optional[DockerRegistry] = None
261
- networks: List[str] = Field(
262
- default_factory=list,
263
- description=(
264
- "A list of strings specifying Docker networks to connect the container to."
265
- ),
266
- )
267
- network_mode: Optional[str] = Field(
268
- default=None,
269
- description=(
270
- "The network mode for the created container (e.g. host, bridge). If"
271
- " 'networks' is set, this cannot be set."
272
- ),
273
- )
274
- auto_remove: bool = Field(
275
- default=False,
276
- description="If set, the container will be removed on completion.",
277
- )
278
- volumes: List[str] = Field(
279
- default_factory=list,
280
- description=(
281
- "A list of volume mount strings in the format of"
282
- ' "local_path:container_path".'
283
- ),
284
- )
285
- stream_output: bool = Field(
286
- default=True,
287
- description=(
288
- "If set, the output will be streamed from the container to local standard"
289
- " output."
290
- ),
291
- )
292
- memswap_limit: Union[int, str] = Field(
293
- default=None,
294
- description=(
295
- "Total memory (memory + swap), -1 to disable swap. Should only be "
296
- "set if `mem_limit` is also set. If `mem_limit` is set, this defaults to"
297
- "allowing the container to use as much swap as memory. For example, if "
298
- "`mem_limit` is 300m and `memswap_limit` is not set, the container can use "
299
- "600m in total of memory and swap."
300
- ),
301
- )
302
- mem_limit: Union[float, str] = Field(
303
- default=None,
304
- description=(
305
- "Memory limit of the created container. Accepts float values to enforce "
306
- "a limit in bytes or a string with a unit e.g. 100000b, 1000k, 128m, 1g. "
307
- "If a string is given without a unit, bytes are assumed."
308
- ),
309
- )
310
- privileged: bool = Field(
311
- default=False,
312
- description="Give extended privileges to this container.",
313
- )
314
-
315
- _block_type_name = "Docker Container"
316
- _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/14a315b79990200db7341e42553e23650b34bb96-250x250.png"
317
- _documentation_url = "https://docs.prefect.io/api-ref/prefect/infrastructure/#prefect.infrastructure.DockerContainer"
318
-
319
- @validator("labels")
320
- def convert_labels_to_docker_format(cls, labels: Dict[str, str]):
321
- labels = labels or {}
322
- new_labels = {}
323
- for name, value in labels.items():
324
- if "/" in name:
325
- namespace, key = name.split("/", maxsplit=1)
326
- new_namespace = ".".join(reversed(namespace.split(".")))
327
- new_labels[f"{new_namespace}.{key}"] = value
328
- else:
329
- new_labels[name] = value
330
- return new_labels
331
-
332
- @validator("volumes")
333
- def check_volume_format(cls, volumes):
334
- for volume in volumes:
335
- if ":" not in volume:
336
- raise ValueError(
337
- "Invalid volume specification. "
338
- f"Expected format 'path:container_path', but got {volume!r}"
339
- )
340
-
341
- return volumes
342
-
343
- @sync_compatible
344
- async def run(
345
- self,
346
- task_status: Optional[anyio.abc.TaskStatus] = None,
347
- ) -> Optional[bool]:
348
- if not self.command:
349
- raise ValueError("Docker container cannot be run with empty command.")
350
-
351
- # The `docker` library uses requests instead of an async http library so it must
352
- # be run in a thread to avoid blocking the event loop.
353
- container = await run_sync_in_worker_thread(self._create_and_start_container)
354
- container_pid = self._get_infrastructure_pid(container_id=container.id)
355
-
356
- # Mark as started and return the infrastructure id
357
- if task_status:
358
- task_status.started(container_pid)
359
-
360
- # Monitor the container
361
- container = await run_sync_in_worker_thread(
362
- self._watch_container_safe, container
363
- )
364
-
365
- exit_code = container.attrs["State"].get("ExitCode")
366
- return DockerContainerResult(
367
- status_code=exit_code if exit_code is not None else -1,
368
- identifier=container_pid,
369
- )
370
-
371
- async def kill(self, infrastructure_pid: str, grace_seconds: int = 30):
372
- docker_client = self._get_client()
373
- base_url, container_id = self._parse_infrastructure_pid(infrastructure_pid)
374
-
375
- if docker_client.api.base_url != base_url:
376
- raise InfrastructureNotAvailable(
377
- "".join(
378
- [
379
- (
380
- f"Unable to stop container {container_id!r}: the current"
381
- " Docker API "
382
- ),
383
- (
384
- f"URL {docker_client.api.base_url!r} does not match the"
385
- " expected "
386
- ),
387
- f"API base URL {base_url}.",
388
- ]
389
- )
390
- )
391
- try:
392
- container = docker_client.containers.get(container_id=container_id)
393
- except docker.errors.NotFound:
394
- raise InfrastructureNotFound(
395
- f"Unable to stop container {container_id!r}: The container was not"
396
- " found."
397
- )
398
-
399
- try:
400
- container.stop(timeout=grace_seconds)
401
- except Exception:
402
- raise
403
-
404
- def preview(self):
405
- # TODO: build and document a more sophisticated preview
406
- docker_client = self._get_client()
407
- try:
408
- return json.dumps(self._build_container_settings(docker_client))
409
- finally:
410
- docker_client.close()
411
-
412
- async def generate_work_pool_base_job_template(self):
413
- from prefect.workers.utilities import (
414
- get_default_base_job_template_for_infrastructure_type,
415
- )
416
-
417
- base_job_template = await get_default_base_job_template_for_infrastructure_type(
418
- self.get_corresponding_worker_type()
419
- )
420
- if base_job_template is None:
421
- return await super().generate_work_pool_base_job_template()
422
- for key, value in self.dict(exclude_unset=True, exclude_defaults=True).items():
423
- if key == "command":
424
- base_job_template["variables"]["properties"]["command"][
425
- "default"
426
- ] = shlex.join(value)
427
- elif key == "image_registry":
428
- self.logger.warning(
429
- "Image registry blocks are not supported by Docker"
430
- " work pools. Please authenticate to your registry using"
431
- " the `docker login` command on your worker instances."
432
- )
433
- elif key in [
434
- "type",
435
- "block_type_slug",
436
- "_block_document_id",
437
- "_block_document_name",
438
- "_is_anonymous",
439
- ]:
440
- continue
441
- elif key == "image_pull_policy":
442
- new_value = None
443
- if value == ImagePullPolicy.ALWAYS:
444
- new_value = "Always"
445
- elif value == ImagePullPolicy.NEVER:
446
- new_value = "Never"
447
- elif value == ImagePullPolicy.IF_NOT_PRESENT:
448
- new_value = "IfNotPresent"
449
-
450
- base_job_template["variables"]["properties"][key]["default"] = new_value
451
- elif key in base_job_template["variables"]["properties"]:
452
- base_job_template["variables"]["properties"][key]["default"] = value
453
- else:
454
- self.logger.warning(
455
- f"Variable {key!r} is not supported by Docker work pools. Skipping."
456
- )
457
-
458
- return base_job_template
459
-
460
- def get_corresponding_worker_type(self):
461
- return "docker"
462
-
463
- def _get_infrastructure_pid(self, container_id: str) -> str:
464
- """Generates a Docker infrastructure_pid string in the form of
465
- `<docker_host_base_url>:<container_id>`.
466
- """
467
- docker_client = self._get_client()
468
- base_url = docker_client.api.base_url
469
- docker_client.close()
470
- return f"{base_url}:{container_id}"
471
-
472
- def _parse_infrastructure_pid(self, infrastructure_pid: str) -> Tuple[str, str]:
473
- """Splits a Docker infrastructure_pid into its component parts"""
474
-
475
- # base_url can contain `:` so we only want the last item of the split
476
- base_url, container_id = infrastructure_pid.rsplit(":", 1)
477
- return base_url, str(container_id)
478
-
479
- def _build_container_settings(
480
- self,
481
- docker_client: "DockerClient",
482
- ) -> Dict:
483
- network_mode = self._get_network_mode()
484
- return dict(
485
- image=self.image,
486
- network=self.networks[0] if self.networks else None,
487
- network_mode=network_mode,
488
- command=self.command,
489
- environment=self._get_environment_variables(network_mode),
490
- auto_remove=self.auto_remove,
491
- labels={**CONTAINER_LABELS, **self.labels},
492
- extra_hosts=self._get_extra_hosts(docker_client),
493
- name=self._get_container_name(),
494
- volumes=self.volumes,
495
- mem_limit=self.mem_limit,
496
- memswap_limit=self.memswap_limit,
497
- privileged=self.privileged,
498
- )
499
-
500
- def _create_and_start_container(self) -> "Container":
501
- if self.image_registry:
502
- # If an image registry block was supplied, load an authenticated Docker
503
- # client from the block. Otherwise, use an unauthenticated client to
504
- # pull images from public registries.
505
- docker_client = self.image_registry.get_docker_client()
506
- else:
507
- docker_client = self._get_client()
508
- container_settings = self._build_container_settings(docker_client)
509
-
510
- if self._should_pull_image(docker_client):
511
- self.logger.info(f"Pulling image {self.image!r}...")
512
- self._pull_image(docker_client)
513
-
514
- container = self._create_container(docker_client, **container_settings)
515
-
516
- # Add additional networks after the container is created; only one network can
517
- # be attached at creation time
518
- if len(self.networks) > 1:
519
- for network_name in self.networks[1:]:
520
- network = docker_client.networks.get(network_name)
521
- network.connect(container)
522
-
523
- # Start the container
524
- container.start()
525
-
526
- docker_client.close()
527
-
528
- return container
529
-
530
- def _get_image_and_tag(self) -> Tuple[str, Optional[str]]:
531
- return parse_image_tag(self.image)
532
-
533
- def _determine_image_pull_policy(self) -> ImagePullPolicy:
534
- """
535
- Determine the appropriate image pull policy.
536
-
537
- 1. If they specified an image pull policy, use that.
538
-
539
- 2. If they did not specify an image pull policy and gave us
540
- the "latest" tag, use ImagePullPolicy.always.
541
-
542
- 3. If they did not specify an image pull policy and did not
543
- specify a tag, use ImagePullPolicy.always.
544
-
545
- 4. If they did not specify an image pull policy and gave us
546
- a tag other than "latest", use ImagePullPolicy.if_not_present.
547
-
548
- This logic matches the behavior of Kubernetes.
549
- See:https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting
550
- """
551
- if not self.image_pull_policy:
552
- _, tag = self._get_image_and_tag()
553
- if tag == "latest" or not tag:
554
- return ImagePullPolicy.ALWAYS
555
- return ImagePullPolicy.IF_NOT_PRESENT
556
- return self.image_pull_policy
557
-
558
- def _get_network_mode(self) -> Optional[str]:
559
- # User's value takes precedence; this may collide with the incompatible options
560
- # mentioned below.
561
- if self.network_mode:
562
- if sys.platform != "linux" and self.network_mode == "host":
563
- warnings.warn(
564
- f"{self.network_mode!r} network mode is not supported on platform "
565
- f"{sys.platform!r} and may not work as intended."
566
- )
567
- return self.network_mode
568
-
569
- # Network mode is not compatible with networks or ports (we do not support ports
570
- # yet though)
571
- if self.networks:
572
- return None
573
-
574
- # Check for a local API connection
575
- api_url = self.env.get("PREFECT_API_URL", PREFECT_API_URL.value())
576
-
577
- if api_url:
578
- try:
579
- _, netloc, _, _, _, _ = urllib.parse.urlparse(api_url)
580
- except Exception as exc:
581
- warnings.warn(
582
- f"Failed to parse host from API URL {api_url!r} with exception: "
583
- f"{exc}\nThe network mode will not be inferred."
584
- )
585
- return None
586
-
587
- host = netloc.split(":")[0]
588
-
589
- # If using a locally hosted API, use a host network on linux
590
- if sys.platform == "linux" and (host == "127.0.0.1" or host == "localhost"):
591
- return "host"
592
-
593
- # Default to unset
594
- return None
595
-
596
- def _should_pull_image(self, docker_client: "DockerClient") -> bool:
597
- """
598
- Decide whether we need to pull the Docker image.
599
- """
600
- image_pull_policy = self._determine_image_pull_policy()
601
-
602
- if image_pull_policy is ImagePullPolicy.ALWAYS:
603
- return True
604
- elif image_pull_policy is ImagePullPolicy.NEVER:
605
- return False
606
- elif image_pull_policy is ImagePullPolicy.IF_NOT_PRESENT:
607
- try:
608
- # NOTE: images.get() wants the tag included with the image
609
- # name, while images.pull() wants them split.
610
- docker_client.images.get(self.image)
611
- except docker.errors.ImageNotFound:
612
- self.logger.debug(f"Could not find Docker image locally: {self.image}")
613
- return True
614
- return False
615
-
616
- def _pull_image(self, docker_client: "DockerClient"):
617
- """
618
- Pull the image we're going to use to create the container.
619
- """
620
- image, tag = self._get_image_and_tag()
621
-
622
- return docker_client.images.pull(image, tag)
623
-
624
- def _create_container(self, docker_client: "DockerClient", **kwargs) -> "Container":
625
- """
626
- Create a docker container with retries on name conflicts.
627
-
628
- If the container already exists with the given name, an incremented index is
629
- added.
630
- """
631
- # Create the container with retries on name conflicts (with an incremented idx)
632
- index = 0
633
- container = None
634
- name = original_name = kwargs.pop("name")
635
-
636
- while not container:
637
- from docker.errors import APIError
638
-
639
- try:
640
- display_name = repr(name) if name else "with auto-generated name"
641
- self.logger.info(f"Creating Docker container {display_name}...")
642
- container = docker_client.containers.create(name=name, **kwargs)
643
- except APIError as exc:
644
- if "Conflict" in str(exc) and "container name" in str(exc):
645
- self.logger.info(
646
- f"Docker container name {display_name} already exists; "
647
- "retrying..."
648
- )
649
- index += 1
650
- name = f"{original_name}-{index}"
651
- else:
652
- raise
653
-
654
- self.logger.info(
655
- f"Docker container {container.name!r} has status {container.status!r}"
656
- )
657
- return container
658
-
659
- def _watch_container_safe(self, container: "Container") -> "Container":
660
- # Monitor the container capturing the latest snapshot while capturing
661
- # not found errors
662
- docker_client = self._get_client()
663
-
664
- try:
665
- for latest_container in self._watch_container(docker_client, container.id):
666
- container = latest_container
667
- except docker.errors.NotFound:
668
- # The container was removed during watching
669
- self.logger.warning(
670
- f"Docker container {container.name} was removed before we could wait "
671
- "for its completion."
672
- )
673
- finally:
674
- docker_client.close()
675
-
676
- return container
677
-
678
- def _watch_container(
679
- self, docker_client: "DockerClient", container_id: str
680
- ) -> Generator[None, None, "Container"]:
681
- container: "Container" = docker_client.containers.get(container_id)
682
-
683
- status = container.status
684
- self.logger.info(
685
- f"Docker container {container.name!r} has status {container.status!r}"
686
- )
687
- yield container
688
-
689
- if self.stream_output:
690
- try:
691
- for log in container.logs(stream=True):
692
- log: bytes
693
- print(log.decode().rstrip())
694
- except docker.errors.APIError as exc:
695
- if "marked for removal" in str(exc):
696
- self.logger.warning(
697
- f"Docker container {container.name} was marked for removal"
698
- " before logs could be retrieved. Output will not be"
699
- " streamed. "
700
- )
701
- else:
702
- self.logger.exception(
703
- "An unexpected Docker API error occurred while streaming"
704
- f" output from container {container.name}."
705
- )
706
-
707
- container.reload()
708
- if container.status != status:
709
- self.logger.info(
710
- f"Docker container {container.name!r} has status"
711
- f" {container.status!r}"
712
- )
713
- yield container
714
-
715
- container.wait()
716
- self.logger.info(
717
- f"Docker container {container.name!r} has status {container.status!r}"
718
- )
719
- yield container
720
-
721
- def _get_client(self):
722
- try:
723
- with warnings.catch_warnings():
724
- # Silence warnings due to use of deprecated methods within dockerpy
725
- # See https://github.com/docker/docker-py/pull/2931
726
- warnings.filterwarnings(
727
- "ignore",
728
- message="distutils Version classes are deprecated.*",
729
- category=DeprecationWarning,
730
- )
731
-
732
- docker_client = docker.from_env()
733
-
734
- except docker.errors.DockerException as exc:
735
- raise RuntimeError("Could not connect to Docker.") from exc
736
-
737
- return docker_client
738
-
739
- def _get_container_name(self) -> Optional[str]:
740
- """
741
- Generates a container name to match the configured name, ensuring it is Docker
742
- compatible.
743
- """
744
- # Must match `/?[a-zA-Z0-9][a-zA-Z0-9_.-]+` in the end
745
- if not self.name:
746
- return None
747
-
748
- return (
749
- slugify(
750
- self.name,
751
- lowercase=False,
752
- # Docker does not limit length but URL limits apply eventually so
753
- # limit the length for safety
754
- max_length=250,
755
- # Docker allows these characters for container names
756
- regex_pattern=r"[^a-zA-Z0-9_.-]+",
757
- ).lstrip(
758
- # Docker does not allow leading underscore, dash, or period
759
- "_-."
760
- )
761
- # Docker does not allow 0 character names so cast to null if the name is
762
- # empty after slufification
763
- or None
764
- )
765
-
766
- def _get_extra_hosts(self, docker_client) -> Dict[str, str]:
767
- """
768
- A host.docker.internal -> host-gateway mapping is necessary for communicating
769
- with the API on Linux machines. Docker Desktop on macOS will automatically
770
- already have this mapping.
771
- """
772
- if sys.platform == "linux" and (
773
- # Do not warn if the user has specified a host manually that does not use
774
- # a local address
775
- "PREFECT_API_URL" not in self.env
776
- or re.search(
777
- ".*(localhost)|(127.0.0.1)|(host.docker.internal).*",
778
- self.env["PREFECT_API_URL"],
779
- )
780
- ):
781
- user_version = packaging.version.parse(
782
- format_outlier_version_name(docker_client.version()["Version"])
783
- )
784
- required_version = packaging.version.parse("20.10.0")
785
-
786
- if user_version < required_version:
787
- warnings.warn(
788
- "`host.docker.internal` could not be automatically resolved to"
789
- " your local ip address. This feature is not supported on Docker"
790
- f" Engine v{user_version}, upgrade to v{required_version}+ if you"
791
- " encounter issues."
792
- )
793
- return {}
794
- else:
795
- # Compatibility for linux -- https://github.com/docker/cli/issues/2290
796
- # Only supported by Docker v20.10.0+ which is our minimum recommend version
797
- return {"host.docker.internal": "host-gateway"}
798
-
799
- def _get_environment_variables(self, network_mode):
800
- # If the API URL has been set by the base environment rather than the by the
801
- # user, update the value to ensure connectivity when using a bridge network by
802
- # updating local connections to use the docker internal host unless the
803
- # network mode is "host" where localhost is available already.
804
- env = {**self._base_environment(), **self.env}
805
-
806
- if (
807
- "PREFECT_API_URL" in env
808
- and "PREFECT_API_URL" not in self.env
809
- and network_mode != "host"
810
- ):
811
- env["PREFECT_API_URL"] = (
812
- env["PREFECT_API_URL"]
813
- .replace("localhost", "host.docker.internal")
814
- .replace("127.0.0.1", "host.docker.internal")
815
- )
816
-
817
- # Drop null values allowing users to "unset" variables
818
- return {key: value for key, value in env.items() if value is not None}