prefect-client 2.20.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (288) hide show
  1. prefect/__init__.py +74 -110
  2. prefect/_internal/compatibility/deprecated.py +6 -115
  3. prefect/_internal/compatibility/experimental.py +4 -79
  4. prefect/_internal/compatibility/migration.py +166 -0
  5. prefect/_internal/concurrency/__init__.py +2 -2
  6. prefect/_internal/concurrency/api.py +1 -35
  7. prefect/_internal/concurrency/calls.py +0 -6
  8. prefect/_internal/concurrency/cancellation.py +0 -3
  9. prefect/_internal/concurrency/event_loop.py +0 -20
  10. prefect/_internal/concurrency/inspection.py +3 -3
  11. prefect/_internal/concurrency/primitives.py +1 -0
  12. prefect/_internal/concurrency/services.py +23 -0
  13. prefect/_internal/concurrency/threads.py +35 -0
  14. prefect/_internal/concurrency/waiters.py +0 -28
  15. prefect/_internal/integrations.py +7 -0
  16. prefect/_internal/pydantic/__init__.py +0 -45
  17. prefect/_internal/pydantic/annotations/pendulum.py +2 -2
  18. prefect/_internal/pydantic/v1_schema.py +21 -22
  19. prefect/_internal/pydantic/v2_schema.py +0 -2
  20. prefect/_internal/pydantic/v2_validated_func.py +18 -23
  21. prefect/_internal/pytz.py +1 -1
  22. prefect/_internal/retries.py +61 -0
  23. prefect/_internal/schemas/bases.py +45 -177
  24. prefect/_internal/schemas/fields.py +1 -43
  25. prefect/_internal/schemas/validators.py +47 -233
  26. prefect/agent.py +3 -695
  27. prefect/artifacts.py +173 -14
  28. prefect/automations.py +39 -4
  29. prefect/blocks/abstract.py +1 -1
  30. prefect/blocks/core.py +405 -153
  31. prefect/blocks/fields.py +2 -57
  32. prefect/blocks/notifications.py +43 -28
  33. prefect/blocks/redis.py +168 -0
  34. prefect/blocks/system.py +67 -20
  35. prefect/blocks/webhook.py +2 -9
  36. prefect/cache_policies.py +239 -0
  37. prefect/client/__init__.py +4 -0
  38. prefect/client/base.py +33 -27
  39. prefect/client/cloud.py +65 -20
  40. prefect/client/collections.py +1 -1
  41. prefect/client/orchestration.py +650 -442
  42. prefect/client/schemas/actions.py +115 -100
  43. prefect/client/schemas/filters.py +46 -52
  44. prefect/client/schemas/objects.py +228 -178
  45. prefect/client/schemas/responses.py +18 -36
  46. prefect/client/schemas/schedules.py +55 -36
  47. prefect/client/schemas/sorting.py +2 -0
  48. prefect/client/subscriptions.py +8 -7
  49. prefect/client/types/flexible_schedule_list.py +11 -0
  50. prefect/client/utilities.py +9 -6
  51. prefect/concurrency/asyncio.py +60 -11
  52. prefect/concurrency/context.py +24 -0
  53. prefect/concurrency/events.py +2 -2
  54. prefect/concurrency/services.py +46 -16
  55. prefect/concurrency/sync.py +51 -7
  56. prefect/concurrency/v1/asyncio.py +143 -0
  57. prefect/concurrency/v1/context.py +27 -0
  58. prefect/concurrency/v1/events.py +61 -0
  59. prefect/concurrency/v1/services.py +116 -0
  60. prefect/concurrency/v1/sync.py +92 -0
  61. prefect/context.py +246 -149
  62. prefect/deployments/__init__.py +33 -18
  63. prefect/deployments/base.py +10 -15
  64. prefect/deployments/deployments.py +2 -1048
  65. prefect/deployments/flow_runs.py +178 -0
  66. prefect/deployments/runner.py +72 -173
  67. prefect/deployments/schedules.py +31 -25
  68. prefect/deployments/steps/__init__.py +0 -1
  69. prefect/deployments/steps/core.py +7 -0
  70. prefect/deployments/steps/pull.py +15 -21
  71. prefect/deployments/steps/utility.py +2 -1
  72. prefect/docker/__init__.py +20 -0
  73. prefect/docker/docker_image.py +82 -0
  74. prefect/engine.py +15 -2475
  75. prefect/events/actions.py +17 -23
  76. prefect/events/cli/automations.py +20 -7
  77. prefect/events/clients.py +142 -80
  78. prefect/events/filters.py +14 -18
  79. prefect/events/related.py +74 -75
  80. prefect/events/schemas/__init__.py +0 -5
  81. prefect/events/schemas/automations.py +55 -46
  82. prefect/events/schemas/deployment_triggers.py +7 -197
  83. prefect/events/schemas/events.py +46 -65
  84. prefect/events/schemas/labelling.py +10 -14
  85. prefect/events/utilities.py +4 -5
  86. prefect/events/worker.py +23 -8
  87. prefect/exceptions.py +15 -0
  88. prefect/filesystems.py +30 -529
  89. prefect/flow_engine.py +827 -0
  90. prefect/flow_runs.py +379 -7
  91. prefect/flows.py +470 -360
  92. prefect/futures.py +382 -331
  93. prefect/infrastructure/__init__.py +5 -26
  94. prefect/infrastructure/base.py +3 -320
  95. prefect/infrastructure/provisioners/__init__.py +5 -3
  96. prefect/infrastructure/provisioners/cloud_run.py +13 -8
  97. prefect/infrastructure/provisioners/container_instance.py +14 -9
  98. prefect/infrastructure/provisioners/ecs.py +10 -8
  99. prefect/infrastructure/provisioners/modal.py +8 -5
  100. prefect/input/__init__.py +4 -0
  101. prefect/input/actions.py +2 -4
  102. prefect/input/run_input.py +9 -9
  103. prefect/logging/formatters.py +2 -4
  104. prefect/logging/handlers.py +9 -14
  105. prefect/logging/loggers.py +5 -5
  106. prefect/main.py +72 -0
  107. prefect/plugins.py +2 -64
  108. prefect/profiles.toml +16 -2
  109. prefect/records/__init__.py +1 -0
  110. prefect/records/base.py +223 -0
  111. prefect/records/filesystem.py +207 -0
  112. prefect/records/memory.py +178 -0
  113. prefect/records/result_store.py +64 -0
  114. prefect/results.py +577 -504
  115. prefect/runner/runner.py +117 -47
  116. prefect/runner/server.py +32 -34
  117. prefect/runner/storage.py +3 -12
  118. prefect/runner/submit.py +2 -10
  119. prefect/runner/utils.py +2 -2
  120. prefect/runtime/__init__.py +1 -0
  121. prefect/runtime/deployment.py +1 -0
  122. prefect/runtime/flow_run.py +40 -5
  123. prefect/runtime/task_run.py +1 -0
  124. prefect/serializers.py +28 -39
  125. prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
  126. prefect/settings.py +209 -332
  127. prefect/states.py +160 -63
  128. prefect/task_engine.py +1478 -57
  129. prefect/task_runners.py +383 -287
  130. prefect/task_runs.py +240 -0
  131. prefect/task_worker.py +463 -0
  132. prefect/tasks.py +684 -374
  133. prefect/transactions.py +410 -0
  134. prefect/types/__init__.py +72 -86
  135. prefect/types/entrypoint.py +13 -0
  136. prefect/utilities/annotations.py +4 -3
  137. prefect/utilities/asyncutils.py +227 -148
  138. prefect/utilities/callables.py +137 -45
  139. prefect/utilities/collections.py +134 -86
  140. prefect/utilities/dispatch.py +27 -14
  141. prefect/utilities/dockerutils.py +11 -4
  142. prefect/utilities/engine.py +186 -32
  143. prefect/utilities/filesystem.py +4 -5
  144. prefect/utilities/importtools.py +26 -27
  145. prefect/utilities/pydantic.py +128 -38
  146. prefect/utilities/schema_tools/hydration.py +18 -1
  147. prefect/utilities/schema_tools/validation.py +30 -0
  148. prefect/utilities/services.py +35 -9
  149. prefect/utilities/templating.py +12 -2
  150. prefect/utilities/timeout.py +20 -5
  151. prefect/utilities/urls.py +195 -0
  152. prefect/utilities/visualization.py +1 -0
  153. prefect/variables.py +78 -59
  154. prefect/workers/__init__.py +0 -1
  155. prefect/workers/base.py +237 -244
  156. prefect/workers/block.py +5 -226
  157. prefect/workers/cloud.py +6 -0
  158. prefect/workers/process.py +265 -12
  159. prefect/workers/server.py +29 -11
  160. {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/METADATA +28 -24
  161. prefect_client-3.0.0.dist-info/RECORD +201 -0
  162. {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/WHEEL +1 -1
  163. prefect/_internal/pydantic/_base_model.py +0 -51
  164. prefect/_internal/pydantic/_compat.py +0 -82
  165. prefect/_internal/pydantic/_flags.py +0 -20
  166. prefect/_internal/pydantic/_types.py +0 -8
  167. prefect/_internal/pydantic/utilities/config_dict.py +0 -72
  168. prefect/_internal/pydantic/utilities/field_validator.py +0 -150
  169. prefect/_internal/pydantic/utilities/model_construct.py +0 -56
  170. prefect/_internal/pydantic/utilities/model_copy.py +0 -55
  171. prefect/_internal/pydantic/utilities/model_dump.py +0 -136
  172. prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
  173. prefect/_internal/pydantic/utilities/model_fields.py +0 -50
  174. prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
  175. prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
  176. prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
  177. prefect/_internal/pydantic/utilities/model_validate.py +0 -75
  178. prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
  179. prefect/_internal/pydantic/utilities/model_validator.py +0 -87
  180. prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
  181. prefect/_vendor/fastapi/__init__.py +0 -25
  182. prefect/_vendor/fastapi/applications.py +0 -946
  183. prefect/_vendor/fastapi/background.py +0 -3
  184. prefect/_vendor/fastapi/concurrency.py +0 -44
  185. prefect/_vendor/fastapi/datastructures.py +0 -58
  186. prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
  187. prefect/_vendor/fastapi/dependencies/models.py +0 -64
  188. prefect/_vendor/fastapi/dependencies/utils.py +0 -877
  189. prefect/_vendor/fastapi/encoders.py +0 -177
  190. prefect/_vendor/fastapi/exception_handlers.py +0 -40
  191. prefect/_vendor/fastapi/exceptions.py +0 -46
  192. prefect/_vendor/fastapi/logger.py +0 -3
  193. prefect/_vendor/fastapi/middleware/__init__.py +0 -1
  194. prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
  195. prefect/_vendor/fastapi/middleware/cors.py +0 -3
  196. prefect/_vendor/fastapi/middleware/gzip.py +0 -3
  197. prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
  198. prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
  199. prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
  200. prefect/_vendor/fastapi/openapi/__init__.py +0 -0
  201. prefect/_vendor/fastapi/openapi/constants.py +0 -2
  202. prefect/_vendor/fastapi/openapi/docs.py +0 -203
  203. prefect/_vendor/fastapi/openapi/models.py +0 -480
  204. prefect/_vendor/fastapi/openapi/utils.py +0 -485
  205. prefect/_vendor/fastapi/param_functions.py +0 -340
  206. prefect/_vendor/fastapi/params.py +0 -453
  207. prefect/_vendor/fastapi/py.typed +0 -0
  208. prefect/_vendor/fastapi/requests.py +0 -4
  209. prefect/_vendor/fastapi/responses.py +0 -40
  210. prefect/_vendor/fastapi/routing.py +0 -1331
  211. prefect/_vendor/fastapi/security/__init__.py +0 -15
  212. prefect/_vendor/fastapi/security/api_key.py +0 -98
  213. prefect/_vendor/fastapi/security/base.py +0 -6
  214. prefect/_vendor/fastapi/security/http.py +0 -172
  215. prefect/_vendor/fastapi/security/oauth2.py +0 -227
  216. prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
  217. prefect/_vendor/fastapi/security/utils.py +0 -10
  218. prefect/_vendor/fastapi/staticfiles.py +0 -1
  219. prefect/_vendor/fastapi/templating.py +0 -3
  220. prefect/_vendor/fastapi/testclient.py +0 -1
  221. prefect/_vendor/fastapi/types.py +0 -3
  222. prefect/_vendor/fastapi/utils.py +0 -235
  223. prefect/_vendor/fastapi/websockets.py +0 -7
  224. prefect/_vendor/starlette/__init__.py +0 -1
  225. prefect/_vendor/starlette/_compat.py +0 -28
  226. prefect/_vendor/starlette/_exception_handler.py +0 -80
  227. prefect/_vendor/starlette/_utils.py +0 -88
  228. prefect/_vendor/starlette/applications.py +0 -261
  229. prefect/_vendor/starlette/authentication.py +0 -159
  230. prefect/_vendor/starlette/background.py +0 -43
  231. prefect/_vendor/starlette/concurrency.py +0 -59
  232. prefect/_vendor/starlette/config.py +0 -151
  233. prefect/_vendor/starlette/convertors.py +0 -87
  234. prefect/_vendor/starlette/datastructures.py +0 -707
  235. prefect/_vendor/starlette/endpoints.py +0 -130
  236. prefect/_vendor/starlette/exceptions.py +0 -60
  237. prefect/_vendor/starlette/formparsers.py +0 -276
  238. prefect/_vendor/starlette/middleware/__init__.py +0 -17
  239. prefect/_vendor/starlette/middleware/authentication.py +0 -52
  240. prefect/_vendor/starlette/middleware/base.py +0 -220
  241. prefect/_vendor/starlette/middleware/cors.py +0 -176
  242. prefect/_vendor/starlette/middleware/errors.py +0 -265
  243. prefect/_vendor/starlette/middleware/exceptions.py +0 -74
  244. prefect/_vendor/starlette/middleware/gzip.py +0 -113
  245. prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
  246. prefect/_vendor/starlette/middleware/sessions.py +0 -82
  247. prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
  248. prefect/_vendor/starlette/middleware/wsgi.py +0 -147
  249. prefect/_vendor/starlette/py.typed +0 -0
  250. prefect/_vendor/starlette/requests.py +0 -328
  251. prefect/_vendor/starlette/responses.py +0 -347
  252. prefect/_vendor/starlette/routing.py +0 -933
  253. prefect/_vendor/starlette/schemas.py +0 -154
  254. prefect/_vendor/starlette/staticfiles.py +0 -248
  255. prefect/_vendor/starlette/status.py +0 -199
  256. prefect/_vendor/starlette/templating.py +0 -231
  257. prefect/_vendor/starlette/testclient.py +0 -804
  258. prefect/_vendor/starlette/types.py +0 -30
  259. prefect/_vendor/starlette/websockets.py +0 -193
  260. prefect/blocks/kubernetes.py +0 -119
  261. prefect/deprecated/__init__.py +0 -0
  262. prefect/deprecated/data_documents.py +0 -350
  263. prefect/deprecated/packaging/__init__.py +0 -12
  264. prefect/deprecated/packaging/base.py +0 -96
  265. prefect/deprecated/packaging/docker.py +0 -146
  266. prefect/deprecated/packaging/file.py +0 -92
  267. prefect/deprecated/packaging/orion.py +0 -80
  268. prefect/deprecated/packaging/serializers.py +0 -171
  269. prefect/events/instrument.py +0 -135
  270. prefect/infrastructure/container.py +0 -824
  271. prefect/infrastructure/kubernetes.py +0 -920
  272. prefect/infrastructure/process.py +0 -289
  273. prefect/manifests.py +0 -20
  274. prefect/new_flow_engine.py +0 -449
  275. prefect/new_task_engine.py +0 -423
  276. prefect/pydantic/__init__.py +0 -76
  277. prefect/pydantic/main.py +0 -39
  278. prefect/software/__init__.py +0 -2
  279. prefect/software/base.py +0 -50
  280. prefect/software/conda.py +0 -199
  281. prefect/software/pip.py +0 -122
  282. prefect/software/python.py +0 -52
  283. prefect/task_server.py +0 -322
  284. prefect_client-2.20.4.dist-info/RECORD +0 -294
  285. /prefect/{_internal/pydantic/utilities → client/types}/__init__.py +0 -0
  286. /prefect/{_vendor → concurrency/v1}/__init__.py +0 -0
  287. {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/LICENSE +0 -0
  288. {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/top_level.txt +0 -0
@@ -1,824 +0,0 @@
1
- """
2
- DEPRECATION WARNING:
3
-
4
- This module is deprecated as of March 2024 and will not be available after September 2024.
5
- It has been replaced by the Docker worker from the prefect-docker package, which offers enhanced functionality and better performance.
6
-
7
- For upgrade instructions, see https://docs.prefect.io/latest/guides/upgrade-guide-agents-to-workers/.
8
- """
9
- import json
10
- import re
11
- import shlex
12
- import sys
13
- import urllib.parse
14
- import warnings
15
- from abc import ABC, abstractmethod
16
- from typing import TYPE_CHECKING, Dict, Generator, List, Optional, Tuple, Union
17
-
18
- import anyio.abc
19
- import packaging.version
20
-
21
- from prefect._internal.compatibility.deprecated import deprecated_class
22
- from prefect._internal.pydantic import HAS_PYDANTIC_V2
23
-
24
- if HAS_PYDANTIC_V2:
25
- from pydantic.v1 import Field, validator
26
- else:
27
- from pydantic import Field, validator
28
-
29
- from typing_extensions import Literal
30
-
31
- import prefect
32
- from prefect.blocks.core import Block, SecretStr
33
- from prefect.exceptions import InfrastructureNotAvailable, InfrastructureNotFound
34
- from prefect.infrastructure.base import Infrastructure, InfrastructureResult
35
- from prefect.settings import PREFECT_API_URL
36
- from prefect.utilities.asyncutils import run_sync_in_worker_thread, sync_compatible
37
- from prefect.utilities.collections import AutoEnum
38
- from prefect.utilities.dockerutils import (
39
- format_outlier_version_name,
40
- get_prefect_image_name,
41
- parse_image_tag,
42
- )
43
- from prefect.utilities.importtools import lazy_import
44
- from prefect.utilities.slugify import slugify
45
-
46
- if TYPE_CHECKING:
47
- import docker
48
- from docker import DockerClient
49
- from docker.models.containers import Container
50
- else:
51
- docker = lazy_import("docker")
52
-
53
-
54
- # Labels to apply to all containers started by Prefect
55
- CONTAINER_LABELS = {
56
- "io.prefect.version": prefect.__version__,
57
- }
58
-
59
-
60
- class ImagePullPolicy(AutoEnum):
61
- IF_NOT_PRESENT = AutoEnum.auto()
62
- ALWAYS = AutoEnum.auto()
63
- NEVER = AutoEnum.auto()
64
-
65
-
66
- @deprecated_class(
67
- start_date="Mar 2024",
68
- help="Use the `DockerRegistryCredentials` class from prefect-docker instead.",
69
- )
70
- class BaseDockerLogin(Block, ABC):
71
- _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/14a315b79990200db7341e42553e23650b34bb96-250x250.png"
72
- _block_schema_capabilities = ["docker-login"]
73
-
74
- @abstractmethod
75
- async def login(self) -> "DockerClient":
76
- """
77
- Log in and return an authenticated `DockerClient`.
78
- (DEPRECATED) Use `get_docker_client` instead of `login`.
79
- """
80
-
81
- @abstractmethod
82
- async def get_docker_client(self) -> "DockerClient":
83
- """
84
- Log in and return an authenticated `DockerClient`.
85
- """
86
-
87
- def _login(self, username, password, registry_url, reauth) -> "DockerClient":
88
- client = self._get_docker_client()
89
-
90
- client.login(
91
- username=username,
92
- password=password,
93
- registry=registry_url,
94
- # See https://github.com/docker/docker-py/issues/2256 for information on
95
- # the default value for reauth.
96
- reauth=reauth,
97
- )
98
-
99
- return client
100
-
101
- @staticmethod
102
- def _get_docker_client():
103
- try:
104
- with warnings.catch_warnings():
105
- # Silence warnings due to use of deprecated methods within dockerpy
106
- # See https://github.com/docker/docker-py/pull/2931
107
- warnings.filterwarnings(
108
- "ignore",
109
- message="distutils Version classes are deprecated.*",
110
- category=DeprecationWarning,
111
- )
112
-
113
- docker_client = docker.from_env()
114
-
115
- except docker.errors.DockerException as exc:
116
- raise RuntimeError("Could not connect to Docker.") from exc
117
-
118
- return docker_client
119
-
120
-
121
- @deprecated_class(
122
- start_date="Mar 2024",
123
- help="Use the `DockerRegistryCredentials` class from prefect-docker instead.",
124
- )
125
- class DockerRegistry(BaseDockerLogin):
126
- """
127
- DEPRECATION WARNING:
128
-
129
- This class is deprecated as of March 2024 and will not be available after September 2024.
130
- It has been replaced by `DockerRegistryCredentials` from the `prefect-docker` package, which
131
- offers enhanced functionality and better a better user experience.
132
-
133
- Connects to a Docker registry.
134
-
135
- Requires a Docker Engine to be connectable.
136
-
137
- Attributes:
138
- username: The username to log into the registry with.
139
- password: The password to log into the registry with.
140
- registry_url: The URL to the registry such as registry.hub.docker.com. Generally, "http" or "https" can be
141
- omitted.
142
- reauth: If already logged into the registry, should login be performed again?
143
- This setting defaults to `True` to support common token authentication
144
- patterns such as ECR.
145
- """
146
-
147
- _block_type_name = "Docker Registry"
148
- _documentation_url = "https://docs.prefect.io/api-ref/prefect/infrastructure/#prefect.infrastructure.docker.DockerRegistry"
149
-
150
- username: str = Field(
151
- default=..., description="The username to log into the registry with."
152
- )
153
- password: SecretStr = Field(
154
- default=..., description="The password to log into the registry with."
155
- )
156
- registry_url: str = Field(
157
- default=...,
158
- description=(
159
- 'The URL to the registry. Generally, "http" or "https" can be omitted.'
160
- ),
161
- )
162
- reauth: bool = Field(
163
- default=True,
164
- description="Whether or not to reauthenticate on each interaction.",
165
- )
166
-
167
- @sync_compatible
168
- async def login(self) -> "DockerClient":
169
- warnings.warn(
170
- (
171
- "`login` is deprecated. Instead, use `get_docker_client` to obtain an"
172
- " authenticated `DockerClient`."
173
- ),
174
- category=DeprecationWarning,
175
- stacklevel=3,
176
- )
177
- return await self.get_docker_client()
178
-
179
- @sync_compatible
180
- async def get_docker_client(self) -> "DockerClient":
181
- client = await run_sync_in_worker_thread(
182
- self._login,
183
- self.username,
184
- self.password.get_secret_value(),
185
- self.registry_url,
186
- self.reauth,
187
- )
188
-
189
- return client
190
-
191
-
192
- class DockerContainerResult(InfrastructureResult):
193
- """Contains information about a completed Docker container"""
194
-
195
-
196
- @deprecated_class(
197
- start_date="Mar 2024",
198
- help="Use the Docker worker from prefect-docker instead."
199
- " Refer to the upgrade guide for more information:"
200
- " https://docs.prefect.io/latest/guides/upgrade-guide-agents-to-workers/.",
201
- )
202
- class DockerContainer(Infrastructure):
203
- """
204
- Runs a command in a container.
205
-
206
- Requires a Docker Engine to be connectable. Docker settings will be retrieved from
207
- the environment.
208
-
209
- Click [here](https://docs.prefect.io/guides/deployment/docker) to see a tutorial.
210
-
211
- Attributes:
212
- auto_remove: If set, the container will be removed on completion. Otherwise,
213
- the container will remain after exit for inspection.
214
- command: A list of strings specifying the command to run in the container to
215
- start the flow run. In most cases you should not override this.
216
- env: Environment variables to set for the container.
217
- image: An optional string specifying the tag of a Docker image to use.
218
- Defaults to the Prefect image.
219
- image_pull_policy: Specifies if the image should be pulled. One of 'ALWAYS',
220
- 'NEVER', 'IF_NOT_PRESENT'.
221
- image_registry: A `DockerRegistry` block containing credentials to use if `image` is stored in a private
222
- image registry.
223
- labels: An optional dictionary of labels, mapping name to value.
224
- name: An optional name for the container.
225
- network_mode: Set the network mode for the created container. Defaults to 'host'
226
- if a local API url is detected, otherwise the Docker default of 'bridge' is
227
- used. If 'networks' is set, this cannot be set.
228
- networks: An optional list of strings specifying Docker networks to connect the
229
- container to.
230
- stream_output: If set, stream output from the container to local standard output.
231
- volumes: An optional list of volume mount strings in the format of
232
- "local_path:container_path".
233
- memswap_limit: Total memory (memory + swap), -1 to disable swap. Should only be
234
- set if `mem_limit` is also set. If `mem_limit` is set, this defaults to
235
- allowing the container to use as much swap as memory. For example, if
236
- `mem_limit` is 300m and `memswap_limit` is not set, the container can use
237
- 600m in total of memory and swap.
238
- mem_limit: Memory limit of the created container. Accepts float values to enforce
239
- a limit in bytes or a string with a unit e.g. 100000b, 1000k, 128m, 1g.
240
- If a string is given without a unit, bytes are assumed.
241
- privileged: Give extended privileges to this container.
242
-
243
- ## Connecting to a locally hosted Prefect API
244
-
245
- If using a local API URL on Linux, we will update the network mode default to 'host'
246
- to enable connectivity. If using another OS or an alternative network mode is used,
247
- we will replace 'localhost' in the API URL with 'host.docker.internal'. Generally,
248
- this will enable connectivity, but the API URL can be provided as an environment
249
- variable to override inference in more complex use-cases.
250
-
251
- Note, if using 'host.docker.internal' in the API URL on Linux, the API must be bound
252
- to 0.0.0.0 or the Docker IP address to allow connectivity. On macOS, this is not
253
- necessary and the API is connectable while bound to localhost.
254
- """
255
-
256
- type: Literal["docker-container"] = Field(
257
- default="docker-container", description="The type of infrastructure."
258
- )
259
- image: str = Field(
260
- default_factory=get_prefect_image_name,
261
- description="Tag of a Docker image to use. Defaults to the Prefect image.",
262
- )
263
- image_pull_policy: Optional[ImagePullPolicy] = Field(
264
- default=None, description="Specifies if the image should be pulled."
265
- )
266
- image_registry: Optional[DockerRegistry] = None
267
- networks: List[str] = Field(
268
- default_factory=list,
269
- description=(
270
- "A list of strings specifying Docker networks to connect the container to."
271
- ),
272
- )
273
- network_mode: Optional[str] = Field(
274
- default=None,
275
- description=(
276
- "The network mode for the created container (e.g. host, bridge). If"
277
- " 'networks' is set, this cannot be set."
278
- ),
279
- )
280
- auto_remove: bool = Field(
281
- default=False,
282
- description="If set, the container will be removed on completion.",
283
- )
284
- volumes: List[str] = Field(
285
- default_factory=list,
286
- description=(
287
- "A list of volume mount strings in the format of"
288
- ' "local_path:container_path".'
289
- ),
290
- )
291
- stream_output: bool = Field(
292
- default=True,
293
- description=(
294
- "If set, the output will be streamed from the container to local standard"
295
- " output."
296
- ),
297
- )
298
- memswap_limit: Union[int, str] = Field(
299
- default=None,
300
- description=(
301
- "Total memory (memory + swap), -1 to disable swap. Should only be "
302
- "set if `mem_limit` is also set. If `mem_limit` is set, this defaults to"
303
- "allowing the container to use as much swap as memory. For example, if "
304
- "`mem_limit` is 300m and `memswap_limit` is not set, the container can use "
305
- "600m in total of memory and swap."
306
- ),
307
- )
308
- mem_limit: Union[float, str] = Field(
309
- default=None,
310
- description=(
311
- "Memory limit of the created container. Accepts float values to enforce "
312
- "a limit in bytes or a string with a unit e.g. 100000b, 1000k, 128m, 1g. "
313
- "If a string is given without a unit, bytes are assumed."
314
- ),
315
- )
316
- privileged: bool = Field(
317
- default=False,
318
- description="Give extended privileges to this container.",
319
- )
320
-
321
- _block_type_name = "Docker Container"
322
- _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/14a315b79990200db7341e42553e23650b34bb96-250x250.png"
323
- _documentation_url = "https://docs.prefect.io/api-ref/prefect/infrastructure/#prefect.infrastructure.DockerContainer"
324
-
325
- @validator("labels")
326
- def convert_labels_to_docker_format(cls, labels: Dict[str, str]):
327
- labels = labels or {}
328
- new_labels = {}
329
- for name, value in labels.items():
330
- if "/" in name:
331
- namespace, key = name.split("/", maxsplit=1)
332
- new_namespace = ".".join(reversed(namespace.split(".")))
333
- new_labels[f"{new_namespace}.{key}"] = value
334
- else:
335
- new_labels[name] = value
336
- return new_labels
337
-
338
- @validator("volumes")
339
- def check_volume_format(cls, volumes):
340
- for volume in volumes:
341
- if ":" not in volume:
342
- raise ValueError(
343
- "Invalid volume specification. "
344
- f"Expected format 'path:container_path', but got {volume!r}"
345
- )
346
-
347
- return volumes
348
-
349
- @sync_compatible
350
- async def run(
351
- self,
352
- task_status: Optional[anyio.abc.TaskStatus] = None,
353
- ) -> Optional[bool]:
354
- if not self.command:
355
- raise ValueError("Docker container cannot be run with empty command.")
356
-
357
- # The `docker` library uses requests instead of an async http library so it must
358
- # be run in a thread to avoid blocking the event loop.
359
- container = await run_sync_in_worker_thread(self._create_and_start_container)
360
- container_pid = self._get_infrastructure_pid(container_id=container.id)
361
-
362
- # Mark as started and return the infrastructure id
363
- if task_status:
364
- task_status.started(container_pid)
365
-
366
- # Monitor the container
367
- container = await run_sync_in_worker_thread(
368
- self._watch_container_safe, container
369
- )
370
-
371
- exit_code = container.attrs["State"].get("ExitCode")
372
- return DockerContainerResult(
373
- status_code=exit_code if exit_code is not None else -1,
374
- identifier=container_pid,
375
- )
376
-
377
- async def kill(self, infrastructure_pid: str, grace_seconds: int = 30):
378
- docker_client = self._get_client()
379
- base_url, container_id = self._parse_infrastructure_pid(infrastructure_pid)
380
-
381
- if docker_client.api.base_url != base_url:
382
- raise InfrastructureNotAvailable(
383
- "".join(
384
- [
385
- (
386
- f"Unable to stop container {container_id!r}: the current"
387
- " Docker API "
388
- ),
389
- (
390
- f"URL {docker_client.api.base_url!r} does not match the"
391
- " expected "
392
- ),
393
- f"API base URL {base_url}.",
394
- ]
395
- )
396
- )
397
- try:
398
- container = docker_client.containers.get(container_id=container_id)
399
- except docker.errors.NotFound:
400
- raise InfrastructureNotFound(
401
- f"Unable to stop container {container_id!r}: The container was not"
402
- " found."
403
- )
404
-
405
- try:
406
- container.stop(timeout=grace_seconds)
407
- except Exception:
408
- raise
409
-
410
- def preview(self):
411
- # TODO: build and document a more sophisticated preview
412
- docker_client = self._get_client()
413
- try:
414
- return json.dumps(self._build_container_settings(docker_client))
415
- finally:
416
- docker_client.close()
417
-
418
- async def generate_work_pool_base_job_template(self):
419
- from prefect.workers.utilities import (
420
- get_default_base_job_template_for_infrastructure_type,
421
- )
422
-
423
- base_job_template = await get_default_base_job_template_for_infrastructure_type(
424
- self.get_corresponding_worker_type()
425
- )
426
- if base_job_template is None:
427
- return await super().generate_work_pool_base_job_template()
428
- for key, value in self.dict(exclude_unset=True, exclude_defaults=True).items():
429
- if key == "command":
430
- base_job_template["variables"]["properties"]["command"][
431
- "default"
432
- ] = shlex.join(value)
433
- elif key == "image_registry":
434
- self.logger.warning(
435
- "Image registry blocks are not supported by Docker"
436
- " work pools. Please authenticate to your registry using"
437
- " the `docker login` command on your worker instances."
438
- )
439
- elif key in [
440
- "type",
441
- "block_type_slug",
442
- "_block_document_id",
443
- "_block_document_name",
444
- "_is_anonymous",
445
- ]:
446
- continue
447
- elif key == "image_pull_policy":
448
- new_value = None
449
- if value == ImagePullPolicy.ALWAYS:
450
- new_value = "Always"
451
- elif value == ImagePullPolicy.NEVER:
452
- new_value = "Never"
453
- elif value == ImagePullPolicy.IF_NOT_PRESENT:
454
- new_value = "IfNotPresent"
455
-
456
- base_job_template["variables"]["properties"][key]["default"] = new_value
457
- elif key in base_job_template["variables"]["properties"]:
458
- base_job_template["variables"]["properties"][key]["default"] = value
459
- else:
460
- self.logger.warning(
461
- f"Variable {key!r} is not supported by Docker work pools. Skipping."
462
- )
463
-
464
- return base_job_template
465
-
466
- def get_corresponding_worker_type(self):
467
- return "docker"
468
-
469
- def _get_infrastructure_pid(self, container_id: str) -> str:
470
- """Generates a Docker infrastructure_pid string in the form of
471
- `<docker_host_base_url>:<container_id>`.
472
- """
473
- docker_client = self._get_client()
474
- base_url = docker_client.api.base_url
475
- docker_client.close()
476
- return f"{base_url}:{container_id}"
477
-
478
- def _parse_infrastructure_pid(self, infrastructure_pid: str) -> Tuple[str, str]:
479
- """Splits a Docker infrastructure_pid into its component parts"""
480
-
481
- # base_url can contain `:` so we only want the last item of the split
482
- base_url, container_id = infrastructure_pid.rsplit(":", 1)
483
- return base_url, str(container_id)
484
-
485
- def _build_container_settings(
486
- self,
487
- docker_client: "DockerClient",
488
- ) -> Dict:
489
- network_mode = self._get_network_mode()
490
- return dict(
491
- image=self.image,
492
- network=self.networks[0] if self.networks else None,
493
- network_mode=network_mode,
494
- command=self.command,
495
- environment=self._get_environment_variables(network_mode),
496
- auto_remove=self.auto_remove,
497
- labels={**CONTAINER_LABELS, **self.labels},
498
- extra_hosts=self._get_extra_hosts(docker_client),
499
- name=self._get_container_name(),
500
- volumes=self.volumes,
501
- mem_limit=self.mem_limit,
502
- memswap_limit=self.memswap_limit,
503
- privileged=self.privileged,
504
- )
505
-
506
- def _create_and_start_container(self) -> "Container":
507
- if self.image_registry:
508
- # If an image registry block was supplied, load an authenticated Docker
509
- # client from the block. Otherwise, use an unauthenticated client to
510
- # pull images from public registries.
511
- docker_client = self.image_registry.get_docker_client()
512
- else:
513
- docker_client = self._get_client()
514
- container_settings = self._build_container_settings(docker_client)
515
-
516
- if self._should_pull_image(docker_client):
517
- self.logger.info(f"Pulling image {self.image!r}...")
518
- self._pull_image(docker_client)
519
-
520
- container = self._create_container(docker_client, **container_settings)
521
-
522
- # Add additional networks after the container is created; only one network can
523
- # be attached at creation time
524
- if len(self.networks) > 1:
525
- for network_name in self.networks[1:]:
526
- network = docker_client.networks.get(network_name)
527
- network.connect(container)
528
-
529
- # Start the container
530
- container.start()
531
-
532
- docker_client.close()
533
-
534
- return container
535
-
536
- def _get_image_and_tag(self) -> Tuple[str, Optional[str]]:
537
- return parse_image_tag(self.image)
538
-
539
- def _determine_image_pull_policy(self) -> ImagePullPolicy:
540
- """
541
- Determine the appropriate image pull policy.
542
-
543
- 1. If they specified an image pull policy, use that.
544
-
545
- 2. If they did not specify an image pull policy and gave us
546
- the "latest" tag, use ImagePullPolicy.always.
547
-
548
- 3. If they did not specify an image pull policy and did not
549
- specify a tag, use ImagePullPolicy.always.
550
-
551
- 4. If they did not specify an image pull policy and gave us
552
- a tag other than "latest", use ImagePullPolicy.if_not_present.
553
-
554
- This logic matches the behavior of Kubernetes.
555
- See:https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting
556
- """
557
- if not self.image_pull_policy:
558
- _, tag = self._get_image_and_tag()
559
- if tag == "latest" or not tag:
560
- return ImagePullPolicy.ALWAYS
561
- return ImagePullPolicy.IF_NOT_PRESENT
562
- return self.image_pull_policy
563
-
564
- def _get_network_mode(self) -> Optional[str]:
565
- # User's value takes precedence; this may collide with the incompatible options
566
- # mentioned below.
567
- if self.network_mode:
568
- if sys.platform != "linux" and self.network_mode == "host":
569
- warnings.warn(
570
- f"{self.network_mode!r} network mode is not supported on platform "
571
- f"{sys.platform!r} and may not work as intended."
572
- )
573
- return self.network_mode
574
-
575
- # Network mode is not compatible with networks or ports (we do not support ports
576
- # yet though)
577
- if self.networks:
578
- return None
579
-
580
- # Check for a local API connection
581
- api_url = self.env.get("PREFECT_API_URL", PREFECT_API_URL.value())
582
-
583
- if api_url:
584
- try:
585
- _, netloc, _, _, _, _ = urllib.parse.urlparse(api_url)
586
- except Exception as exc:
587
- warnings.warn(
588
- f"Failed to parse host from API URL {api_url!r} with exception: "
589
- f"{exc}\nThe network mode will not be inferred."
590
- )
591
- return None
592
-
593
- host = netloc.split(":")[0]
594
-
595
- # If using a locally hosted API, use a host network on linux
596
- if sys.platform == "linux" and (host == "127.0.0.1" or host == "localhost"):
597
- return "host"
598
-
599
- # Default to unset
600
- return None
601
-
602
- def _should_pull_image(self, docker_client: "DockerClient") -> bool:
603
- """
604
- Decide whether we need to pull the Docker image.
605
- """
606
- image_pull_policy = self._determine_image_pull_policy()
607
-
608
- if image_pull_policy is ImagePullPolicy.ALWAYS:
609
- return True
610
- elif image_pull_policy is ImagePullPolicy.NEVER:
611
- return False
612
- elif image_pull_policy is ImagePullPolicy.IF_NOT_PRESENT:
613
- try:
614
- # NOTE: images.get() wants the tag included with the image
615
- # name, while images.pull() wants them split.
616
- docker_client.images.get(self.image)
617
- except docker.errors.ImageNotFound:
618
- self.logger.debug(f"Could not find Docker image locally: {self.image}")
619
- return True
620
- return False
621
-
622
- def _pull_image(self, docker_client: "DockerClient"):
623
- """
624
- Pull the image we're going to use to create the container.
625
- """
626
- image, tag = self._get_image_and_tag()
627
-
628
- return docker_client.images.pull(image, tag)
629
-
630
- def _create_container(self, docker_client: "DockerClient", **kwargs) -> "Container":
631
- """
632
- Create a docker container with retries on name conflicts.
633
-
634
- If the container already exists with the given name, an incremented index is
635
- added.
636
- """
637
- # Create the container with retries on name conflicts (with an incremented idx)
638
- index = 0
639
- container = None
640
- name = original_name = kwargs.pop("name")
641
-
642
- while not container:
643
- from docker.errors import APIError
644
-
645
- try:
646
- display_name = repr(name) if name else "with auto-generated name"
647
- self.logger.info(f"Creating Docker container {display_name}...")
648
- container = docker_client.containers.create(name=name, **kwargs)
649
- except APIError as exc:
650
- if "Conflict" in str(exc) and "container name" in str(exc):
651
- self.logger.info(
652
- f"Docker container name {display_name} already exists; "
653
- "retrying..."
654
- )
655
- index += 1
656
- name = f"{original_name}-{index}"
657
- else:
658
- raise
659
-
660
- self.logger.info(
661
- f"Docker container {container.name!r} has status {container.status!r}"
662
- )
663
- return container
664
-
665
- def _watch_container_safe(self, container: "Container") -> "Container":
666
- # Monitor the container capturing the latest snapshot while capturing
667
- # not found errors
668
- docker_client = self._get_client()
669
-
670
- try:
671
- for latest_container in self._watch_container(docker_client, container.id):
672
- container = latest_container
673
- except docker.errors.NotFound:
674
- # The container was removed during watching
675
- self.logger.warning(
676
- f"Docker container {container.name} was removed before we could wait "
677
- "for its completion."
678
- )
679
- finally:
680
- docker_client.close()
681
-
682
- return container
683
-
684
- def _watch_container(
685
- self, docker_client: "DockerClient", container_id: str
686
- ) -> Generator[None, None, "Container"]:
687
- container: "Container" = docker_client.containers.get(container_id)
688
-
689
- status = container.status
690
- self.logger.info(
691
- f"Docker container {container.name!r} has status {container.status!r}"
692
- )
693
- yield container
694
-
695
- if self.stream_output:
696
- try:
697
- for log in container.logs(stream=True):
698
- log: bytes
699
- print(log.decode().rstrip())
700
- except docker.errors.APIError as exc:
701
- if "marked for removal" in str(exc):
702
- self.logger.warning(
703
- f"Docker container {container.name} was marked for removal"
704
- " before logs could be retrieved. Output will not be"
705
- " streamed. "
706
- )
707
- else:
708
- self.logger.exception(
709
- "An unexpected Docker API error occurred while streaming"
710
- f" output from container {container.name}."
711
- )
712
-
713
- container.reload()
714
- if container.status != status:
715
- self.logger.info(
716
- f"Docker container {container.name!r} has status"
717
- f" {container.status!r}"
718
- )
719
- yield container
720
-
721
- container.wait()
722
- self.logger.info(
723
- f"Docker container {container.name!r} has status {container.status!r}"
724
- )
725
- yield container
726
-
727
- def _get_client(self):
728
- try:
729
- with warnings.catch_warnings():
730
- # Silence warnings due to use of deprecated methods within dockerpy
731
- # See https://github.com/docker/docker-py/pull/2931
732
- warnings.filterwarnings(
733
- "ignore",
734
- message="distutils Version classes are deprecated.*",
735
- category=DeprecationWarning,
736
- )
737
-
738
- docker_client = docker.from_env()
739
-
740
- except docker.errors.DockerException as exc:
741
- raise RuntimeError("Could not connect to Docker.") from exc
742
-
743
- return docker_client
744
-
745
- def _get_container_name(self) -> Optional[str]:
746
- """
747
- Generates a container name to match the configured name, ensuring it is Docker
748
- compatible.
749
- """
750
- # Must match `/?[a-zA-Z0-9][a-zA-Z0-9_.-]+` in the end
751
- if not self.name:
752
- return None
753
-
754
- return (
755
- slugify(
756
- self.name,
757
- lowercase=False,
758
- # Docker does not limit length but URL limits apply eventually so
759
- # limit the length for safety
760
- max_length=250,
761
- # Docker allows these characters for container names
762
- regex_pattern=r"[^a-zA-Z0-9_.-]+",
763
- ).lstrip(
764
- # Docker does not allow leading underscore, dash, or period
765
- "_-."
766
- )
767
- # Docker does not allow 0 character names so cast to null if the name is
768
- # empty after slufification
769
- or None
770
- )
771
-
772
- def _get_extra_hosts(self, docker_client) -> Dict[str, str]:
773
- """
774
- A host.docker.internal -> host-gateway mapping is necessary for communicating
775
- with the API on Linux machines. Docker Desktop on macOS will automatically
776
- already have this mapping.
777
- """
778
- if sys.platform == "linux" and (
779
- # Do not warn if the user has specified a host manually that does not use
780
- # a local address
781
- "PREFECT_API_URL" not in self.env
782
- or re.search(
783
- ".*(localhost)|(127.0.0.1)|(host.docker.internal).*",
784
- self.env["PREFECT_API_URL"],
785
- )
786
- ):
787
- user_version = packaging.version.parse(
788
- format_outlier_version_name(docker_client.version()["Version"])
789
- )
790
- required_version = packaging.version.parse("20.10.0")
791
-
792
- if user_version < required_version:
793
- warnings.warn(
794
- "`host.docker.internal` could not be automatically resolved to"
795
- " your local ip address. This feature is not supported on Docker"
796
- f" Engine v{user_version}, upgrade to v{required_version}+ if you"
797
- " encounter issues."
798
- )
799
- return {}
800
- else:
801
- # Compatibility for linux -- https://github.com/docker/cli/issues/2290
802
- # Only supported by Docker v20.10.0+ which is our minimum recommend version
803
- return {"host.docker.internal": "host-gateway"}
804
-
805
- def _get_environment_variables(self, network_mode):
806
- # If the API URL has been set by the base environment rather than the by the
807
- # user, update the value to ensure connectivity when using a bridge network by
808
- # updating local connections to use the docker internal host unless the
809
- # network mode is "host" where localhost is available already.
810
- env = {**self._base_environment(), **self.env}
811
-
812
- if (
813
- "PREFECT_API_URL" in env
814
- and "PREFECT_API_URL" not in self.env
815
- and network_mode != "host"
816
- ):
817
- env["PREFECT_API_URL"] = (
818
- env["PREFECT_API_URL"]
819
- .replace("localhost", "host.docker.internal")
820
- .replace("127.0.0.1", "host.docker.internal")
821
- )
822
-
823
- # Drop null values allowing users to "unset" variables
824
- return {key: value for key, value in env.items() if value is not None}