prefect-client 3.1.4__py3-none-any.whl → 3.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +3 -0
- prefect/_internal/compatibility/migration.py +1 -1
- prefect/_internal/concurrency/api.py +52 -52
- prefect/_internal/concurrency/calls.py +59 -35
- prefect/_internal/concurrency/cancellation.py +34 -18
- prefect/_internal/concurrency/event_loop.py +7 -6
- prefect/_internal/concurrency/threads.py +41 -33
- prefect/_internal/concurrency/waiters.py +28 -21
- prefect/_internal/pydantic/v1_schema.py +2 -2
- prefect/_internal/pydantic/v2_schema.py +10 -9
- prefect/_internal/schemas/bases.py +10 -11
- prefect/_internal/schemas/validators.py +2 -1
- prefect/_version.py +3 -3
- prefect/automations.py +53 -47
- prefect/blocks/abstract.py +12 -10
- prefect/blocks/core.py +4 -2
- prefect/cache_policies.py +11 -11
- prefect/client/__init__.py +3 -1
- prefect/client/base.py +36 -37
- prefect/client/cloud.py +26 -19
- prefect/client/collections.py +2 -2
- prefect/client/orchestration.py +366 -277
- prefect/client/schemas/__init__.py +24 -0
- prefect/client/schemas/actions.py +132 -120
- prefect/client/schemas/filters.py +5 -0
- prefect/client/schemas/objects.py +113 -85
- prefect/client/schemas/responses.py +21 -18
- prefect/client/schemas/schedules.py +136 -93
- prefect/client/subscriptions.py +28 -14
- prefect/client/utilities.py +32 -36
- prefect/concurrency/asyncio.py +6 -9
- prefect/concurrency/services.py +3 -0
- prefect/concurrency/sync.py +35 -5
- prefect/context.py +39 -31
- prefect/deployments/flow_runs.py +3 -5
- prefect/docker/__init__.py +1 -1
- prefect/events/schemas/events.py +25 -20
- prefect/events/utilities.py +1 -2
- prefect/filesystems.py +3 -3
- prefect/flow_engine.py +755 -138
- prefect/flow_runs.py +3 -3
- prefect/flows.py +214 -170
- prefect/logging/configuration.py +1 -1
- prefect/logging/highlighters.py +1 -2
- prefect/logging/loggers.py +30 -20
- prefect/main.py +17 -24
- prefect/runner/runner.py +43 -21
- prefect/runner/server.py +30 -32
- prefect/runner/submit.py +3 -6
- prefect/runner/utils.py +6 -6
- prefect/runtime/flow_run.py +7 -0
- prefect/settings/constants.py +2 -2
- prefect/settings/legacy.py +1 -1
- prefect/settings/models/server/events.py +10 -0
- prefect/settings/sources.py +9 -2
- prefect/task_engine.py +72 -19
- prefect/task_runners.py +2 -2
- prefect/tasks.py +46 -33
- prefect/telemetry/bootstrap.py +15 -2
- prefect/telemetry/run_telemetry.py +107 -0
- prefect/transactions.py +14 -14
- prefect/types/__init__.py +20 -3
- prefect/utilities/_engine.py +96 -0
- prefect/utilities/annotations.py +25 -18
- prefect/utilities/asyncutils.py +126 -140
- prefect/utilities/callables.py +87 -78
- prefect/utilities/collections.py +278 -117
- prefect/utilities/compat.py +13 -21
- prefect/utilities/context.py +6 -5
- prefect/utilities/dispatch.py +23 -12
- prefect/utilities/dockerutils.py +33 -32
- prefect/utilities/engine.py +126 -239
- prefect/utilities/filesystem.py +18 -15
- prefect/utilities/hashing.py +10 -11
- prefect/utilities/importtools.py +40 -27
- prefect/utilities/math.py +9 -5
- prefect/utilities/names.py +3 -3
- prefect/utilities/processutils.py +121 -57
- prefect/utilities/pydantic.py +41 -36
- prefect/utilities/render_swagger.py +22 -12
- prefect/utilities/schema_tools/__init__.py +2 -1
- prefect/utilities/schema_tools/hydration.py +50 -43
- prefect/utilities/schema_tools/validation.py +52 -42
- prefect/utilities/services.py +13 -12
- prefect/utilities/templating.py +45 -45
- prefect/utilities/text.py +2 -1
- prefect/utilities/timeout.py +4 -4
- prefect/utilities/urls.py +9 -4
- prefect/utilities/visualization.py +46 -24
- prefect/variables.py +9 -8
- prefect/workers/base.py +18 -10
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/METADATA +5 -5
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/RECORD +96 -94
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/WHEEL +1 -1
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/top_level.txt +0 -0
prefect/utilities/context.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
|
+
from collections.abc import Generator
|
1
2
|
from contextlib import contextmanager
|
2
3
|
from contextvars import Context, ContextVar, Token
|
3
|
-
from typing import TYPE_CHECKING,
|
4
|
+
from typing import TYPE_CHECKING, Any, Optional, cast
|
4
5
|
from uuid import UUID
|
5
6
|
|
6
7
|
if TYPE_CHECKING:
|
@@ -8,8 +9,8 @@ if TYPE_CHECKING:
|
|
8
9
|
|
9
10
|
|
10
11
|
@contextmanager
|
11
|
-
def temporary_context(context: Context):
|
12
|
-
tokens:
|
12
|
+
def temporary_context(context: Context) -> Generator[None, Any, None]:
|
13
|
+
tokens: dict[ContextVar[Any], Token[Any]] = {}
|
13
14
|
for key, value in context.items():
|
14
15
|
token = key.set(value)
|
15
16
|
tokens[key] = token
|
@@ -38,11 +39,11 @@ def get_flow_run_id() -> Optional[UUID]:
|
|
38
39
|
return None
|
39
40
|
|
40
41
|
|
41
|
-
def get_task_and_flow_run_ids() ->
|
42
|
+
def get_task_and_flow_run_ids() -> tuple[Optional[UUID], Optional[UUID]]:
|
42
43
|
"""
|
43
44
|
Get the task run and flow run ids from the context, if available.
|
44
45
|
|
45
46
|
Returns:
|
46
|
-
|
47
|
+
tuple[Optional[UUID], Optional[UUID]]: a tuple of the task run id and flow run id
|
47
48
|
"""
|
48
49
|
return get_task_run_id(), get_flow_run_id()
|
prefect/utilities/dispatch.py
CHANGED
@@ -23,28 +23,39 @@ lookup_type(Base, key) # Foo
|
|
23
23
|
import abc
|
24
24
|
import inspect
|
25
25
|
import warnings
|
26
|
-
from typing import Any,
|
26
|
+
from typing import Any, Literal, Optional, TypeVar, overload
|
27
27
|
|
28
|
-
T = TypeVar("T", bound=
|
28
|
+
T = TypeVar("T", bound=type[Any])
|
29
29
|
|
30
|
-
_TYPE_REGISTRIES:
|
30
|
+
_TYPE_REGISTRIES: dict[Any, dict[str, Any]] = {}
|
31
31
|
|
32
32
|
|
33
|
-
def get_registry_for_type(cls: T) -> Optional[
|
33
|
+
def get_registry_for_type(cls: T) -> Optional[dict[str, T]]:
|
34
34
|
"""
|
35
35
|
Get the first matching registry for a class or any of its base classes.
|
36
36
|
|
37
37
|
If not found, `None` is returned.
|
38
38
|
"""
|
39
39
|
return next(
|
40
|
-
|
41
|
-
lambda registry: registry is not None,
|
42
|
-
(_TYPE_REGISTRIES.get(cls) for cls in cls.mro()),
|
43
|
-
),
|
40
|
+
(reg for cls in cls.mro() if (reg := _TYPE_REGISTRIES.get(cls)) is not None),
|
44
41
|
None,
|
45
42
|
)
|
46
43
|
|
47
44
|
|
45
|
+
@overload
|
46
|
+
def get_dispatch_key(
|
47
|
+
cls_or_instance: Any, allow_missing: Literal[False] = False
|
48
|
+
) -> str:
|
49
|
+
...
|
50
|
+
|
51
|
+
|
52
|
+
@overload
|
53
|
+
def get_dispatch_key(
|
54
|
+
cls_or_instance: Any, allow_missing: Literal[True] = ...
|
55
|
+
) -> Optional[str]:
|
56
|
+
...
|
57
|
+
|
58
|
+
|
48
59
|
def get_dispatch_key(
|
49
60
|
cls_or_instance: Any, allow_missing: bool = False
|
50
61
|
) -> Optional[str]:
|
@@ -89,14 +100,14 @@ def get_dispatch_key(
|
|
89
100
|
|
90
101
|
|
91
102
|
@classmethod
|
92
|
-
def _register_subclass_of_base_type(cls, **kwargs):
|
103
|
+
def _register_subclass_of_base_type(cls: type[Any], **kwargs: Any) -> None:
|
93
104
|
if hasattr(cls, "__init_subclass_original__"):
|
94
105
|
cls.__init_subclass_original__(**kwargs)
|
95
106
|
elif hasattr(cls, "__pydantic_init_subclass_original__"):
|
96
107
|
cls.__pydantic_init_subclass_original__(**kwargs)
|
97
108
|
|
98
109
|
# Do not register abstract base classes
|
99
|
-
if abc.ABC in
|
110
|
+
if abc.ABC in cls.__bases__:
|
100
111
|
return
|
101
112
|
|
102
113
|
register_type(cls)
|
@@ -123,7 +134,7 @@ def register_base_type(cls: T) -> T:
|
|
123
134
|
cls.__pydantic_init_subclass__ = _register_subclass_of_base_type
|
124
135
|
else:
|
125
136
|
cls.__init_subclass_original__ = getattr(cls, "__init_subclass__")
|
126
|
-
cls
|
137
|
+
setattr(cls, "__init_subclass__", _register_subclass_of_base_type)
|
127
138
|
|
128
139
|
return cls
|
129
140
|
|
@@ -190,7 +201,7 @@ def lookup_type(cls: T, dispatch_key: str) -> T:
|
|
190
201
|
Look up a dispatch key in the type registry for the given class.
|
191
202
|
"""
|
192
203
|
# Get the first matching registry for the class or one of its bases
|
193
|
-
registry = get_registry_for_type(cls)
|
204
|
+
registry = get_registry_for_type(cls) or {}
|
194
205
|
|
195
206
|
# Look up this type in the registry
|
196
207
|
subcls = registry.get(dispatch_key)
|
prefect/utilities/dockerutils.py
CHANGED
@@ -3,22 +3,12 @@ import os
|
|
3
3
|
import shutil
|
4
4
|
import sys
|
5
5
|
import warnings
|
6
|
+
from collections.abc import Generator, Iterable, Iterator
|
6
7
|
from contextlib import contextmanager
|
7
8
|
from pathlib import Path, PurePosixPath
|
8
9
|
from tempfile import TemporaryDirectory
|
9
10
|
from types import TracebackType
|
10
|
-
from typing import
|
11
|
-
TYPE_CHECKING,
|
12
|
-
Any,
|
13
|
-
Generator,
|
14
|
-
Iterable,
|
15
|
-
List,
|
16
|
-
Optional,
|
17
|
-
TextIO,
|
18
|
-
Tuple,
|
19
|
-
Type,
|
20
|
-
Union,
|
21
|
-
)
|
11
|
+
from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast
|
22
12
|
from urllib.parse import urlsplit
|
23
13
|
|
24
14
|
import pendulum
|
@@ -29,6 +19,12 @@ import prefect
|
|
29
19
|
from prefect.utilities.importtools import lazy_import
|
30
20
|
from prefect.utilities.slugify import slugify
|
31
21
|
|
22
|
+
if TYPE_CHECKING:
|
23
|
+
import docker
|
24
|
+
import docker.errors
|
25
|
+
from docker import DockerClient
|
26
|
+
from docker.models.images import Image
|
27
|
+
|
32
28
|
CONTAINER_LABELS = {
|
33
29
|
"io.prefect.version": prefect.__version__,
|
34
30
|
}
|
@@ -102,10 +98,7 @@ def silence_docker_warnings() -> Generator[None, None, None]:
|
|
102
98
|
# want to have those popping up in various modules and test suites. Instead,
|
103
99
|
# consolidate the imports we need here, and expose them via this module.
|
104
100
|
with silence_docker_warnings():
|
105
|
-
if TYPE_CHECKING:
|
106
|
-
import docker
|
107
|
-
from docker import DockerClient
|
108
|
-
else:
|
101
|
+
if not TYPE_CHECKING:
|
109
102
|
docker = lazy_import("docker")
|
110
103
|
|
111
104
|
|
@@ -123,7 +116,8 @@ def docker_client() -> Generator["DockerClient", None, None]:
|
|
123
116
|
"This error is often thrown because Docker is not running. Please ensure Docker is running."
|
124
117
|
) from exc
|
125
118
|
finally:
|
126
|
-
client is not None
|
119
|
+
if client is not None:
|
120
|
+
client.close() # type: ignore # typing stub is not complete
|
127
121
|
|
128
122
|
|
129
123
|
class BuildError(Exception):
|
@@ -207,14 +201,15 @@ class ImageBuilder:
|
|
207
201
|
base_directory: Path
|
208
202
|
context: Optional[Path]
|
209
203
|
platform: Optional[str]
|
210
|
-
dockerfile_lines:
|
204
|
+
dockerfile_lines: list[str]
|
205
|
+
temporary_directory: Optional[TemporaryDirectory[str]]
|
211
206
|
|
212
207
|
def __init__(
|
213
208
|
self,
|
214
209
|
base_image: str,
|
215
|
-
base_directory: Path = None,
|
210
|
+
base_directory: Optional[Path] = None,
|
216
211
|
platform: Optional[str] = None,
|
217
|
-
context: Path = None,
|
212
|
+
context: Optional[Path] = None,
|
218
213
|
):
|
219
214
|
"""Create an ImageBuilder
|
220
215
|
|
@@ -250,7 +245,7 @@ class ImageBuilder:
|
|
250
245
|
return self
|
251
246
|
|
252
247
|
def __exit__(
|
253
|
-
self, exc:
|
248
|
+
self, exc: type[BaseException], value: BaseException, traceback: TracebackType
|
254
249
|
) -> None:
|
255
250
|
if not self.temporary_directory:
|
256
251
|
return
|
@@ -267,7 +262,9 @@ class ImageBuilder:
|
|
267
262
|
"""Add lines to this image's Dockerfile"""
|
268
263
|
self.dockerfile_lines.extend(lines)
|
269
264
|
|
270
|
-
def copy(
|
265
|
+
def copy(
|
266
|
+
self, source: Union[str, Path], destination: Union[str, PurePosixPath]
|
267
|
+
) -> None:
|
271
268
|
"""Copy a file to this image"""
|
272
269
|
if not self.context:
|
273
270
|
raise Exception("No context available")
|
@@ -291,7 +288,7 @@ class ImageBuilder:
|
|
291
288
|
|
292
289
|
self.add_line(f"COPY {source} {destination}")
|
293
290
|
|
294
|
-
def write_text(self, text: str, destination: Union[str, PurePosixPath]):
|
291
|
+
def write_text(self, text: str, destination: Union[str, PurePosixPath]) -> None:
|
295
292
|
if not self.context:
|
296
293
|
raise Exception("No context available")
|
297
294
|
|
@@ -315,6 +312,7 @@ class ImageBuilder:
|
|
315
312
|
Returns:
|
316
313
|
The image ID
|
317
314
|
"""
|
315
|
+
assert self.context is not None
|
318
316
|
dockerfile_path: Path = self.context / "Dockerfile"
|
319
317
|
|
320
318
|
with dockerfile_path.open("w") as dockerfile:
|
@@ -436,9 +434,12 @@ def push_image(
|
|
436
434
|
repository = f"{registry}/{name}"
|
437
435
|
|
438
436
|
with docker_client() as client:
|
439
|
-
image: "
|
440
|
-
image.tag(repository, tag=tag)
|
441
|
-
events =
|
437
|
+
image: "Image" = client.images.get(image_id)
|
438
|
+
image.tag(repository, tag=tag) # type: ignore # typing stub is not complete
|
439
|
+
events = cast(
|
440
|
+
Iterator[dict[str, Any]],
|
441
|
+
client.api.push(repository, tag=tag, stream=True, decode=True), # type: ignore # typing stub is not complete
|
442
|
+
)
|
442
443
|
try:
|
443
444
|
for event in events:
|
444
445
|
if "status" in event:
|
@@ -452,12 +453,12 @@ def push_image(
|
|
452
453
|
elif "error" in event:
|
453
454
|
raise PushError(event["error"])
|
454
455
|
finally:
|
455
|
-
client.api.remove_image(f"{repository}:{tag}", noprune=True)
|
456
|
+
client.api.remove_image(f"{repository}:{tag}", noprune=True) # type: ignore # typing stub is not complete
|
456
457
|
|
457
458
|
return f"{repository}:{tag}"
|
458
459
|
|
459
460
|
|
460
|
-
def to_run_command(command:
|
461
|
+
def to_run_command(command: list[str]) -> str:
|
461
462
|
"""
|
462
463
|
Convert a process-style list of command arguments to a single Dockerfile RUN
|
463
464
|
instruction.
|
@@ -481,7 +482,7 @@ def to_run_command(command: List[str]) -> str:
|
|
481
482
|
return run_command
|
482
483
|
|
483
484
|
|
484
|
-
def parse_image_tag(name: str) ->
|
485
|
+
def parse_image_tag(name: str) -> tuple[str, Optional[str]]:
|
485
486
|
"""
|
486
487
|
Parse Docker Image String
|
487
488
|
|
@@ -519,7 +520,7 @@ def parse_image_tag(name: str) -> Tuple[str, Optional[str]]:
|
|
519
520
|
return image_name, tag
|
520
521
|
|
521
522
|
|
522
|
-
def split_repository_path(repository_path: str) ->
|
523
|
+
def split_repository_path(repository_path: str) -> tuple[Optional[str], str]:
|
523
524
|
"""
|
524
525
|
Splits a Docker repository path into its namespace and repository components.
|
525
526
|
|
@@ -550,7 +551,7 @@ def split_repository_path(repository_path: str) -> Tuple[Optional[str], str]:
|
|
550
551
|
return namespace, repository
|
551
552
|
|
552
553
|
|
553
|
-
def format_outlier_version_name(version: str):
|
554
|
+
def format_outlier_version_name(version: str) -> str:
|
554
555
|
"""
|
555
556
|
Formats outlier docker version names to pass `packaging.version.parse` validation
|
556
557
|
- Current cases are simple, but creates stub for more complicated formatting if eventually needed.
|
@@ -580,7 +581,7 @@ def generate_default_dockerfile(context: Optional[Path] = None):
|
|
580
581
|
"""
|
581
582
|
if not context:
|
582
583
|
context = Path.cwd()
|
583
|
-
lines = []
|
584
|
+
lines: list[str] = []
|
584
585
|
base_image = get_prefect_image_name()
|
585
586
|
lines.append(f"FROM {base_image}")
|
586
587
|
dir_name = context.name
|