prefect-client 3.0.0rc4__py3-none-any.whl → 3.0.0rc6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +0 -2
- prefect/{records/cache_policies.py → cache_policies.py} +78 -23
- prefect/client/schemas/schedules.py +9 -2
- prefect/client/types/__init__.py +0 -0
- prefect/client/types/flexible_schedule_list.py +11 -0
- prefect/concurrency/asyncio.py +14 -4
- prefect/concurrency/services.py +29 -22
- prefect/concurrency/sync.py +3 -5
- prefect/context.py +0 -114
- prefect/deployments/__init__.py +1 -1
- prefect/deployments/runner.py +11 -93
- prefect/deployments/schedules.py +5 -7
- prefect/docker/__init__.py +20 -0
- prefect/docker/docker_image.py +82 -0
- prefect/flow_engine.py +14 -18
- prefect/flows.py +24 -93
- prefect/futures.py +13 -1
- prefect/infrastructure/provisioners/cloud_run.py +2 -2
- prefect/infrastructure/provisioners/container_instance.py +2 -2
- prefect/infrastructure/provisioners/ecs.py +2 -2
- prefect/records/result_store.py +5 -1
- prefect/results.py +78 -11
- prefect/runner/runner.py +5 -3
- prefect/runner/server.py +6 -2
- prefect/states.py +13 -3
- prefect/task_engine.py +10 -1
- prefect/tasks.py +8 -6
- prefect/transactions.py +2 -2
- prefect/types/entrypoint.py +13 -0
- prefect/utilities/dockerutils.py +2 -1
- {prefect_client-3.0.0rc4.dist-info → prefect_client-3.0.0rc6.dist-info}/METADATA +1 -1
- {prefect_client-3.0.0rc4.dist-info → prefect_client-3.0.0rc6.dist-info}/RECORD +35 -30
- {prefect_client-3.0.0rc4.dist-info → prefect_client-3.0.0rc6.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.0rc4.dist-info → prefect_client-3.0.0rc6.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.0rc4.dist-info → prefect_client-3.0.0rc6.dist-info}/top_level.txt +0 -0
prefect/deployments/runner.py
CHANGED
@@ -29,7 +29,6 @@ Example:
|
|
29
29
|
|
30
30
|
"""
|
31
31
|
|
32
|
-
import enum
|
33
32
|
import importlib
|
34
33
|
import tempfile
|
35
34
|
from datetime import datetime, timedelta
|
@@ -37,7 +36,6 @@ from pathlib import Path
|
|
37
36
|
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
38
37
|
from uuid import UUID
|
39
38
|
|
40
|
-
import pendulum
|
41
39
|
from pydantic import (
|
42
40
|
BaseModel,
|
43
41
|
ConfigDict,
|
@@ -61,9 +59,9 @@ from prefect.client.schemas.schedules import (
|
|
61
59
|
construct_schedule,
|
62
60
|
)
|
63
61
|
from prefect.deployments.schedules import (
|
64
|
-
FlexibleScheduleList,
|
65
62
|
create_deployment_schedule_create,
|
66
63
|
)
|
64
|
+
from prefect.docker.docker_image import DockerImage
|
67
65
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
68
66
|
from prefect.exceptions import (
|
69
67
|
ObjectNotFound,
|
@@ -71,24 +69,19 @@ from prefect.exceptions import (
|
|
71
69
|
)
|
72
70
|
from prefect.runner.storage import RunnerStorage
|
73
71
|
from prefect.settings import (
|
74
|
-
PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE,
|
75
72
|
PREFECT_DEFAULT_WORK_POOL_NAME,
|
76
73
|
PREFECT_UI_URL,
|
77
74
|
)
|
75
|
+
from prefect.types.entrypoint import EntrypointType
|
78
76
|
from prefect.utilities.asyncutils import sync_compatible
|
79
77
|
from prefect.utilities.callables import ParameterSchema, parameter_schema
|
80
78
|
from prefect.utilities.collections import get_from_dict, isiterable
|
81
79
|
from prefect.utilities.dockerutils import (
|
82
|
-
PushError,
|
83
|
-
build_image,
|
84
|
-
docker_client,
|
85
|
-
generate_default_dockerfile,
|
86
80
|
parse_image_tag,
|
87
|
-
split_repository_path,
|
88
81
|
)
|
89
|
-
from prefect.utilities.slugify import slugify
|
90
82
|
|
91
83
|
if TYPE_CHECKING:
|
84
|
+
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
92
85
|
from prefect.flows import Flow
|
93
86
|
|
94
87
|
__all__ = ["RunnerDeployment"]
|
@@ -100,18 +93,6 @@ class DeploymentApplyError(RuntimeError):
|
|
100
93
|
"""
|
101
94
|
|
102
95
|
|
103
|
-
class EntrypointType(enum.Enum):
|
104
|
-
"""
|
105
|
-
Enum representing a entrypoint type.
|
106
|
-
|
107
|
-
File path entrypoints are in the format: `path/to/file.py:function_name`.
|
108
|
-
Module path entrypoints are in the format: `path.to.module.function_name`.
|
109
|
-
"""
|
110
|
-
|
111
|
-
FILE_PATH = "file_path"
|
112
|
-
MODULE_PATH = "module_path"
|
113
|
-
|
114
|
-
|
115
96
|
class RunnerDeployment(BaseModel):
|
116
97
|
"""
|
117
98
|
A Prefect RunnerDeployment definition, used for specifying and building deployments.
|
@@ -363,8 +344,8 @@ class RunnerDeployment(BaseModel):
|
|
363
344
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
364
345
|
timezone: Optional[str] = None,
|
365
346
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
366
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
367
|
-
) -> Union[List[DeploymentScheduleCreate], FlexibleScheduleList]:
|
347
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
348
|
+
) -> Union[List[DeploymentScheduleCreate], "FlexibleScheduleList"]:
|
368
349
|
"""
|
369
350
|
Construct a schedule or schedules from the provided arguments.
|
370
351
|
|
@@ -455,7 +436,7 @@ class RunnerDeployment(BaseModel):
|
|
455
436
|
cron: Optional[Union[Iterable[str], str]] = None,
|
456
437
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
457
438
|
paused: Optional[bool] = None,
|
458
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
439
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
459
440
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
460
441
|
is_schedule_active: Optional[bool] = None,
|
461
442
|
parameters: Optional[dict] = None,
|
@@ -591,7 +572,7 @@ class RunnerDeployment(BaseModel):
|
|
591
572
|
cron: Optional[Union[Iterable[str], str]] = None,
|
592
573
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
593
574
|
paused: Optional[bool] = None,
|
594
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
575
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
595
576
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
596
577
|
is_schedule_active: Optional[bool] = None,
|
597
578
|
parameters: Optional[dict] = None,
|
@@ -689,7 +670,7 @@ class RunnerDeployment(BaseModel):
|
|
689
670
|
cron: Optional[Union[Iterable[str], str]] = None,
|
690
671
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
691
672
|
paused: Optional[bool] = None,
|
692
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
673
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
693
674
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
694
675
|
is_schedule_active: Optional[bool] = None,
|
695
676
|
parameters: Optional[dict] = None,
|
@@ -786,74 +767,11 @@ class RunnerDeployment(BaseModel):
|
|
786
767
|
return deployment
|
787
768
|
|
788
769
|
|
789
|
-
class DeploymentImage:
|
790
|
-
"""
|
791
|
-
Configuration used to build and push a Docker image for a deployment.
|
792
|
-
|
793
|
-
Attributes:
|
794
|
-
name: The name of the Docker image to build, including the registry and
|
795
|
-
repository.
|
796
|
-
tag: The tag to apply to the built image.
|
797
|
-
dockerfile: The path to the Dockerfile to use for building the image. If
|
798
|
-
not provided, a default Dockerfile will be generated.
|
799
|
-
**build_kwargs: Additional keyword arguments to pass to the Docker build request.
|
800
|
-
See the [`docker-py` documentation](https://docker-py.readthedocs.io/en/stable/images.html#docker.models.images.ImageCollection.build)
|
801
|
-
for more information.
|
802
|
-
|
803
|
-
"""
|
804
|
-
|
805
|
-
def __init__(self, name, tag=None, dockerfile="auto", **build_kwargs):
|
806
|
-
image_name, image_tag = parse_image_tag(name)
|
807
|
-
if tag and image_tag:
|
808
|
-
raise ValueError(
|
809
|
-
f"Only one tag can be provided - both {image_tag!r} and {tag!r} were"
|
810
|
-
" provided as tags."
|
811
|
-
)
|
812
|
-
namespace, repository = split_repository_path(image_name)
|
813
|
-
# if the provided image name does not include a namespace (registry URL or user/org name),
|
814
|
-
# use the default namespace
|
815
|
-
if not namespace:
|
816
|
-
namespace = PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE.value()
|
817
|
-
# join the namespace and repository to create the full image name
|
818
|
-
# ignore namespace if it is None
|
819
|
-
self.name = "/".join(filter(None, [namespace, repository]))
|
820
|
-
self.tag = tag or image_tag or slugify(pendulum.now("utc").isoformat())
|
821
|
-
self.dockerfile = dockerfile
|
822
|
-
self.build_kwargs = build_kwargs
|
823
|
-
|
824
|
-
@property
|
825
|
-
def reference(self):
|
826
|
-
return f"{self.name}:{self.tag}"
|
827
|
-
|
828
|
-
def build(self):
|
829
|
-
full_image_name = self.reference
|
830
|
-
build_kwargs = self.build_kwargs.copy()
|
831
|
-
build_kwargs["context"] = Path.cwd()
|
832
|
-
build_kwargs["tag"] = full_image_name
|
833
|
-
build_kwargs["pull"] = build_kwargs.get("pull", True)
|
834
|
-
|
835
|
-
if self.dockerfile == "auto":
|
836
|
-
with generate_default_dockerfile():
|
837
|
-
build_image(**build_kwargs)
|
838
|
-
else:
|
839
|
-
build_kwargs["dockerfile"] = self.dockerfile
|
840
|
-
build_image(**build_kwargs)
|
841
|
-
|
842
|
-
def push(self):
|
843
|
-
with docker_client() as client:
|
844
|
-
events = client.api.push(
|
845
|
-
repository=self.name, tag=self.tag, stream=True, decode=True
|
846
|
-
)
|
847
|
-
for event in events:
|
848
|
-
if "error" in event:
|
849
|
-
raise PushError(event["error"])
|
850
|
-
|
851
|
-
|
852
770
|
@sync_compatible
|
853
771
|
async def deploy(
|
854
772
|
*deployments: RunnerDeployment,
|
855
773
|
work_pool_name: Optional[str] = None,
|
856
|
-
image: Optional[Union[str,
|
774
|
+
image: Optional[Union[str, DockerImage]] = None,
|
857
775
|
build: bool = True,
|
858
776
|
push: bool = True,
|
859
777
|
print_next_steps_message: bool = True,
|
@@ -875,7 +793,7 @@ async def deploy(
|
|
875
793
|
work_pool_name: The name of the work pool to use for these deployments. Defaults to
|
876
794
|
the value of `PREFECT_DEFAULT_WORK_POOL_NAME`.
|
877
795
|
image: The name of the Docker image to build, including the registry and
|
878
|
-
repository. Pass a
|
796
|
+
repository. Pass a DockerImage instance to customize the Dockerfile used
|
879
797
|
and build arguments.
|
880
798
|
build: Whether or not to build a new image for the flow. If False, the provided
|
881
799
|
image will be used as-is and pulled at runtime.
|
@@ -930,7 +848,7 @@ async def deploy(
|
|
930
848
|
|
931
849
|
if image and isinstance(image, str):
|
932
850
|
image_name, image_tag = parse_image_tag(image)
|
933
|
-
image =
|
851
|
+
image = DockerImage(name=image_name, tag=image_tag)
|
934
852
|
|
935
853
|
try:
|
936
854
|
async with get_client() as client:
|
prefect/deployments/schedules.py
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
from typing import TYPE_CHECKING, Any, List, Optional
|
1
|
+
from typing import TYPE_CHECKING, Any, List, Optional
|
2
2
|
|
3
3
|
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
4
|
+
from prefect.client.schemas.schedules import is_schedule_type
|
4
5
|
|
5
6
|
if TYPE_CHECKING:
|
6
7
|
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
7
|
-
|
8
|
-
FlexibleScheduleList = Sequence[Union[DeploymentScheduleCreate, dict, "SCHEDULE_TYPES"]]
|
8
|
+
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
9
9
|
|
10
10
|
|
11
11
|
def create_deployment_schedule_create(
|
@@ -26,12 +26,10 @@ def create_deployment_schedule_create(
|
|
26
26
|
def normalize_to_deployment_schedule_create(
|
27
27
|
schedules: Optional["FlexibleScheduleList"],
|
28
28
|
) -> List[DeploymentScheduleCreate]:
|
29
|
-
|
30
|
-
|
31
|
-
normalized = []
|
29
|
+
normalized: list[DeploymentScheduleCreate] = []
|
32
30
|
if schedules is not None:
|
33
31
|
for obj in schedules:
|
34
|
-
if
|
32
|
+
if is_schedule_type(obj):
|
35
33
|
normalized.append(create_deployment_schedule_create(obj))
|
36
34
|
elif isinstance(obj, dict):
|
37
35
|
normalized.append(create_deployment_schedule_create(**obj))
|
@@ -0,0 +1,20 @@
|
|
1
|
+
from typing import TYPE_CHECKING
|
2
|
+
|
3
|
+
if TYPE_CHECKING:
|
4
|
+
from prefect.docker.docker_image import DockerImage
|
5
|
+
|
6
|
+
__all__ = ["DockerImage"]
|
7
|
+
|
8
|
+
_public_api: dict[str, tuple[str, str]] = {
|
9
|
+
"DockerImage": ("prefect.docker.docker_image", "DockerImage"),
|
10
|
+
}
|
11
|
+
|
12
|
+
|
13
|
+
def __getattr__(name: str) -> object:
|
14
|
+
from importlib import import_module
|
15
|
+
|
16
|
+
if name in _public_api:
|
17
|
+
module, attr = _public_api[name]
|
18
|
+
return getattr(import_module(module), attr)
|
19
|
+
|
20
|
+
raise ImportError(f"module {__name__!r} has no attribute {name!r}")
|
@@ -0,0 +1,82 @@
|
|
1
|
+
from pathlib import Path
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
from pendulum import now as pendulum_now
|
5
|
+
|
6
|
+
from prefect.settings import (
|
7
|
+
PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE,
|
8
|
+
)
|
9
|
+
from prefect.utilities.dockerutils import (
|
10
|
+
PushError,
|
11
|
+
build_image,
|
12
|
+
docker_client,
|
13
|
+
generate_default_dockerfile,
|
14
|
+
parse_image_tag,
|
15
|
+
split_repository_path,
|
16
|
+
)
|
17
|
+
from prefect.utilities.slugify import slugify
|
18
|
+
|
19
|
+
|
20
|
+
class DockerImage:
|
21
|
+
"""
|
22
|
+
Configuration used to build and push a Docker image for a deployment.
|
23
|
+
|
24
|
+
Attributes:
|
25
|
+
name: The name of the Docker image to build, including the registry and
|
26
|
+
repository.
|
27
|
+
tag: The tag to apply to the built image.
|
28
|
+
dockerfile: The path to the Dockerfile to use for building the image. If
|
29
|
+
not provided, a default Dockerfile will be generated.
|
30
|
+
**build_kwargs: Additional keyword arguments to pass to the Docker build request.
|
31
|
+
See the [`docker-py` documentation](https://docker-py.readthedocs.io/en/stable/images.html#docker.models.images.ImageCollection.build)
|
32
|
+
for more information.
|
33
|
+
|
34
|
+
"""
|
35
|
+
|
36
|
+
def __init__(
|
37
|
+
self, name: str, tag: Optional[str] = None, dockerfile="auto", **build_kwargs
|
38
|
+
):
|
39
|
+
image_name, image_tag = parse_image_tag(name)
|
40
|
+
if tag and image_tag:
|
41
|
+
raise ValueError(
|
42
|
+
f"Only one tag can be provided - both {image_tag!r} and {tag!r} were"
|
43
|
+
" provided as tags."
|
44
|
+
)
|
45
|
+
namespace, repository = split_repository_path(image_name)
|
46
|
+
# if the provided image name does not include a namespace (registry URL or user/org name),
|
47
|
+
# use the default namespace
|
48
|
+
if not namespace:
|
49
|
+
namespace = PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE.value()
|
50
|
+
# join the namespace and repository to create the full image name
|
51
|
+
# ignore namespace if it is None
|
52
|
+
self.name = "/".join(filter(None, [namespace, repository]))
|
53
|
+
self.tag = tag or image_tag or slugify(pendulum_now("utc").isoformat())
|
54
|
+
self.dockerfile = dockerfile
|
55
|
+
self.build_kwargs = build_kwargs
|
56
|
+
|
57
|
+
@property
|
58
|
+
def reference(self):
|
59
|
+
return f"{self.name}:{self.tag}"
|
60
|
+
|
61
|
+
def build(self):
|
62
|
+
full_image_name = self.reference
|
63
|
+
build_kwargs = self.build_kwargs.copy()
|
64
|
+
build_kwargs["context"] = Path.cwd()
|
65
|
+
build_kwargs["tag"] = full_image_name
|
66
|
+
build_kwargs["pull"] = build_kwargs.get("pull", True)
|
67
|
+
|
68
|
+
if self.dockerfile == "auto":
|
69
|
+
with generate_default_dockerfile():
|
70
|
+
build_image(**build_kwargs)
|
71
|
+
else:
|
72
|
+
build_kwargs["dockerfile"] = self.dockerfile
|
73
|
+
build_image(**build_kwargs)
|
74
|
+
|
75
|
+
def push(self):
|
76
|
+
with docker_client() as client:
|
77
|
+
events = client.api.push(
|
78
|
+
repository=self.name, tag=self.tag, stream=True, decode=True
|
79
|
+
)
|
80
|
+
for event in events:
|
81
|
+
if "error" in event:
|
82
|
+
raise PushError(event["error"])
|
prefect/flow_engine.py
CHANGED
@@ -168,6 +168,20 @@ class FlowRunEngine(Generic[P, R]):
|
|
168
168
|
)
|
169
169
|
return state
|
170
170
|
|
171
|
+
# validate prior to context so that context receives validated params
|
172
|
+
if self.flow.should_validate_parameters:
|
173
|
+
try:
|
174
|
+
self.parameters = self.flow.validate_parameters(self.parameters or {})
|
175
|
+
except Exception as exc:
|
176
|
+
message = "Validation of flow parameters failed with error:"
|
177
|
+
self.logger.error("%s %s", message, exc)
|
178
|
+
self.handle_exception(
|
179
|
+
exc,
|
180
|
+
msg=message,
|
181
|
+
result_factory=run_coro_as_sync(ResultFactory.from_flow(self.flow)),
|
182
|
+
)
|
183
|
+
self.short_circuit = True
|
184
|
+
|
171
185
|
new_state = Running()
|
172
186
|
state = self.set_state(new_state)
|
173
187
|
while state.is_pending():
|
@@ -484,24 +498,6 @@ class FlowRunEngine(Generic[P, R]):
|
|
484
498
|
flow_version=self.flow.version,
|
485
499
|
empirical_policy=self.flow_run.empirical_policy,
|
486
500
|
)
|
487
|
-
|
488
|
-
# validate prior to context so that context receives validated params
|
489
|
-
if self.flow.should_validate_parameters:
|
490
|
-
try:
|
491
|
-
self.parameters = self.flow.validate_parameters(
|
492
|
-
self.parameters or {}
|
493
|
-
)
|
494
|
-
except Exception as exc:
|
495
|
-
message = "Validation of flow parameters failed with error:"
|
496
|
-
self.logger.error("%s %s", message, exc)
|
497
|
-
self.handle_exception(
|
498
|
-
exc,
|
499
|
-
msg=message,
|
500
|
-
result_factory=run_coro_as_sync(
|
501
|
-
ResultFactory.from_flow(self.flow)
|
502
|
-
),
|
503
|
-
)
|
504
|
-
self.short_circuit = True
|
505
501
|
try:
|
506
502
|
yield self
|
507
503
|
except Exception:
|
prefect/flows.py
CHANGED
@@ -17,11 +17,9 @@ import warnings
|
|
17
17
|
from copy import copy
|
18
18
|
from functools import partial, update_wrapper
|
19
19
|
from pathlib import Path
|
20
|
-
from tempfile import NamedTemporaryFile
|
21
20
|
from typing import (
|
22
21
|
TYPE_CHECKING,
|
23
22
|
Any,
|
24
|
-
AnyStr,
|
25
23
|
Awaitable,
|
26
24
|
Callable,
|
27
25
|
Coroutine,
|
@@ -56,9 +54,9 @@ from prefect.client.schemas.objects import Flow as FlowSchema
|
|
56
54
|
from prefect.client.schemas.objects import FlowRun
|
57
55
|
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
58
56
|
from prefect.client.utilities import client_injector
|
59
|
-
from prefect.
|
60
|
-
from prefect.deployments.runner import DeploymentImage, EntrypointType, deploy
|
57
|
+
from prefect.deployments.runner import deploy
|
61
58
|
from prefect.deployments.steps.core import run_steps
|
59
|
+
from prefect.docker.docker_image import DockerImage
|
62
60
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
63
61
|
from prefect.exceptions import (
|
64
62
|
InvalidNameError,
|
@@ -87,6 +85,7 @@ from prefect.settings import (
|
|
87
85
|
from prefect.states import State
|
88
86
|
from prefect.task_runners import TaskRunner, ThreadPoolTaskRunner
|
89
87
|
from prefect.types import BANNED_CHARACTERS, WITHOUT_BANNED_CHARACTERS
|
88
|
+
from prefect.types.entrypoint import EntrypointType
|
90
89
|
from prefect.utilities.annotations import NotSet
|
91
90
|
from prefect.utilities.asyncutils import (
|
92
91
|
run_sync_in_worker_thread,
|
@@ -118,11 +117,11 @@ logger = get_logger("flows")
|
|
118
117
|
|
119
118
|
if TYPE_CHECKING:
|
120
119
|
from prefect.client.orchestration import PrefectClient
|
121
|
-
from prefect.
|
120
|
+
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
121
|
+
from prefect.deployments.runner import RunnerDeployment
|
122
122
|
from prefect.flows import FlowRun
|
123
123
|
|
124
124
|
|
125
|
-
@PrefectObjectRegistry.register_instances
|
126
125
|
class Flow(Generic[P, R]):
|
127
126
|
"""
|
128
127
|
A Prefect workflow definition.
|
@@ -145,7 +144,7 @@ class Flow(Generic[P, R]):
|
|
145
144
|
be provided as a string template with the flow's parameters as variables,
|
146
145
|
or a function that returns a string.
|
147
146
|
task_runner: An optional task runner to use for task execution within the flow;
|
148
|
-
if not provided, a `
|
147
|
+
if not provided, a `ThreadPoolTaskRunner` will be used.
|
149
148
|
description: An optional string description for the flow; if not provided, the
|
150
149
|
description will be pulled from the docstring for the decorated function.
|
151
150
|
timeout_seconds: An optional number of seconds indicating a maximum runtime for
|
@@ -998,7 +997,7 @@ class Flow(Generic[P, R]):
|
|
998
997
|
self,
|
999
998
|
name: str,
|
1000
999
|
work_pool_name: Optional[str] = None,
|
1001
|
-
image: Optional[Union[str,
|
1000
|
+
image: Optional[Union[str, DockerImage]] = None,
|
1002
1001
|
build: bool = True,
|
1003
1002
|
push: bool = True,
|
1004
1003
|
work_queue_name: Optional[str] = None,
|
@@ -1034,7 +1033,7 @@ class Flow(Generic[P, R]):
|
|
1034
1033
|
work_pool_name: The name of the work pool to use for this deployment. Defaults to
|
1035
1034
|
the value of `PREFECT_DEFAULT_WORK_POOL_NAME`.
|
1036
1035
|
image: The name of the Docker image to build, including the registry and
|
1037
|
-
repository. Pass a
|
1036
|
+
repository. Pass a DockerImage instance to customize the Dockerfile used
|
1038
1037
|
and build arguments.
|
1039
1038
|
build: Whether or not to build a new image for the flow. If False, the provided
|
1040
1039
|
image will be used as-is and pulled at runtime.
|
@@ -1638,47 +1637,6 @@ def select_flow(
|
|
1638
1637
|
return list(flows_dict.values())[0]
|
1639
1638
|
|
1640
1639
|
|
1641
|
-
def load_flows_from_script(path: str) -> List[Flow]:
|
1642
|
-
"""
|
1643
|
-
Load all flow objects from the given python script. All of the code in the file
|
1644
|
-
will be executed.
|
1645
|
-
|
1646
|
-
Returns:
|
1647
|
-
A list of flows
|
1648
|
-
|
1649
|
-
Raises:
|
1650
|
-
FlowScriptError: If an exception is encountered while running the script
|
1651
|
-
"""
|
1652
|
-
return registry_from_script(path).get_instances(Flow)
|
1653
|
-
|
1654
|
-
|
1655
|
-
def load_flow_from_script(path: str, flow_name: Optional[str] = None) -> Flow:
|
1656
|
-
"""
|
1657
|
-
Extract a flow object from a script by running all of the code in the file.
|
1658
|
-
|
1659
|
-
If the script has multiple flows in it, a flow name must be provided to specify
|
1660
|
-
the flow to return.
|
1661
|
-
|
1662
|
-
Args:
|
1663
|
-
path: A path to a Python script containing flows
|
1664
|
-
flow_name: An optional flow name to look for in the script
|
1665
|
-
|
1666
|
-
Returns:
|
1667
|
-
The flow object from the script
|
1668
|
-
|
1669
|
-
Raises:
|
1670
|
-
FlowScriptError: If an exception is encountered while running the script
|
1671
|
-
MissingFlowError: If no flows exist in the iterable
|
1672
|
-
MissingFlowError: If a flow name is provided and that flow does not exist
|
1673
|
-
UnspecifiedFlowError: If multiple flows exist but no flow name was provided
|
1674
|
-
"""
|
1675
|
-
return select_flow(
|
1676
|
-
load_flows_from_script(path),
|
1677
|
-
flow_name=flow_name,
|
1678
|
-
from_message=f"in script '{path}'",
|
1679
|
-
)
|
1680
|
-
|
1681
|
-
|
1682
1640
|
def load_flow_from_entrypoint(
|
1683
1641
|
entrypoint: str,
|
1684
1642
|
) -> Flow:
|
@@ -1696,52 +1654,25 @@ def load_flow_from_entrypoint(
|
|
1696
1654
|
FlowScriptError: If an exception is encountered while running the script
|
1697
1655
|
MissingFlowError: If the flow function specified in the entrypoint does not exist
|
1698
1656
|
"""
|
1699
|
-
with PrefectObjectRegistry( # type: ignore
|
1700
|
-
block_code_execution=True,
|
1701
|
-
capture_failures=True,
|
1702
|
-
):
|
1703
|
-
if ":" in entrypoint:
|
1704
|
-
# split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
|
1705
|
-
path, func_name = entrypoint.rsplit(":", maxsplit=1)
|
1706
|
-
else:
|
1707
|
-
path, func_name = entrypoint.rsplit(".", maxsplit=1)
|
1708
|
-
try:
|
1709
|
-
flow = import_object(entrypoint)
|
1710
|
-
except AttributeError as exc:
|
1711
|
-
raise MissingFlowError(
|
1712
|
-
f"Flow function with name {func_name!r} not found in {path!r}. "
|
1713
|
-
) from exc
|
1714
|
-
|
1715
|
-
if not isinstance(flow, Flow):
|
1716
|
-
raise MissingFlowError(
|
1717
|
-
f"Function with name {func_name!r} is not a flow. Make sure that it is "
|
1718
|
-
"decorated with '@flow'."
|
1719
|
-
)
|
1720
|
-
|
1721
|
-
return flow
|
1722
1657
|
|
1658
|
+
if ":" in entrypoint:
|
1659
|
+
# split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
|
1660
|
+
path, func_name = entrypoint.rsplit(":", maxsplit=1)
|
1661
|
+
else:
|
1662
|
+
path, func_name = entrypoint.rsplit(".", maxsplit=1)
|
1663
|
+
try:
|
1664
|
+
flow = import_object(entrypoint)
|
1665
|
+
except AttributeError as exc:
|
1666
|
+
raise MissingFlowError(
|
1667
|
+
f"Flow function with name {func_name!r} not found in {path!r}. "
|
1668
|
+
) from exc
|
1723
1669
|
|
1724
|
-
|
1725
|
-
|
1726
|
-
|
1670
|
+
if not isinstance(flow, Flow):
|
1671
|
+
raise MissingFlowError(
|
1672
|
+
f"Function with name {func_name!r} is not a flow. Make sure that it is "
|
1673
|
+
"decorated with '@flow'."
|
1674
|
+
)
|
1727
1675
|
|
1728
|
-
The script will be written to a temporary local file path so errors can refer
|
1729
|
-
to line numbers and contextual tracebacks can be provided.
|
1730
|
-
"""
|
1731
|
-
with NamedTemporaryFile(
|
1732
|
-
mode="wt" if isinstance(script_contents, str) else "wb",
|
1733
|
-
prefix=f"flow-script-{flow_name}",
|
1734
|
-
suffix=".py",
|
1735
|
-
delete=False,
|
1736
|
-
) as tmpfile:
|
1737
|
-
tmpfile.write(script_contents)
|
1738
|
-
tmpfile.flush()
|
1739
|
-
try:
|
1740
|
-
flow = load_flow_from_script(tmpfile.name, flow_name=flow_name)
|
1741
|
-
finally:
|
1742
|
-
# windows compat
|
1743
|
-
tmpfile.close()
|
1744
|
-
os.remove(tmpfile.name)
|
1745
1676
|
return flow
|
1746
1677
|
|
1747
1678
|
|
prefect/futures.py
CHANGED
@@ -10,7 +10,7 @@ from typing_extensions import TypeVar
|
|
10
10
|
from prefect.client.orchestration import get_client
|
11
11
|
from prefect.client.schemas.objects import TaskRun
|
12
12
|
from prefect.exceptions import ObjectNotFound
|
13
|
-
from prefect.logging.loggers import get_logger
|
13
|
+
from prefect.logging.loggers import get_logger, get_run_logger
|
14
14
|
from prefect.states import Pending, State
|
15
15
|
from prefect.task_runs import TaskRunWaiter
|
16
16
|
from prefect.utilities.annotations import quote
|
@@ -143,6 +143,18 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[concurrent.futures.Future]):
|
|
143
143
|
_result = run_coro_as_sync(_result)
|
144
144
|
return _result
|
145
145
|
|
146
|
+
def __del__(self):
|
147
|
+
if self._final_state or self._wrapped_future.done():
|
148
|
+
return
|
149
|
+
try:
|
150
|
+
local_logger = get_run_logger()
|
151
|
+
except Exception:
|
152
|
+
local_logger = logger
|
153
|
+
local_logger.warning(
|
154
|
+
"A future was garbage collected before it resolved."
|
155
|
+
" Please call `.wait()` or `.result()` on futures to ensure they resolve.",
|
156
|
+
)
|
157
|
+
|
146
158
|
|
147
159
|
class PrefectDistributedFuture(PrefectFuture):
|
148
160
|
"""
|
@@ -404,7 +404,7 @@ class CloudRunPushProvisioner:
|
|
404
404
|
dedent(
|
405
405
|
f"""\
|
406
406
|
from prefect import flow
|
407
|
-
from prefect.
|
407
|
+
from prefect.docker import DockerImage
|
408
408
|
|
409
409
|
|
410
410
|
@flow(log_prints=True)
|
@@ -416,7 +416,7 @@ class CloudRunPushProvisioner:
|
|
416
416
|
my_flow.deploy(
|
417
417
|
name="my-deployment",
|
418
418
|
work_pool_name="{work_pool_name}",
|
419
|
-
image=
|
419
|
+
image=DockerImage(
|
420
420
|
name="my-image:latest",
|
421
421
|
platform="linux/amd64",
|
422
422
|
)
|
@@ -1042,7 +1042,7 @@ class ContainerInstancePushProvisioner:
|
|
1042
1042
|
dedent(
|
1043
1043
|
f"""\
|
1044
1044
|
from prefect import flow
|
1045
|
-
from prefect.
|
1045
|
+
from prefect.docker import DockerImage
|
1046
1046
|
|
1047
1047
|
|
1048
1048
|
@flow(log_prints=True)
|
@@ -1054,7 +1054,7 @@ class ContainerInstancePushProvisioner:
|
|
1054
1054
|
my_flow.deploy(
|
1055
1055
|
name="my-deployment",
|
1056
1056
|
work_pool_name="{work_pool_name}",
|
1057
|
-
image=
|
1057
|
+
image=DockerImage(
|
1058
1058
|
name="my-image:latest",
|
1059
1059
|
platform="linux/amd64",
|
1060
1060
|
)
|
@@ -950,7 +950,7 @@ class ContainerRepositoryResource:
|
|
950
950
|
dedent(
|
951
951
|
f"""\
|
952
952
|
from prefect import flow
|
953
|
-
from prefect.
|
953
|
+
from prefect.docker import DockerImage
|
954
954
|
|
955
955
|
|
956
956
|
@flow(log_prints=True)
|
@@ -962,7 +962,7 @@ class ContainerRepositoryResource:
|
|
962
962
|
my_flow.deploy(
|
963
963
|
name="my-deployment",
|
964
964
|
work_pool_name="{self._work_pool_name}",
|
965
|
-
image=
|
965
|
+
image=DockerImage(
|
966
966
|
name="{self._repository_name}:latest",
|
967
967
|
platform="linux/amd64",
|
968
968
|
)
|
prefect/records/result_store.py
CHANGED
@@ -44,6 +44,10 @@ class ResultFactoryStore(RecordStore):
|
|
44
44
|
raise ValueError("Result could not be read")
|
45
45
|
|
46
46
|
def write(self, key: str, value: Any) -> BaseResult:
|
47
|
-
if isinstance(value,
|
47
|
+
if isinstance(value, PersistedResult):
|
48
|
+
# if the value is already a persisted result, write it
|
49
|
+
value.write(_sync=True)
|
50
|
+
return value
|
51
|
+
elif isinstance(value, BaseResult):
|
48
52
|
return value
|
49
53
|
return run_coro_as_sync(self.result_factory.create_result(obj=value, key=key))
|