prefect-client 3.1.5__py3-none-any.whl → 3.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +3 -0
- prefect/_experimental/__init__.py +0 -0
- prefect/_experimental/lineage.py +181 -0
- prefect/_internal/compatibility/async_dispatch.py +38 -9
- prefect/_internal/compatibility/migration.py +1 -1
- prefect/_internal/concurrency/api.py +52 -52
- prefect/_internal/concurrency/calls.py +59 -35
- prefect/_internal/concurrency/cancellation.py +34 -18
- prefect/_internal/concurrency/event_loop.py +7 -6
- prefect/_internal/concurrency/threads.py +41 -33
- prefect/_internal/concurrency/waiters.py +28 -21
- prefect/_internal/pydantic/v1_schema.py +2 -2
- prefect/_internal/pydantic/v2_schema.py +10 -9
- prefect/_internal/pydantic/v2_validated_func.py +15 -10
- prefect/_internal/retries.py +15 -6
- prefect/_internal/schemas/bases.py +11 -8
- prefect/_internal/schemas/validators.py +7 -5
- prefect/_version.py +3 -3
- prefect/automations.py +53 -47
- prefect/blocks/abstract.py +12 -10
- prefect/blocks/core.py +148 -19
- prefect/blocks/system.py +2 -1
- prefect/cache_policies.py +11 -11
- prefect/client/__init__.py +3 -1
- prefect/client/base.py +36 -37
- prefect/client/cloud.py +26 -19
- prefect/client/collections.py +2 -2
- prefect/client/orchestration.py +430 -273
- prefect/client/schemas/__init__.py +24 -0
- prefect/client/schemas/actions.py +128 -121
- prefect/client/schemas/filters.py +1 -1
- prefect/client/schemas/objects.py +114 -85
- prefect/client/schemas/responses.py +19 -20
- prefect/client/schemas/schedules.py +136 -93
- prefect/client/subscriptions.py +30 -15
- prefect/client/utilities.py +46 -36
- prefect/concurrency/asyncio.py +6 -9
- prefect/concurrency/sync.py +35 -5
- prefect/context.py +40 -32
- prefect/deployments/flow_runs.py +6 -8
- prefect/deployments/runner.py +14 -14
- prefect/deployments/steps/core.py +3 -1
- prefect/deployments/steps/pull.py +60 -12
- prefect/docker/__init__.py +1 -1
- prefect/events/clients.py +55 -4
- prefect/events/filters.py +1 -1
- prefect/events/related.py +2 -1
- prefect/events/schemas/events.py +26 -21
- prefect/events/utilities.py +3 -2
- prefect/events/worker.py +8 -0
- prefect/filesystems.py +3 -3
- prefect/flow_engine.py +87 -87
- prefect/flow_runs.py +7 -5
- prefect/flows.py +218 -176
- prefect/logging/configuration.py +1 -1
- prefect/logging/highlighters.py +1 -2
- prefect/logging/loggers.py +30 -20
- prefect/main.py +17 -24
- prefect/results.py +43 -22
- prefect/runner/runner.py +43 -21
- prefect/runner/server.py +30 -32
- prefect/runner/storage.py +3 -3
- prefect/runner/submit.py +3 -6
- prefect/runner/utils.py +6 -6
- prefect/runtime/flow_run.py +7 -0
- prefect/serializers.py +28 -24
- prefect/settings/constants.py +2 -2
- prefect/settings/legacy.py +1 -1
- prefect/settings/models/experiments.py +5 -0
- prefect/settings/models/server/events.py +10 -0
- prefect/task_engine.py +87 -26
- prefect/task_runners.py +2 -2
- prefect/task_worker.py +43 -25
- prefect/tasks.py +148 -142
- prefect/telemetry/bootstrap.py +15 -2
- prefect/telemetry/instrumentation.py +1 -1
- prefect/telemetry/processors.py +10 -7
- prefect/telemetry/run_telemetry.py +231 -0
- prefect/transactions.py +14 -14
- prefect/types/__init__.py +5 -5
- prefect/utilities/_engine.py +96 -0
- prefect/utilities/annotations.py +25 -18
- prefect/utilities/asyncutils.py +126 -140
- prefect/utilities/callables.py +87 -78
- prefect/utilities/collections.py +278 -117
- prefect/utilities/compat.py +13 -21
- prefect/utilities/context.py +6 -5
- prefect/utilities/dispatch.py +23 -12
- prefect/utilities/dockerutils.py +33 -32
- prefect/utilities/engine.py +126 -239
- prefect/utilities/filesystem.py +18 -15
- prefect/utilities/hashing.py +10 -11
- prefect/utilities/importtools.py +40 -27
- prefect/utilities/math.py +9 -5
- prefect/utilities/names.py +3 -3
- prefect/utilities/processutils.py +121 -57
- prefect/utilities/pydantic.py +41 -36
- prefect/utilities/render_swagger.py +22 -12
- prefect/utilities/schema_tools/__init__.py +2 -1
- prefect/utilities/schema_tools/hydration.py +50 -43
- prefect/utilities/schema_tools/validation.py +52 -42
- prefect/utilities/services.py +13 -12
- prefect/utilities/templating.py +45 -45
- prefect/utilities/text.py +2 -1
- prefect/utilities/timeout.py +4 -4
- prefect/utilities/urls.py +9 -4
- prefect/utilities/visualization.py +46 -24
- prefect/variables.py +136 -27
- prefect/workers/base.py +15 -8
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/METADATA +5 -2
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/RECORD +114 -110
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/top_level.txt +0 -0
prefect/flows.py
CHANGED
@@ -23,13 +23,10 @@ from typing import (
|
|
23
23
|
Awaitable,
|
24
24
|
Callable,
|
25
25
|
Coroutine,
|
26
|
-
Dict,
|
27
26
|
Generic,
|
28
27
|
Iterable,
|
29
|
-
List,
|
30
28
|
NoReturn,
|
31
29
|
Optional,
|
32
|
-
Set,
|
33
30
|
Tuple,
|
34
31
|
Type,
|
35
32
|
TypeVar,
|
@@ -45,7 +42,7 @@ from pydantic.v1 import BaseModel as V1BaseModel
|
|
45
42
|
from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
|
46
43
|
from pydantic.v1.errors import ConfigError # TODO
|
47
44
|
from rich.console import Console
|
48
|
-
from typing_extensions import Literal, ParamSpec,
|
45
|
+
from typing_extensions import Literal, ParamSpec, TypeAlias
|
49
46
|
|
50
47
|
from prefect._internal.concurrency.api import create_call, from_async
|
51
48
|
from prefect.blocks.core import Block
|
@@ -97,6 +94,7 @@ from prefect.utilities.filesystem import relative_path_to_current_platform
|
|
97
94
|
from prefect.utilities.hashing import file_hash
|
98
95
|
from prefect.utilities.importtools import import_object, safe_load_namespace
|
99
96
|
|
97
|
+
from ._internal.compatibility.async_dispatch import is_in_async_context
|
100
98
|
from ._internal.pydantic.v2_schema import is_v2_type
|
101
99
|
from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
|
102
100
|
from ._internal.pydantic.v2_validated_func import (
|
@@ -106,7 +104,11 @@ from ._internal.pydantic.v2_validated_func import (
|
|
106
104
|
T = TypeVar("T") # Generic type var for capturing the inner return type of async funcs
|
107
105
|
R = TypeVar("R") # The return type of the user's function
|
108
106
|
P = ParamSpec("P") # The parameters of the flow
|
109
|
-
F = TypeVar("F", bound="Flow") # The type of the flow
|
107
|
+
F = TypeVar("F", bound="Flow[Any, Any]") # The type of the flow
|
108
|
+
|
109
|
+
StateHookCallable: TypeAlias = Callable[
|
110
|
+
[FlowSchema, FlowRun, State], Union[Awaitable[None], None]
|
111
|
+
]
|
110
112
|
|
111
113
|
logger = get_logger("flows")
|
112
114
|
|
@@ -185,7 +187,9 @@ class Flow(Generic[P, R]):
|
|
185
187
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
186
188
|
retries: Optional[int] = None,
|
187
189
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
188
|
-
task_runner: Union[
|
190
|
+
task_runner: Union[
|
191
|
+
Type[TaskRunner[PrefectFuture[R]]], TaskRunner[PrefectFuture[R]], None
|
192
|
+
] = None,
|
189
193
|
description: Optional[str] = None,
|
190
194
|
timeout_seconds: Union[int, float, None] = None,
|
191
195
|
validate_parameters: bool = True,
|
@@ -194,15 +198,11 @@ class Flow(Generic[P, R]):
|
|
194
198
|
result_serializer: Optional[ResultSerializer] = None,
|
195
199
|
cache_result_in_memory: bool = True,
|
196
200
|
log_prints: Optional[bool] = None,
|
197
|
-
on_completion: Optional[
|
198
|
-
|
199
|
-
] = None,
|
200
|
-
|
201
|
-
|
202
|
-
List[Callable[[FlowSchema, FlowRun, State], None]]
|
203
|
-
] = None,
|
204
|
-
on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
205
|
-
on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
201
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
202
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
203
|
+
on_cancellation: Optional[list[StateHookCallable]] = None,
|
204
|
+
on_crashed: Optional[list[StateHookCallable]] = None,
|
205
|
+
on_running: Optional[list[StateHookCallable]] = None,
|
206
206
|
):
|
207
207
|
if name is not None and not isinstance(name, str):
|
208
208
|
raise TypeError(
|
@@ -374,7 +374,7 @@ class Flow(Generic[P, R]):
|
|
374
374
|
def ismethod(self) -> bool:
|
375
375
|
return hasattr(self.fn, "__prefect_self__")
|
376
376
|
|
377
|
-
def __get__(self, instance, owner):
|
377
|
+
def __get__(self, instance: Any, owner: Any):
|
378
378
|
"""
|
379
379
|
Implement the descriptor protocol so that the flow can be used as an instance method.
|
380
380
|
When an instance method is loaded, this method is called with the "self" instance as
|
@@ -401,7 +401,9 @@ class Flow(Generic[P, R]):
|
|
401
401
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
402
402
|
description: Optional[str] = None,
|
403
403
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
404
|
-
task_runner: Union[
|
404
|
+
task_runner: Union[
|
405
|
+
Type[TaskRunner[PrefectFuture[R]]], TaskRunner[PrefectFuture[R]], None
|
406
|
+
] = None,
|
405
407
|
timeout_seconds: Union[int, float, None] = None,
|
406
408
|
validate_parameters: Optional[bool] = None,
|
407
409
|
persist_result: Optional[bool] = NotSet, # type: ignore
|
@@ -409,16 +411,12 @@ class Flow(Generic[P, R]):
|
|
409
411
|
result_serializer: Optional[ResultSerializer] = NotSet, # type: ignore
|
410
412
|
cache_result_in_memory: Optional[bool] = None,
|
411
413
|
log_prints: Optional[bool] = NotSet, # type: ignore
|
412
|
-
on_completion: Optional[
|
413
|
-
|
414
|
-
] = None,
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
] = None,
|
419
|
-
on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
420
|
-
on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
421
|
-
) -> Self:
|
414
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
415
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
416
|
+
on_cancellation: Optional[list[StateHookCallable]] = None,
|
417
|
+
on_crashed: Optional[list[StateHookCallable]] = None,
|
418
|
+
on_running: Optional[list[StateHookCallable]] = None,
|
419
|
+
) -> "Flow[P, R]":
|
422
420
|
"""
|
423
421
|
Create a new flow from the current object, updating provided options.
|
424
422
|
|
@@ -521,7 +519,7 @@ class Flow(Generic[P, R]):
|
|
521
519
|
new_flow._entrypoint = self._entrypoint
|
522
520
|
return new_flow
|
523
521
|
|
524
|
-
def validate_parameters(self, parameters:
|
522
|
+
def validate_parameters(self, parameters: dict[str, Any]) -> dict[str, Any]:
|
525
523
|
"""
|
526
524
|
Validate parameters for compatibility with the flow by attempting to cast the inputs to the
|
527
525
|
associated types specified by the function's type annotations.
|
@@ -566,14 +564,12 @@ class Flow(Generic[P, R]):
|
|
566
564
|
"Cannot mix Pydantic v1 and v2 types as arguments to a flow."
|
567
565
|
)
|
568
566
|
|
567
|
+
validated_fn_kwargs = dict(arbitrary_types_allowed=True)
|
568
|
+
|
569
569
|
if has_v1_models:
|
570
|
-
validated_fn = V1ValidatedFunction(
|
571
|
-
self.fn, config={"arbitrary_types_allowed": True}
|
572
|
-
)
|
570
|
+
validated_fn = V1ValidatedFunction(self.fn, config=validated_fn_kwargs)
|
573
571
|
else:
|
574
|
-
validated_fn = V2ValidatedFunction(
|
575
|
-
self.fn, config=pydantic.ConfigDict(arbitrary_types_allowed=True)
|
576
|
-
)
|
572
|
+
validated_fn = V2ValidatedFunction(self.fn, config=validated_fn_kwargs)
|
577
573
|
|
578
574
|
try:
|
579
575
|
with warnings.catch_warnings():
|
@@ -598,7 +594,7 @@ class Flow(Generic[P, R]):
|
|
598
594
|
}
|
599
595
|
return cast_parameters
|
600
596
|
|
601
|
-
def serialize_parameters(self, parameters:
|
597
|
+
def serialize_parameters(self, parameters: dict[str, Any]) -> dict[str, Any]:
|
602
598
|
"""
|
603
599
|
Convert parameters to a serializable form.
|
604
600
|
|
@@ -644,15 +640,15 @@ class Flow(Generic[P, R]):
|
|
644
640
|
paused: Optional[bool] = None,
|
645
641
|
schedules: Optional["FlexibleScheduleList"] = None,
|
646
642
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
647
|
-
parameters: Optional[dict] = None,
|
648
|
-
triggers: Optional[
|
643
|
+
parameters: Optional[dict[str, Any]] = None,
|
644
|
+
triggers: Optional[list[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
649
645
|
description: Optional[str] = None,
|
650
|
-
tags: Optional[
|
646
|
+
tags: Optional[list[str]] = None,
|
651
647
|
version: Optional[str] = None,
|
652
648
|
enforce_parameter_schema: bool = True,
|
653
649
|
work_pool_name: Optional[str] = None,
|
654
650
|
work_queue_name: Optional[str] = None,
|
655
|
-
job_variables: Optional[
|
651
|
+
job_variables: Optional[dict[str, Any]] = None,
|
656
652
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
657
653
|
) -> "RunnerDeployment":
|
658
654
|
"""
|
@@ -754,33 +750,23 @@ class Flow(Generic[P, R]):
|
|
754
750
|
entrypoint_type=entrypoint_type,
|
755
751
|
)
|
756
752
|
|
757
|
-
def on_completion(
|
758
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
759
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
753
|
+
def on_completion(self, fn: StateHookCallable) -> StateHookCallable:
|
760
754
|
self.on_completion_hooks.append(fn)
|
761
755
|
return fn
|
762
756
|
|
763
|
-
def on_cancellation(
|
764
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
765
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
757
|
+
def on_cancellation(self, fn: StateHookCallable) -> StateHookCallable:
|
766
758
|
self.on_cancellation_hooks.append(fn)
|
767
759
|
return fn
|
768
760
|
|
769
|
-
def on_crashed(
|
770
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
771
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
761
|
+
def on_crashed(self, fn: StateHookCallable) -> StateHookCallable:
|
772
762
|
self.on_crashed_hooks.append(fn)
|
773
763
|
return fn
|
774
764
|
|
775
|
-
def on_running(
|
776
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
777
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
765
|
+
def on_running(self, fn: StateHookCallable) -> StateHookCallable:
|
778
766
|
self.on_running_hooks.append(fn)
|
779
767
|
return fn
|
780
768
|
|
781
|
-
def on_failure(
|
782
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
783
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
769
|
+
def on_failure(self, fn: StateHookCallable) -> StateHookCallable:
|
784
770
|
self.on_failure_hooks.append(fn)
|
785
771
|
return fn
|
786
772
|
|
@@ -800,10 +786,10 @@ class Flow(Generic[P, R]):
|
|
800
786
|
paused: Optional[bool] = None,
|
801
787
|
schedules: Optional["FlexibleScheduleList"] = None,
|
802
788
|
global_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
803
|
-
triggers: Optional[
|
804
|
-
parameters: Optional[dict] = None,
|
789
|
+
triggers: Optional[list[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
790
|
+
parameters: Optional[dict[str, Any]] = None,
|
805
791
|
description: Optional[str] = None,
|
806
|
-
tags: Optional[
|
792
|
+
tags: Optional[list[str]] = None,
|
807
793
|
version: Optional[str] = None,
|
808
794
|
enforce_parameter_schema: bool = True,
|
809
795
|
pause_on_shutdown: bool = True,
|
@@ -1038,8 +1024,11 @@ class Flow(Generic[P, R]):
|
|
1038
1024
|
await storage.pull_code()
|
1039
1025
|
|
1040
1026
|
full_entrypoint = str(storage.destination / entrypoint)
|
1041
|
-
flow
|
1042
|
-
|
1027
|
+
flow = cast(
|
1028
|
+
Flow[P, R],
|
1029
|
+
await from_async.wait_for_call_in_new_thread(
|
1030
|
+
create_call(load_flow_from_entrypoint, full_entrypoint)
|
1031
|
+
),
|
1043
1032
|
)
|
1044
1033
|
flow._storage = storage
|
1045
1034
|
flow._entrypoint = entrypoint
|
@@ -1055,17 +1044,17 @@ class Flow(Generic[P, R]):
|
|
1055
1044
|
build: bool = True,
|
1056
1045
|
push: bool = True,
|
1057
1046
|
work_queue_name: Optional[str] = None,
|
1058
|
-
job_variables: Optional[dict] = None,
|
1047
|
+
job_variables: Optional[dict[str, Any]] = None,
|
1059
1048
|
interval: Optional[Union[int, float, datetime.timedelta]] = None,
|
1060
1049
|
cron: Optional[str] = None,
|
1061
1050
|
rrule: Optional[str] = None,
|
1062
1051
|
paused: Optional[bool] = None,
|
1063
|
-
schedules: Optional[
|
1052
|
+
schedules: Optional[list[DeploymentScheduleCreate]] = None,
|
1064
1053
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
1065
|
-
triggers: Optional[
|
1066
|
-
parameters: Optional[dict] = None,
|
1054
|
+
triggers: Optional[list[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
1055
|
+
parameters: Optional[dict[str, Any]] = None,
|
1067
1056
|
description: Optional[str] = None,
|
1068
|
-
tags: Optional[
|
1057
|
+
tags: Optional[list[str]] = None,
|
1069
1058
|
version: Optional[str] = None,
|
1070
1059
|
enforce_parameter_schema: bool = True,
|
1071
1060
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
@@ -1288,7 +1277,7 @@ class Flow(Generic[P, R]):
|
|
1288
1277
|
self,
|
1289
1278
|
*args: "P.args",
|
1290
1279
|
return_state: bool = False,
|
1291
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
1280
|
+
wait_for: Optional[Iterable[PrefectFuture[Any]]] = None,
|
1292
1281
|
**kwargs: "P.kwargs",
|
1293
1282
|
):
|
1294
1283
|
"""
|
@@ -1360,7 +1349,7 @@ class Flow(Generic[P, R]):
|
|
1360
1349
|
)
|
1361
1350
|
|
1362
1351
|
@sync_compatible
|
1363
|
-
async def visualize(self, *args, **kwargs):
|
1352
|
+
async def visualize(self, *args: "P.args", **kwargs: "P.kwargs"):
|
1364
1353
|
"""
|
1365
1354
|
Generates a graphviz object representing the current flow. In IPython notebooks,
|
1366
1355
|
it's rendered inline, otherwise in a new window as a PNG.
|
@@ -1389,7 +1378,7 @@ class Flow(Generic[P, R]):
|
|
1389
1378
|
try:
|
1390
1379
|
with TaskVizTracker() as tracker:
|
1391
1380
|
if self.isasync:
|
1392
|
-
await self.fn(*args, **kwargs)
|
1381
|
+
await self.fn(*args, **kwargs) # type: ignore[reportGeneralTypeIssues]
|
1393
1382
|
else:
|
1394
1383
|
self.fn(*args, **kwargs)
|
1395
1384
|
|
@@ -1432,7 +1421,7 @@ def flow(
|
|
1432
1421
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1433
1422
|
retries: Optional[int] = None,
|
1434
1423
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
1435
|
-
task_runner: Optional[TaskRunner] = None,
|
1424
|
+
task_runner: Optional[TaskRunner[PrefectFuture[R]]] = None,
|
1436
1425
|
description: Optional[str] = None,
|
1437
1426
|
timeout_seconds: Union[int, float, None] = None,
|
1438
1427
|
validate_parameters: bool = True,
|
@@ -1441,30 +1430,24 @@ def flow(
|
|
1441
1430
|
result_serializer: Optional[ResultSerializer] = None,
|
1442
1431
|
cache_result_in_memory: bool = True,
|
1443
1432
|
log_prints: Optional[bool] = None,
|
1444
|
-
on_completion: Optional[
|
1445
|
-
|
1446
|
-
] = None,
|
1447
|
-
|
1448
|
-
|
1449
|
-
] = None,
|
1450
|
-
on_cancellation: Optional[
|
1451
|
-
List[Callable[[FlowSchema, FlowRun, State], None]]
|
1452
|
-
] = None,
|
1453
|
-
on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
1454
|
-
on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
1433
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
1434
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
1435
|
+
on_cancellation: Optional[list[StateHookCallable]] = None,
|
1436
|
+
on_crashed: Optional[list[StateHookCallable]] = None,
|
1437
|
+
on_running: Optional[list[StateHookCallable]] = None,
|
1455
1438
|
) -> Callable[[Callable[P, R]], Flow[P, R]]:
|
1456
1439
|
...
|
1457
1440
|
|
1458
1441
|
|
1459
1442
|
def flow(
|
1460
|
-
__fn=None,
|
1443
|
+
__fn: Optional[Callable[P, R]] = None,
|
1461
1444
|
*,
|
1462
1445
|
name: Optional[str] = None,
|
1463
1446
|
version: Optional[str] = None,
|
1464
1447
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1465
1448
|
retries: Optional[int] = None,
|
1466
1449
|
retry_delay_seconds: Union[int, float, None] = None,
|
1467
|
-
task_runner: Optional[TaskRunner] = None,
|
1450
|
+
task_runner: Optional[TaskRunner[PrefectFuture[R]]] = None,
|
1468
1451
|
description: Optional[str] = None,
|
1469
1452
|
timeout_seconds: Union[int, float, None] = None,
|
1470
1453
|
validate_parameters: bool = True,
|
@@ -1473,17 +1456,11 @@ def flow(
|
|
1473
1456
|
result_serializer: Optional[ResultSerializer] = None,
|
1474
1457
|
cache_result_in_memory: bool = True,
|
1475
1458
|
log_prints: Optional[bool] = None,
|
1476
|
-
on_completion: Optional[
|
1477
|
-
|
1478
|
-
] = None,
|
1479
|
-
|
1480
|
-
|
1481
|
-
] = None,
|
1482
|
-
on_cancellation: Optional[
|
1483
|
-
List[Callable[[FlowSchema, FlowRun, State], None]]
|
1484
|
-
] = None,
|
1485
|
-
on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
1486
|
-
on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
1459
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
1460
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
1461
|
+
on_cancellation: Optional[list[StateHookCallable]] = None,
|
1462
|
+
on_crashed: Optional[list[StateHookCallable]] = None,
|
1463
|
+
on_running: Optional[list[StateHookCallable]] = None,
|
1487
1464
|
):
|
1488
1465
|
"""
|
1489
1466
|
Decorator to designate a function as a Prefect workflow.
|
@@ -1592,30 +1569,27 @@ def flow(
|
|
1592
1569
|
if isinstance(__fn, (classmethod, staticmethod)):
|
1593
1570
|
method_decorator = type(__fn).__name__
|
1594
1571
|
raise TypeError(f"@{method_decorator} should be applied on top of @flow")
|
1595
|
-
return
|
1596
|
-
|
1597
|
-
|
1598
|
-
|
1599
|
-
|
1600
|
-
|
1601
|
-
|
1602
|
-
|
1603
|
-
|
1604
|
-
|
1605
|
-
|
1606
|
-
|
1607
|
-
|
1608
|
-
|
1609
|
-
|
1610
|
-
|
1611
|
-
|
1612
|
-
|
1613
|
-
|
1614
|
-
|
1615
|
-
|
1616
|
-
on_crashed=on_crashed,
|
1617
|
-
on_running=on_running,
|
1618
|
-
),
|
1572
|
+
return Flow(
|
1573
|
+
fn=__fn,
|
1574
|
+
name=name,
|
1575
|
+
version=version,
|
1576
|
+
flow_run_name=flow_run_name,
|
1577
|
+
task_runner=task_runner,
|
1578
|
+
description=description,
|
1579
|
+
timeout_seconds=timeout_seconds,
|
1580
|
+
validate_parameters=validate_parameters,
|
1581
|
+
retries=retries,
|
1582
|
+
retry_delay_seconds=retry_delay_seconds,
|
1583
|
+
persist_result=persist_result,
|
1584
|
+
result_storage=result_storage,
|
1585
|
+
result_serializer=result_serializer,
|
1586
|
+
cache_result_in_memory=cache_result_in_memory,
|
1587
|
+
log_prints=log_prints,
|
1588
|
+
on_completion=on_completion,
|
1589
|
+
on_failure=on_failure,
|
1590
|
+
on_cancellation=on_cancellation,
|
1591
|
+
on_crashed=on_crashed,
|
1592
|
+
on_running=on_running,
|
1619
1593
|
)
|
1620
1594
|
else:
|
1621
1595
|
return cast(
|
@@ -1667,10 +1641,10 @@ flow.from_source = Flow.from_source
|
|
1667
1641
|
|
1668
1642
|
|
1669
1643
|
def select_flow(
|
1670
|
-
flows: Iterable[Flow],
|
1644
|
+
flows: Iterable[Flow[P, R]],
|
1671
1645
|
flow_name: Optional[str] = None,
|
1672
1646
|
from_message: Optional[str] = None,
|
1673
|
-
) -> Flow:
|
1647
|
+
) -> Flow[P, R]:
|
1674
1648
|
"""
|
1675
1649
|
Select the only flow in an iterable or a flow specified by name.
|
1676
1650
|
|
@@ -1715,7 +1689,7 @@ def select_flow(
|
|
1715
1689
|
def load_flow_from_entrypoint(
|
1716
1690
|
entrypoint: str,
|
1717
1691
|
use_placeholder_flow: bool = True,
|
1718
|
-
) -> Flow:
|
1692
|
+
) -> Flow[P, Any]:
|
1719
1693
|
"""
|
1720
1694
|
Extract a flow object from a script at an entrypoint by running all of the code in the file.
|
1721
1695
|
|
@@ -1739,7 +1713,7 @@ def load_flow_from_entrypoint(
|
|
1739
1713
|
else:
|
1740
1714
|
path, func_name = entrypoint.rsplit(".", maxsplit=1)
|
1741
1715
|
try:
|
1742
|
-
flow = import_object(entrypoint)
|
1716
|
+
flow: Flow[P, Any] = import_object(entrypoint) # pyright: ignore[reportRedeclaration]
|
1743
1717
|
except AttributeError as exc:
|
1744
1718
|
raise MissingFlowError(
|
1745
1719
|
f"Flow function with name {func_name!r} not found in {path!r}. "
|
@@ -1748,13 +1722,13 @@ def load_flow_from_entrypoint(
|
|
1748
1722
|
# If the flow has dependencies that are not installed in the current
|
1749
1723
|
# environment, fallback to loading the flow via AST parsing.
|
1750
1724
|
if use_placeholder_flow:
|
1751
|
-
flow = safe_load_flow_from_entrypoint(entrypoint)
|
1725
|
+
flow: Optional[Flow[P, Any]] = safe_load_flow_from_entrypoint(entrypoint)
|
1752
1726
|
if flow is None:
|
1753
1727
|
raise
|
1754
1728
|
else:
|
1755
1729
|
raise
|
1756
1730
|
|
1757
|
-
if not isinstance(flow, Flow):
|
1731
|
+
if not isinstance(flow, Flow): # pyright: ignore[reportUnnecessaryIsInstance]
|
1758
1732
|
raise MissingFlowError(
|
1759
1733
|
f"Function with name {func_name!r} is not a flow. Make sure that it is "
|
1760
1734
|
"decorated with '@flow'."
|
@@ -1768,8 +1742,8 @@ def serve(
|
|
1768
1742
|
pause_on_shutdown: bool = True,
|
1769
1743
|
print_starting_message: bool = True,
|
1770
1744
|
limit: Optional[int] = None,
|
1771
|
-
**kwargs,
|
1772
|
-
):
|
1745
|
+
**kwargs: Any,
|
1746
|
+
) -> None:
|
1773
1747
|
"""
|
1774
1748
|
Serve the provided list of deployments.
|
1775
1749
|
|
@@ -1812,61 +1786,129 @@ def serve(
|
|
1812
1786
|
serve(hello_deploy, bye_deploy)
|
1813
1787
|
```
|
1814
1788
|
"""
|
1815
|
-
from rich.console import Console, Group
|
1816
|
-
from rich.table import Table
|
1817
1789
|
|
1818
1790
|
from prefect.runner import Runner
|
1819
1791
|
|
1792
|
+
if is_in_async_context():
|
1793
|
+
raise RuntimeError(
|
1794
|
+
"Cannot call `serve` in an asynchronous context. Use `aserve` instead."
|
1795
|
+
)
|
1796
|
+
|
1820
1797
|
runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
|
1821
1798
|
for deployment in args:
|
1822
1799
|
runner.add_deployment(deployment)
|
1823
1800
|
|
1824
1801
|
if print_starting_message:
|
1825
|
-
|
1826
|
-
|
1827
|
-
|
1828
|
-
)
|
1802
|
+
_display_serve_start_message(*args)
|
1803
|
+
|
1804
|
+
try:
|
1805
|
+
asyncio.run(runner.start())
|
1806
|
+
except (KeyboardInterrupt, TerminationSignal) as exc:
|
1807
|
+
logger.info(f"Received {type(exc).__name__}, shutting down...")
|
1829
1808
|
|
1830
|
-
table = Table(title="Deployments", show_header=False)
|
1831
1809
|
|
1832
|
-
|
1810
|
+
async def aserve(
|
1811
|
+
*args: "RunnerDeployment",
|
1812
|
+
pause_on_shutdown: bool = True,
|
1813
|
+
print_starting_message: bool = True,
|
1814
|
+
limit: Optional[int] = None,
|
1815
|
+
**kwargs: Any,
|
1816
|
+
) -> None:
|
1817
|
+
"""
|
1818
|
+
Asynchronously serve the provided list of deployments.
|
1833
1819
|
|
1834
|
-
|
1835
|
-
table.add_row(f"{deployment.flow_name}/{deployment.name}")
|
1820
|
+
Use `serve` instead if calling from a synchronous context.
|
1836
1821
|
|
1837
|
-
|
1838
|
-
|
1839
|
-
|
1840
|
-
|
1841
|
-
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1845
|
-
|
1822
|
+
Args:
|
1823
|
+
*args: A list of deployments to serve.
|
1824
|
+
pause_on_shutdown: A boolean for whether or not to automatically pause
|
1825
|
+
deployment schedules on shutdown.
|
1826
|
+
print_starting_message: Whether or not to print message to the console
|
1827
|
+
on startup.
|
1828
|
+
limit: The maximum number of runs that can be executed concurrently.
|
1829
|
+
**kwargs: Additional keyword arguments to pass to the runner.
|
1830
|
+
|
1831
|
+
Examples:
|
1832
|
+
Prepare deployment and asynchronous initialization function and serve them:
|
1833
|
+
|
1834
|
+
```python
|
1835
|
+
import asyncio
|
1836
|
+
import datetime
|
1837
|
+
|
1838
|
+
from prefect import flow, aserve, get_client
|
1839
|
+
|
1840
|
+
|
1841
|
+
async def init():
|
1842
|
+
await set_concurrency_limit()
|
1843
|
+
|
1844
|
+
|
1845
|
+
async def set_concurrency_limit():
|
1846
|
+
async with get_client() as client:
|
1847
|
+
await client.create_concurrency_limit(tag='dev', concurrency_limit=3)
|
1848
|
+
|
1849
|
+
|
1850
|
+
@flow
|
1851
|
+
async def my_flow(name):
|
1852
|
+
print(f"hello {name}")
|
1853
|
+
|
1854
|
+
|
1855
|
+
async def main():
|
1856
|
+
# Initialization function
|
1857
|
+
await init()
|
1858
|
+
|
1859
|
+
# Run once a day
|
1860
|
+
hello_deploy = await my_flow.to_deployment(
|
1861
|
+
"hello", tags=["dev"], interval=datetime.timedelta(days=1)
|
1846
1862
|
)
|
1847
1863
|
|
1848
|
-
|
1849
|
-
console.print(
|
1850
|
-
Group(help_message_top, table, help_message_bottom), soft_wrap=True
|
1851
|
-
)
|
1864
|
+
await aserve(hello_deploy)
|
1852
1865
|
|
1853
|
-
try:
|
1854
|
-
loop = asyncio.get_running_loop()
|
1855
|
-
except RuntimeError as exc:
|
1856
|
-
if "no running event loop" in str(exc):
|
1857
|
-
loop = None
|
1858
|
-
else:
|
1859
|
-
raise
|
1860
1866
|
|
1861
|
-
|
1862
|
-
|
1863
|
-
|
1864
|
-
|
1865
|
-
|
1866
|
-
|
1867
|
-
|
1868
|
-
|
1869
|
-
|
1867
|
+
if __name__ == "__main__":
|
1868
|
+
asyncio.run(main())
|
1869
|
+
"""
|
1870
|
+
|
1871
|
+
from prefect.runner import Runner
|
1872
|
+
|
1873
|
+
runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
|
1874
|
+
for deployment in args:
|
1875
|
+
await runner.add_deployment(deployment)
|
1876
|
+
|
1877
|
+
if print_starting_message:
|
1878
|
+
_display_serve_start_message(*args)
|
1879
|
+
|
1880
|
+
await runner.start()
|
1881
|
+
|
1882
|
+
|
1883
|
+
def _display_serve_start_message(*args: "RunnerDeployment"):
|
1884
|
+
from rich.console import Console, Group
|
1885
|
+
from rich.table import Table
|
1886
|
+
|
1887
|
+
help_message_top = (
|
1888
|
+
"[green]Your deployments are being served and polling for"
|
1889
|
+
" scheduled runs!\n[/]"
|
1890
|
+
)
|
1891
|
+
|
1892
|
+
table = Table(title="Deployments", show_header=False)
|
1893
|
+
|
1894
|
+
table.add_column(style="blue", no_wrap=True)
|
1895
|
+
|
1896
|
+
for deployment in args:
|
1897
|
+
table.add_row(f"{deployment.flow_name}/{deployment.name}")
|
1898
|
+
|
1899
|
+
help_message_bottom = (
|
1900
|
+
"\nTo trigger any of these deployments, use the"
|
1901
|
+
" following command:\n[blue]\n\t$ prefect deployment run"
|
1902
|
+
" [DEPLOYMENT_NAME]\n[/]"
|
1903
|
+
)
|
1904
|
+
if PREFECT_UI_URL:
|
1905
|
+
help_message_bottom += (
|
1906
|
+
"\nYou can also trigger your deployments via the Prefect UI:"
|
1907
|
+
f" [blue]{PREFECT_UI_URL.value()}/deployments[/]\n"
|
1908
|
+
)
|
1909
|
+
|
1910
|
+
console = Console()
|
1911
|
+
console.print(Group(help_message_top, table, help_message_bottom), soft_wrap=True)
|
1870
1912
|
|
1871
1913
|
|
1872
1914
|
@client_injector
|
@@ -1876,7 +1918,7 @@ async def load_flow_from_flow_run(
|
|
1876
1918
|
ignore_storage: bool = False,
|
1877
1919
|
storage_base_path: Optional[str] = None,
|
1878
1920
|
use_placeholder_flow: bool = True,
|
1879
|
-
) -> Flow:
|
1921
|
+
) -> Flow[P, Any]:
|
1880
1922
|
"""
|
1881
1923
|
Load a flow from the location/script provided in a deployment's storage document.
|
1882
1924
|
|
@@ -1955,7 +1997,7 @@ async def load_flow_from_flow_run(
|
|
1955
1997
|
return flow
|
1956
1998
|
|
1957
1999
|
|
1958
|
-
def load_placeholder_flow(entrypoint: str, raises: Exception):
|
2000
|
+
def load_placeholder_flow(entrypoint: str, raises: Exception) -> Flow[P, Any]:
|
1959
2001
|
"""
|
1960
2002
|
Load a placeholder flow that is initialized with the same arguments as the
|
1961
2003
|
flow specified in the entrypoint. If called the flow will raise `raises`.
|
@@ -1972,10 +2014,10 @@ def load_placeholder_flow(entrypoint: str, raises: Exception):
|
|
1972
2014
|
def _base_placeholder():
|
1973
2015
|
raise raises
|
1974
2016
|
|
1975
|
-
def sync_placeholder_flow(*args, **kwargs):
|
2017
|
+
def sync_placeholder_flow(*args: "P.args", **kwargs: "P.kwargs"):
|
1976
2018
|
_base_placeholder()
|
1977
2019
|
|
1978
|
-
async def async_placeholder_flow(*args, **kwargs):
|
2020
|
+
async def async_placeholder_flow(*args: "P.args", **kwargs: "P.kwargs"):
|
1979
2021
|
_base_placeholder()
|
1980
2022
|
|
1981
2023
|
placeholder_flow = (
|
@@ -1990,7 +2032,7 @@ def load_placeholder_flow(entrypoint: str, raises: Exception):
|
|
1990
2032
|
return Flow(**arguments)
|
1991
2033
|
|
1992
2034
|
|
1993
|
-
def safe_load_flow_from_entrypoint(entrypoint: str) -> Optional[Flow]:
|
2035
|
+
def safe_load_flow_from_entrypoint(entrypoint: str) -> Optional[Flow[P, Any]]:
|
1994
2036
|
"""
|
1995
2037
|
Load a flow from an entrypoint and return None if an exception is raised.
|
1996
2038
|
|
@@ -2015,8 +2057,8 @@ def safe_load_flow_from_entrypoint(entrypoint: str) -> Optional[Flow]:
|
|
2015
2057
|
|
2016
2058
|
|
2017
2059
|
def _sanitize_and_load_flow(
|
2018
|
-
func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], namespace:
|
2019
|
-
) -> Optional[Flow]:
|
2060
|
+
func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], namespace: dict[str, Any]
|
2061
|
+
) -> Optional[Flow[P, Any]]:
|
2020
2062
|
"""
|
2021
2063
|
Attempt to load a flow from the function definition after sanitizing the annotations
|
2022
2064
|
and defaults that can't be compiled.
|
@@ -2053,7 +2095,7 @@ def _sanitize_and_load_flow(
|
|
2053
2095
|
arg.annotation = None
|
2054
2096
|
|
2055
2097
|
# Remove defaults that can't be compiled
|
2056
|
-
new_defaults = []
|
2098
|
+
new_defaults: list[Any] = []
|
2057
2099
|
for default in func_def.args.defaults:
|
2058
2100
|
try:
|
2059
2101
|
code = compile(ast.Expression(default), "<ast>", "eval")
|
@@ -2073,7 +2115,7 @@ def _sanitize_and_load_flow(
|
|
2073
2115
|
func_def.args.defaults = new_defaults
|
2074
2116
|
|
2075
2117
|
# Remove kw_defaults that can't be compiled
|
2076
|
-
new_kw_defaults = []
|
2118
|
+
new_kw_defaults: list[Any] = []
|
2077
2119
|
for default in func_def.args.kw_defaults:
|
2078
2120
|
if default is not None:
|
2079
2121
|
try:
|
@@ -2132,7 +2174,7 @@ def _sanitize_and_load_flow(
|
|
2132
2174
|
|
2133
2175
|
|
2134
2176
|
def load_flow_arguments_from_entrypoint(
|
2135
|
-
entrypoint: str, arguments: Optional[Union[
|
2177
|
+
entrypoint: str, arguments: Optional[Union[list[str], set[str]]] = None
|
2136
2178
|
) -> dict[str, Any]:
|
2137
2179
|
"""
|
2138
2180
|
Extract flow arguments from an entrypoint string.
|
@@ -2166,7 +2208,7 @@ def load_flow_arguments_from_entrypoint(
|
|
2166
2208
|
"log_prints",
|
2167
2209
|
}
|
2168
2210
|
|
2169
|
-
result = {}
|
2211
|
+
result: dict[str, Any] = {}
|
2170
2212
|
|
2171
2213
|
for decorator in func_def.decorator_list:
|
2172
2214
|
if (
|
@@ -2179,7 +2221,7 @@ def load_flow_arguments_from_entrypoint(
|
|
2179
2221
|
|
2180
2222
|
if isinstance(keyword.value, ast.Constant):
|
2181
2223
|
# Use the string value of the argument
|
2182
|
-
result[keyword.arg] = str(keyword.value.value)
|
2224
|
+
result[cast(str, keyword.arg)] = str(keyword.value.value)
|
2183
2225
|
continue
|
2184
2226
|
|
2185
2227
|
# if the arg value is not a raw str (i.e. a variable or expression),
|
@@ -2192,7 +2234,7 @@ def load_flow_arguments_from_entrypoint(
|
|
2192
2234
|
|
2193
2235
|
try:
|
2194
2236
|
evaluated_value = eval(cleaned_value, namespace) # type: ignore
|
2195
|
-
result[keyword.arg] = str(evaluated_value)
|
2237
|
+
result[cast(str, keyword.arg)] = str(evaluated_value)
|
2196
2238
|
except Exception as e:
|
2197
2239
|
logger.info(
|
2198
2240
|
"Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",
|