prefect-client 3.1.5__py3-none-any.whl → 3.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +3 -0
- prefect/_internal/compatibility/migration.py +1 -1
- prefect/_internal/concurrency/api.py +52 -52
- prefect/_internal/concurrency/calls.py +59 -35
- prefect/_internal/concurrency/cancellation.py +34 -18
- prefect/_internal/concurrency/event_loop.py +7 -6
- prefect/_internal/concurrency/threads.py +41 -33
- prefect/_internal/concurrency/waiters.py +28 -21
- prefect/_internal/pydantic/v1_schema.py +2 -2
- prefect/_internal/pydantic/v2_schema.py +10 -9
- prefect/_internal/schemas/bases.py +9 -7
- prefect/_internal/schemas/validators.py +2 -1
- prefect/_version.py +3 -3
- prefect/automations.py +53 -47
- prefect/blocks/abstract.py +12 -10
- prefect/blocks/core.py +4 -2
- prefect/cache_policies.py +11 -11
- prefect/client/__init__.py +3 -1
- prefect/client/base.py +36 -37
- prefect/client/cloud.py +26 -19
- prefect/client/collections.py +2 -2
- prefect/client/orchestration.py +342 -273
- prefect/client/schemas/__init__.py +24 -0
- prefect/client/schemas/actions.py +123 -116
- prefect/client/schemas/objects.py +110 -81
- prefect/client/schemas/responses.py +18 -18
- prefect/client/schemas/schedules.py +136 -93
- prefect/client/subscriptions.py +28 -14
- prefect/client/utilities.py +32 -36
- prefect/concurrency/asyncio.py +6 -9
- prefect/concurrency/sync.py +35 -5
- prefect/context.py +39 -31
- prefect/deployments/flow_runs.py +3 -5
- prefect/docker/__init__.py +1 -1
- prefect/events/schemas/events.py +25 -20
- prefect/events/utilities.py +1 -2
- prefect/filesystems.py +3 -3
- prefect/flow_engine.py +61 -21
- prefect/flow_runs.py +3 -3
- prefect/flows.py +214 -170
- prefect/logging/configuration.py +1 -1
- prefect/logging/highlighters.py +1 -2
- prefect/logging/loggers.py +30 -20
- prefect/main.py +17 -24
- prefect/runner/runner.py +43 -21
- prefect/runner/server.py +30 -32
- prefect/runner/submit.py +3 -6
- prefect/runner/utils.py +6 -6
- prefect/runtime/flow_run.py +7 -0
- prefect/settings/constants.py +2 -2
- prefect/settings/legacy.py +1 -1
- prefect/settings/models/server/events.py +10 -0
- prefect/task_engine.py +72 -19
- prefect/task_runners.py +2 -2
- prefect/tasks.py +46 -33
- prefect/telemetry/bootstrap.py +15 -2
- prefect/telemetry/run_telemetry.py +107 -0
- prefect/transactions.py +14 -14
- prefect/types/__init__.py +1 -4
- prefect/utilities/_engine.py +96 -0
- prefect/utilities/annotations.py +25 -18
- prefect/utilities/asyncutils.py +126 -140
- prefect/utilities/callables.py +87 -78
- prefect/utilities/collections.py +278 -117
- prefect/utilities/compat.py +13 -21
- prefect/utilities/context.py +6 -5
- prefect/utilities/dispatch.py +23 -12
- prefect/utilities/dockerutils.py +33 -32
- prefect/utilities/engine.py +126 -239
- prefect/utilities/filesystem.py +18 -15
- prefect/utilities/hashing.py +10 -11
- prefect/utilities/importtools.py +40 -27
- prefect/utilities/math.py +9 -5
- prefect/utilities/names.py +3 -3
- prefect/utilities/processutils.py +121 -57
- prefect/utilities/pydantic.py +41 -36
- prefect/utilities/render_swagger.py +22 -12
- prefect/utilities/schema_tools/__init__.py +2 -1
- prefect/utilities/schema_tools/hydration.py +50 -43
- prefect/utilities/schema_tools/validation.py +52 -42
- prefect/utilities/services.py +13 -12
- prefect/utilities/templating.py +45 -45
- prefect/utilities/text.py +2 -1
- prefect/utilities/timeout.py +4 -4
- prefect/utilities/urls.py +9 -4
- prefect/utilities/visualization.py +46 -24
- prefect/variables.py +9 -8
- prefect/workers/base.py +15 -8
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/METADATA +4 -2
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/RECORD +93 -91
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/top_level.txt +0 -0
prefect/flows.py
CHANGED
@@ -23,13 +23,10 @@ from typing import (
|
|
23
23
|
Awaitable,
|
24
24
|
Callable,
|
25
25
|
Coroutine,
|
26
|
-
Dict,
|
27
26
|
Generic,
|
28
27
|
Iterable,
|
29
|
-
List,
|
30
28
|
NoReturn,
|
31
29
|
Optional,
|
32
|
-
Set,
|
33
30
|
Tuple,
|
34
31
|
Type,
|
35
32
|
TypeVar,
|
@@ -45,7 +42,7 @@ from pydantic.v1 import BaseModel as V1BaseModel
|
|
45
42
|
from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
|
46
43
|
from pydantic.v1.errors import ConfigError # TODO
|
47
44
|
from rich.console import Console
|
48
|
-
from typing_extensions import Literal, ParamSpec,
|
45
|
+
from typing_extensions import Literal, ParamSpec, TypeAlias
|
49
46
|
|
50
47
|
from prefect._internal.concurrency.api import create_call, from_async
|
51
48
|
from prefect.blocks.core import Block
|
@@ -97,6 +94,7 @@ from prefect.utilities.filesystem import relative_path_to_current_platform
|
|
97
94
|
from prefect.utilities.hashing import file_hash
|
98
95
|
from prefect.utilities.importtools import import_object, safe_load_namespace
|
99
96
|
|
97
|
+
from ._internal.compatibility.async_dispatch import is_in_async_context
|
100
98
|
from ._internal.pydantic.v2_schema import is_v2_type
|
101
99
|
from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
|
102
100
|
from ._internal.pydantic.v2_validated_func import (
|
@@ -106,7 +104,11 @@ from ._internal.pydantic.v2_validated_func import (
|
|
106
104
|
T = TypeVar("T") # Generic type var for capturing the inner return type of async funcs
|
107
105
|
R = TypeVar("R") # The return type of the user's function
|
108
106
|
P = ParamSpec("P") # The parameters of the flow
|
109
|
-
F = TypeVar("F", bound="Flow") # The type of the flow
|
107
|
+
F = TypeVar("F", bound="Flow[Any, Any]") # The type of the flow
|
108
|
+
|
109
|
+
StateHookCallable: TypeAlias = Callable[
|
110
|
+
[FlowSchema, FlowRun, State], Union[Awaitable[None], None]
|
111
|
+
]
|
110
112
|
|
111
113
|
logger = get_logger("flows")
|
112
114
|
|
@@ -185,7 +187,9 @@ class Flow(Generic[P, R]):
|
|
185
187
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
186
188
|
retries: Optional[int] = None,
|
187
189
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
188
|
-
task_runner: Union[
|
190
|
+
task_runner: Union[
|
191
|
+
Type[TaskRunner[PrefectFuture[R]]], TaskRunner[PrefectFuture[R]], None
|
192
|
+
] = None,
|
189
193
|
description: Optional[str] = None,
|
190
194
|
timeout_seconds: Union[int, float, None] = None,
|
191
195
|
validate_parameters: bool = True,
|
@@ -194,15 +198,11 @@ class Flow(Generic[P, R]):
|
|
194
198
|
result_serializer: Optional[ResultSerializer] = None,
|
195
199
|
cache_result_in_memory: bool = True,
|
196
200
|
log_prints: Optional[bool] = None,
|
197
|
-
on_completion: Optional[
|
198
|
-
|
199
|
-
] = None,
|
200
|
-
|
201
|
-
|
202
|
-
List[Callable[[FlowSchema, FlowRun, State], None]]
|
203
|
-
] = None,
|
204
|
-
on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
205
|
-
on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
201
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
202
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
203
|
+
on_cancellation: Optional[list[StateHookCallable]] = None,
|
204
|
+
on_crashed: Optional[list[StateHookCallable]] = None,
|
205
|
+
on_running: Optional[list[StateHookCallable]] = None,
|
206
206
|
):
|
207
207
|
if name is not None and not isinstance(name, str):
|
208
208
|
raise TypeError(
|
@@ -374,7 +374,7 @@ class Flow(Generic[P, R]):
|
|
374
374
|
def ismethod(self) -> bool:
|
375
375
|
return hasattr(self.fn, "__prefect_self__")
|
376
376
|
|
377
|
-
def __get__(self, instance, owner):
|
377
|
+
def __get__(self, instance: Any, owner: Any):
|
378
378
|
"""
|
379
379
|
Implement the descriptor protocol so that the flow can be used as an instance method.
|
380
380
|
When an instance method is loaded, this method is called with the "self" instance as
|
@@ -401,7 +401,9 @@ class Flow(Generic[P, R]):
|
|
401
401
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
402
402
|
description: Optional[str] = None,
|
403
403
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
404
|
-
task_runner: Union[
|
404
|
+
task_runner: Union[
|
405
|
+
Type[TaskRunner[PrefectFuture[R]]], TaskRunner[PrefectFuture[R]], None
|
406
|
+
] = None,
|
405
407
|
timeout_seconds: Union[int, float, None] = None,
|
406
408
|
validate_parameters: Optional[bool] = None,
|
407
409
|
persist_result: Optional[bool] = NotSet, # type: ignore
|
@@ -409,16 +411,12 @@ class Flow(Generic[P, R]):
|
|
409
411
|
result_serializer: Optional[ResultSerializer] = NotSet, # type: ignore
|
410
412
|
cache_result_in_memory: Optional[bool] = None,
|
411
413
|
log_prints: Optional[bool] = NotSet, # type: ignore
|
412
|
-
on_completion: Optional[
|
413
|
-
|
414
|
-
] = None,
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
] = None,
|
419
|
-
on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
420
|
-
on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
421
|
-
) -> Self:
|
414
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
415
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
416
|
+
on_cancellation: Optional[list[StateHookCallable]] = None,
|
417
|
+
on_crashed: Optional[list[StateHookCallable]] = None,
|
418
|
+
on_running: Optional[list[StateHookCallable]] = None,
|
419
|
+
) -> "Flow[P, R]":
|
422
420
|
"""
|
423
421
|
Create a new flow from the current object, updating provided options.
|
424
422
|
|
@@ -521,7 +519,7 @@ class Flow(Generic[P, R]):
|
|
521
519
|
new_flow._entrypoint = self._entrypoint
|
522
520
|
return new_flow
|
523
521
|
|
524
|
-
def validate_parameters(self, parameters:
|
522
|
+
def validate_parameters(self, parameters: dict[str, Any]) -> dict[str, Any]:
|
525
523
|
"""
|
526
524
|
Validate parameters for compatibility with the flow by attempting to cast the inputs to the
|
527
525
|
associated types specified by the function's type annotations.
|
@@ -598,7 +596,7 @@ class Flow(Generic[P, R]):
|
|
598
596
|
}
|
599
597
|
return cast_parameters
|
600
598
|
|
601
|
-
def serialize_parameters(self, parameters:
|
599
|
+
def serialize_parameters(self, parameters: dict[str, Any]) -> dict[str, Any]:
|
602
600
|
"""
|
603
601
|
Convert parameters to a serializable form.
|
604
602
|
|
@@ -644,15 +642,15 @@ class Flow(Generic[P, R]):
|
|
644
642
|
paused: Optional[bool] = None,
|
645
643
|
schedules: Optional["FlexibleScheduleList"] = None,
|
646
644
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
647
|
-
parameters: Optional[dict] = None,
|
648
|
-
triggers: Optional[
|
645
|
+
parameters: Optional[dict[str, Any]] = None,
|
646
|
+
triggers: Optional[list[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
649
647
|
description: Optional[str] = None,
|
650
|
-
tags: Optional[
|
648
|
+
tags: Optional[list[str]] = None,
|
651
649
|
version: Optional[str] = None,
|
652
650
|
enforce_parameter_schema: bool = True,
|
653
651
|
work_pool_name: Optional[str] = None,
|
654
652
|
work_queue_name: Optional[str] = None,
|
655
|
-
job_variables: Optional[
|
653
|
+
job_variables: Optional[dict[str, Any]] = None,
|
656
654
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
657
655
|
) -> "RunnerDeployment":
|
658
656
|
"""
|
@@ -754,33 +752,23 @@ class Flow(Generic[P, R]):
|
|
754
752
|
entrypoint_type=entrypoint_type,
|
755
753
|
)
|
756
754
|
|
757
|
-
def on_completion(
|
758
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
759
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
755
|
+
def on_completion(self, fn: StateHookCallable) -> StateHookCallable:
|
760
756
|
self.on_completion_hooks.append(fn)
|
761
757
|
return fn
|
762
758
|
|
763
|
-
def on_cancellation(
|
764
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
765
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
759
|
+
def on_cancellation(self, fn: StateHookCallable) -> StateHookCallable:
|
766
760
|
self.on_cancellation_hooks.append(fn)
|
767
761
|
return fn
|
768
762
|
|
769
|
-
def on_crashed(
|
770
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
771
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
763
|
+
def on_crashed(self, fn: StateHookCallable) -> StateHookCallable:
|
772
764
|
self.on_crashed_hooks.append(fn)
|
773
765
|
return fn
|
774
766
|
|
775
|
-
def on_running(
|
776
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
777
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
767
|
+
def on_running(self, fn: StateHookCallable) -> StateHookCallable:
|
778
768
|
self.on_running_hooks.append(fn)
|
779
769
|
return fn
|
780
770
|
|
781
|
-
def on_failure(
|
782
|
-
self, fn: Callable[["Flow", FlowRun, State], None]
|
783
|
-
) -> Callable[["Flow", FlowRun, State], None]:
|
771
|
+
def on_failure(self, fn: StateHookCallable) -> StateHookCallable:
|
784
772
|
self.on_failure_hooks.append(fn)
|
785
773
|
return fn
|
786
774
|
|
@@ -800,10 +788,10 @@ class Flow(Generic[P, R]):
|
|
800
788
|
paused: Optional[bool] = None,
|
801
789
|
schedules: Optional["FlexibleScheduleList"] = None,
|
802
790
|
global_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
803
|
-
triggers: Optional[
|
804
|
-
parameters: Optional[dict] = None,
|
791
|
+
triggers: Optional[list[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
792
|
+
parameters: Optional[dict[str, Any]] = None,
|
805
793
|
description: Optional[str] = None,
|
806
|
-
tags: Optional[
|
794
|
+
tags: Optional[list[str]] = None,
|
807
795
|
version: Optional[str] = None,
|
808
796
|
enforce_parameter_schema: bool = True,
|
809
797
|
pause_on_shutdown: bool = True,
|
@@ -1038,8 +1026,11 @@ class Flow(Generic[P, R]):
|
|
1038
1026
|
await storage.pull_code()
|
1039
1027
|
|
1040
1028
|
full_entrypoint = str(storage.destination / entrypoint)
|
1041
|
-
flow
|
1042
|
-
|
1029
|
+
flow = cast(
|
1030
|
+
Flow[P, R],
|
1031
|
+
await from_async.wait_for_call_in_new_thread(
|
1032
|
+
create_call(load_flow_from_entrypoint, full_entrypoint)
|
1033
|
+
),
|
1043
1034
|
)
|
1044
1035
|
flow._storage = storage
|
1045
1036
|
flow._entrypoint = entrypoint
|
@@ -1055,17 +1046,17 @@ class Flow(Generic[P, R]):
|
|
1055
1046
|
build: bool = True,
|
1056
1047
|
push: bool = True,
|
1057
1048
|
work_queue_name: Optional[str] = None,
|
1058
|
-
job_variables: Optional[dict] = None,
|
1049
|
+
job_variables: Optional[dict[str, Any]] = None,
|
1059
1050
|
interval: Optional[Union[int, float, datetime.timedelta]] = None,
|
1060
1051
|
cron: Optional[str] = None,
|
1061
1052
|
rrule: Optional[str] = None,
|
1062
1053
|
paused: Optional[bool] = None,
|
1063
|
-
schedules: Optional[
|
1054
|
+
schedules: Optional[list[DeploymentScheduleCreate]] = None,
|
1064
1055
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
1065
|
-
triggers: Optional[
|
1066
|
-
parameters: Optional[dict] = None,
|
1056
|
+
triggers: Optional[list[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
1057
|
+
parameters: Optional[dict[str, Any]] = None,
|
1067
1058
|
description: Optional[str] = None,
|
1068
|
-
tags: Optional[
|
1059
|
+
tags: Optional[list[str]] = None,
|
1069
1060
|
version: Optional[str] = None,
|
1070
1061
|
enforce_parameter_schema: bool = True,
|
1071
1062
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
@@ -1288,7 +1279,7 @@ class Flow(Generic[P, R]):
|
|
1288
1279
|
self,
|
1289
1280
|
*args: "P.args",
|
1290
1281
|
return_state: bool = False,
|
1291
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
1282
|
+
wait_for: Optional[Iterable[PrefectFuture[Any]]] = None,
|
1292
1283
|
**kwargs: "P.kwargs",
|
1293
1284
|
):
|
1294
1285
|
"""
|
@@ -1360,7 +1351,7 @@ class Flow(Generic[P, R]):
|
|
1360
1351
|
)
|
1361
1352
|
|
1362
1353
|
@sync_compatible
|
1363
|
-
async def visualize(self, *args, **kwargs):
|
1354
|
+
async def visualize(self, *args: "P.args", **kwargs: "P.kwargs"):
|
1364
1355
|
"""
|
1365
1356
|
Generates a graphviz object representing the current flow. In IPython notebooks,
|
1366
1357
|
it's rendered inline, otherwise in a new window as a PNG.
|
@@ -1389,7 +1380,7 @@ class Flow(Generic[P, R]):
|
|
1389
1380
|
try:
|
1390
1381
|
with TaskVizTracker() as tracker:
|
1391
1382
|
if self.isasync:
|
1392
|
-
await self.fn(*args, **kwargs)
|
1383
|
+
await self.fn(*args, **kwargs) # type: ignore[reportGeneralTypeIssues]
|
1393
1384
|
else:
|
1394
1385
|
self.fn(*args, **kwargs)
|
1395
1386
|
|
@@ -1432,7 +1423,7 @@ def flow(
|
|
1432
1423
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1433
1424
|
retries: Optional[int] = None,
|
1434
1425
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
1435
|
-
task_runner: Optional[TaskRunner] = None,
|
1426
|
+
task_runner: Optional[TaskRunner[PrefectFuture[R]]] = None,
|
1436
1427
|
description: Optional[str] = None,
|
1437
1428
|
timeout_seconds: Union[int, float, None] = None,
|
1438
1429
|
validate_parameters: bool = True,
|
@@ -1441,30 +1432,24 @@ def flow(
|
|
1441
1432
|
result_serializer: Optional[ResultSerializer] = None,
|
1442
1433
|
cache_result_in_memory: bool = True,
|
1443
1434
|
log_prints: Optional[bool] = None,
|
1444
|
-
on_completion: Optional[
|
1445
|
-
|
1446
|
-
] = None,
|
1447
|
-
|
1448
|
-
|
1449
|
-
] = None,
|
1450
|
-
on_cancellation: Optional[
|
1451
|
-
List[Callable[[FlowSchema, FlowRun, State], None]]
|
1452
|
-
] = None,
|
1453
|
-
on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
1454
|
-
on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
1435
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
1436
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
1437
|
+
on_cancellation: Optional[list[StateHookCallable]] = None,
|
1438
|
+
on_crashed: Optional[list[StateHookCallable]] = None,
|
1439
|
+
on_running: Optional[list[StateHookCallable]] = None,
|
1455
1440
|
) -> Callable[[Callable[P, R]], Flow[P, R]]:
|
1456
1441
|
...
|
1457
1442
|
|
1458
1443
|
|
1459
1444
|
def flow(
|
1460
|
-
__fn=None,
|
1445
|
+
__fn: Optional[Callable[P, R]] = None,
|
1461
1446
|
*,
|
1462
1447
|
name: Optional[str] = None,
|
1463
1448
|
version: Optional[str] = None,
|
1464
1449
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1465
1450
|
retries: Optional[int] = None,
|
1466
1451
|
retry_delay_seconds: Union[int, float, None] = None,
|
1467
|
-
task_runner: Optional[TaskRunner] = None,
|
1452
|
+
task_runner: Optional[TaskRunner[PrefectFuture[R]]] = None,
|
1468
1453
|
description: Optional[str] = None,
|
1469
1454
|
timeout_seconds: Union[int, float, None] = None,
|
1470
1455
|
validate_parameters: bool = True,
|
@@ -1473,17 +1458,11 @@ def flow(
|
|
1473
1458
|
result_serializer: Optional[ResultSerializer] = None,
|
1474
1459
|
cache_result_in_memory: bool = True,
|
1475
1460
|
log_prints: Optional[bool] = None,
|
1476
|
-
on_completion: Optional[
|
1477
|
-
|
1478
|
-
] = None,
|
1479
|
-
|
1480
|
-
|
1481
|
-
] = None,
|
1482
|
-
on_cancellation: Optional[
|
1483
|
-
List[Callable[[FlowSchema, FlowRun, State], None]]
|
1484
|
-
] = None,
|
1485
|
-
on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
1486
|
-
on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
|
1461
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
1462
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
1463
|
+
on_cancellation: Optional[list[StateHookCallable]] = None,
|
1464
|
+
on_crashed: Optional[list[StateHookCallable]] = None,
|
1465
|
+
on_running: Optional[list[StateHookCallable]] = None,
|
1487
1466
|
):
|
1488
1467
|
"""
|
1489
1468
|
Decorator to designate a function as a Prefect workflow.
|
@@ -1592,30 +1571,27 @@ def flow(
|
|
1592
1571
|
if isinstance(__fn, (classmethod, staticmethod)):
|
1593
1572
|
method_decorator = type(__fn).__name__
|
1594
1573
|
raise TypeError(f"@{method_decorator} should be applied on top of @flow")
|
1595
|
-
return
|
1596
|
-
|
1597
|
-
|
1598
|
-
|
1599
|
-
|
1600
|
-
|
1601
|
-
|
1602
|
-
|
1603
|
-
|
1604
|
-
|
1605
|
-
|
1606
|
-
|
1607
|
-
|
1608
|
-
|
1609
|
-
|
1610
|
-
|
1611
|
-
|
1612
|
-
|
1613
|
-
|
1614
|
-
|
1615
|
-
|
1616
|
-
on_crashed=on_crashed,
|
1617
|
-
on_running=on_running,
|
1618
|
-
),
|
1574
|
+
return Flow(
|
1575
|
+
fn=__fn,
|
1576
|
+
name=name,
|
1577
|
+
version=version,
|
1578
|
+
flow_run_name=flow_run_name,
|
1579
|
+
task_runner=task_runner,
|
1580
|
+
description=description,
|
1581
|
+
timeout_seconds=timeout_seconds,
|
1582
|
+
validate_parameters=validate_parameters,
|
1583
|
+
retries=retries,
|
1584
|
+
retry_delay_seconds=retry_delay_seconds,
|
1585
|
+
persist_result=persist_result,
|
1586
|
+
result_storage=result_storage,
|
1587
|
+
result_serializer=result_serializer,
|
1588
|
+
cache_result_in_memory=cache_result_in_memory,
|
1589
|
+
log_prints=log_prints,
|
1590
|
+
on_completion=on_completion,
|
1591
|
+
on_failure=on_failure,
|
1592
|
+
on_cancellation=on_cancellation,
|
1593
|
+
on_crashed=on_crashed,
|
1594
|
+
on_running=on_running,
|
1619
1595
|
)
|
1620
1596
|
else:
|
1621
1597
|
return cast(
|
@@ -1667,10 +1643,10 @@ flow.from_source = Flow.from_source
|
|
1667
1643
|
|
1668
1644
|
|
1669
1645
|
def select_flow(
|
1670
|
-
flows: Iterable[Flow],
|
1646
|
+
flows: Iterable[Flow[P, R]],
|
1671
1647
|
flow_name: Optional[str] = None,
|
1672
1648
|
from_message: Optional[str] = None,
|
1673
|
-
) -> Flow:
|
1649
|
+
) -> Flow[P, R]:
|
1674
1650
|
"""
|
1675
1651
|
Select the only flow in an iterable or a flow specified by name.
|
1676
1652
|
|
@@ -1715,7 +1691,7 @@ def select_flow(
|
|
1715
1691
|
def load_flow_from_entrypoint(
|
1716
1692
|
entrypoint: str,
|
1717
1693
|
use_placeholder_flow: bool = True,
|
1718
|
-
) -> Flow:
|
1694
|
+
) -> Flow[P, Any]:
|
1719
1695
|
"""
|
1720
1696
|
Extract a flow object from a script at an entrypoint by running all of the code in the file.
|
1721
1697
|
|
@@ -1739,7 +1715,7 @@ def load_flow_from_entrypoint(
|
|
1739
1715
|
else:
|
1740
1716
|
path, func_name = entrypoint.rsplit(".", maxsplit=1)
|
1741
1717
|
try:
|
1742
|
-
flow = import_object(entrypoint)
|
1718
|
+
flow: Flow[P, Any] = import_object(entrypoint) # pyright: ignore[reportRedeclaration]
|
1743
1719
|
except AttributeError as exc:
|
1744
1720
|
raise MissingFlowError(
|
1745
1721
|
f"Flow function with name {func_name!r} not found in {path!r}. "
|
@@ -1748,13 +1724,13 @@ def load_flow_from_entrypoint(
|
|
1748
1724
|
# If the flow has dependencies that are not installed in the current
|
1749
1725
|
# environment, fallback to loading the flow via AST parsing.
|
1750
1726
|
if use_placeholder_flow:
|
1751
|
-
flow = safe_load_flow_from_entrypoint(entrypoint)
|
1727
|
+
flow: Optional[Flow[P, Any]] = safe_load_flow_from_entrypoint(entrypoint)
|
1752
1728
|
if flow is None:
|
1753
1729
|
raise
|
1754
1730
|
else:
|
1755
1731
|
raise
|
1756
1732
|
|
1757
|
-
if not isinstance(flow, Flow):
|
1733
|
+
if not isinstance(flow, Flow): # pyright: ignore[reportUnnecessaryIsInstance]
|
1758
1734
|
raise MissingFlowError(
|
1759
1735
|
f"Function with name {func_name!r} is not a flow. Make sure that it is "
|
1760
1736
|
"decorated with '@flow'."
|
@@ -1768,8 +1744,8 @@ def serve(
|
|
1768
1744
|
pause_on_shutdown: bool = True,
|
1769
1745
|
print_starting_message: bool = True,
|
1770
1746
|
limit: Optional[int] = None,
|
1771
|
-
**kwargs,
|
1772
|
-
):
|
1747
|
+
**kwargs: Any,
|
1748
|
+
) -> None:
|
1773
1749
|
"""
|
1774
1750
|
Serve the provided list of deployments.
|
1775
1751
|
|
@@ -1812,61 +1788,129 @@ def serve(
|
|
1812
1788
|
serve(hello_deploy, bye_deploy)
|
1813
1789
|
```
|
1814
1790
|
"""
|
1815
|
-
from rich.console import Console, Group
|
1816
|
-
from rich.table import Table
|
1817
1791
|
|
1818
1792
|
from prefect.runner import Runner
|
1819
1793
|
|
1794
|
+
if is_in_async_context():
|
1795
|
+
raise RuntimeError(
|
1796
|
+
"Cannot call `serve` in an asynchronous context. Use `aserve` instead."
|
1797
|
+
)
|
1798
|
+
|
1820
1799
|
runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
|
1821
1800
|
for deployment in args:
|
1822
1801
|
runner.add_deployment(deployment)
|
1823
1802
|
|
1824
1803
|
if print_starting_message:
|
1825
|
-
|
1826
|
-
"[green]Your deployments are being served and polling for"
|
1827
|
-
" scheduled runs!\n[/]"
|
1828
|
-
)
|
1804
|
+
_display_serve_start_message(*args)
|
1829
1805
|
|
1830
|
-
|
1806
|
+
try:
|
1807
|
+
asyncio.run(runner.start())
|
1808
|
+
except (KeyboardInterrupt, TerminationSignal) as exc:
|
1809
|
+
logger.info(f"Received {type(exc).__name__}, shutting down...")
|
1831
1810
|
|
1832
|
-
table.add_column(style="blue", no_wrap=True)
|
1833
1811
|
|
1834
|
-
|
1835
|
-
|
1812
|
+
async def aserve(
|
1813
|
+
*args: "RunnerDeployment",
|
1814
|
+
pause_on_shutdown: bool = True,
|
1815
|
+
print_starting_message: bool = True,
|
1816
|
+
limit: Optional[int] = None,
|
1817
|
+
**kwargs: Any,
|
1818
|
+
) -> None:
|
1819
|
+
"""
|
1820
|
+
Asynchronously serve the provided list of deployments.
|
1836
1821
|
|
1837
|
-
|
1838
|
-
|
1839
|
-
|
1840
|
-
|
1841
|
-
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1845
|
-
|
1822
|
+
Use `serve` instead if calling from a synchronous context.
|
1823
|
+
|
1824
|
+
Args:
|
1825
|
+
*args: A list of deployments to serve.
|
1826
|
+
pause_on_shutdown: A boolean for whether or not to automatically pause
|
1827
|
+
deployment schedules on shutdown.
|
1828
|
+
print_starting_message: Whether or not to print message to the console
|
1829
|
+
on startup.
|
1830
|
+
limit: The maximum number of runs that can be executed concurrently.
|
1831
|
+
**kwargs: Additional keyword arguments to pass to the runner.
|
1832
|
+
|
1833
|
+
Examples:
|
1834
|
+
Prepare deployment and asynchronous initialization function and serve them:
|
1835
|
+
|
1836
|
+
```python
|
1837
|
+
import asyncio
|
1838
|
+
import datetime
|
1839
|
+
|
1840
|
+
from prefect import flow, aserve, get_client
|
1841
|
+
|
1842
|
+
|
1843
|
+
async def init():
|
1844
|
+
await set_concurrency_limit()
|
1845
|
+
|
1846
|
+
|
1847
|
+
async def set_concurrency_limit():
|
1848
|
+
async with get_client() as client:
|
1849
|
+
await client.create_concurrency_limit(tag='dev', concurrency_limit=3)
|
1850
|
+
|
1851
|
+
|
1852
|
+
@flow
|
1853
|
+
async def my_flow(name):
|
1854
|
+
print(f"hello {name}")
|
1855
|
+
|
1856
|
+
|
1857
|
+
async def main():
|
1858
|
+
# Initialization function
|
1859
|
+
await init()
|
1860
|
+
|
1861
|
+
# Run once a day
|
1862
|
+
hello_deploy = await my_flow.to_deployment(
|
1863
|
+
"hello", tags=["dev"], interval=datetime.timedelta(days=1)
|
1846
1864
|
)
|
1847
1865
|
|
1848
|
-
|
1849
|
-
console.print(
|
1850
|
-
Group(help_message_top, table, help_message_bottom), soft_wrap=True
|
1851
|
-
)
|
1866
|
+
await aserve(hello_deploy)
|
1852
1867
|
|
1853
|
-
try:
|
1854
|
-
loop = asyncio.get_running_loop()
|
1855
|
-
except RuntimeError as exc:
|
1856
|
-
if "no running event loop" in str(exc):
|
1857
|
-
loop = None
|
1858
|
-
else:
|
1859
|
-
raise
|
1860
1868
|
|
1861
|
-
|
1862
|
-
|
1863
|
-
|
1864
|
-
|
1865
|
-
|
1866
|
-
|
1867
|
-
|
1868
|
-
|
1869
|
-
|
1869
|
+
if __name__ == "__main__":
|
1870
|
+
asyncio.run(main())
|
1871
|
+
"""
|
1872
|
+
|
1873
|
+
from prefect.runner import Runner
|
1874
|
+
|
1875
|
+
runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
|
1876
|
+
for deployment in args:
|
1877
|
+
await runner.add_deployment(deployment)
|
1878
|
+
|
1879
|
+
if print_starting_message:
|
1880
|
+
_display_serve_start_message(*args)
|
1881
|
+
|
1882
|
+
await runner.start()
|
1883
|
+
|
1884
|
+
|
1885
|
+
def _display_serve_start_message(*args: "RunnerDeployment"):
|
1886
|
+
from rich.console import Console, Group
|
1887
|
+
from rich.table import Table
|
1888
|
+
|
1889
|
+
help_message_top = (
|
1890
|
+
"[green]Your deployments are being served and polling for"
|
1891
|
+
" scheduled runs!\n[/]"
|
1892
|
+
)
|
1893
|
+
|
1894
|
+
table = Table(title="Deployments", show_header=False)
|
1895
|
+
|
1896
|
+
table.add_column(style="blue", no_wrap=True)
|
1897
|
+
|
1898
|
+
for deployment in args:
|
1899
|
+
table.add_row(f"{deployment.flow_name}/{deployment.name}")
|
1900
|
+
|
1901
|
+
help_message_bottom = (
|
1902
|
+
"\nTo trigger any of these deployments, use the"
|
1903
|
+
" following command:\n[blue]\n\t$ prefect deployment run"
|
1904
|
+
" [DEPLOYMENT_NAME]\n[/]"
|
1905
|
+
)
|
1906
|
+
if PREFECT_UI_URL:
|
1907
|
+
help_message_bottom += (
|
1908
|
+
"\nYou can also trigger your deployments via the Prefect UI:"
|
1909
|
+
f" [blue]{PREFECT_UI_URL.value()}/deployments[/]\n"
|
1910
|
+
)
|
1911
|
+
|
1912
|
+
console = Console()
|
1913
|
+
console.print(Group(help_message_top, table, help_message_bottom), soft_wrap=True)
|
1870
1914
|
|
1871
1915
|
|
1872
1916
|
@client_injector
|
@@ -1876,7 +1920,7 @@ async def load_flow_from_flow_run(
|
|
1876
1920
|
ignore_storage: bool = False,
|
1877
1921
|
storage_base_path: Optional[str] = None,
|
1878
1922
|
use_placeholder_flow: bool = True,
|
1879
|
-
) -> Flow:
|
1923
|
+
) -> Flow[P, Any]:
|
1880
1924
|
"""
|
1881
1925
|
Load a flow from the location/script provided in a deployment's storage document.
|
1882
1926
|
|
@@ -1955,7 +1999,7 @@ async def load_flow_from_flow_run(
|
|
1955
1999
|
return flow
|
1956
2000
|
|
1957
2001
|
|
1958
|
-
def load_placeholder_flow(entrypoint: str, raises: Exception):
|
2002
|
+
def load_placeholder_flow(entrypoint: str, raises: Exception) -> Flow[P, Any]:
|
1959
2003
|
"""
|
1960
2004
|
Load a placeholder flow that is initialized with the same arguments as the
|
1961
2005
|
flow specified in the entrypoint. If called the flow will raise `raises`.
|
@@ -1972,10 +2016,10 @@ def load_placeholder_flow(entrypoint: str, raises: Exception):
|
|
1972
2016
|
def _base_placeholder():
|
1973
2017
|
raise raises
|
1974
2018
|
|
1975
|
-
def sync_placeholder_flow(*args, **kwargs):
|
2019
|
+
def sync_placeholder_flow(*args: "P.args", **kwargs: "P.kwargs"):
|
1976
2020
|
_base_placeholder()
|
1977
2021
|
|
1978
|
-
async def async_placeholder_flow(*args, **kwargs):
|
2022
|
+
async def async_placeholder_flow(*args: "P.args", **kwargs: "P.kwargs"):
|
1979
2023
|
_base_placeholder()
|
1980
2024
|
|
1981
2025
|
placeholder_flow = (
|
@@ -1990,7 +2034,7 @@ def load_placeholder_flow(entrypoint: str, raises: Exception):
|
|
1990
2034
|
return Flow(**arguments)
|
1991
2035
|
|
1992
2036
|
|
1993
|
-
def safe_load_flow_from_entrypoint(entrypoint: str) -> Optional[Flow]:
|
2037
|
+
def safe_load_flow_from_entrypoint(entrypoint: str) -> Optional[Flow[P, Any]]:
|
1994
2038
|
"""
|
1995
2039
|
Load a flow from an entrypoint and return None if an exception is raised.
|
1996
2040
|
|
@@ -2015,8 +2059,8 @@ def safe_load_flow_from_entrypoint(entrypoint: str) -> Optional[Flow]:
|
|
2015
2059
|
|
2016
2060
|
|
2017
2061
|
def _sanitize_and_load_flow(
|
2018
|
-
func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], namespace:
|
2019
|
-
) -> Optional[Flow]:
|
2062
|
+
func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], namespace: dict[str, Any]
|
2063
|
+
) -> Optional[Flow[P, Any]]:
|
2020
2064
|
"""
|
2021
2065
|
Attempt to load a flow from the function definition after sanitizing the annotations
|
2022
2066
|
and defaults that can't be compiled.
|
@@ -2053,7 +2097,7 @@ def _sanitize_and_load_flow(
|
|
2053
2097
|
arg.annotation = None
|
2054
2098
|
|
2055
2099
|
# Remove defaults that can't be compiled
|
2056
|
-
new_defaults = []
|
2100
|
+
new_defaults: list[Any] = []
|
2057
2101
|
for default in func_def.args.defaults:
|
2058
2102
|
try:
|
2059
2103
|
code = compile(ast.Expression(default), "<ast>", "eval")
|
@@ -2073,7 +2117,7 @@ def _sanitize_and_load_flow(
|
|
2073
2117
|
func_def.args.defaults = new_defaults
|
2074
2118
|
|
2075
2119
|
# Remove kw_defaults that can't be compiled
|
2076
|
-
new_kw_defaults = []
|
2120
|
+
new_kw_defaults: list[Any] = []
|
2077
2121
|
for default in func_def.args.kw_defaults:
|
2078
2122
|
if default is not None:
|
2079
2123
|
try:
|
@@ -2132,7 +2176,7 @@ def _sanitize_and_load_flow(
|
|
2132
2176
|
|
2133
2177
|
|
2134
2178
|
def load_flow_arguments_from_entrypoint(
|
2135
|
-
entrypoint: str, arguments: Optional[Union[
|
2179
|
+
entrypoint: str, arguments: Optional[Union[list[str], set[str]]] = None
|
2136
2180
|
) -> dict[str, Any]:
|
2137
2181
|
"""
|
2138
2182
|
Extract flow arguments from an entrypoint string.
|
@@ -2166,7 +2210,7 @@ def load_flow_arguments_from_entrypoint(
|
|
2166
2210
|
"log_prints",
|
2167
2211
|
}
|
2168
2212
|
|
2169
|
-
result = {}
|
2213
|
+
result: dict[str, Any] = {}
|
2170
2214
|
|
2171
2215
|
for decorator in func_def.decorator_list:
|
2172
2216
|
if (
|
@@ -2179,7 +2223,7 @@ def load_flow_arguments_from_entrypoint(
|
|
2179
2223
|
|
2180
2224
|
if isinstance(keyword.value, ast.Constant):
|
2181
2225
|
# Use the string value of the argument
|
2182
|
-
result[keyword.arg] = str(keyword.value.value)
|
2226
|
+
result[cast(str, keyword.arg)] = str(keyword.value.value)
|
2183
2227
|
continue
|
2184
2228
|
|
2185
2229
|
# if the arg value is not a raw str (i.e. a variable or expression),
|
@@ -2192,7 +2236,7 @@ def load_flow_arguments_from_entrypoint(
|
|
2192
2236
|
|
2193
2237
|
try:
|
2194
2238
|
evaluated_value = eval(cleaned_value, namespace) # type: ignore
|
2195
|
-
result[keyword.arg] = str(evaluated_value)
|
2239
|
+
result[cast(str, keyword.arg)] = str(evaluated_value)
|
2196
2240
|
except Exception as e:
|
2197
2241
|
logger.info(
|
2198
2242
|
"Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",
|