prefect-client 3.1.5__py3-none-any.whl → 3.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +3 -0
- prefect/_experimental/__init__.py +0 -0
- prefect/_experimental/lineage.py +181 -0
- prefect/_internal/compatibility/async_dispatch.py +38 -9
- prefect/_internal/compatibility/migration.py +1 -1
- prefect/_internal/concurrency/api.py +52 -52
- prefect/_internal/concurrency/calls.py +59 -35
- prefect/_internal/concurrency/cancellation.py +34 -18
- prefect/_internal/concurrency/event_loop.py +7 -6
- prefect/_internal/concurrency/threads.py +41 -33
- prefect/_internal/concurrency/waiters.py +28 -21
- prefect/_internal/pydantic/v1_schema.py +2 -2
- prefect/_internal/pydantic/v2_schema.py +10 -9
- prefect/_internal/pydantic/v2_validated_func.py +15 -10
- prefect/_internal/retries.py +15 -6
- prefect/_internal/schemas/bases.py +11 -8
- prefect/_internal/schemas/validators.py +7 -5
- prefect/_version.py +3 -3
- prefect/automations.py +53 -47
- prefect/blocks/abstract.py +12 -10
- prefect/blocks/core.py +148 -19
- prefect/blocks/system.py +2 -1
- prefect/cache_policies.py +11 -11
- prefect/client/__init__.py +3 -1
- prefect/client/base.py +36 -37
- prefect/client/cloud.py +26 -19
- prefect/client/collections.py +2 -2
- prefect/client/orchestration.py +430 -273
- prefect/client/schemas/__init__.py +24 -0
- prefect/client/schemas/actions.py +128 -121
- prefect/client/schemas/filters.py +1 -1
- prefect/client/schemas/objects.py +114 -85
- prefect/client/schemas/responses.py +19 -20
- prefect/client/schemas/schedules.py +136 -93
- prefect/client/subscriptions.py +30 -15
- prefect/client/utilities.py +46 -36
- prefect/concurrency/asyncio.py +6 -9
- prefect/concurrency/sync.py +35 -5
- prefect/context.py +40 -32
- prefect/deployments/flow_runs.py +6 -8
- prefect/deployments/runner.py +14 -14
- prefect/deployments/steps/core.py +3 -1
- prefect/deployments/steps/pull.py +60 -12
- prefect/docker/__init__.py +1 -1
- prefect/events/clients.py +55 -4
- prefect/events/filters.py +1 -1
- prefect/events/related.py +2 -1
- prefect/events/schemas/events.py +26 -21
- prefect/events/utilities.py +3 -2
- prefect/events/worker.py +8 -0
- prefect/filesystems.py +3 -3
- prefect/flow_engine.py +87 -87
- prefect/flow_runs.py +7 -5
- prefect/flows.py +218 -176
- prefect/logging/configuration.py +1 -1
- prefect/logging/highlighters.py +1 -2
- prefect/logging/loggers.py +30 -20
- prefect/main.py +17 -24
- prefect/results.py +43 -22
- prefect/runner/runner.py +43 -21
- prefect/runner/server.py +30 -32
- prefect/runner/storage.py +3 -3
- prefect/runner/submit.py +3 -6
- prefect/runner/utils.py +6 -6
- prefect/runtime/flow_run.py +7 -0
- prefect/serializers.py +28 -24
- prefect/settings/constants.py +2 -2
- prefect/settings/legacy.py +1 -1
- prefect/settings/models/experiments.py +5 -0
- prefect/settings/models/server/events.py +10 -0
- prefect/task_engine.py +87 -26
- prefect/task_runners.py +2 -2
- prefect/task_worker.py +43 -25
- prefect/tasks.py +148 -142
- prefect/telemetry/bootstrap.py +15 -2
- prefect/telemetry/instrumentation.py +1 -1
- prefect/telemetry/processors.py +10 -7
- prefect/telemetry/run_telemetry.py +231 -0
- prefect/transactions.py +14 -14
- prefect/types/__init__.py +5 -5
- prefect/utilities/_engine.py +96 -0
- prefect/utilities/annotations.py +25 -18
- prefect/utilities/asyncutils.py +126 -140
- prefect/utilities/callables.py +87 -78
- prefect/utilities/collections.py +278 -117
- prefect/utilities/compat.py +13 -21
- prefect/utilities/context.py +6 -5
- prefect/utilities/dispatch.py +23 -12
- prefect/utilities/dockerutils.py +33 -32
- prefect/utilities/engine.py +126 -239
- prefect/utilities/filesystem.py +18 -15
- prefect/utilities/hashing.py +10 -11
- prefect/utilities/importtools.py +40 -27
- prefect/utilities/math.py +9 -5
- prefect/utilities/names.py +3 -3
- prefect/utilities/processutils.py +121 -57
- prefect/utilities/pydantic.py +41 -36
- prefect/utilities/render_swagger.py +22 -12
- prefect/utilities/schema_tools/__init__.py +2 -1
- prefect/utilities/schema_tools/hydration.py +50 -43
- prefect/utilities/schema_tools/validation.py +52 -42
- prefect/utilities/services.py +13 -12
- prefect/utilities/templating.py +45 -45
- prefect/utilities/text.py +2 -1
- prefect/utilities/timeout.py +4 -4
- prefect/utilities/urls.py +9 -4
- prefect/utilities/visualization.py +46 -24
- prefect/variables.py +136 -27
- prefect/workers/base.py +15 -8
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/METADATA +5 -2
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/RECORD +114 -110
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/top_level.txt +0 -0
prefect/tasks.py
CHANGED
@@ -15,15 +15,11 @@ from typing import (
|
|
15
15
|
Awaitable,
|
16
16
|
Callable,
|
17
17
|
Coroutine,
|
18
|
-
Dict,
|
19
18
|
Generic,
|
20
19
|
Iterable,
|
21
|
-
List,
|
22
20
|
NoReturn,
|
23
21
|
Optional,
|
24
|
-
|
25
|
-
Tuple,
|
26
|
-
Type,
|
22
|
+
Protocol,
|
27
23
|
TypeVar,
|
28
24
|
Union,
|
29
25
|
cast,
|
@@ -31,7 +27,7 @@ from typing import (
|
|
31
27
|
)
|
32
28
|
from uuid import UUID, uuid4
|
33
29
|
|
34
|
-
from typing_extensions import Literal, ParamSpec
|
30
|
+
from typing_extensions import Literal, ParamSpec, Self, TypeAlias, TypeIs
|
35
31
|
|
36
32
|
import prefect.states
|
37
33
|
from prefect.cache_policies import DEFAULT, NONE, CachePolicy
|
@@ -63,10 +59,7 @@ from prefect.settings import (
|
|
63
59
|
)
|
64
60
|
from prefect.states import Pending, Scheduled, State
|
65
61
|
from prefect.utilities.annotations import NotSet
|
66
|
-
from prefect.utilities.asyncutils import
|
67
|
-
run_coro_as_sync,
|
68
|
-
sync_compatible,
|
69
|
-
)
|
62
|
+
from prefect.utilities.asyncutils import run_coro_as_sync, sync_compatible
|
70
63
|
from prefect.utilities.callables import (
|
71
64
|
expand_mapping_parameters,
|
72
65
|
get_call_parameters,
|
@@ -91,7 +84,7 @@ logger = get_logger("tasks")
|
|
91
84
|
|
92
85
|
|
93
86
|
def task_input_hash(
|
94
|
-
context: "TaskRunContext", arguments:
|
87
|
+
context: "TaskRunContext", arguments: dict[str, Any]
|
95
88
|
) -> Optional[str]:
|
96
89
|
"""
|
97
90
|
A task cache key implementation which hashes all inputs to the task using a JSON or
|
@@ -116,7 +109,7 @@ def task_input_hash(
|
|
116
109
|
)
|
117
110
|
|
118
111
|
|
119
|
-
def exponential_backoff(backoff_factor: float) -> Callable[[int],
|
112
|
+
def exponential_backoff(backoff_factor: float) -> Callable[[int], list[float]]:
|
120
113
|
"""
|
121
114
|
A task retry backoff utility that configures exponential backoff for task retries.
|
122
115
|
The exponential backoff design matches the urllib3 implementation.
|
@@ -129,7 +122,7 @@ def exponential_backoff(backoff_factor: float) -> Callable[[int], List[float]]:
|
|
129
122
|
a callable that can be passed to the task constructor
|
130
123
|
"""
|
131
124
|
|
132
|
-
def retry_backoff_callable(retries: int) ->
|
125
|
+
def retry_backoff_callable(retries: int) -> list[float]:
|
133
126
|
# no more than 50 retry delays can be configured on a task
|
134
127
|
retries = min(retries, 50)
|
135
128
|
|
@@ -141,8 +134,8 @@ def exponential_backoff(backoff_factor: float) -> Callable[[int], List[float]]:
|
|
141
134
|
def _infer_parent_task_runs(
|
142
135
|
flow_run_context: Optional[FlowRunContext],
|
143
136
|
task_run_context: Optional[TaskRunContext],
|
144
|
-
parameters:
|
145
|
-
):
|
137
|
+
parameters: dict[str, Any],
|
138
|
+
) -> list[TaskRunResult]:
|
146
139
|
"""
|
147
140
|
Attempt to infer the parent task runs for this task run based on the
|
148
141
|
provided flow run and task run contexts, as well as any parameters. It is
|
@@ -151,7 +144,7 @@ def _infer_parent_task_runs(
|
|
151
144
|
a parent. This is expected to happen when task inputs are yielded from
|
152
145
|
generator tasks.
|
153
146
|
"""
|
154
|
-
parents = []
|
147
|
+
parents: list[TaskRunResult] = []
|
155
148
|
|
156
149
|
# check if this task has a parent task run based on running in another
|
157
150
|
# task run's existing context. A task run is only considered a parent if
|
@@ -223,6 +216,24 @@ def _generate_task_key(fn: Callable[..., Any]) -> str:
|
|
223
216
|
return f"{qualname}-{code_hash}"
|
224
217
|
|
225
218
|
|
219
|
+
class TaskRunNameCallbackWithParameters(Protocol):
|
220
|
+
@classmethod
|
221
|
+
def is_callback_with_parameters(cls, callable: Callable[..., str]) -> TypeIs[Self]:
|
222
|
+
sig = inspect.signature(callable)
|
223
|
+
return "parameters" in sig.parameters
|
224
|
+
|
225
|
+
def __call__(self, parameters: dict[str, Any]) -> str:
|
226
|
+
...
|
227
|
+
|
228
|
+
|
229
|
+
StateHookCallable: TypeAlias = Callable[
|
230
|
+
["Task[..., Any]", TaskRun, State], Union[Awaitable[None], None]
|
231
|
+
]
|
232
|
+
TaskRunNameValueOrCallable: TypeAlias = Union[
|
233
|
+
Callable[[], str], TaskRunNameCallbackWithParameters, str
|
234
|
+
]
|
235
|
+
|
236
|
+
|
226
237
|
class Task(Generic[P, R]):
|
227
238
|
"""
|
228
239
|
A Prefect task definition.
|
@@ -305,21 +316,19 @@ class Task(Generic[P, R]):
|
|
305
316
|
description: Optional[str] = None,
|
306
317
|
tags: Optional[Iterable[str]] = None,
|
307
318
|
version: Optional[str] = None,
|
308
|
-
cache_policy: Union[CachePolicy,
|
319
|
+
cache_policy: Union[CachePolicy, type[NotSet]] = NotSet,
|
309
320
|
cache_key_fn: Optional[
|
310
|
-
Callable[["TaskRunContext",
|
321
|
+
Callable[["TaskRunContext", dict[str, Any]], Optional[str]]
|
311
322
|
] = None,
|
312
323
|
cache_expiration: Optional[datetime.timedelta] = None,
|
313
|
-
task_run_name: Optional[
|
314
|
-
Union[Callable[[], str], Callable[[Dict[str, Any]], str], str]
|
315
|
-
] = None,
|
324
|
+
task_run_name: Optional[TaskRunNameValueOrCallable] = None,
|
316
325
|
retries: Optional[int] = None,
|
317
326
|
retry_delay_seconds: Optional[
|
318
327
|
Union[
|
319
328
|
float,
|
320
329
|
int,
|
321
|
-
|
322
|
-
Callable[[int],
|
330
|
+
list[float],
|
331
|
+
Callable[[int], list[float]],
|
323
332
|
]
|
324
333
|
] = None,
|
325
334
|
retry_jitter_factor: Optional[float] = None,
|
@@ -331,11 +340,13 @@ class Task(Generic[P, R]):
|
|
331
340
|
timeout_seconds: Union[int, float, None] = None,
|
332
341
|
log_prints: Optional[bool] = False,
|
333
342
|
refresh_cache: Optional[bool] = None,
|
334
|
-
on_completion: Optional[
|
335
|
-
on_failure: Optional[
|
336
|
-
on_rollback: Optional[
|
337
|
-
on_commit: Optional[
|
338
|
-
retry_condition_fn: Optional[
|
343
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
344
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
345
|
+
on_rollback: Optional[list[Callable[["Transaction"], None]]] = None,
|
346
|
+
on_commit: Optional[list[Callable[["Transaction"], None]]] = None,
|
347
|
+
retry_condition_fn: Optional[
|
348
|
+
Callable[["Task[..., Any]", TaskRun, State], bool]
|
349
|
+
] = None,
|
339
350
|
viz_return_value: Optional[Any] = None,
|
340
351
|
):
|
341
352
|
# Validate if hook passed is list and contains callables
|
@@ -460,6 +471,10 @@ class Task(Generic[P, R]):
|
|
460
471
|
|
461
472
|
if callable(retry_delay_seconds):
|
462
473
|
self.retry_delay_seconds = retry_delay_seconds(retries)
|
474
|
+
elif not isinstance(retry_delay_seconds, (list, int, float, type(None))):
|
475
|
+
raise TypeError(
|
476
|
+
f"Invalid `retry_delay_seconds` provided; must be an int, float, list or callable. Received type {type(retry_delay_seconds)}"
|
477
|
+
)
|
463
478
|
else:
|
464
479
|
self.retry_delay_seconds = retry_delay_seconds
|
465
480
|
|
@@ -505,7 +520,7 @@ class Task(Generic[P, R]):
|
|
505
520
|
def ismethod(self) -> bool:
|
506
521
|
return hasattr(self.fn, "__prefect_self__")
|
507
522
|
|
508
|
-
def __get__(self, instance, owner):
|
523
|
+
def __get__(self, instance: Any, owner: Any):
|
509
524
|
"""
|
510
525
|
Implement the descriptor protocol so that the task can be used as an instance method.
|
511
526
|
When an instance method is loaded, this method is called with the "self" instance as
|
@@ -520,7 +535,7 @@ class Task(Generic[P, R]):
|
|
520
535
|
# of the task's function. This will allow it to be automatically added to the task's parameters
|
521
536
|
else:
|
522
537
|
bound_task = copy(self)
|
523
|
-
bound_task.fn.__prefect_self__ = instance
|
538
|
+
bound_task.fn.__prefect_self__ = instance # type: ignore[attr-defined]
|
524
539
|
return bound_task
|
525
540
|
|
526
541
|
def with_options(
|
@@ -529,38 +544,36 @@ class Task(Generic[P, R]):
|
|
529
544
|
name: Optional[str] = None,
|
530
545
|
description: Optional[str] = None,
|
531
546
|
tags: Optional[Iterable[str]] = None,
|
532
|
-
cache_policy: Union[CachePolicy,
|
547
|
+
cache_policy: Union[CachePolicy, type[NotSet]] = NotSet,
|
533
548
|
cache_key_fn: Optional[
|
534
|
-
Callable[["TaskRunContext",
|
549
|
+
Callable[["TaskRunContext", dict[str, Any]], Optional[str]]
|
535
550
|
] = None,
|
536
551
|
task_run_name: Optional[
|
537
|
-
Union[
|
552
|
+
Union[TaskRunNameValueOrCallable, type[NotSet]]
|
538
553
|
] = NotSet,
|
539
554
|
cache_expiration: Optional[datetime.timedelta] = None,
|
540
|
-
retries: Union[int,
|
555
|
+
retries: Union[int, type[NotSet]] = NotSet,
|
541
556
|
retry_delay_seconds: Union[
|
542
557
|
float,
|
543
558
|
int,
|
544
|
-
|
545
|
-
Callable[[int],
|
546
|
-
|
559
|
+
list[float],
|
560
|
+
Callable[[int], list[float]],
|
561
|
+
type[NotSet],
|
547
562
|
] = NotSet,
|
548
|
-
retry_jitter_factor: Union[float,
|
549
|
-
persist_result: Union[bool,
|
550
|
-
result_storage: Union[ResultStorage,
|
551
|
-
result_serializer: Union[ResultSerializer,
|
552
|
-
result_storage_key: Union[str,
|
563
|
+
retry_jitter_factor: Union[float, type[NotSet]] = NotSet,
|
564
|
+
persist_result: Union[bool, type[NotSet]] = NotSet,
|
565
|
+
result_storage: Union[ResultStorage, type[NotSet]] = NotSet,
|
566
|
+
result_serializer: Union[ResultSerializer, type[NotSet]] = NotSet,
|
567
|
+
result_storage_key: Union[str, type[NotSet]] = NotSet,
|
553
568
|
cache_result_in_memory: Optional[bool] = None,
|
554
569
|
timeout_seconds: Union[int, float, None] = None,
|
555
|
-
log_prints: Union[bool,
|
556
|
-
refresh_cache: Union[bool,
|
557
|
-
on_completion: Optional[
|
558
|
-
|
570
|
+
log_prints: Union[bool, type[NotSet]] = NotSet,
|
571
|
+
refresh_cache: Union[bool, type[NotSet]] = NotSet,
|
572
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
573
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
574
|
+
retry_condition_fn: Optional[
|
575
|
+
Callable[["Task[..., Any]", TaskRun, State], bool]
|
559
576
|
] = None,
|
560
|
-
on_failure: Optional[
|
561
|
-
List[Callable[["Task", TaskRun, State], Union[Awaitable[None], None]]]
|
562
|
-
] = None,
|
563
|
-
retry_condition_fn: Optional[Callable[["Task", TaskRun, State], bool]] = None,
|
564
577
|
viz_return_value: Optional[Any] = None,
|
565
578
|
):
|
566
579
|
"""
|
@@ -697,15 +710,11 @@ class Task(Generic[P, R]):
|
|
697
710
|
viz_return_value=viz_return_value or self.viz_return_value,
|
698
711
|
)
|
699
712
|
|
700
|
-
def on_completion(
|
701
|
-
self, fn: Callable[["Task", TaskRun, State], None]
|
702
|
-
) -> Callable[["Task", TaskRun, State], None]:
|
713
|
+
def on_completion(self, fn: StateHookCallable) -> StateHookCallable:
|
703
714
|
self.on_completion_hooks.append(fn)
|
704
715
|
return fn
|
705
716
|
|
706
|
-
def on_failure(
|
707
|
-
self, fn: Callable[["Task", TaskRun, State], None]
|
708
|
-
) -> Callable[["Task", TaskRun, State], None]:
|
717
|
+
def on_failure(self, fn: StateHookCallable) -> StateHookCallable:
|
709
718
|
self.on_failure_hooks.append(fn)
|
710
719
|
return fn
|
711
720
|
|
@@ -725,17 +734,15 @@ class Task(Generic[P, R]):
|
|
725
734
|
self,
|
726
735
|
client: Optional["PrefectClient"] = None,
|
727
736
|
id: Optional[UUID] = None,
|
728
|
-
parameters: Optional[
|
737
|
+
parameters: Optional[dict[str, Any]] = None,
|
729
738
|
flow_run_context: Optional[FlowRunContext] = None,
|
730
739
|
parent_task_run_context: Optional[TaskRunContext] = None,
|
731
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
732
|
-
extra_task_inputs: Optional[
|
740
|
+
wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
|
741
|
+
extra_task_inputs: Optional[dict[str, set[TaskRunInput]]] = None,
|
733
742
|
deferred: bool = False,
|
734
743
|
) -> TaskRun:
|
735
|
-
from prefect.utilities.
|
736
|
-
|
737
|
-
collect_task_run_inputs_sync,
|
738
|
-
)
|
744
|
+
from prefect.utilities._engine import dynamic_key_for_task_run
|
745
|
+
from prefect.utilities.engine import collect_task_run_inputs_sync
|
739
746
|
|
740
747
|
if flow_run_context is None:
|
741
748
|
flow_run_context = FlowRunContext.get()
|
@@ -751,7 +758,7 @@ class Task(Generic[P, R]):
|
|
751
758
|
dynamic_key = f"{self.task_key}-{str(uuid4().hex)}"
|
752
759
|
task_run_name = self.name
|
753
760
|
else:
|
754
|
-
dynamic_key =
|
761
|
+
dynamic_key = dynamic_key_for_task_run(
|
755
762
|
context=flow_run_context, task=self
|
756
763
|
)
|
757
764
|
task_run_name = f"{self.name}-{dynamic_key}"
|
@@ -775,7 +782,7 @@ class Task(Generic[P, R]):
|
|
775
782
|
result_storage=await get_or_create_default_task_scheduling_storage()
|
776
783
|
).update_for_task(self)
|
777
784
|
context = serialize_context()
|
778
|
-
data:
|
785
|
+
data: dict[str, Any] = {"context": context}
|
779
786
|
if parameters:
|
780
787
|
data["parameters"] = parameters
|
781
788
|
if wait_for:
|
@@ -828,17 +835,15 @@ class Task(Generic[P, R]):
|
|
828
835
|
self,
|
829
836
|
client: Optional["PrefectClient"] = None,
|
830
837
|
id: Optional[UUID] = None,
|
831
|
-
parameters: Optional[
|
838
|
+
parameters: Optional[dict[str, Any]] = None,
|
832
839
|
flow_run_context: Optional[FlowRunContext] = None,
|
833
840
|
parent_task_run_context: Optional[TaskRunContext] = None,
|
834
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
835
|
-
extra_task_inputs: Optional[
|
841
|
+
wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
|
842
|
+
extra_task_inputs: Optional[dict[str, set[TaskRunInput]]] = None,
|
836
843
|
deferred: bool = False,
|
837
844
|
) -> TaskRun:
|
838
|
-
from prefect.utilities.
|
839
|
-
|
840
|
-
collect_task_run_inputs_sync,
|
841
|
-
)
|
845
|
+
from prefect.utilities._engine import dynamic_key_for_task_run
|
846
|
+
from prefect.utilities.engine import collect_task_run_inputs_sync
|
842
847
|
|
843
848
|
if flow_run_context is None:
|
844
849
|
flow_run_context = FlowRunContext.get()
|
@@ -854,7 +859,7 @@ class Task(Generic[P, R]):
|
|
854
859
|
dynamic_key = f"{self.task_key}-{str(uuid4().hex)}"
|
855
860
|
task_run_name = self.name
|
856
861
|
else:
|
857
|
-
dynamic_key =
|
862
|
+
dynamic_key = dynamic_key_for_task_run(
|
858
863
|
context=flow_run_context, task=self, stable=False
|
859
864
|
)
|
860
865
|
task_run_name = f"{self.name}-{dynamic_key[:3]}"
|
@@ -878,7 +883,7 @@ class Task(Generic[P, R]):
|
|
878
883
|
result_storage=await get_or_create_default_task_scheduling_storage()
|
879
884
|
).update_for_task(task)
|
880
885
|
context = serialize_context()
|
881
|
-
data:
|
886
|
+
data: dict[str, Any] = {"context": context}
|
882
887
|
if parameters:
|
883
888
|
data["parameters"] = parameters
|
884
889
|
if wait_for:
|
@@ -974,7 +979,7 @@ class Task(Generic[P, R]):
|
|
974
979
|
self,
|
975
980
|
*args: P.args,
|
976
981
|
return_state: bool = False,
|
977
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
982
|
+
wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
|
978
983
|
**kwargs: P.kwargs,
|
979
984
|
):
|
980
985
|
"""
|
@@ -1054,7 +1059,7 @@ class Task(Generic[P, R]):
|
|
1054
1059
|
self,
|
1055
1060
|
*args: Any,
|
1056
1061
|
return_state: bool = False,
|
1057
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
1062
|
+
wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
|
1058
1063
|
**kwargs: Any,
|
1059
1064
|
):
|
1060
1065
|
"""
|
@@ -1179,17 +1184,17 @@ class Task(Generic[P, R]):
|
|
1179
1184
|
self: "Task[P, R]",
|
1180
1185
|
*args: Any,
|
1181
1186
|
return_state: Literal[True],
|
1182
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1187
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1183
1188
|
deferred: bool = ...,
|
1184
1189
|
**kwargs: Any,
|
1185
|
-
) ->
|
1190
|
+
) -> list[State[R]]:
|
1186
1191
|
...
|
1187
1192
|
|
1188
1193
|
@overload
|
1189
1194
|
def map(
|
1190
1195
|
self: "Task[P, R]",
|
1191
1196
|
*args: Any,
|
1192
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1197
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1193
1198
|
deferred: bool = ...,
|
1194
1199
|
**kwargs: Any,
|
1195
1200
|
) -> PrefectFutureList[R]:
|
@@ -1200,17 +1205,17 @@ class Task(Generic[P, R]):
|
|
1200
1205
|
self: "Task[P, R]",
|
1201
1206
|
*args: Any,
|
1202
1207
|
return_state: Literal[True],
|
1203
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1208
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1204
1209
|
deferred: bool = ...,
|
1205
1210
|
**kwargs: Any,
|
1206
|
-
) ->
|
1211
|
+
) -> list[State[R]]:
|
1207
1212
|
...
|
1208
1213
|
|
1209
1214
|
@overload
|
1210
1215
|
def map(
|
1211
1216
|
self: "Task[P, R]",
|
1212
1217
|
*args: Any,
|
1213
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1218
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1214
1219
|
deferred: bool = ...,
|
1215
1220
|
**kwargs: Any,
|
1216
1221
|
) -> PrefectFutureList[R]:
|
@@ -1221,10 +1226,10 @@ class Task(Generic[P, R]):
|
|
1221
1226
|
self: "Task[P, Coroutine[Any, Any, R]]",
|
1222
1227
|
*args: Any,
|
1223
1228
|
return_state: Literal[True],
|
1224
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1229
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1225
1230
|
deferred: bool = ...,
|
1226
1231
|
**kwargs: Any,
|
1227
|
-
) ->
|
1232
|
+
) -> list[State[R]]:
|
1228
1233
|
...
|
1229
1234
|
|
1230
1235
|
@overload
|
@@ -1232,7 +1237,7 @@ class Task(Generic[P, R]):
|
|
1232
1237
|
self: "Task[P, Coroutine[Any, Any, R]]",
|
1233
1238
|
*args: Any,
|
1234
1239
|
return_state: Literal[False],
|
1235
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1240
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1236
1241
|
deferred: bool = ...,
|
1237
1242
|
**kwargs: Any,
|
1238
1243
|
) -> PrefectFutureList[R]:
|
@@ -1242,10 +1247,10 @@ class Task(Generic[P, R]):
|
|
1242
1247
|
self,
|
1243
1248
|
*args: Any,
|
1244
1249
|
return_state: bool = False,
|
1245
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1250
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = None,
|
1246
1251
|
deferred: bool = False,
|
1247
1252
|
**kwargs: Any,
|
1248
|
-
):
|
1253
|
+
) -> Union[list[State[R]], PrefectFutureList[R]]:
|
1249
1254
|
"""
|
1250
1255
|
Submit a mapped run of the task to a worker.
|
1251
1256
|
|
@@ -1394,7 +1399,7 @@ class Task(Generic[P, R]):
|
|
1394
1399
|
" execution."
|
1395
1400
|
)
|
1396
1401
|
if return_state:
|
1397
|
-
states = []
|
1402
|
+
states: list[State[R]] = []
|
1398
1403
|
for future in futures:
|
1399
1404
|
future.wait()
|
1400
1405
|
states.append(future.state)
|
@@ -1404,11 +1409,11 @@ class Task(Generic[P, R]):
|
|
1404
1409
|
|
1405
1410
|
def apply_async(
|
1406
1411
|
self,
|
1407
|
-
args: Optional[
|
1408
|
-
kwargs: Optional[
|
1409
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
1410
|
-
dependencies: Optional[
|
1411
|
-
) -> PrefectDistributedFuture:
|
1412
|
+
args: Optional[tuple[Any, ...]] = None,
|
1413
|
+
kwargs: Optional[dict[str, Any]] = None,
|
1414
|
+
wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
|
1415
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1416
|
+
) -> PrefectDistributedFuture[R]:
|
1412
1417
|
"""
|
1413
1418
|
Create a pending task run for a task worker to execute.
|
1414
1419
|
|
@@ -1507,7 +1512,7 @@ class Task(Generic[P, R]):
|
|
1507
1512
|
|
1508
1513
|
return PrefectDistributedFuture(task_run_id=task_run.id)
|
1509
1514
|
|
1510
|
-
def delay(self, *args: P.args, **kwargs: P.kwargs) -> PrefectDistributedFuture:
|
1515
|
+
def delay(self, *args: P.args, **kwargs: P.kwargs) -> PrefectDistributedFuture[R]:
|
1511
1516
|
"""
|
1512
1517
|
An alias for `apply_async` with simpler calling semantics.
|
1513
1518
|
|
@@ -1575,6 +1580,14 @@ def task(__fn: Callable[P, R]) -> Task[P, R]:
|
|
1575
1580
|
...
|
1576
1581
|
|
1577
1582
|
|
1583
|
+
# see https://github.com/PrefectHQ/prefect/issues/16380
|
1584
|
+
@overload
|
1585
|
+
def task(
|
1586
|
+
__fn: Literal[None] = None,
|
1587
|
+
) -> Callable[[Callable[P, R]], Task[P, R]]:
|
1588
|
+
...
|
1589
|
+
|
1590
|
+
|
1578
1591
|
@overload
|
1579
1592
|
def task(
|
1580
1593
|
*,
|
@@ -1582,20 +1595,18 @@ def task(
|
|
1582
1595
|
description: Optional[str] = None,
|
1583
1596
|
tags: Optional[Iterable[str]] = None,
|
1584
1597
|
version: Optional[str] = None,
|
1585
|
-
cache_policy: Union[CachePolicy,
|
1598
|
+
cache_policy: Union[CachePolicy, type[NotSet]] = NotSet,
|
1586
1599
|
cache_key_fn: Optional[
|
1587
|
-
Callable[["TaskRunContext",
|
1600
|
+
Callable[["TaskRunContext", dict[str, Any]], Optional[str]]
|
1588
1601
|
] = None,
|
1589
1602
|
cache_expiration: Optional[datetime.timedelta] = None,
|
1590
|
-
task_run_name: Optional[
|
1591
|
-
Union[Callable[[], str], Callable[[Dict[str, Any]], str], str]
|
1592
|
-
] = None,
|
1603
|
+
task_run_name: Optional[TaskRunNameValueOrCallable] = None,
|
1593
1604
|
retries: int = 0,
|
1594
1605
|
retry_delay_seconds: Union[
|
1595
1606
|
float,
|
1596
1607
|
int,
|
1597
|
-
|
1598
|
-
Callable[[int],
|
1608
|
+
list[float],
|
1609
|
+
Callable[[int], list[float]],
|
1599
1610
|
] = 0,
|
1600
1611
|
retry_jitter_factor: Optional[float] = None,
|
1601
1612
|
persist_result: Optional[bool] = None,
|
@@ -1606,32 +1617,30 @@ def task(
|
|
1606
1617
|
timeout_seconds: Union[int, float, None] = None,
|
1607
1618
|
log_prints: Optional[bool] = None,
|
1608
1619
|
refresh_cache: Optional[bool] = None,
|
1609
|
-
on_completion: Optional[
|
1610
|
-
on_failure: Optional[
|
1611
|
-
retry_condition_fn: Optional[Callable[["Task", TaskRun, State], bool]] = None,
|
1620
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
1621
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
1622
|
+
retry_condition_fn: Optional[Callable[["Task[P, R]", TaskRun, State], bool]] = None,
|
1612
1623
|
viz_return_value: Any = None,
|
1613
1624
|
) -> Callable[[Callable[P, R]], Task[P, R]]:
|
1614
1625
|
...
|
1615
1626
|
|
1616
1627
|
|
1617
1628
|
def task(
|
1618
|
-
__fn=None,
|
1629
|
+
__fn: Optional[Callable[P, R]] = None,
|
1619
1630
|
*,
|
1620
1631
|
name: Optional[str] = None,
|
1621
1632
|
description: Optional[str] = None,
|
1622
1633
|
tags: Optional[Iterable[str]] = None,
|
1623
1634
|
version: Optional[str] = None,
|
1624
|
-
cache_policy: Union[CachePolicy,
|
1635
|
+
cache_policy: Union[CachePolicy, type[NotSet]] = NotSet,
|
1625
1636
|
cache_key_fn: Union[
|
1626
|
-
Callable[["TaskRunContext",
|
1637
|
+
Callable[["TaskRunContext", dict[str, Any]], Optional[str]], None
|
1627
1638
|
] = None,
|
1628
1639
|
cache_expiration: Optional[datetime.timedelta] = None,
|
1629
|
-
task_run_name: Optional[
|
1630
|
-
Union[Callable[[], str], Callable[[Dict[str, Any]], str], str]
|
1631
|
-
] = None,
|
1640
|
+
task_run_name: Optional[TaskRunNameValueOrCallable] = None,
|
1632
1641
|
retries: Optional[int] = None,
|
1633
1642
|
retry_delay_seconds: Union[
|
1634
|
-
float, int,
|
1643
|
+
float, int, list[float], Callable[[int], list[float]], None
|
1635
1644
|
] = None,
|
1636
1645
|
retry_jitter_factor: Optional[float] = None,
|
1637
1646
|
persist_result: Optional[bool] = None,
|
@@ -1642,9 +1651,9 @@ def task(
|
|
1642
1651
|
timeout_seconds: Union[int, float, None] = None,
|
1643
1652
|
log_prints: Optional[bool] = None,
|
1644
1653
|
refresh_cache: Optional[bool] = None,
|
1645
|
-
on_completion: Optional[
|
1646
|
-
on_failure: Optional[
|
1647
|
-
retry_condition_fn: Optional[Callable[["Task", TaskRun, State], bool]] = None,
|
1654
|
+
on_completion: Optional[list[StateHookCallable]] = None,
|
1655
|
+
on_failure: Optional[list[StateHookCallable]] = None,
|
1656
|
+
retry_condition_fn: Optional[Callable[["Task[P, R]", TaskRun, State], bool]] = None,
|
1648
1657
|
viz_return_value: Any = None,
|
1649
1658
|
):
|
1650
1659
|
"""
|
@@ -1760,34 +1769,31 @@ def task(
|
|
1760
1769
|
if isinstance(__fn, (classmethod, staticmethod)):
|
1761
1770
|
method_decorator = type(__fn).__name__
|
1762
1771
|
raise TypeError(f"@{method_decorator} should be applied on top of @task")
|
1763
|
-
return
|
1764
|
-
|
1765
|
-
|
1766
|
-
|
1767
|
-
|
1768
|
-
|
1769
|
-
|
1770
|
-
|
1771
|
-
|
1772
|
-
|
1773
|
-
|
1774
|
-
|
1775
|
-
|
1776
|
-
|
1777
|
-
|
1778
|
-
|
1779
|
-
|
1780
|
-
|
1781
|
-
|
1782
|
-
|
1783
|
-
|
1784
|
-
|
1785
|
-
|
1786
|
-
|
1787
|
-
|
1788
|
-
retry_condition_fn=retry_condition_fn,
|
1789
|
-
viz_return_value=viz_return_value,
|
1790
|
-
),
|
1772
|
+
return Task(
|
1773
|
+
fn=__fn,
|
1774
|
+
name=name,
|
1775
|
+
description=description,
|
1776
|
+
tags=tags,
|
1777
|
+
version=version,
|
1778
|
+
cache_policy=cache_policy,
|
1779
|
+
cache_key_fn=cache_key_fn,
|
1780
|
+
cache_expiration=cache_expiration,
|
1781
|
+
task_run_name=task_run_name,
|
1782
|
+
retries=retries,
|
1783
|
+
retry_delay_seconds=retry_delay_seconds,
|
1784
|
+
retry_jitter_factor=retry_jitter_factor,
|
1785
|
+
persist_result=persist_result,
|
1786
|
+
result_storage=result_storage,
|
1787
|
+
result_storage_key=result_storage_key,
|
1788
|
+
result_serializer=result_serializer,
|
1789
|
+
cache_result_in_memory=cache_result_in_memory,
|
1790
|
+
timeout_seconds=timeout_seconds,
|
1791
|
+
log_prints=log_prints,
|
1792
|
+
refresh_cache=refresh_cache,
|
1793
|
+
on_completion=on_completion,
|
1794
|
+
on_failure=on_failure,
|
1795
|
+
retry_condition_fn=retry_condition_fn,
|
1796
|
+
viz_return_value=viz_return_value,
|
1791
1797
|
)
|
1792
1798
|
else:
|
1793
1799
|
return cast(
|
prefect/telemetry/bootstrap.py
CHANGED
@@ -23,10 +23,23 @@ def setup_telemetry() -> (
|
|
23
23
|
if server_type != ServerType.CLOUD:
|
24
24
|
return None, None, None
|
25
25
|
|
26
|
-
|
26
|
+
if not settings.api.key:
|
27
|
+
raise ValueError(
|
28
|
+
"A Prefect Cloud API key is required to enable telemetry. Please set "
|
29
|
+
"the `PREFECT_API_KEY` environment variable or authenticate with "
|
30
|
+
"Prefect Cloud via the `prefect cloud login` command."
|
31
|
+
)
|
32
|
+
|
27
33
|
assert settings.api.url
|
28
34
|
|
29
35
|
# This import is here to defer importing of the `opentelemetry` packages.
|
30
|
-
|
36
|
+
try:
|
37
|
+
from .instrumentation import setup_exporters
|
38
|
+
except ImportError as exc:
|
39
|
+
raise ValueError(
|
40
|
+
"Unable to import OpenTelemetry instrumentation libraries. Please "
|
41
|
+
"ensure you have installed the `otel` extra when installing Prefect: "
|
42
|
+
"`pip install 'prefect[otel]'`"
|
43
|
+
) from exc
|
31
44
|
|
32
45
|
return setup_exporters(settings.api.url, settings.api.key.get_secret_value())
|
@@ -55,7 +55,7 @@ def _url_join(base_url: str, path: str) -> str:
|
|
55
55
|
|
56
56
|
def setup_exporters(
|
57
57
|
api_url: str, api_key: str
|
58
|
-
) -> tuple[TracerProvider, MeterProvider,
|
58
|
+
) -> "tuple[TracerProvider, MeterProvider, LoggerProvider]":
|
59
59
|
account_id, workspace_id = extract_account_and_workspace_id(api_url)
|
60
60
|
telemetry_url = _url_join(api_url, "telemetry/")
|
61
61
|
|