prefect-client 3.1.4__py3-none-any.whl → 3.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +3 -0
- prefect/_internal/compatibility/migration.py +1 -1
- prefect/_internal/concurrency/api.py +52 -52
- prefect/_internal/concurrency/calls.py +59 -35
- prefect/_internal/concurrency/cancellation.py +34 -18
- prefect/_internal/concurrency/event_loop.py +7 -6
- prefect/_internal/concurrency/threads.py +41 -33
- prefect/_internal/concurrency/waiters.py +28 -21
- prefect/_internal/pydantic/v1_schema.py +2 -2
- prefect/_internal/pydantic/v2_schema.py +10 -9
- prefect/_internal/schemas/bases.py +10 -11
- prefect/_internal/schemas/validators.py +2 -1
- prefect/_version.py +3 -3
- prefect/automations.py +53 -47
- prefect/blocks/abstract.py +12 -10
- prefect/blocks/core.py +4 -2
- prefect/cache_policies.py +11 -11
- prefect/client/__init__.py +3 -1
- prefect/client/base.py +36 -37
- prefect/client/cloud.py +26 -19
- prefect/client/collections.py +2 -2
- prefect/client/orchestration.py +366 -277
- prefect/client/schemas/__init__.py +24 -0
- prefect/client/schemas/actions.py +132 -120
- prefect/client/schemas/filters.py +5 -0
- prefect/client/schemas/objects.py +113 -85
- prefect/client/schemas/responses.py +21 -18
- prefect/client/schemas/schedules.py +136 -93
- prefect/client/subscriptions.py +28 -14
- prefect/client/utilities.py +32 -36
- prefect/concurrency/asyncio.py +6 -9
- prefect/concurrency/services.py +3 -0
- prefect/concurrency/sync.py +35 -5
- prefect/context.py +39 -31
- prefect/deployments/flow_runs.py +3 -5
- prefect/docker/__init__.py +1 -1
- prefect/events/schemas/events.py +25 -20
- prefect/events/utilities.py +1 -2
- prefect/filesystems.py +3 -3
- prefect/flow_engine.py +755 -138
- prefect/flow_runs.py +3 -3
- prefect/flows.py +214 -170
- prefect/logging/configuration.py +1 -1
- prefect/logging/highlighters.py +1 -2
- prefect/logging/loggers.py +30 -20
- prefect/main.py +17 -24
- prefect/runner/runner.py +43 -21
- prefect/runner/server.py +30 -32
- prefect/runner/submit.py +3 -6
- prefect/runner/utils.py +6 -6
- prefect/runtime/flow_run.py +7 -0
- prefect/settings/constants.py +2 -2
- prefect/settings/legacy.py +1 -1
- prefect/settings/models/server/events.py +10 -0
- prefect/settings/sources.py +9 -2
- prefect/task_engine.py +72 -19
- prefect/task_runners.py +2 -2
- prefect/tasks.py +46 -33
- prefect/telemetry/bootstrap.py +15 -2
- prefect/telemetry/run_telemetry.py +107 -0
- prefect/transactions.py +14 -14
- prefect/types/__init__.py +20 -3
- prefect/utilities/_engine.py +96 -0
- prefect/utilities/annotations.py +25 -18
- prefect/utilities/asyncutils.py +126 -140
- prefect/utilities/callables.py +87 -78
- prefect/utilities/collections.py +278 -117
- prefect/utilities/compat.py +13 -21
- prefect/utilities/context.py +6 -5
- prefect/utilities/dispatch.py +23 -12
- prefect/utilities/dockerutils.py +33 -32
- prefect/utilities/engine.py +126 -239
- prefect/utilities/filesystem.py +18 -15
- prefect/utilities/hashing.py +10 -11
- prefect/utilities/importtools.py +40 -27
- prefect/utilities/math.py +9 -5
- prefect/utilities/names.py +3 -3
- prefect/utilities/processutils.py +121 -57
- prefect/utilities/pydantic.py +41 -36
- prefect/utilities/render_swagger.py +22 -12
- prefect/utilities/schema_tools/__init__.py +2 -1
- prefect/utilities/schema_tools/hydration.py +50 -43
- prefect/utilities/schema_tools/validation.py +52 -42
- prefect/utilities/services.py +13 -12
- prefect/utilities/templating.py +45 -45
- prefect/utilities/text.py +2 -1
- prefect/utilities/timeout.py +4 -4
- prefect/utilities/urls.py +9 -4
- prefect/utilities/visualization.py +46 -24
- prefect/variables.py +9 -8
- prefect/workers/base.py +18 -10
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/METADATA +5 -5
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/RECORD +96 -94
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/WHEEL +1 -1
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.4.dist-info → prefect_client-3.1.6.dist-info}/top_level.txt +0 -0
prefect/task_engine.py
CHANGED
@@ -29,6 +29,7 @@ from uuid import UUID
|
|
29
29
|
|
30
30
|
import anyio
|
31
31
|
import pendulum
|
32
|
+
from opentelemetry import trace
|
32
33
|
from typing_extensions import ParamSpec
|
33
34
|
|
34
35
|
from prefect import Task
|
@@ -79,13 +80,14 @@ from prefect.states import (
|
|
79
80
|
exception_to_failed_state,
|
80
81
|
return_value_to_state,
|
81
82
|
)
|
83
|
+
from prefect.telemetry.run_telemetry import RunTelemetry
|
82
84
|
from prefect.transactions import IsolationLevel, Transaction, transaction
|
85
|
+
from prefect.utilities._engine import get_hook_name
|
83
86
|
from prefect.utilities.annotations import NotSet
|
84
87
|
from prefect.utilities.asyncutils import run_coro_as_sync
|
85
88
|
from prefect.utilities.callables import call_with_parameters, parameters_to_args_kwargs
|
86
89
|
from prefect.utilities.collections import visit_collection
|
87
90
|
from prefect.utilities.engine import (
|
88
|
-
_get_hook_name,
|
89
91
|
emit_task_run_state_change_event,
|
90
92
|
link_state_to_result,
|
91
93
|
resolve_to_final_result,
|
@@ -120,6 +122,7 @@ class BaseTaskRunEngine(Generic[P, R]):
|
|
120
122
|
_is_started: bool = False
|
121
123
|
_task_name_set: bool = False
|
122
124
|
_last_event: Optional[PrefectEvent] = None
|
125
|
+
_telemetry: RunTelemetry = field(default_factory=RunTelemetry)
|
123
126
|
|
124
127
|
def __post_init__(self):
|
125
128
|
if self.parameters is None:
|
@@ -193,11 +196,11 @@ class BaseTaskRunEngine(Generic[P, R]):
|
|
193
196
|
self.parameters = resolved_parameters
|
194
197
|
|
195
198
|
def _set_custom_task_run_name(self):
|
196
|
-
from prefect.utilities.
|
199
|
+
from prefect.utilities._engine import resolve_custom_task_run_name
|
197
200
|
|
198
201
|
# update the task run name if necessary
|
199
202
|
if not self._task_name_set and self.task.task_run_name:
|
200
|
-
task_run_name =
|
203
|
+
task_run_name = resolve_custom_task_run_name(
|
201
204
|
task=self.task, parameters=self.parameters or {}
|
202
205
|
)
|
203
206
|
|
@@ -351,7 +354,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
351
354
|
hooks = None
|
352
355
|
|
353
356
|
for hook in hooks or []:
|
354
|
-
hook_name =
|
357
|
+
hook_name = get_hook_name(hook)
|
355
358
|
|
356
359
|
try:
|
357
360
|
self.logger.info(
|
@@ -427,6 +430,11 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
427
430
|
|
428
431
|
self.task_run.state = new_state = state
|
429
432
|
|
433
|
+
if last_state.timestamp == new_state.timestamp:
|
434
|
+
# Ensure that the state timestamp is unique, or at least not equal to the last state.
|
435
|
+
# This might occur especially on Windows where the timestamp resolution is limited.
|
436
|
+
new_state.timestamp += pendulum.duration(microseconds=1)
|
437
|
+
|
430
438
|
# Ensure that the state_details are populated with the current run IDs
|
431
439
|
new_state.state_details.task_run_id = self.task_run.id
|
432
440
|
new_state.state_details.flow_run_id = self.task_run.flow_run_id
|
@@ -460,7 +468,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
460
468
|
validated_state=self.task_run.state,
|
461
469
|
follows=self._last_event,
|
462
470
|
)
|
463
|
-
|
471
|
+
self._telemetry.update_state(new_state)
|
464
472
|
return new_state
|
465
473
|
|
466
474
|
def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
|
@@ -514,6 +522,8 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
514
522
|
self.record_terminal_state_timing(terminal_state)
|
515
523
|
self.set_state(terminal_state)
|
516
524
|
self._return_value = result
|
525
|
+
|
526
|
+
self._telemetry.end_span_on_success(terminal_state.message)
|
517
527
|
return result
|
518
528
|
|
519
529
|
def handle_retry(self, exc: Exception) -> bool:
|
@@ -562,6 +572,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
562
572
|
|
563
573
|
def handle_exception(self, exc: Exception) -> None:
|
564
574
|
# If the task fails, and we have retries left, set the task to retrying.
|
575
|
+
self._telemetry.record_exception(exc)
|
565
576
|
if not self.handle_retry(exc):
|
566
577
|
# If the task has no retries left, or the retry condition is not met, set the task to failed.
|
567
578
|
state = run_coro_as_sync(
|
@@ -575,6 +586,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
575
586
|
self.record_terminal_state_timing(state)
|
576
587
|
self.set_state(state)
|
577
588
|
self._raised = exc
|
589
|
+
self._telemetry.end_span_on_failure(state.message)
|
578
590
|
|
579
591
|
def handle_timeout(self, exc: TimeoutError) -> None:
|
580
592
|
if not self.handle_retry(exc):
|
@@ -598,6 +610,8 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
598
610
|
self.record_terminal_state_timing(state)
|
599
611
|
self.set_state(state, force=True)
|
600
612
|
self._raised = exc
|
613
|
+
self._telemetry.record_exception(exc)
|
614
|
+
self._telemetry.end_span_on_failure(state.message)
|
601
615
|
|
602
616
|
@contextmanager
|
603
617
|
def setup_run_context(self, client: Optional[SyncPrefectClient] = None):
|
@@ -655,14 +669,17 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
655
669
|
with SyncClientContext.get_or_create() as client_ctx:
|
656
670
|
self._client = client_ctx.client
|
657
671
|
self._is_started = True
|
672
|
+
flow_run_context = FlowRunContext.get()
|
673
|
+
parent_task_run_context = TaskRunContext.get()
|
674
|
+
|
658
675
|
try:
|
659
676
|
if not self.task_run:
|
660
677
|
self.task_run = run_coro_as_sync(
|
661
678
|
self.task.create_local_run(
|
662
679
|
id=task_run_id,
|
663
680
|
parameters=self.parameters,
|
664
|
-
flow_run_context=
|
665
|
-
parent_task_run_context=
|
681
|
+
flow_run_context=flow_run_context,
|
682
|
+
parent_task_run_context=parent_task_run_context,
|
666
683
|
wait_for=self.wait_for,
|
667
684
|
extra_task_inputs=dependencies,
|
668
685
|
)
|
@@ -679,6 +696,13 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
679
696
|
self.logger.debug(
|
680
697
|
f"Created task run {self.task_run.name!r} for task {self.task.name!r}"
|
681
698
|
)
|
699
|
+
labels = (
|
700
|
+
flow_run_context.flow_run.labels if flow_run_context else {}
|
701
|
+
)
|
702
|
+
self._telemetry.start_span(
|
703
|
+
self.task_run, self.parameters, labels
|
704
|
+
)
|
705
|
+
|
682
706
|
yield self
|
683
707
|
|
684
708
|
except TerminationSignal as exc:
|
@@ -730,11 +754,12 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
730
754
|
dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
|
731
755
|
) -> Generator[None, None, None]:
|
732
756
|
with self.initialize_run(task_run_id=task_run_id, dependencies=dependencies):
|
733
|
-
self.
|
734
|
-
|
735
|
-
|
736
|
-
|
737
|
-
|
757
|
+
with trace.use_span(self._telemetry._span):
|
758
|
+
self.begin_run()
|
759
|
+
try:
|
760
|
+
yield
|
761
|
+
finally:
|
762
|
+
self.call_hooks()
|
738
763
|
|
739
764
|
@contextmanager
|
740
765
|
def transaction_context(self) -> Generator[Transaction, None, None]:
|
@@ -866,7 +891,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
866
891
|
hooks = None
|
867
892
|
|
868
893
|
for hook in hooks or []:
|
869
|
-
hook_name =
|
894
|
+
hook_name = get_hook_name(hook)
|
870
895
|
|
871
896
|
try:
|
872
897
|
self.logger.info(
|
@@ -942,6 +967,11 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
942
967
|
|
943
968
|
self.task_run.state = new_state = state
|
944
969
|
|
970
|
+
if last_state.timestamp == new_state.timestamp:
|
971
|
+
# Ensure that the state timestamp is unique, or at least not equal to the last state.
|
972
|
+
# This might occur especially on Windows where the timestamp resolution is limited.
|
973
|
+
new_state.timestamp += pendulum.duration(microseconds=1)
|
974
|
+
|
945
975
|
# Ensure that the state_details are populated with the current run IDs
|
946
976
|
new_state.state_details.task_run_id = self.task_run.id
|
947
977
|
new_state.state_details.flow_run_id = self.task_run.flow_run_id
|
@@ -977,6 +1007,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
977
1007
|
follows=self._last_event,
|
978
1008
|
)
|
979
1009
|
|
1010
|
+
self._telemetry.update_state(new_state)
|
980
1011
|
return new_state
|
981
1012
|
|
982
1013
|
async def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
|
@@ -1025,6 +1056,9 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1025
1056
|
self.record_terminal_state_timing(terminal_state)
|
1026
1057
|
await self.set_state(terminal_state)
|
1027
1058
|
self._return_value = result
|
1059
|
+
|
1060
|
+
self._telemetry.end_span_on_success(terminal_state.message)
|
1061
|
+
|
1028
1062
|
return result
|
1029
1063
|
|
1030
1064
|
async def handle_retry(self, exc: Exception) -> bool:
|
@@ -1073,6 +1107,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1073
1107
|
|
1074
1108
|
async def handle_exception(self, exc: Exception) -> None:
|
1075
1109
|
# If the task fails, and we have retries left, set the task to retrying.
|
1110
|
+
self._telemetry.record_exception(exc)
|
1076
1111
|
if not await self.handle_retry(exc):
|
1077
1112
|
# If the task has no retries left, or the retry condition is not met, set the task to failed.
|
1078
1113
|
state = await exception_to_failed_state(
|
@@ -1084,7 +1119,10 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1084
1119
|
await self.set_state(state)
|
1085
1120
|
self._raised = exc
|
1086
1121
|
|
1122
|
+
self._telemetry.end_span_on_failure(state.message)
|
1123
|
+
|
1087
1124
|
async def handle_timeout(self, exc: TimeoutError) -> None:
|
1125
|
+
self._telemetry.record_exception(exc)
|
1088
1126
|
if not await self.handle_retry(exc):
|
1089
1127
|
if isinstance(exc, TaskRunTimeoutError):
|
1090
1128
|
message = f"Task run exceeded timeout of {self.task.timeout_seconds} second(s)"
|
@@ -1098,6 +1136,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1098
1136
|
)
|
1099
1137
|
await self.set_state(state)
|
1100
1138
|
self._raised = exc
|
1139
|
+
self._telemetry.end_span_on_failure(state.message)
|
1101
1140
|
|
1102
1141
|
async def handle_crash(self, exc: BaseException) -> None:
|
1103
1142
|
state = await exception_to_crashed_state(exc)
|
@@ -1107,6 +1146,9 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1107
1146
|
await self.set_state(state, force=True)
|
1108
1147
|
self._raised = exc
|
1109
1148
|
|
1149
|
+
self._telemetry.record_exception(exc)
|
1150
|
+
self._telemetry.end_span_on_failure(state.message)
|
1151
|
+
|
1110
1152
|
@asynccontextmanager
|
1111
1153
|
async def setup_run_context(self, client: Optional[PrefectClient] = None):
|
1112
1154
|
from prefect.utilities.engine import (
|
@@ -1162,12 +1204,14 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1162
1204
|
async with AsyncClientContext.get_or_create():
|
1163
1205
|
self._client = get_client()
|
1164
1206
|
self._is_started = True
|
1207
|
+
flow_run_context = FlowRunContext.get()
|
1208
|
+
|
1165
1209
|
try:
|
1166
1210
|
if not self.task_run:
|
1167
1211
|
self.task_run = await self.task.create_local_run(
|
1168
1212
|
id=task_run_id,
|
1169
1213
|
parameters=self.parameters,
|
1170
|
-
flow_run_context=
|
1214
|
+
flow_run_context=flow_run_context,
|
1171
1215
|
parent_task_run_context=TaskRunContext.get(),
|
1172
1216
|
wait_for=self.wait_for,
|
1173
1217
|
extra_task_inputs=dependencies,
|
@@ -1184,6 +1228,14 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1184
1228
|
self.logger.debug(
|
1185
1229
|
f"Created task run {self.task_run.name!r} for task {self.task.name!r}"
|
1186
1230
|
)
|
1231
|
+
|
1232
|
+
labels = (
|
1233
|
+
flow_run_context.flow_run.labels if flow_run_context else {}
|
1234
|
+
)
|
1235
|
+
self._telemetry.start_span(
|
1236
|
+
self.task_run, self.parameters, labels
|
1237
|
+
)
|
1238
|
+
|
1187
1239
|
yield self
|
1188
1240
|
|
1189
1241
|
except TerminationSignal as exc:
|
@@ -1237,11 +1289,12 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1237
1289
|
async with self.initialize_run(
|
1238
1290
|
task_run_id=task_run_id, dependencies=dependencies
|
1239
1291
|
):
|
1240
|
-
|
1241
|
-
|
1242
|
-
|
1243
|
-
|
1244
|
-
|
1292
|
+
with trace.use_span(self._telemetry._span):
|
1293
|
+
await self.begin_run()
|
1294
|
+
try:
|
1295
|
+
yield
|
1296
|
+
finally:
|
1297
|
+
await self.call_hooks()
|
1245
1298
|
|
1246
1299
|
@asynccontextmanager
|
1247
1300
|
async def transaction_context(self) -> AsyncGenerator[Transaction, None]:
|
prefect/task_runners.py
CHANGED
@@ -97,9 +97,9 @@ class TaskRunner(abc.ABC, Generic[F]):
|
|
97
97
|
|
98
98
|
def map(
|
99
99
|
self,
|
100
|
-
task: "Task",
|
100
|
+
task: "Task[P, R]",
|
101
101
|
parameters: Dict[str, Any],
|
102
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
102
|
+
wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
|
103
103
|
) -> PrefectFutureList[F]:
|
104
104
|
"""
|
105
105
|
Submit multiple tasks to the task run engine.
|
prefect/tasks.py
CHANGED
@@ -21,6 +21,7 @@ from typing import (
|
|
21
21
|
List,
|
22
22
|
NoReturn,
|
23
23
|
Optional,
|
24
|
+
Protocol,
|
24
25
|
Set,
|
25
26
|
Tuple,
|
26
27
|
Type,
|
@@ -31,7 +32,7 @@ from typing import (
|
|
31
32
|
)
|
32
33
|
from uuid import UUID, uuid4
|
33
34
|
|
34
|
-
from typing_extensions import Literal, ParamSpec
|
35
|
+
from typing_extensions import Literal, ParamSpec, Self, TypeIs
|
35
36
|
|
36
37
|
import prefect.states
|
37
38
|
from prefect.cache_policies import DEFAULT, NONE, CachePolicy
|
@@ -223,6 +224,16 @@ def _generate_task_key(fn: Callable[..., Any]) -> str:
|
|
223
224
|
return f"{qualname}-{code_hash}"
|
224
225
|
|
225
226
|
|
227
|
+
class TaskRunNameCallbackWithParameters(Protocol):
|
228
|
+
@classmethod
|
229
|
+
def is_callback_with_parameters(cls, callable: Callable[..., str]) -> TypeIs[Self]:
|
230
|
+
sig = inspect.signature(callable)
|
231
|
+
return "parameters" in sig.parameters
|
232
|
+
|
233
|
+
def __call__(self, parameters: dict[str, Any]) -> str:
|
234
|
+
...
|
235
|
+
|
236
|
+
|
226
237
|
class Task(Generic[P, R]):
|
227
238
|
"""
|
228
239
|
A Prefect task definition.
|
@@ -311,7 +322,7 @@ class Task(Generic[P, R]):
|
|
311
322
|
] = None,
|
312
323
|
cache_expiration: Optional[datetime.timedelta] = None,
|
313
324
|
task_run_name: Optional[
|
314
|
-
Union[Callable[[], str],
|
325
|
+
Union[Callable[[], str], TaskRunNameCallbackWithParameters, str]
|
315
326
|
] = None,
|
316
327
|
retries: Optional[int] = None,
|
317
328
|
retry_delay_seconds: Optional[
|
@@ -534,7 +545,9 @@ class Task(Generic[P, R]):
|
|
534
545
|
Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
|
535
546
|
] = None,
|
536
547
|
task_run_name: Optional[
|
537
|
-
Union[
|
548
|
+
Union[
|
549
|
+
Callable[[], str], TaskRunNameCallbackWithParameters, str, Type[NotSet]
|
550
|
+
]
|
538
551
|
] = NotSet,
|
539
552
|
cache_expiration: Optional[datetime.timedelta] = None,
|
540
553
|
retries: Union[int, Type[NotSet]] = NotSet,
|
@@ -732,10 +745,8 @@ class Task(Generic[P, R]):
|
|
732
745
|
extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
|
733
746
|
deferred: bool = False,
|
734
747
|
) -> TaskRun:
|
735
|
-
from prefect.utilities.
|
736
|
-
|
737
|
-
collect_task_run_inputs_sync,
|
738
|
-
)
|
748
|
+
from prefect.utilities._engine import dynamic_key_for_task_run
|
749
|
+
from prefect.utilities.engine import collect_task_run_inputs_sync
|
739
750
|
|
740
751
|
if flow_run_context is None:
|
741
752
|
flow_run_context = FlowRunContext.get()
|
@@ -751,7 +762,7 @@ class Task(Generic[P, R]):
|
|
751
762
|
dynamic_key = f"{self.task_key}-{str(uuid4().hex)}"
|
752
763
|
task_run_name = self.name
|
753
764
|
else:
|
754
|
-
dynamic_key =
|
765
|
+
dynamic_key = dynamic_key_for_task_run(
|
755
766
|
context=flow_run_context, task=self
|
756
767
|
)
|
757
768
|
task_run_name = f"{self.name}-{dynamic_key}"
|
@@ -835,10 +846,8 @@ class Task(Generic[P, R]):
|
|
835
846
|
extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
|
836
847
|
deferred: bool = False,
|
837
848
|
) -> TaskRun:
|
838
|
-
from prefect.utilities.
|
839
|
-
|
840
|
-
collect_task_run_inputs_sync,
|
841
|
-
)
|
849
|
+
from prefect.utilities._engine import dynamic_key_for_task_run
|
850
|
+
from prefect.utilities.engine import collect_task_run_inputs_sync
|
842
851
|
|
843
852
|
if flow_run_context is None:
|
844
853
|
flow_run_context = FlowRunContext.get()
|
@@ -854,7 +863,7 @@ class Task(Generic[P, R]):
|
|
854
863
|
dynamic_key = f"{self.task_key}-{str(uuid4().hex)}"
|
855
864
|
task_run_name = self.name
|
856
865
|
else:
|
857
|
-
dynamic_key =
|
866
|
+
dynamic_key = dynamic_key_for_task_run(
|
858
867
|
context=flow_run_context, task=self, stable=False
|
859
868
|
)
|
860
869
|
task_run_name = f"{self.name}-{dynamic_key[:3]}"
|
@@ -1179,7 +1188,7 @@ class Task(Generic[P, R]):
|
|
1179
1188
|
self: "Task[P, R]",
|
1180
1189
|
*args: Any,
|
1181
1190
|
return_state: Literal[True],
|
1182
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1191
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1183
1192
|
deferred: bool = ...,
|
1184
1193
|
**kwargs: Any,
|
1185
1194
|
) -> List[State[R]]:
|
@@ -1189,7 +1198,7 @@ class Task(Generic[P, R]):
|
|
1189
1198
|
def map(
|
1190
1199
|
self: "Task[P, R]",
|
1191
1200
|
*args: Any,
|
1192
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1201
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1193
1202
|
deferred: bool = ...,
|
1194
1203
|
**kwargs: Any,
|
1195
1204
|
) -> PrefectFutureList[R]:
|
@@ -1200,7 +1209,7 @@ class Task(Generic[P, R]):
|
|
1200
1209
|
self: "Task[P, R]",
|
1201
1210
|
*args: Any,
|
1202
1211
|
return_state: Literal[True],
|
1203
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1212
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1204
1213
|
deferred: bool = ...,
|
1205
1214
|
**kwargs: Any,
|
1206
1215
|
) -> List[State[R]]:
|
@@ -1210,7 +1219,7 @@ class Task(Generic[P, R]):
|
|
1210
1219
|
def map(
|
1211
1220
|
self: "Task[P, R]",
|
1212
1221
|
*args: Any,
|
1213
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1222
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1214
1223
|
deferred: bool = ...,
|
1215
1224
|
**kwargs: Any,
|
1216
1225
|
) -> PrefectFutureList[R]:
|
@@ -1221,7 +1230,7 @@ class Task(Generic[P, R]):
|
|
1221
1230
|
self: "Task[P, Coroutine[Any, Any, R]]",
|
1222
1231
|
*args: Any,
|
1223
1232
|
return_state: Literal[True],
|
1224
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1233
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1225
1234
|
deferred: bool = ...,
|
1226
1235
|
**kwargs: Any,
|
1227
1236
|
) -> List[State[R]]:
|
@@ -1232,7 +1241,7 @@ class Task(Generic[P, R]):
|
|
1232
1241
|
self: "Task[P, Coroutine[Any, Any, R]]",
|
1233
1242
|
*args: Any,
|
1234
1243
|
return_state: Literal[False],
|
1235
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1244
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
|
1236
1245
|
deferred: bool = ...,
|
1237
1246
|
**kwargs: Any,
|
1238
1247
|
) -> PrefectFutureList[R]:
|
@@ -1242,10 +1251,10 @@ class Task(Generic[P, R]):
|
|
1242
1251
|
self,
|
1243
1252
|
*args: Any,
|
1244
1253
|
return_state: bool = False,
|
1245
|
-
wait_for: Optional[Iterable[Union[PrefectFuture[
|
1254
|
+
wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = None,
|
1246
1255
|
deferred: bool = False,
|
1247
1256
|
**kwargs: Any,
|
1248
|
-
):
|
1257
|
+
) -> Union[List[State[R]], PrefectFutureList[R]]:
|
1249
1258
|
"""
|
1250
1259
|
Submit a mapped run of the task to a worker.
|
1251
1260
|
|
@@ -1394,7 +1403,7 @@ class Task(Generic[P, R]):
|
|
1394
1403
|
" execution."
|
1395
1404
|
)
|
1396
1405
|
if return_state:
|
1397
|
-
states = []
|
1406
|
+
states: list[State[R]] = []
|
1398
1407
|
for future in futures:
|
1399
1408
|
future.wait()
|
1400
1409
|
states.append(future.state)
|
@@ -1406,9 +1415,9 @@ class Task(Generic[P, R]):
|
|
1406
1415
|
self,
|
1407
1416
|
args: Optional[Tuple[Any, ...]] = None,
|
1408
1417
|
kwargs: Optional[Dict[str, Any]] = None,
|
1409
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
1418
|
+
wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
|
1410
1419
|
dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
|
1411
|
-
) -> PrefectDistributedFuture:
|
1420
|
+
) -> PrefectDistributedFuture[R]:
|
1412
1421
|
"""
|
1413
1422
|
Create a pending task run for a task worker to execute.
|
1414
1423
|
|
@@ -1588,7 +1597,7 @@ def task(
|
|
1588
1597
|
] = None,
|
1589
1598
|
cache_expiration: Optional[datetime.timedelta] = None,
|
1590
1599
|
task_run_name: Optional[
|
1591
|
-
Union[Callable[[], str],
|
1600
|
+
Union[Callable[[], str], TaskRunNameCallbackWithParameters, str]
|
1592
1601
|
] = None,
|
1593
1602
|
retries: int = 0,
|
1594
1603
|
retry_delay_seconds: Union[
|
@@ -1606,16 +1615,18 @@ def task(
|
|
1606
1615
|
timeout_seconds: Union[int, float, None] = None,
|
1607
1616
|
log_prints: Optional[bool] = None,
|
1608
1617
|
refresh_cache: Optional[bool] = None,
|
1609
|
-
on_completion: Optional[
|
1610
|
-
|
1611
|
-
|
1618
|
+
on_completion: Optional[
|
1619
|
+
List[Callable[["Task[P, R]", TaskRun, State], None]]
|
1620
|
+
] = None,
|
1621
|
+
on_failure: Optional[List[Callable[["Task[P, R]", TaskRun, State], None]]] = None,
|
1622
|
+
retry_condition_fn: Optional[Callable[["Task[P, R]", TaskRun, State], bool]] = None,
|
1612
1623
|
viz_return_value: Any = None,
|
1613
1624
|
) -> Callable[[Callable[P, R]], Task[P, R]]:
|
1614
1625
|
...
|
1615
1626
|
|
1616
1627
|
|
1617
1628
|
def task(
|
1618
|
-
__fn=None,
|
1629
|
+
__fn: Optional[Callable[P, R]] = None,
|
1619
1630
|
*,
|
1620
1631
|
name: Optional[str] = None,
|
1621
1632
|
description: Optional[str] = None,
|
@@ -1627,7 +1638,7 @@ def task(
|
|
1627
1638
|
] = None,
|
1628
1639
|
cache_expiration: Optional[datetime.timedelta] = None,
|
1629
1640
|
task_run_name: Optional[
|
1630
|
-
Union[Callable[[], str],
|
1641
|
+
Union[Callable[[], str], TaskRunNameCallbackWithParameters, str]
|
1631
1642
|
] = None,
|
1632
1643
|
retries: Optional[int] = None,
|
1633
1644
|
retry_delay_seconds: Union[
|
@@ -1642,9 +1653,11 @@ def task(
|
|
1642
1653
|
timeout_seconds: Union[int, float, None] = None,
|
1643
1654
|
log_prints: Optional[bool] = None,
|
1644
1655
|
refresh_cache: Optional[bool] = None,
|
1645
|
-
on_completion: Optional[
|
1646
|
-
|
1647
|
-
|
1656
|
+
on_completion: Optional[
|
1657
|
+
List[Callable[["Task[P, R]", TaskRun, State], None]]
|
1658
|
+
] = None,
|
1659
|
+
on_failure: Optional[List[Callable[["Task[P, R]", TaskRun, State], None]]] = None,
|
1660
|
+
retry_condition_fn: Optional[Callable[["Task[P, R]", TaskRun, State], bool]] = None,
|
1648
1661
|
viz_return_value: Any = None,
|
1649
1662
|
):
|
1650
1663
|
"""
|
prefect/telemetry/bootstrap.py
CHANGED
@@ -23,10 +23,23 @@ def setup_telemetry() -> (
|
|
23
23
|
if server_type != ServerType.CLOUD:
|
24
24
|
return None, None, None
|
25
25
|
|
26
|
-
|
26
|
+
if not settings.api.key:
|
27
|
+
raise ValueError(
|
28
|
+
"A Prefect Cloud API key is required to enable telemetry. Please set "
|
29
|
+
"the `PREFECT_API_KEY` environment variable or authenticate with "
|
30
|
+
"Prefect Cloud via the `prefect cloud login` command."
|
31
|
+
)
|
32
|
+
|
27
33
|
assert settings.api.url
|
28
34
|
|
29
35
|
# This import is here to defer importing of the `opentelemetry` packages.
|
30
|
-
|
36
|
+
try:
|
37
|
+
from .instrumentation import setup_exporters
|
38
|
+
except ImportError as exc:
|
39
|
+
raise ValueError(
|
40
|
+
"Unable to import OpenTelemetry instrumentation libraries. Please "
|
41
|
+
"ensure you have installed the `otel` extra when installing Prefect: "
|
42
|
+
"`pip install 'prefect[otel]'`"
|
43
|
+
) from exc
|
31
44
|
|
32
45
|
return setup_exporters(settings.api.url, settings.api.key.get_secret_value())
|
@@ -0,0 +1,107 @@
|
|
1
|
+
import time
|
2
|
+
from dataclasses import dataclass, field
|
3
|
+
from typing import TYPE_CHECKING, Any, Dict, Optional
|
4
|
+
|
5
|
+
from opentelemetry.propagators.textmap import Setter
|
6
|
+
from opentelemetry.trace import (
|
7
|
+
Status,
|
8
|
+
StatusCode,
|
9
|
+
get_tracer,
|
10
|
+
)
|
11
|
+
|
12
|
+
import prefect
|
13
|
+
from prefect.client.schemas import TaskRun
|
14
|
+
from prefect.client.schemas.objects import State
|
15
|
+
from prefect.types import KeyValueLabels
|
16
|
+
|
17
|
+
if TYPE_CHECKING:
|
18
|
+
from opentelemetry.trace import Tracer
|
19
|
+
|
20
|
+
|
21
|
+
class OTELSetter(Setter[KeyValueLabels]):
|
22
|
+
"""
|
23
|
+
A setter for OpenTelemetry that supports Prefect's custom labels.
|
24
|
+
"""
|
25
|
+
|
26
|
+
def set(self, carrier: KeyValueLabels, key: str, value: str) -> None:
|
27
|
+
carrier[key] = value
|
28
|
+
|
29
|
+
|
30
|
+
@dataclass
|
31
|
+
class RunTelemetry:
|
32
|
+
"""
|
33
|
+
A class for managing the telemetry of runs.
|
34
|
+
"""
|
35
|
+
|
36
|
+
_tracer: "Tracer" = field(
|
37
|
+
default_factory=lambda: get_tracer("prefect", prefect.__version__)
|
38
|
+
)
|
39
|
+
_span = None
|
40
|
+
|
41
|
+
def start_span(
|
42
|
+
self,
|
43
|
+
task_run: TaskRun,
|
44
|
+
parameters: Optional[Dict[str, Any]] = None,
|
45
|
+
labels: Optional[Dict[str, Any]] = None,
|
46
|
+
):
|
47
|
+
"""
|
48
|
+
Start a span for a task run.
|
49
|
+
"""
|
50
|
+
if parameters is None:
|
51
|
+
parameters = {}
|
52
|
+
if labels is None:
|
53
|
+
labels = {}
|
54
|
+
parameter_attributes = {
|
55
|
+
f"prefect.run.parameter.{k}": type(v).__name__
|
56
|
+
for k, v in parameters.items()
|
57
|
+
}
|
58
|
+
self._span = self._tracer.start_span(
|
59
|
+
name=task_run.name,
|
60
|
+
attributes={
|
61
|
+
"prefect.run.type": "task",
|
62
|
+
"prefect.run.id": str(task_run.id),
|
63
|
+
"prefect.tags": task_run.tags,
|
64
|
+
**parameter_attributes,
|
65
|
+
**labels,
|
66
|
+
},
|
67
|
+
)
|
68
|
+
|
69
|
+
def end_span_on_success(self, terminal_message: str) -> None:
|
70
|
+
"""
|
71
|
+
End a span for a task run on success.
|
72
|
+
"""
|
73
|
+
if self._span:
|
74
|
+
self._span.set_status(Status(StatusCode.OK), terminal_message)
|
75
|
+
self._span.end(time.time_ns())
|
76
|
+
self._span = None
|
77
|
+
|
78
|
+
def end_span_on_failure(self, terminal_message: str) -> None:
|
79
|
+
"""
|
80
|
+
End a span for a task run on failure.
|
81
|
+
"""
|
82
|
+
if self._span:
|
83
|
+
self._span.set_status(Status(StatusCode.ERROR, terminal_message))
|
84
|
+
self._span.end(time.time_ns())
|
85
|
+
self._span = None
|
86
|
+
|
87
|
+
def record_exception(self, exc: Exception) -> None:
|
88
|
+
"""
|
89
|
+
Record an exception on a span.
|
90
|
+
"""
|
91
|
+
if self._span:
|
92
|
+
self._span.record_exception(exc)
|
93
|
+
|
94
|
+
def update_state(self, new_state: State) -> None:
|
95
|
+
"""
|
96
|
+
Update a span with the state of a task run.
|
97
|
+
"""
|
98
|
+
if self._span:
|
99
|
+
self._span.add_event(
|
100
|
+
new_state.name or new_state.type,
|
101
|
+
{
|
102
|
+
"prefect.state.message": new_state.message or "",
|
103
|
+
"prefect.state.type": new_state.type,
|
104
|
+
"prefect.state.name": new_state.name or new_state.type,
|
105
|
+
"prefect.state.id": str(new_state.id),
|
106
|
+
},
|
107
|
+
)
|