prefect-client 3.1.5__py3-none-any.whl → 3.1.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. prefect/__init__.py +3 -0
  2. prefect/_internal/compatibility/migration.py +1 -1
  3. prefect/_internal/concurrency/api.py +52 -52
  4. prefect/_internal/concurrency/calls.py +59 -35
  5. prefect/_internal/concurrency/cancellation.py +34 -18
  6. prefect/_internal/concurrency/event_loop.py +7 -6
  7. prefect/_internal/concurrency/threads.py +41 -33
  8. prefect/_internal/concurrency/waiters.py +28 -21
  9. prefect/_internal/pydantic/v1_schema.py +2 -2
  10. prefect/_internal/pydantic/v2_schema.py +10 -9
  11. prefect/_internal/schemas/bases.py +9 -7
  12. prefect/_internal/schemas/validators.py +2 -1
  13. prefect/_version.py +3 -3
  14. prefect/automations.py +53 -47
  15. prefect/blocks/abstract.py +12 -10
  16. prefect/blocks/core.py +4 -2
  17. prefect/cache_policies.py +11 -11
  18. prefect/client/__init__.py +3 -1
  19. prefect/client/base.py +36 -37
  20. prefect/client/cloud.py +26 -19
  21. prefect/client/collections.py +2 -2
  22. prefect/client/orchestration.py +342 -273
  23. prefect/client/schemas/__init__.py +24 -0
  24. prefect/client/schemas/actions.py +123 -116
  25. prefect/client/schemas/objects.py +110 -81
  26. prefect/client/schemas/responses.py +18 -18
  27. prefect/client/schemas/schedules.py +136 -93
  28. prefect/client/subscriptions.py +28 -14
  29. prefect/client/utilities.py +32 -36
  30. prefect/concurrency/asyncio.py +6 -9
  31. prefect/concurrency/sync.py +35 -5
  32. prefect/context.py +39 -31
  33. prefect/deployments/flow_runs.py +3 -5
  34. prefect/docker/__init__.py +1 -1
  35. prefect/events/schemas/events.py +25 -20
  36. prefect/events/utilities.py +1 -2
  37. prefect/filesystems.py +3 -3
  38. prefect/flow_engine.py +61 -21
  39. prefect/flow_runs.py +3 -3
  40. prefect/flows.py +214 -170
  41. prefect/logging/configuration.py +1 -1
  42. prefect/logging/highlighters.py +1 -2
  43. prefect/logging/loggers.py +30 -20
  44. prefect/main.py +17 -24
  45. prefect/runner/runner.py +43 -21
  46. prefect/runner/server.py +30 -32
  47. prefect/runner/submit.py +3 -6
  48. prefect/runner/utils.py +6 -6
  49. prefect/runtime/flow_run.py +7 -0
  50. prefect/settings/constants.py +2 -2
  51. prefect/settings/legacy.py +1 -1
  52. prefect/settings/models/server/events.py +10 -0
  53. prefect/task_engine.py +72 -19
  54. prefect/task_runners.py +2 -2
  55. prefect/tasks.py +46 -33
  56. prefect/telemetry/bootstrap.py +15 -2
  57. prefect/telemetry/run_telemetry.py +107 -0
  58. prefect/transactions.py +14 -14
  59. prefect/types/__init__.py +1 -4
  60. prefect/utilities/_engine.py +96 -0
  61. prefect/utilities/annotations.py +25 -18
  62. prefect/utilities/asyncutils.py +126 -140
  63. prefect/utilities/callables.py +87 -78
  64. prefect/utilities/collections.py +278 -117
  65. prefect/utilities/compat.py +13 -21
  66. prefect/utilities/context.py +6 -5
  67. prefect/utilities/dispatch.py +23 -12
  68. prefect/utilities/dockerutils.py +33 -32
  69. prefect/utilities/engine.py +126 -239
  70. prefect/utilities/filesystem.py +18 -15
  71. prefect/utilities/hashing.py +10 -11
  72. prefect/utilities/importtools.py +40 -27
  73. prefect/utilities/math.py +9 -5
  74. prefect/utilities/names.py +3 -3
  75. prefect/utilities/processutils.py +121 -57
  76. prefect/utilities/pydantic.py +41 -36
  77. prefect/utilities/render_swagger.py +22 -12
  78. prefect/utilities/schema_tools/__init__.py +2 -1
  79. prefect/utilities/schema_tools/hydration.py +50 -43
  80. prefect/utilities/schema_tools/validation.py +52 -42
  81. prefect/utilities/services.py +13 -12
  82. prefect/utilities/templating.py +45 -45
  83. prefect/utilities/text.py +2 -1
  84. prefect/utilities/timeout.py +4 -4
  85. prefect/utilities/urls.py +9 -4
  86. prefect/utilities/visualization.py +46 -24
  87. prefect/variables.py +9 -8
  88. prefect/workers/base.py +15 -8
  89. {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/METADATA +4 -2
  90. {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/RECORD +93 -91
  91. {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/LICENSE +0 -0
  92. {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/WHEEL +0 -0
  93. {prefect_client-3.1.5.dist-info → prefect_client-3.1.6.dist-info}/top_level.txt +0 -0
prefect/task_engine.py CHANGED
@@ -29,6 +29,7 @@ from uuid import UUID
29
29
 
30
30
  import anyio
31
31
  import pendulum
32
+ from opentelemetry import trace
32
33
  from typing_extensions import ParamSpec
33
34
 
34
35
  from prefect import Task
@@ -79,13 +80,14 @@ from prefect.states import (
79
80
  exception_to_failed_state,
80
81
  return_value_to_state,
81
82
  )
83
+ from prefect.telemetry.run_telemetry import RunTelemetry
82
84
  from prefect.transactions import IsolationLevel, Transaction, transaction
85
+ from prefect.utilities._engine import get_hook_name
83
86
  from prefect.utilities.annotations import NotSet
84
87
  from prefect.utilities.asyncutils import run_coro_as_sync
85
88
  from prefect.utilities.callables import call_with_parameters, parameters_to_args_kwargs
86
89
  from prefect.utilities.collections import visit_collection
87
90
  from prefect.utilities.engine import (
88
- _get_hook_name,
89
91
  emit_task_run_state_change_event,
90
92
  link_state_to_result,
91
93
  resolve_to_final_result,
@@ -120,6 +122,7 @@ class BaseTaskRunEngine(Generic[P, R]):
120
122
  _is_started: bool = False
121
123
  _task_name_set: bool = False
122
124
  _last_event: Optional[PrefectEvent] = None
125
+ _telemetry: RunTelemetry = field(default_factory=RunTelemetry)
123
126
 
124
127
  def __post_init__(self):
125
128
  if self.parameters is None:
@@ -193,11 +196,11 @@ class BaseTaskRunEngine(Generic[P, R]):
193
196
  self.parameters = resolved_parameters
194
197
 
195
198
  def _set_custom_task_run_name(self):
196
- from prefect.utilities.engine import _resolve_custom_task_run_name
199
+ from prefect.utilities._engine import resolve_custom_task_run_name
197
200
 
198
201
  # update the task run name if necessary
199
202
  if not self._task_name_set and self.task.task_run_name:
200
- task_run_name = _resolve_custom_task_run_name(
203
+ task_run_name = resolve_custom_task_run_name(
201
204
  task=self.task, parameters=self.parameters or {}
202
205
  )
203
206
 
@@ -351,7 +354,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
351
354
  hooks = None
352
355
 
353
356
  for hook in hooks or []:
354
- hook_name = _get_hook_name(hook)
357
+ hook_name = get_hook_name(hook)
355
358
 
356
359
  try:
357
360
  self.logger.info(
@@ -427,6 +430,11 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
427
430
 
428
431
  self.task_run.state = new_state = state
429
432
 
433
+ if last_state.timestamp == new_state.timestamp:
434
+ # Ensure that the state timestamp is unique, or at least not equal to the last state.
435
+ # This might occur especially on Windows where the timestamp resolution is limited.
436
+ new_state.timestamp += pendulum.duration(microseconds=1)
437
+
430
438
  # Ensure that the state_details are populated with the current run IDs
431
439
  new_state.state_details.task_run_id = self.task_run.id
432
440
  new_state.state_details.flow_run_id = self.task_run.flow_run_id
@@ -460,7 +468,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
460
468
  validated_state=self.task_run.state,
461
469
  follows=self._last_event,
462
470
  )
463
-
471
+ self._telemetry.update_state(new_state)
464
472
  return new_state
465
473
 
466
474
  def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
@@ -514,6 +522,8 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
514
522
  self.record_terminal_state_timing(terminal_state)
515
523
  self.set_state(terminal_state)
516
524
  self._return_value = result
525
+
526
+ self._telemetry.end_span_on_success(terminal_state.message)
517
527
  return result
518
528
 
519
529
  def handle_retry(self, exc: Exception) -> bool:
@@ -562,6 +572,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
562
572
 
563
573
  def handle_exception(self, exc: Exception) -> None:
564
574
  # If the task fails, and we have retries left, set the task to retrying.
575
+ self._telemetry.record_exception(exc)
565
576
  if not self.handle_retry(exc):
566
577
  # If the task has no retries left, or the retry condition is not met, set the task to failed.
567
578
  state = run_coro_as_sync(
@@ -575,6 +586,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
575
586
  self.record_terminal_state_timing(state)
576
587
  self.set_state(state)
577
588
  self._raised = exc
589
+ self._telemetry.end_span_on_failure(state.message)
578
590
 
579
591
  def handle_timeout(self, exc: TimeoutError) -> None:
580
592
  if not self.handle_retry(exc):
@@ -598,6 +610,8 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
598
610
  self.record_terminal_state_timing(state)
599
611
  self.set_state(state, force=True)
600
612
  self._raised = exc
613
+ self._telemetry.record_exception(exc)
614
+ self._telemetry.end_span_on_failure(state.message)
601
615
 
602
616
  @contextmanager
603
617
  def setup_run_context(self, client: Optional[SyncPrefectClient] = None):
@@ -655,14 +669,17 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
655
669
  with SyncClientContext.get_or_create() as client_ctx:
656
670
  self._client = client_ctx.client
657
671
  self._is_started = True
672
+ flow_run_context = FlowRunContext.get()
673
+ parent_task_run_context = TaskRunContext.get()
674
+
658
675
  try:
659
676
  if not self.task_run:
660
677
  self.task_run = run_coro_as_sync(
661
678
  self.task.create_local_run(
662
679
  id=task_run_id,
663
680
  parameters=self.parameters,
664
- flow_run_context=FlowRunContext.get(),
665
- parent_task_run_context=TaskRunContext.get(),
681
+ flow_run_context=flow_run_context,
682
+ parent_task_run_context=parent_task_run_context,
666
683
  wait_for=self.wait_for,
667
684
  extra_task_inputs=dependencies,
668
685
  )
@@ -679,6 +696,13 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
679
696
  self.logger.debug(
680
697
  f"Created task run {self.task_run.name!r} for task {self.task.name!r}"
681
698
  )
699
+ labels = (
700
+ flow_run_context.flow_run.labels if flow_run_context else {}
701
+ )
702
+ self._telemetry.start_span(
703
+ self.task_run, self.parameters, labels
704
+ )
705
+
682
706
  yield self
683
707
 
684
708
  except TerminationSignal as exc:
@@ -730,11 +754,12 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
730
754
  dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
731
755
  ) -> Generator[None, None, None]:
732
756
  with self.initialize_run(task_run_id=task_run_id, dependencies=dependencies):
733
- self.begin_run()
734
- try:
735
- yield
736
- finally:
737
- self.call_hooks()
757
+ with trace.use_span(self._telemetry._span):
758
+ self.begin_run()
759
+ try:
760
+ yield
761
+ finally:
762
+ self.call_hooks()
738
763
 
739
764
  @contextmanager
740
765
  def transaction_context(self) -> Generator[Transaction, None, None]:
@@ -866,7 +891,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
866
891
  hooks = None
867
892
 
868
893
  for hook in hooks or []:
869
- hook_name = _get_hook_name(hook)
894
+ hook_name = get_hook_name(hook)
870
895
 
871
896
  try:
872
897
  self.logger.info(
@@ -942,6 +967,11 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
942
967
 
943
968
  self.task_run.state = new_state = state
944
969
 
970
+ if last_state.timestamp == new_state.timestamp:
971
+ # Ensure that the state timestamp is unique, or at least not equal to the last state.
972
+ # This might occur especially on Windows where the timestamp resolution is limited.
973
+ new_state.timestamp += pendulum.duration(microseconds=1)
974
+
945
975
  # Ensure that the state_details are populated with the current run IDs
946
976
  new_state.state_details.task_run_id = self.task_run.id
947
977
  new_state.state_details.flow_run_id = self.task_run.flow_run_id
@@ -977,6 +1007,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
977
1007
  follows=self._last_event,
978
1008
  )
979
1009
 
1010
+ self._telemetry.update_state(new_state)
980
1011
  return new_state
981
1012
 
982
1013
  async def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
@@ -1025,6 +1056,9 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1025
1056
  self.record_terminal_state_timing(terminal_state)
1026
1057
  await self.set_state(terminal_state)
1027
1058
  self._return_value = result
1059
+
1060
+ self._telemetry.end_span_on_success(terminal_state.message)
1061
+
1028
1062
  return result
1029
1063
 
1030
1064
  async def handle_retry(self, exc: Exception) -> bool:
@@ -1073,6 +1107,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1073
1107
 
1074
1108
  async def handle_exception(self, exc: Exception) -> None:
1075
1109
  # If the task fails, and we have retries left, set the task to retrying.
1110
+ self._telemetry.record_exception(exc)
1076
1111
  if not await self.handle_retry(exc):
1077
1112
  # If the task has no retries left, or the retry condition is not met, set the task to failed.
1078
1113
  state = await exception_to_failed_state(
@@ -1084,7 +1119,10 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1084
1119
  await self.set_state(state)
1085
1120
  self._raised = exc
1086
1121
 
1122
+ self._telemetry.end_span_on_failure(state.message)
1123
+
1087
1124
  async def handle_timeout(self, exc: TimeoutError) -> None:
1125
+ self._telemetry.record_exception(exc)
1088
1126
  if not await self.handle_retry(exc):
1089
1127
  if isinstance(exc, TaskRunTimeoutError):
1090
1128
  message = f"Task run exceeded timeout of {self.task.timeout_seconds} second(s)"
@@ -1098,6 +1136,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1098
1136
  )
1099
1137
  await self.set_state(state)
1100
1138
  self._raised = exc
1139
+ self._telemetry.end_span_on_failure(state.message)
1101
1140
 
1102
1141
  async def handle_crash(self, exc: BaseException) -> None:
1103
1142
  state = await exception_to_crashed_state(exc)
@@ -1107,6 +1146,9 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1107
1146
  await self.set_state(state, force=True)
1108
1147
  self._raised = exc
1109
1148
 
1149
+ self._telemetry.record_exception(exc)
1150
+ self._telemetry.end_span_on_failure(state.message)
1151
+
1110
1152
  @asynccontextmanager
1111
1153
  async def setup_run_context(self, client: Optional[PrefectClient] = None):
1112
1154
  from prefect.utilities.engine import (
@@ -1162,12 +1204,14 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1162
1204
  async with AsyncClientContext.get_or_create():
1163
1205
  self._client = get_client()
1164
1206
  self._is_started = True
1207
+ flow_run_context = FlowRunContext.get()
1208
+
1165
1209
  try:
1166
1210
  if not self.task_run:
1167
1211
  self.task_run = await self.task.create_local_run(
1168
1212
  id=task_run_id,
1169
1213
  parameters=self.parameters,
1170
- flow_run_context=FlowRunContext.get(),
1214
+ flow_run_context=flow_run_context,
1171
1215
  parent_task_run_context=TaskRunContext.get(),
1172
1216
  wait_for=self.wait_for,
1173
1217
  extra_task_inputs=dependencies,
@@ -1184,6 +1228,14 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1184
1228
  self.logger.debug(
1185
1229
  f"Created task run {self.task_run.name!r} for task {self.task.name!r}"
1186
1230
  )
1231
+
1232
+ labels = (
1233
+ flow_run_context.flow_run.labels if flow_run_context else {}
1234
+ )
1235
+ self._telemetry.start_span(
1236
+ self.task_run, self.parameters, labels
1237
+ )
1238
+
1187
1239
  yield self
1188
1240
 
1189
1241
  except TerminationSignal as exc:
@@ -1237,11 +1289,12 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1237
1289
  async with self.initialize_run(
1238
1290
  task_run_id=task_run_id, dependencies=dependencies
1239
1291
  ):
1240
- await self.begin_run()
1241
- try:
1242
- yield
1243
- finally:
1244
- await self.call_hooks()
1292
+ with trace.use_span(self._telemetry._span):
1293
+ await self.begin_run()
1294
+ try:
1295
+ yield
1296
+ finally:
1297
+ await self.call_hooks()
1245
1298
 
1246
1299
  @asynccontextmanager
1247
1300
  async def transaction_context(self) -> AsyncGenerator[Transaction, None]:
prefect/task_runners.py CHANGED
@@ -97,9 +97,9 @@ class TaskRunner(abc.ABC, Generic[F]):
97
97
 
98
98
  def map(
99
99
  self,
100
- task: "Task",
100
+ task: "Task[P, R]",
101
101
  parameters: Dict[str, Any],
102
- wait_for: Optional[Iterable[PrefectFuture]] = None,
102
+ wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
103
103
  ) -> PrefectFutureList[F]:
104
104
  """
105
105
  Submit multiple tasks to the task run engine.
prefect/tasks.py CHANGED
@@ -21,6 +21,7 @@ from typing import (
21
21
  List,
22
22
  NoReturn,
23
23
  Optional,
24
+ Protocol,
24
25
  Set,
25
26
  Tuple,
26
27
  Type,
@@ -31,7 +32,7 @@ from typing import (
31
32
  )
32
33
  from uuid import UUID, uuid4
33
34
 
34
- from typing_extensions import Literal, ParamSpec
35
+ from typing_extensions import Literal, ParamSpec, Self, TypeIs
35
36
 
36
37
  import prefect.states
37
38
  from prefect.cache_policies import DEFAULT, NONE, CachePolicy
@@ -223,6 +224,16 @@ def _generate_task_key(fn: Callable[..., Any]) -> str:
223
224
  return f"{qualname}-{code_hash}"
224
225
 
225
226
 
227
+ class TaskRunNameCallbackWithParameters(Protocol):
228
+ @classmethod
229
+ def is_callback_with_parameters(cls, callable: Callable[..., str]) -> TypeIs[Self]:
230
+ sig = inspect.signature(callable)
231
+ return "parameters" in sig.parameters
232
+
233
+ def __call__(self, parameters: dict[str, Any]) -> str:
234
+ ...
235
+
236
+
226
237
  class Task(Generic[P, R]):
227
238
  """
228
239
  A Prefect task definition.
@@ -311,7 +322,7 @@ class Task(Generic[P, R]):
311
322
  ] = None,
312
323
  cache_expiration: Optional[datetime.timedelta] = None,
313
324
  task_run_name: Optional[
314
- Union[Callable[[], str], Callable[[Dict[str, Any]], str], str]
325
+ Union[Callable[[], str], TaskRunNameCallbackWithParameters, str]
315
326
  ] = None,
316
327
  retries: Optional[int] = None,
317
328
  retry_delay_seconds: Optional[
@@ -534,7 +545,9 @@ class Task(Generic[P, R]):
534
545
  Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
535
546
  ] = None,
536
547
  task_run_name: Optional[
537
- Union[Callable[[], str], Callable[[Dict[str, Any]], str], str, Type[NotSet]]
548
+ Union[
549
+ Callable[[], str], TaskRunNameCallbackWithParameters, str, Type[NotSet]
550
+ ]
538
551
  ] = NotSet,
539
552
  cache_expiration: Optional[datetime.timedelta] = None,
540
553
  retries: Union[int, Type[NotSet]] = NotSet,
@@ -732,10 +745,8 @@ class Task(Generic[P, R]):
732
745
  extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
733
746
  deferred: bool = False,
734
747
  ) -> TaskRun:
735
- from prefect.utilities.engine import (
736
- _dynamic_key_for_task_run,
737
- collect_task_run_inputs_sync,
738
- )
748
+ from prefect.utilities._engine import dynamic_key_for_task_run
749
+ from prefect.utilities.engine import collect_task_run_inputs_sync
739
750
 
740
751
  if flow_run_context is None:
741
752
  flow_run_context = FlowRunContext.get()
@@ -751,7 +762,7 @@ class Task(Generic[P, R]):
751
762
  dynamic_key = f"{self.task_key}-{str(uuid4().hex)}"
752
763
  task_run_name = self.name
753
764
  else:
754
- dynamic_key = _dynamic_key_for_task_run(
765
+ dynamic_key = dynamic_key_for_task_run(
755
766
  context=flow_run_context, task=self
756
767
  )
757
768
  task_run_name = f"{self.name}-{dynamic_key}"
@@ -835,10 +846,8 @@ class Task(Generic[P, R]):
835
846
  extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
836
847
  deferred: bool = False,
837
848
  ) -> TaskRun:
838
- from prefect.utilities.engine import (
839
- _dynamic_key_for_task_run,
840
- collect_task_run_inputs_sync,
841
- )
849
+ from prefect.utilities._engine import dynamic_key_for_task_run
850
+ from prefect.utilities.engine import collect_task_run_inputs_sync
842
851
 
843
852
  if flow_run_context is None:
844
853
  flow_run_context = FlowRunContext.get()
@@ -854,7 +863,7 @@ class Task(Generic[P, R]):
854
863
  dynamic_key = f"{self.task_key}-{str(uuid4().hex)}"
855
864
  task_run_name = self.name
856
865
  else:
857
- dynamic_key = _dynamic_key_for_task_run(
866
+ dynamic_key = dynamic_key_for_task_run(
858
867
  context=flow_run_context, task=self, stable=False
859
868
  )
860
869
  task_run_name = f"{self.name}-{dynamic_key[:3]}"
@@ -1179,7 +1188,7 @@ class Task(Generic[P, R]):
1179
1188
  self: "Task[P, R]",
1180
1189
  *args: Any,
1181
1190
  return_state: Literal[True],
1182
- wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
1191
+ wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
1183
1192
  deferred: bool = ...,
1184
1193
  **kwargs: Any,
1185
1194
  ) -> List[State[R]]:
@@ -1189,7 +1198,7 @@ class Task(Generic[P, R]):
1189
1198
  def map(
1190
1199
  self: "Task[P, R]",
1191
1200
  *args: Any,
1192
- wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
1201
+ wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
1193
1202
  deferred: bool = ...,
1194
1203
  **kwargs: Any,
1195
1204
  ) -> PrefectFutureList[R]:
@@ -1200,7 +1209,7 @@ class Task(Generic[P, R]):
1200
1209
  self: "Task[P, R]",
1201
1210
  *args: Any,
1202
1211
  return_state: Literal[True],
1203
- wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
1212
+ wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
1204
1213
  deferred: bool = ...,
1205
1214
  **kwargs: Any,
1206
1215
  ) -> List[State[R]]:
@@ -1210,7 +1219,7 @@ class Task(Generic[P, R]):
1210
1219
  def map(
1211
1220
  self: "Task[P, R]",
1212
1221
  *args: Any,
1213
- wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
1222
+ wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
1214
1223
  deferred: bool = ...,
1215
1224
  **kwargs: Any,
1216
1225
  ) -> PrefectFutureList[R]:
@@ -1221,7 +1230,7 @@ class Task(Generic[P, R]):
1221
1230
  self: "Task[P, Coroutine[Any, Any, R]]",
1222
1231
  *args: Any,
1223
1232
  return_state: Literal[True],
1224
- wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
1233
+ wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
1225
1234
  deferred: bool = ...,
1226
1235
  **kwargs: Any,
1227
1236
  ) -> List[State[R]]:
@@ -1232,7 +1241,7 @@ class Task(Generic[P, R]):
1232
1241
  self: "Task[P, Coroutine[Any, Any, R]]",
1233
1242
  *args: Any,
1234
1243
  return_state: Literal[False],
1235
- wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
1244
+ wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = ...,
1236
1245
  deferred: bool = ...,
1237
1246
  **kwargs: Any,
1238
1247
  ) -> PrefectFutureList[R]:
@@ -1242,10 +1251,10 @@ class Task(Generic[P, R]):
1242
1251
  self,
1243
1252
  *args: Any,
1244
1253
  return_state: bool = False,
1245
- wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = None,
1254
+ wait_for: Optional[Iterable[Union[PrefectFuture[R], R]]] = None,
1246
1255
  deferred: bool = False,
1247
1256
  **kwargs: Any,
1248
- ):
1257
+ ) -> Union[List[State[R]], PrefectFutureList[R]]:
1249
1258
  """
1250
1259
  Submit a mapped run of the task to a worker.
1251
1260
 
@@ -1394,7 +1403,7 @@ class Task(Generic[P, R]):
1394
1403
  " execution."
1395
1404
  )
1396
1405
  if return_state:
1397
- states = []
1406
+ states: list[State[R]] = []
1398
1407
  for future in futures:
1399
1408
  future.wait()
1400
1409
  states.append(future.state)
@@ -1406,9 +1415,9 @@ class Task(Generic[P, R]):
1406
1415
  self,
1407
1416
  args: Optional[Tuple[Any, ...]] = None,
1408
1417
  kwargs: Optional[Dict[str, Any]] = None,
1409
- wait_for: Optional[Iterable[PrefectFuture]] = None,
1418
+ wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
1410
1419
  dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
1411
- ) -> PrefectDistributedFuture:
1420
+ ) -> PrefectDistributedFuture[R]:
1412
1421
  """
1413
1422
  Create a pending task run for a task worker to execute.
1414
1423
 
@@ -1588,7 +1597,7 @@ def task(
1588
1597
  ] = None,
1589
1598
  cache_expiration: Optional[datetime.timedelta] = None,
1590
1599
  task_run_name: Optional[
1591
- Union[Callable[[], str], Callable[[Dict[str, Any]], str], str]
1600
+ Union[Callable[[], str], TaskRunNameCallbackWithParameters, str]
1592
1601
  ] = None,
1593
1602
  retries: int = 0,
1594
1603
  retry_delay_seconds: Union[
@@ -1606,16 +1615,18 @@ def task(
1606
1615
  timeout_seconds: Union[int, float, None] = None,
1607
1616
  log_prints: Optional[bool] = None,
1608
1617
  refresh_cache: Optional[bool] = None,
1609
- on_completion: Optional[List[Callable[["Task", TaskRun, State], None]]] = None,
1610
- on_failure: Optional[List[Callable[["Task", TaskRun, State], None]]] = None,
1611
- retry_condition_fn: Optional[Callable[["Task", TaskRun, State], bool]] = None,
1618
+ on_completion: Optional[
1619
+ List[Callable[["Task[P, R]", TaskRun, State], None]]
1620
+ ] = None,
1621
+ on_failure: Optional[List[Callable[["Task[P, R]", TaskRun, State], None]]] = None,
1622
+ retry_condition_fn: Optional[Callable[["Task[P, R]", TaskRun, State], bool]] = None,
1612
1623
  viz_return_value: Any = None,
1613
1624
  ) -> Callable[[Callable[P, R]], Task[P, R]]:
1614
1625
  ...
1615
1626
 
1616
1627
 
1617
1628
  def task(
1618
- __fn=None,
1629
+ __fn: Optional[Callable[P, R]] = None,
1619
1630
  *,
1620
1631
  name: Optional[str] = None,
1621
1632
  description: Optional[str] = None,
@@ -1627,7 +1638,7 @@ def task(
1627
1638
  ] = None,
1628
1639
  cache_expiration: Optional[datetime.timedelta] = None,
1629
1640
  task_run_name: Optional[
1630
- Union[Callable[[], str], Callable[[Dict[str, Any]], str], str]
1641
+ Union[Callable[[], str], TaskRunNameCallbackWithParameters, str]
1631
1642
  ] = None,
1632
1643
  retries: Optional[int] = None,
1633
1644
  retry_delay_seconds: Union[
@@ -1642,9 +1653,11 @@ def task(
1642
1653
  timeout_seconds: Union[int, float, None] = None,
1643
1654
  log_prints: Optional[bool] = None,
1644
1655
  refresh_cache: Optional[bool] = None,
1645
- on_completion: Optional[List[Callable[["Task", TaskRun, State], None]]] = None,
1646
- on_failure: Optional[List[Callable[["Task", TaskRun, State], None]]] = None,
1647
- retry_condition_fn: Optional[Callable[["Task", TaskRun, State], bool]] = None,
1656
+ on_completion: Optional[
1657
+ List[Callable[["Task[P, R]", TaskRun, State], None]]
1658
+ ] = None,
1659
+ on_failure: Optional[List[Callable[["Task[P, R]", TaskRun, State], None]]] = None,
1660
+ retry_condition_fn: Optional[Callable[["Task[P, R]", TaskRun, State], bool]] = None,
1648
1661
  viz_return_value: Any = None,
1649
1662
  ):
1650
1663
  """
@@ -23,10 +23,23 @@ def setup_telemetry() -> (
23
23
  if server_type != ServerType.CLOUD:
24
24
  return None, None, None
25
25
 
26
- assert settings.api.key
26
+ if not settings.api.key:
27
+ raise ValueError(
28
+ "A Prefect Cloud API key is required to enable telemetry. Please set "
29
+ "the `PREFECT_API_KEY` environment variable or authenticate with "
30
+ "Prefect Cloud via the `prefect cloud login` command."
31
+ )
32
+
27
33
  assert settings.api.url
28
34
 
29
35
  # This import is here to defer importing of the `opentelemetry` packages.
30
- from .instrumentation import setup_exporters
36
+ try:
37
+ from .instrumentation import setup_exporters
38
+ except ImportError as exc:
39
+ raise ValueError(
40
+ "Unable to import OpenTelemetry instrumentation libraries. Please "
41
+ "ensure you have installed the `otel` extra when installing Prefect: "
42
+ "`pip install 'prefect[otel]'`"
43
+ ) from exc
31
44
 
32
45
  return setup_exporters(settings.api.url, settings.api.key.get_secret_value())
@@ -0,0 +1,107 @@
1
+ import time
2
+ from dataclasses import dataclass, field
3
+ from typing import TYPE_CHECKING, Any, Dict, Optional
4
+
5
+ from opentelemetry.propagators.textmap import Setter
6
+ from opentelemetry.trace import (
7
+ Status,
8
+ StatusCode,
9
+ get_tracer,
10
+ )
11
+
12
+ import prefect
13
+ from prefect.client.schemas import TaskRun
14
+ from prefect.client.schemas.objects import State
15
+ from prefect.types import KeyValueLabels
16
+
17
+ if TYPE_CHECKING:
18
+ from opentelemetry.trace import Tracer
19
+
20
+
21
+ class OTELSetter(Setter[KeyValueLabels]):
22
+ """
23
+ A setter for OpenTelemetry that supports Prefect's custom labels.
24
+ """
25
+
26
+ def set(self, carrier: KeyValueLabels, key: str, value: str) -> None:
27
+ carrier[key] = value
28
+
29
+
30
+ @dataclass
31
+ class RunTelemetry:
32
+ """
33
+ A class for managing the telemetry of runs.
34
+ """
35
+
36
+ _tracer: "Tracer" = field(
37
+ default_factory=lambda: get_tracer("prefect", prefect.__version__)
38
+ )
39
+ _span = None
40
+
41
+ def start_span(
42
+ self,
43
+ task_run: TaskRun,
44
+ parameters: Optional[Dict[str, Any]] = None,
45
+ labels: Optional[Dict[str, Any]] = None,
46
+ ):
47
+ """
48
+ Start a span for a task run.
49
+ """
50
+ if parameters is None:
51
+ parameters = {}
52
+ if labels is None:
53
+ labels = {}
54
+ parameter_attributes = {
55
+ f"prefect.run.parameter.{k}": type(v).__name__
56
+ for k, v in parameters.items()
57
+ }
58
+ self._span = self._tracer.start_span(
59
+ name=task_run.name,
60
+ attributes={
61
+ "prefect.run.type": "task",
62
+ "prefect.run.id": str(task_run.id),
63
+ "prefect.tags": task_run.tags,
64
+ **parameter_attributes,
65
+ **labels,
66
+ },
67
+ )
68
+
69
+ def end_span_on_success(self, terminal_message: str) -> None:
70
+ """
71
+ End a span for a task run on success.
72
+ """
73
+ if self._span:
74
+ self._span.set_status(Status(StatusCode.OK), terminal_message)
75
+ self._span.end(time.time_ns())
76
+ self._span = None
77
+
78
+ def end_span_on_failure(self, terminal_message: str) -> None:
79
+ """
80
+ End a span for a task run on failure.
81
+ """
82
+ if self._span:
83
+ self._span.set_status(Status(StatusCode.ERROR, terminal_message))
84
+ self._span.end(time.time_ns())
85
+ self._span = None
86
+
87
+ def record_exception(self, exc: Exception) -> None:
88
+ """
89
+ Record an exception on a span.
90
+ """
91
+ if self._span:
92
+ self._span.record_exception(exc)
93
+
94
+ def update_state(self, new_state: State) -> None:
95
+ """
96
+ Update a span with the state of a task run.
97
+ """
98
+ if self._span:
99
+ self._span.add_event(
100
+ new_state.name or new_state.type,
101
+ {
102
+ "prefect.state.message": new_state.message or "",
103
+ "prefect.state.type": new_state.type,
104
+ "prefect.state.name": new_state.name or new_state.type,
105
+ "prefect.state.id": str(new_state.id),
106
+ },
107
+ )