prefect-client 3.1.10__py3-none-any.whl → 3.1.12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. prefect/_experimental/lineage.py +7 -8
  2. prefect/_experimental/sla/__init__.py +0 -0
  3. prefect/_experimental/sla/client.py +66 -0
  4. prefect/_experimental/sla/objects.py +53 -0
  5. prefect/_internal/_logging.py +15 -3
  6. prefect/_internal/compatibility/async_dispatch.py +22 -16
  7. prefect/_internal/compatibility/deprecated.py +42 -18
  8. prefect/_internal/compatibility/migration.py +2 -2
  9. prefect/_internal/concurrency/inspection.py +12 -14
  10. prefect/_internal/concurrency/primitives.py +2 -2
  11. prefect/_internal/concurrency/services.py +154 -80
  12. prefect/_internal/concurrency/waiters.py +13 -9
  13. prefect/_internal/pydantic/annotations/pendulum.py +7 -7
  14. prefect/_internal/pytz.py +4 -3
  15. prefect/_internal/retries.py +10 -5
  16. prefect/_internal/schemas/bases.py +19 -10
  17. prefect/_internal/schemas/validators.py +227 -388
  18. prefect/_version.py +3 -3
  19. prefect/automations.py +236 -30
  20. prefect/blocks/__init__.py +3 -3
  21. prefect/blocks/abstract.py +53 -30
  22. prefect/blocks/core.py +183 -84
  23. prefect/blocks/notifications.py +133 -73
  24. prefect/blocks/redis.py +13 -9
  25. prefect/blocks/system.py +24 -11
  26. prefect/blocks/webhook.py +7 -5
  27. prefect/cache_policies.py +3 -2
  28. prefect/client/orchestration/__init__.py +1957 -0
  29. prefect/client/orchestration/_artifacts/__init__.py +0 -0
  30. prefect/client/orchestration/_artifacts/client.py +239 -0
  31. prefect/client/orchestration/_automations/__init__.py +0 -0
  32. prefect/client/orchestration/_automations/client.py +329 -0
  33. prefect/client/orchestration/_blocks_documents/__init__.py +0 -0
  34. prefect/client/orchestration/_blocks_documents/client.py +334 -0
  35. prefect/client/orchestration/_blocks_schemas/__init__.py +0 -0
  36. prefect/client/orchestration/_blocks_schemas/client.py +200 -0
  37. prefect/client/orchestration/_blocks_types/__init__.py +0 -0
  38. prefect/client/orchestration/_blocks_types/client.py +380 -0
  39. prefect/client/orchestration/_concurrency_limits/__init__.py +0 -0
  40. prefect/client/orchestration/_concurrency_limits/client.py +762 -0
  41. prefect/client/orchestration/_deployments/__init__.py +0 -0
  42. prefect/client/orchestration/_deployments/client.py +1128 -0
  43. prefect/client/orchestration/_flow_runs/__init__.py +0 -0
  44. prefect/client/orchestration/_flow_runs/client.py +903 -0
  45. prefect/client/orchestration/_flows/__init__.py +0 -0
  46. prefect/client/orchestration/_flows/client.py +343 -0
  47. prefect/client/orchestration/_logs/__init__.py +0 -0
  48. prefect/client/orchestration/_logs/client.py +97 -0
  49. prefect/client/orchestration/_variables/__init__.py +0 -0
  50. prefect/client/orchestration/_variables/client.py +157 -0
  51. prefect/client/orchestration/base.py +46 -0
  52. prefect/client/orchestration/routes.py +145 -0
  53. prefect/client/schemas/__init__.py +68 -28
  54. prefect/client/schemas/actions.py +2 -2
  55. prefect/client/schemas/filters.py +5 -0
  56. prefect/client/schemas/objects.py +8 -15
  57. prefect/client/schemas/schedules.py +22 -10
  58. prefect/concurrency/_asyncio.py +87 -0
  59. prefect/concurrency/{events.py → _events.py} +10 -10
  60. prefect/concurrency/asyncio.py +20 -104
  61. prefect/concurrency/context.py +6 -4
  62. prefect/concurrency/services.py +26 -74
  63. prefect/concurrency/sync.py +23 -44
  64. prefect/concurrency/v1/_asyncio.py +63 -0
  65. prefect/concurrency/v1/{events.py → _events.py} +13 -15
  66. prefect/concurrency/v1/asyncio.py +27 -80
  67. prefect/concurrency/v1/context.py +6 -4
  68. prefect/concurrency/v1/services.py +33 -79
  69. prefect/concurrency/v1/sync.py +18 -37
  70. prefect/context.py +66 -45
  71. prefect/deployments/base.py +10 -144
  72. prefect/deployments/flow_runs.py +12 -2
  73. prefect/deployments/runner.py +53 -4
  74. prefect/deployments/steps/pull.py +13 -0
  75. prefect/engine.py +17 -4
  76. prefect/events/clients.py +7 -1
  77. prefect/events/schemas/events.py +3 -2
  78. prefect/filesystems.py +6 -2
  79. prefect/flow_engine.py +101 -85
  80. prefect/flows.py +10 -1
  81. prefect/input/run_input.py +2 -1
  82. prefect/logging/logging.yml +1 -1
  83. prefect/main.py +1 -3
  84. prefect/results.py +2 -307
  85. prefect/runner/runner.py +4 -2
  86. prefect/runner/storage.py +87 -21
  87. prefect/serializers.py +32 -25
  88. prefect/settings/legacy.py +4 -4
  89. prefect/settings/models/api.py +3 -3
  90. prefect/settings/models/cli.py +3 -3
  91. prefect/settings/models/client.py +5 -3
  92. prefect/settings/models/cloud.py +8 -3
  93. prefect/settings/models/deployments.py +3 -3
  94. prefect/settings/models/experiments.py +4 -7
  95. prefect/settings/models/flows.py +3 -3
  96. prefect/settings/models/internal.py +4 -2
  97. prefect/settings/models/logging.py +4 -3
  98. prefect/settings/models/results.py +3 -3
  99. prefect/settings/models/root.py +3 -2
  100. prefect/settings/models/runner.py +4 -4
  101. prefect/settings/models/server/api.py +3 -3
  102. prefect/settings/models/server/database.py +11 -4
  103. prefect/settings/models/server/deployments.py +6 -2
  104. prefect/settings/models/server/ephemeral.py +4 -2
  105. prefect/settings/models/server/events.py +3 -2
  106. prefect/settings/models/server/flow_run_graph.py +6 -2
  107. prefect/settings/models/server/root.py +3 -3
  108. prefect/settings/models/server/services.py +26 -11
  109. prefect/settings/models/server/tasks.py +6 -3
  110. prefect/settings/models/server/ui.py +3 -3
  111. prefect/settings/models/tasks.py +5 -5
  112. prefect/settings/models/testing.py +3 -3
  113. prefect/settings/models/worker.py +5 -3
  114. prefect/settings/profiles.py +15 -2
  115. prefect/states.py +61 -45
  116. prefect/task_engine.py +54 -75
  117. prefect/task_runners.py +56 -55
  118. prefect/task_worker.py +2 -2
  119. prefect/tasks.py +90 -36
  120. prefect/telemetry/bootstrap.py +10 -9
  121. prefect/telemetry/run_telemetry.py +13 -8
  122. prefect/telemetry/services.py +4 -0
  123. prefect/transactions.py +4 -15
  124. prefect/utilities/_git.py +34 -0
  125. prefect/utilities/asyncutils.py +1 -1
  126. prefect/utilities/engine.py +3 -19
  127. prefect/utilities/generics.py +18 -0
  128. prefect/utilities/templating.py +25 -1
  129. prefect/workers/base.py +6 -3
  130. prefect/workers/process.py +1 -1
  131. {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/METADATA +2 -2
  132. {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/RECORD +135 -109
  133. prefect/client/orchestration.py +0 -4523
  134. prefect/records/__init__.py +0 -1
  135. prefect/records/base.py +0 -235
  136. prefect/records/filesystem.py +0 -213
  137. prefect/records/memory.py +0 -184
  138. prefect/records/result_store.py +0 -70
  139. {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/LICENSE +0 -0
  140. {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/WHEEL +0 -0
  141. {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/top_level.txt +0 -0
prefect/task_engine.py CHANGED
@@ -9,18 +9,16 @@ from dataclasses import dataclass, field
9
9
  from functools import partial
10
10
  from textwrap import dedent
11
11
  from typing import (
12
+ TYPE_CHECKING,
12
13
  Any,
13
14
  AsyncGenerator,
14
15
  Callable,
15
16
  Coroutine,
16
- Dict,
17
17
  Generator,
18
18
  Generic,
19
- Iterable,
20
19
  Literal,
21
20
  Optional,
22
21
  Sequence,
23
- Set,
24
22
  Type,
25
23
  TypeVar,
26
24
  Union,
@@ -32,7 +30,6 @@ import pendulum
32
30
  from opentelemetry import trace
33
31
  from typing_extensions import ParamSpec
34
32
 
35
- from prefect import Task
36
33
  from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_client
37
34
  from prefect.client.schemas import TaskRun
38
35
  from prefect.client.schemas.objects import State, TaskRunInput
@@ -55,10 +52,8 @@ from prefect.exceptions import (
55
52
  TerminationSignal,
56
53
  UpstreamTaskError,
57
54
  )
58
- from prefect.futures import PrefectFuture
59
55
  from prefect.logging.loggers import get_logger, patch_print, task_run_logger
60
56
  from prefect.results import (
61
- BaseResult,
62
57
  ResultRecord,
63
58
  _format_user_supplied_storage_key,
64
59
  get_result_store,
@@ -95,6 +90,9 @@ from prefect.utilities.engine import (
95
90
  from prefect.utilities.math import clamped_poisson_interval
96
91
  from prefect.utilities.timeout import timeout, timeout_async
97
92
 
93
+ if TYPE_CHECKING:
94
+ from prefect.tasks import OneOrManyFutureOrResult, Task
95
+
98
96
  P = ParamSpec("P")
99
97
  R = TypeVar("R")
100
98
 
@@ -107,13 +105,13 @@ class TaskRunTimeoutError(TimeoutError):
107
105
 
108
106
  @dataclass
109
107
  class BaseTaskRunEngine(Generic[P, R]):
110
- task: Union[Task[P, R], Task[P, Coroutine[Any, Any, R]]]
108
+ task: Union["Task[P, R]", "Task[P, Coroutine[Any, Any, R]]"]
111
109
  logger: logging.Logger = field(default_factory=lambda: get_logger("engine"))
112
- parameters: Optional[Dict[str, Any]] = None
110
+ parameters: Optional[dict[str, Any]] = None
113
111
  task_run: Optional[TaskRun] = None
114
112
  retries: int = 0
115
- wait_for: Optional[Iterable[PrefectFuture]] = None
116
- context: Optional[Dict[str, Any]] = None
113
+ wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None
114
+ context: Optional[dict[str, Any]] = None
117
115
  # holds the return value from the user code
118
116
  _return_value: Union[R, Type[NotSet]] = NotSet
119
117
  # holds the exception raised by the user code, if any
@@ -210,6 +208,7 @@ class BaseTaskRunEngine(Generic[P, R]):
210
208
  )
211
209
  self.task_run.name = task_run_name
212
210
  self._task_name_set = True
211
+ self._telemetry.update_run_name(name=task_run_name)
213
212
 
214
213
  def _wait_for_dependencies(self):
215
214
  if not self.wait_for:
@@ -305,7 +304,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
305
304
 
306
305
  def can_retry(self, exc: Exception) -> bool:
307
306
  retry_condition: Optional[
308
- Callable[[Task[P, Coroutine[Any, Any, R]], TaskRun, State], bool]
307
+ Callable[["Task[P, Coroutine[Any, Any, R]]", TaskRun, State], bool]
309
308
  ] = self.task.retry_condition_fn
310
309
  if not self.task_run:
311
310
  raise ValueError("Task run is not set")
@@ -448,13 +447,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
448
447
  self.task_run.run_count += 1
449
448
 
450
449
  if new_state.is_final():
451
- if isinstance(state.data, BaseResult) and state.data.has_cached_object():
452
- # Avoid fetching the result unless it is cached, otherwise we defeat
453
- # the purpose of disabling `cache_result_in_memory`
454
- result = state.result(raise_on_failure=False, fetch=True)
455
- if asyncio.iscoroutine(result):
456
- result = run_coro_as_sync(result)
457
- elif isinstance(state.data, ResultRecord):
450
+ if isinstance(state.data, ResultRecord):
458
451
  result = state.data.result
459
452
  else:
460
453
  result = state.data
@@ -473,13 +466,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
473
466
 
474
467
  def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
475
468
  if self._return_value is not NotSet:
476
- # if the return value is a BaseResult, we need to fetch it
477
- if isinstance(self._return_value, BaseResult):
478
- _result = self._return_value.get()
479
- if asyncio.iscoroutine(_result):
480
- _result = run_coro_as_sync(_result)
481
- return _result
482
- elif isinstance(self._return_value, ResultRecord):
469
+ if isinstance(self._return_value, ResultRecord):
483
470
  return self._return_value.result
484
471
  # otherwise, return the value as is
485
472
  return self._return_value
@@ -660,7 +647,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
660
647
  def initialize_run(
661
648
  self,
662
649
  task_run_id: Optional[UUID] = None,
663
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
650
+ dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
664
651
  ) -> Generator["SyncTaskRunEngine", Any, Any]:
665
652
  """
666
653
  Enters a client context and creates a task run if needed.
@@ -700,7 +687,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
700
687
 
701
688
  self._telemetry.start_span(
702
689
  run=self.task_run,
703
- name=self.task.name,
704
690
  client=self.client,
705
691
  parameters=self.parameters,
706
692
  )
@@ -753,12 +739,14 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
753
739
  def start(
754
740
  self,
755
741
  task_run_id: Optional[UUID] = None,
756
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
742
+ dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
757
743
  ) -> Generator[None, None, None]:
758
744
  with self.initialize_run(task_run_id=task_run_id, dependencies=dependencies):
759
- with trace.use_span(
760
- self._telemetry.span
761
- ) if self._telemetry.span else nullcontext():
745
+ with (
746
+ trace.use_span(self._telemetry.span)
747
+ if self._telemetry.span
748
+ else nullcontext()
749
+ ):
762
750
  self.begin_run()
763
751
  try:
764
752
  yield
@@ -847,7 +835,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
847
835
 
848
836
  async def can_retry(self, exc: Exception) -> bool:
849
837
  retry_condition: Optional[
850
- Callable[[Task[P, Coroutine[Any, Any, R]], TaskRun, State], bool]
838
+ Callable[["Task[P, Coroutine[Any, Any, R]]", TaskRun, State], bool]
851
839
  ] = self.task.retry_condition_fn
852
840
  if not self.task_run:
853
841
  raise ValueError("Task run is not set")
@@ -989,14 +977,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
989
977
  self.task_run.run_count += 1
990
978
 
991
979
  if new_state.is_final():
992
- if (
993
- isinstance(new_state.data, BaseResult)
994
- and new_state.data.has_cached_object()
995
- ):
996
- # Avoid fetching the result unless it is cached, otherwise we defeat
997
- # the purpose of disabling `cache_result_in_memory`
998
- result = await new_state.result(raise_on_failure=False, fetch=True)
999
- elif isinstance(new_state.data, ResultRecord):
980
+ if isinstance(new_state.data, ResultRecord):
1000
981
  result = new_state.data.result
1001
982
  else:
1002
983
  result = new_state.data
@@ -1016,10 +997,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1016
997
 
1017
998
  async def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
1018
999
  if self._return_value is not NotSet:
1019
- # if the return value is a BaseResult, we need to fetch it
1020
- if isinstance(self._return_value, BaseResult):
1021
- return await self._return_value.get()
1022
- elif isinstance(self._return_value, ResultRecord):
1000
+ if isinstance(self._return_value, ResultRecord):
1023
1001
  return self._return_value.result
1024
1002
  # otherwise, return the value as is
1025
1003
  return self._return_value
@@ -1199,7 +1177,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1199
1177
  async def initialize_run(
1200
1178
  self,
1201
1179
  task_run_id: Optional[UUID] = None,
1202
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
1180
+ dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
1203
1181
  ) -> AsyncGenerator["AsyncTaskRunEngine", Any]:
1204
1182
  """
1205
1183
  Enters a client context and creates a task run if needed.
@@ -1237,7 +1215,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1237
1215
 
1238
1216
  await self._telemetry.async_start_span(
1239
1217
  run=self.task_run,
1240
- name=self.task.name,
1241
1218
  client=self.client,
1242
1219
  parameters=self.parameters,
1243
1220
  )
@@ -1290,14 +1267,16 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1290
1267
  async def start(
1291
1268
  self,
1292
1269
  task_run_id: Optional[UUID] = None,
1293
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
1270
+ dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
1294
1271
  ) -> AsyncGenerator[None, None]:
1295
1272
  async with self.initialize_run(
1296
1273
  task_run_id=task_run_id, dependencies=dependencies
1297
1274
  ):
1298
- with trace.use_span(
1299
- self._telemetry.span
1300
- ) if self._telemetry.span else nullcontext():
1275
+ with (
1276
+ trace.use_span(self._telemetry.span)
1277
+ if self._telemetry.span
1278
+ else nullcontext()
1279
+ ):
1301
1280
  await self.begin_run()
1302
1281
  try:
1303
1282
  yield
@@ -1374,14 +1353,14 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
1374
1353
 
1375
1354
 
1376
1355
  def run_task_sync(
1377
- task: Task[P, R],
1356
+ task: "Task[P, R]",
1378
1357
  task_run_id: Optional[UUID] = None,
1379
1358
  task_run: Optional[TaskRun] = None,
1380
- parameters: Optional[Dict[str, Any]] = None,
1381
- wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
1359
+ parameters: Optional[dict[str, Any]] = None,
1360
+ wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
1382
1361
  return_type: Literal["state", "result"] = "result",
1383
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
1384
- context: Optional[Dict[str, Any]] = None,
1362
+ dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
1363
+ context: Optional[dict[str, Any]] = None,
1385
1364
  ) -> Union[R, State, None]:
1386
1365
  engine = SyncTaskRunEngine[P, R](
1387
1366
  task=task,
@@ -1401,14 +1380,14 @@ def run_task_sync(
1401
1380
 
1402
1381
 
1403
1382
  async def run_task_async(
1404
- task: Task[P, R],
1383
+ task: "Task[P, R]",
1405
1384
  task_run_id: Optional[UUID] = None,
1406
1385
  task_run: Optional[TaskRun] = None,
1407
- parameters: Optional[Dict[str, Any]] = None,
1408
- wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
1386
+ parameters: Optional[dict[str, Any]] = None,
1387
+ wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
1409
1388
  return_type: Literal["state", "result"] = "result",
1410
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
1411
- context: Optional[Dict[str, Any]] = None,
1389
+ dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
1390
+ context: Optional[dict[str, Any]] = None,
1412
1391
  ) -> Union[R, State, None]:
1413
1392
  engine = AsyncTaskRunEngine[P, R](
1414
1393
  task=task,
@@ -1428,14 +1407,14 @@ async def run_task_async(
1428
1407
 
1429
1408
 
1430
1409
  def run_generator_task_sync(
1431
- task: Task[P, R],
1410
+ task: "Task[P, R]",
1432
1411
  task_run_id: Optional[UUID] = None,
1433
1412
  task_run: Optional[TaskRun] = None,
1434
- parameters: Optional[Dict[str, Any]] = None,
1435
- wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
1413
+ parameters: Optional[dict[str, Any]] = None,
1414
+ wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
1436
1415
  return_type: Literal["state", "result"] = "result",
1437
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
1438
- context: Optional[Dict[str, Any]] = None,
1416
+ dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
1417
+ context: Optional[dict[str, Any]] = None,
1439
1418
  ) -> Generator[R, None, None]:
1440
1419
  if return_type != "result":
1441
1420
  raise ValueError("The return_type for a generator task must be 'result'")
@@ -1483,14 +1462,14 @@ def run_generator_task_sync(
1483
1462
 
1484
1463
 
1485
1464
  async def run_generator_task_async(
1486
- task: Task[P, R],
1465
+ task: "Task[P, R]",
1487
1466
  task_run_id: Optional[UUID] = None,
1488
1467
  task_run: Optional[TaskRun] = None,
1489
- parameters: Optional[Dict[str, Any]] = None,
1490
- wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
1468
+ parameters: Optional[dict[str, Any]] = None,
1469
+ wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
1491
1470
  return_type: Literal["state", "result"] = "result",
1492
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
1493
- context: Optional[Dict[str, Any]] = None,
1471
+ dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
1472
+ context: Optional[dict[str, Any]] = None,
1494
1473
  ) -> AsyncGenerator[R, None]:
1495
1474
  if return_type != "result":
1496
1475
  raise ValueError("The return_type for a generator task must be 'result'")
@@ -1539,14 +1518,14 @@ async def run_generator_task_async(
1539
1518
 
1540
1519
 
1541
1520
  def run_task(
1542
- task: Task[P, Union[R, Coroutine[Any, Any, R]]],
1521
+ task: "Task[P, Union[R, Coroutine[Any, Any, R]]]",
1543
1522
  task_run_id: Optional[UUID] = None,
1544
1523
  task_run: Optional[TaskRun] = None,
1545
- parameters: Optional[Dict[str, Any]] = None,
1546
- wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
1524
+ parameters: Optional[dict[str, Any]] = None,
1525
+ wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
1547
1526
  return_type: Literal["state", "result"] = "result",
1548
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
1549
- context: Optional[Dict[str, Any]] = None,
1527
+ dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
1528
+ context: Optional[dict[str, Any]] = None,
1550
1529
  ) -> Union[R, State, None, Coroutine[Any, Any, Union[R, State, None]]]:
1551
1530
  """
1552
1531
  Runs the provided task.
prefect/task_runners.py CHANGED
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import abc
2
4
  import asyncio
3
5
  import sys
@@ -14,7 +16,6 @@ from typing import (
14
16
  Iterable,
15
17
  List,
16
18
  Optional,
17
- Set,
18
19
  overload,
19
20
  )
20
21
 
@@ -44,7 +45,7 @@ if TYPE_CHECKING:
44
45
  P = ParamSpec("P")
45
46
  T = TypeVar("T")
46
47
  R = TypeVar("R")
47
- F = TypeVar("F", bound=PrefectFuture, default=PrefectConcurrentFuture)
48
+ F = TypeVar("F", bound=PrefectFuture[Any], default=PrefectConcurrentFuture[Any])
48
49
 
49
50
 
50
51
  class TaskRunner(abc.ABC, Generic[F]):
@@ -76,10 +77,10 @@ class TaskRunner(abc.ABC, Generic[F]):
76
77
  @abc.abstractmethod
77
78
  def submit(
78
79
  self,
79
- task: "Task",
80
- parameters: Dict[str, Any],
81
- wait_for: Optional[Iterable[PrefectFuture]] = None,
82
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
80
+ task: "Task[P, R]",
81
+ parameters: dict[str, Any],
82
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
83
+ dependencies: dict[str, set[TaskRunInput]] | None = None,
83
84
  ) -> F:
84
85
  """
85
86
  Submit a task to the task run engine.
@@ -98,7 +99,7 @@ class TaskRunner(abc.ABC, Generic[F]):
98
99
  def map(
99
100
  self,
100
101
  task: "Task[P, R]",
101
- parameters: Dict[str, Any],
102
+ parameters: dict[str, Any],
102
103
  wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
103
104
  ) -> PrefectFutureList[F]:
104
105
  """
@@ -138,9 +139,9 @@ class TaskRunner(abc.ABC, Generic[F]):
138
139
  # Ensure that any parameters in kwargs are expanded before this check
139
140
  parameters = explode_variadic_parameter(task.fn, parameters)
140
141
 
141
- iterable_parameters = {}
142
- static_parameters = {}
143
- annotated_parameters = {}
142
+ iterable_parameters: dict[str, Any] = {}
143
+ static_parameters: dict[str, Any] = {}
144
+ annotated_parameters: dict[str, Any] = {}
144
145
  for key, val in parameters.items():
145
146
  if isinstance(val, (allow_failure, quote)):
146
147
  # Unwrap annotated parameters to determine if they are iterable
@@ -172,9 +173,9 @@ class TaskRunner(abc.ABC, Generic[F]):
172
173
 
173
174
  map_length = list(lengths)[0]
174
175
 
175
- futures: List[PrefectFuture] = []
176
+ futures: List[PrefectFuture[Any]] = []
176
177
  for i in range(map_length):
177
- call_parameters = {
178
+ call_parameters: dict[str, Any] = {
178
179
  key: value[i] for key, value in iterable_parameters.items()
179
180
  }
180
181
  call_parameters.update(
@@ -212,12 +213,12 @@ class TaskRunner(abc.ABC, Generic[F]):
212
213
  self._started = True
213
214
  return self
214
215
 
215
- def __exit__(self, exc_type, exc_value, traceback):
216
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any):
216
217
  self.logger.debug("Stopping task runner")
217
218
  self._started = False
218
219
 
219
220
 
220
- class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
221
+ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture[Any]]):
221
222
  def __init__(self, max_workers: Optional[int] = None):
222
223
  super().__init__()
223
224
  self._executor: Optional[ThreadPoolExecutor] = None
@@ -235,9 +236,9 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
235
236
  def submit(
236
237
  self,
237
238
  task: "Task[P, Coroutine[Any, Any, R]]",
238
- parameters: Dict[str, Any],
239
- wait_for: Optional[Iterable[PrefectFuture]] = None,
240
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
239
+ parameters: dict[str, Any],
240
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
241
+ dependencies: dict[str, set[TaskRunInput]] | None = None,
241
242
  ) -> PrefectConcurrentFuture[R]:
242
243
  ...
243
244
 
@@ -245,19 +246,19 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
245
246
  def submit(
246
247
  self,
247
248
  task: "Task[Any, R]",
248
- parameters: Dict[str, Any],
249
- wait_for: Optional[Iterable[PrefectFuture]] = None,
250
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
249
+ parameters: dict[str, Any],
250
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
251
+ dependencies: dict[str, set[TaskRunInput]] | None = None,
251
252
  ) -> PrefectConcurrentFuture[R]:
252
253
  ...
253
254
 
254
255
  def submit(
255
256
  self,
256
- task: "Task",
257
- parameters: Dict[str, Any],
258
- wait_for: Optional[Iterable[PrefectFuture]] = None,
259
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
260
- ):
257
+ task: "Task[P, R]",
258
+ parameters: dict[str, Any],
259
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
260
+ dependencies: dict[str, set[TaskRunInput]] | None = None,
261
+ ) -> PrefectConcurrentFuture[R]:
261
262
  """
262
263
  Submit a task to the task run engine running in a separate thread.
263
264
 
@@ -289,7 +290,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
289
290
  else:
290
291
  self.logger.debug(f"Submitting task {task.name} to thread pool executor...")
291
292
 
292
- submit_kwargs = dict(
293
+ submit_kwargs: dict[str, Any] = dict(
293
294
  task=task,
294
295
  task_run_id=task_run_id,
295
296
  parameters=parameters,
@@ -322,8 +323,8 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
322
323
  def map(
323
324
  self,
324
325
  task: "Task[P, Coroutine[Any, Any, R]]",
325
- parameters: Dict[str, Any],
326
- wait_for: Optional[Iterable[PrefectFuture]] = None,
326
+ parameters: dict[str, Any],
327
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
327
328
  ) -> PrefectFutureList[PrefectConcurrentFuture[R]]:
328
329
  ...
329
330
 
@@ -331,17 +332,17 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
331
332
  def map(
332
333
  self,
333
334
  task: "Task[Any, R]",
334
- parameters: Dict[str, Any],
335
- wait_for: Optional[Iterable[PrefectFuture]] = None,
335
+ parameters: dict[str, Any],
336
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
336
337
  ) -> PrefectFutureList[PrefectConcurrentFuture[R]]:
337
338
  ...
338
339
 
339
340
  def map(
340
341
  self,
341
- task: "Task",
342
- parameters: Dict[str, Any],
343
- wait_for: Optional[Iterable[PrefectFuture]] = None,
344
- ):
342
+ task: "Task[P, R]",
343
+ parameters: dict[str, Any],
344
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
345
+ ) -> PrefectFutureList[PrefectConcurrentFuture[R]]:
345
346
  return super().map(task, parameters, wait_for)
346
347
 
347
348
  def cancel_all(self):
@@ -358,7 +359,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
358
359
  self._executor = ThreadPoolExecutor(max_workers=self._max_workers)
359
360
  return self
360
361
 
361
- def __exit__(self, exc_type, exc_value, traceback):
362
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any):
362
363
  self.cancel_all()
363
364
  if self._executor is not None:
364
365
  self._executor.shutdown(cancel_futures=True)
@@ -375,7 +376,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
375
376
  ConcurrentTaskRunner = ThreadPoolTaskRunner
376
377
 
377
378
 
378
- class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
379
+ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture[R]]):
379
380
  def __init__(self):
380
381
  super().__init__()
381
382
 
@@ -386,9 +387,9 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
386
387
  def submit(
387
388
  self,
388
389
  task: "Task[P, Coroutine[Any, Any, R]]",
389
- parameters: Dict[str, Any],
390
- wait_for: Optional[Iterable[PrefectFuture]] = None,
391
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
390
+ parameters: dict[str, Any],
391
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
392
+ dependencies: dict[str, set[TaskRunInput]] | None = None,
392
393
  ) -> PrefectDistributedFuture[R]:
393
394
  ...
394
395
 
@@ -396,19 +397,19 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
396
397
  def submit(
397
398
  self,
398
399
  task: "Task[Any, R]",
399
- parameters: Dict[str, Any],
400
- wait_for: Optional[Iterable[PrefectFuture]] = None,
401
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
400
+ parameters: dict[str, Any],
401
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
402
+ dependencies: dict[str, set[TaskRunInput]] | None = None,
402
403
  ) -> PrefectDistributedFuture[R]:
403
404
  ...
404
405
 
405
406
  def submit(
406
407
  self,
407
- task: "Task",
408
- parameters: Dict[str, Any],
409
- wait_for: Optional[Iterable[PrefectFuture]] = None,
410
- dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
411
- ):
408
+ task: "Task[P, R]",
409
+ parameters: dict[str, Any],
410
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
411
+ dependencies: dict[str, set[TaskRunInput]] | None = None,
412
+ ) -> PrefectDistributedFuture[R]:
412
413
  """
413
414
  Submit a task to the task run engine running in a separate thread.
414
415
 
@@ -443,8 +444,8 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
443
444
  def map(
444
445
  self,
445
446
  task: "Task[P, Coroutine[Any, Any, R]]",
446
- parameters: Dict[str, Any],
447
- wait_for: Optional[Iterable[PrefectFuture]] = None,
447
+ parameters: dict[str, Any],
448
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
448
449
  ) -> PrefectFutureList[PrefectDistributedFuture[R]]:
449
450
  ...
450
451
 
@@ -452,15 +453,15 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
452
453
  def map(
453
454
  self,
454
455
  task: "Task[Any, R]",
455
- parameters: Dict[str, Any],
456
- wait_for: Optional[Iterable[PrefectFuture]] = None,
456
+ parameters: dict[str, Any],
457
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
457
458
  ) -> PrefectFutureList[PrefectDistributedFuture[R]]:
458
459
  ...
459
460
 
460
461
  def map(
461
462
  self,
462
- task: "Task",
463
- parameters: Dict[str, Any],
464
- wait_for: Optional[Iterable[PrefectFuture]] = None,
465
- ):
463
+ task: "Task[P, R]",
464
+ parameters: dict[str, Any],
465
+ wait_for: Iterable[PrefectFuture[Any]] | None = None,
466
+ ) -> PrefectFutureList[PrefectDistributedFuture[R]]:
466
467
  return super().map(task, parameters, wait_for)
prefect/task_worker.py CHANGED
@@ -21,7 +21,7 @@ from websockets.exceptions import InvalidStatusCode
21
21
 
22
22
  from prefect import Task
23
23
  from prefect._internal.concurrency.api import create_call, from_sync
24
- from prefect.cache_policies import DEFAULT, NONE
24
+ from prefect.cache_policies import DEFAULT, NO_CACHE
25
25
  from prefect.client.orchestration import get_client
26
26
  from prefect.client.schemas.objects import TaskRun
27
27
  from prefect.client.subscriptions import Subscription
@@ -93,7 +93,7 @@ class TaskWorker:
93
93
  if not isinstance(t, Task):
94
94
  continue
95
95
 
96
- if t.cache_policy in [None, NONE, NotSet]:
96
+ if t.cache_policy in [None, NO_CACHE, NotSet]:
97
97
  self.tasks.append(
98
98
  t.with_options(persist_result=True, cache_policy=DEFAULT)
99
99
  )