prefect-client 3.1.10__py3-none-any.whl → 3.1.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_experimental/lineage.py +7 -8
- prefect/_experimental/sla/__init__.py +0 -0
- prefect/_experimental/sla/client.py +66 -0
- prefect/_experimental/sla/objects.py +53 -0
- prefect/_internal/_logging.py +15 -3
- prefect/_internal/compatibility/async_dispatch.py +22 -16
- prefect/_internal/compatibility/deprecated.py +42 -18
- prefect/_internal/compatibility/migration.py +2 -2
- prefect/_internal/concurrency/inspection.py +12 -14
- prefect/_internal/concurrency/primitives.py +2 -2
- prefect/_internal/concurrency/services.py +154 -80
- prefect/_internal/concurrency/waiters.py +13 -9
- prefect/_internal/pydantic/annotations/pendulum.py +7 -7
- prefect/_internal/pytz.py +4 -3
- prefect/_internal/retries.py +10 -5
- prefect/_internal/schemas/bases.py +19 -10
- prefect/_internal/schemas/validators.py +227 -388
- prefect/_version.py +3 -3
- prefect/automations.py +236 -30
- prefect/blocks/__init__.py +3 -3
- prefect/blocks/abstract.py +53 -30
- prefect/blocks/core.py +183 -84
- prefect/blocks/notifications.py +133 -73
- prefect/blocks/redis.py +13 -9
- prefect/blocks/system.py +24 -11
- prefect/blocks/webhook.py +7 -5
- prefect/cache_policies.py +3 -2
- prefect/client/orchestration/__init__.py +1957 -0
- prefect/client/orchestration/_artifacts/__init__.py +0 -0
- prefect/client/orchestration/_artifacts/client.py +239 -0
- prefect/client/orchestration/_automations/__init__.py +0 -0
- prefect/client/orchestration/_automations/client.py +329 -0
- prefect/client/orchestration/_blocks_documents/__init__.py +0 -0
- prefect/client/orchestration/_blocks_documents/client.py +334 -0
- prefect/client/orchestration/_blocks_schemas/__init__.py +0 -0
- prefect/client/orchestration/_blocks_schemas/client.py +200 -0
- prefect/client/orchestration/_blocks_types/__init__.py +0 -0
- prefect/client/orchestration/_blocks_types/client.py +380 -0
- prefect/client/orchestration/_concurrency_limits/__init__.py +0 -0
- prefect/client/orchestration/_concurrency_limits/client.py +762 -0
- prefect/client/orchestration/_deployments/__init__.py +0 -0
- prefect/client/orchestration/_deployments/client.py +1128 -0
- prefect/client/orchestration/_flow_runs/__init__.py +0 -0
- prefect/client/orchestration/_flow_runs/client.py +903 -0
- prefect/client/orchestration/_flows/__init__.py +0 -0
- prefect/client/orchestration/_flows/client.py +343 -0
- prefect/client/orchestration/_logs/__init__.py +0 -0
- prefect/client/orchestration/_logs/client.py +97 -0
- prefect/client/orchestration/_variables/__init__.py +0 -0
- prefect/client/orchestration/_variables/client.py +157 -0
- prefect/client/orchestration/base.py +46 -0
- prefect/client/orchestration/routes.py +145 -0
- prefect/client/schemas/__init__.py +68 -28
- prefect/client/schemas/actions.py +2 -2
- prefect/client/schemas/filters.py +5 -0
- prefect/client/schemas/objects.py +8 -15
- prefect/client/schemas/schedules.py +22 -10
- prefect/concurrency/_asyncio.py +87 -0
- prefect/concurrency/{events.py → _events.py} +10 -10
- prefect/concurrency/asyncio.py +20 -104
- prefect/concurrency/context.py +6 -4
- prefect/concurrency/services.py +26 -74
- prefect/concurrency/sync.py +23 -44
- prefect/concurrency/v1/_asyncio.py +63 -0
- prefect/concurrency/v1/{events.py → _events.py} +13 -15
- prefect/concurrency/v1/asyncio.py +27 -80
- prefect/concurrency/v1/context.py +6 -4
- prefect/concurrency/v1/services.py +33 -79
- prefect/concurrency/v1/sync.py +18 -37
- prefect/context.py +66 -45
- prefect/deployments/base.py +10 -144
- prefect/deployments/flow_runs.py +12 -2
- prefect/deployments/runner.py +53 -4
- prefect/deployments/steps/pull.py +13 -0
- prefect/engine.py +17 -4
- prefect/events/clients.py +7 -1
- prefect/events/schemas/events.py +3 -2
- prefect/filesystems.py +6 -2
- prefect/flow_engine.py +101 -85
- prefect/flows.py +10 -1
- prefect/input/run_input.py +2 -1
- prefect/logging/logging.yml +1 -1
- prefect/main.py +1 -3
- prefect/results.py +2 -307
- prefect/runner/runner.py +4 -2
- prefect/runner/storage.py +87 -21
- prefect/serializers.py +32 -25
- prefect/settings/legacy.py +4 -4
- prefect/settings/models/api.py +3 -3
- prefect/settings/models/cli.py +3 -3
- prefect/settings/models/client.py +5 -3
- prefect/settings/models/cloud.py +8 -3
- prefect/settings/models/deployments.py +3 -3
- prefect/settings/models/experiments.py +4 -7
- prefect/settings/models/flows.py +3 -3
- prefect/settings/models/internal.py +4 -2
- prefect/settings/models/logging.py +4 -3
- prefect/settings/models/results.py +3 -3
- prefect/settings/models/root.py +3 -2
- prefect/settings/models/runner.py +4 -4
- prefect/settings/models/server/api.py +3 -3
- prefect/settings/models/server/database.py +11 -4
- prefect/settings/models/server/deployments.py +6 -2
- prefect/settings/models/server/ephemeral.py +4 -2
- prefect/settings/models/server/events.py +3 -2
- prefect/settings/models/server/flow_run_graph.py +6 -2
- prefect/settings/models/server/root.py +3 -3
- prefect/settings/models/server/services.py +26 -11
- prefect/settings/models/server/tasks.py +6 -3
- prefect/settings/models/server/ui.py +3 -3
- prefect/settings/models/tasks.py +5 -5
- prefect/settings/models/testing.py +3 -3
- prefect/settings/models/worker.py +5 -3
- prefect/settings/profiles.py +15 -2
- prefect/states.py +61 -45
- prefect/task_engine.py +54 -75
- prefect/task_runners.py +56 -55
- prefect/task_worker.py +2 -2
- prefect/tasks.py +90 -36
- prefect/telemetry/bootstrap.py +10 -9
- prefect/telemetry/run_telemetry.py +13 -8
- prefect/telemetry/services.py +4 -0
- prefect/transactions.py +4 -15
- prefect/utilities/_git.py +34 -0
- prefect/utilities/asyncutils.py +1 -1
- prefect/utilities/engine.py +3 -19
- prefect/utilities/generics.py +18 -0
- prefect/utilities/templating.py +25 -1
- prefect/workers/base.py +6 -3
- prefect/workers/process.py +1 -1
- {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/METADATA +2 -2
- {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/RECORD +135 -109
- prefect/client/orchestration.py +0 -4523
- prefect/records/__init__.py +0 -1
- prefect/records/base.py +0 -235
- prefect/records/filesystem.py +0 -213
- prefect/records/memory.py +0 -184
- prefect/records/result_store.py +0 -70
- {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.10.dist-info → prefect_client-3.1.12.dist-info}/top_level.txt +0 -0
prefect/task_engine.py
CHANGED
@@ -9,18 +9,16 @@ from dataclasses import dataclass, field
|
|
9
9
|
from functools import partial
|
10
10
|
from textwrap import dedent
|
11
11
|
from typing import (
|
12
|
+
TYPE_CHECKING,
|
12
13
|
Any,
|
13
14
|
AsyncGenerator,
|
14
15
|
Callable,
|
15
16
|
Coroutine,
|
16
|
-
Dict,
|
17
17
|
Generator,
|
18
18
|
Generic,
|
19
|
-
Iterable,
|
20
19
|
Literal,
|
21
20
|
Optional,
|
22
21
|
Sequence,
|
23
|
-
Set,
|
24
22
|
Type,
|
25
23
|
TypeVar,
|
26
24
|
Union,
|
@@ -32,7 +30,6 @@ import pendulum
|
|
32
30
|
from opentelemetry import trace
|
33
31
|
from typing_extensions import ParamSpec
|
34
32
|
|
35
|
-
from prefect import Task
|
36
33
|
from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_client
|
37
34
|
from prefect.client.schemas import TaskRun
|
38
35
|
from prefect.client.schemas.objects import State, TaskRunInput
|
@@ -55,10 +52,8 @@ from prefect.exceptions import (
|
|
55
52
|
TerminationSignal,
|
56
53
|
UpstreamTaskError,
|
57
54
|
)
|
58
|
-
from prefect.futures import PrefectFuture
|
59
55
|
from prefect.logging.loggers import get_logger, patch_print, task_run_logger
|
60
56
|
from prefect.results import (
|
61
|
-
BaseResult,
|
62
57
|
ResultRecord,
|
63
58
|
_format_user_supplied_storage_key,
|
64
59
|
get_result_store,
|
@@ -95,6 +90,9 @@ from prefect.utilities.engine import (
|
|
95
90
|
from prefect.utilities.math import clamped_poisson_interval
|
96
91
|
from prefect.utilities.timeout import timeout, timeout_async
|
97
92
|
|
93
|
+
if TYPE_CHECKING:
|
94
|
+
from prefect.tasks import OneOrManyFutureOrResult, Task
|
95
|
+
|
98
96
|
P = ParamSpec("P")
|
99
97
|
R = TypeVar("R")
|
100
98
|
|
@@ -107,13 +105,13 @@ class TaskRunTimeoutError(TimeoutError):
|
|
107
105
|
|
108
106
|
@dataclass
|
109
107
|
class BaseTaskRunEngine(Generic[P, R]):
|
110
|
-
task: Union[Task[P, R], Task[P, Coroutine[Any, Any, R]]]
|
108
|
+
task: Union["Task[P, R]", "Task[P, Coroutine[Any, Any, R]]"]
|
111
109
|
logger: logging.Logger = field(default_factory=lambda: get_logger("engine"))
|
112
|
-
parameters: Optional[
|
110
|
+
parameters: Optional[dict[str, Any]] = None
|
113
111
|
task_run: Optional[TaskRun] = None
|
114
112
|
retries: int = 0
|
115
|
-
wait_for: Optional[
|
116
|
-
context: Optional[
|
113
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None
|
114
|
+
context: Optional[dict[str, Any]] = None
|
117
115
|
# holds the return value from the user code
|
118
116
|
_return_value: Union[R, Type[NotSet]] = NotSet
|
119
117
|
# holds the exception raised by the user code, if any
|
@@ -210,6 +208,7 @@ class BaseTaskRunEngine(Generic[P, R]):
|
|
210
208
|
)
|
211
209
|
self.task_run.name = task_run_name
|
212
210
|
self._task_name_set = True
|
211
|
+
self._telemetry.update_run_name(name=task_run_name)
|
213
212
|
|
214
213
|
def _wait_for_dependencies(self):
|
215
214
|
if not self.wait_for:
|
@@ -305,7 +304,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
305
304
|
|
306
305
|
def can_retry(self, exc: Exception) -> bool:
|
307
306
|
retry_condition: Optional[
|
308
|
-
Callable[[Task[P, Coroutine[Any, Any, R]], TaskRun, State], bool]
|
307
|
+
Callable[["Task[P, Coroutine[Any, Any, R]]", TaskRun, State], bool]
|
309
308
|
] = self.task.retry_condition_fn
|
310
309
|
if not self.task_run:
|
311
310
|
raise ValueError("Task run is not set")
|
@@ -448,13 +447,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
448
447
|
self.task_run.run_count += 1
|
449
448
|
|
450
449
|
if new_state.is_final():
|
451
|
-
if isinstance(state.data,
|
452
|
-
# Avoid fetching the result unless it is cached, otherwise we defeat
|
453
|
-
# the purpose of disabling `cache_result_in_memory`
|
454
|
-
result = state.result(raise_on_failure=False, fetch=True)
|
455
|
-
if asyncio.iscoroutine(result):
|
456
|
-
result = run_coro_as_sync(result)
|
457
|
-
elif isinstance(state.data, ResultRecord):
|
450
|
+
if isinstance(state.data, ResultRecord):
|
458
451
|
result = state.data.result
|
459
452
|
else:
|
460
453
|
result = state.data
|
@@ -473,13 +466,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
473
466
|
|
474
467
|
def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
|
475
468
|
if self._return_value is not NotSet:
|
476
|
-
|
477
|
-
if isinstance(self._return_value, BaseResult):
|
478
|
-
_result = self._return_value.get()
|
479
|
-
if asyncio.iscoroutine(_result):
|
480
|
-
_result = run_coro_as_sync(_result)
|
481
|
-
return _result
|
482
|
-
elif isinstance(self._return_value, ResultRecord):
|
469
|
+
if isinstance(self._return_value, ResultRecord):
|
483
470
|
return self._return_value.result
|
484
471
|
# otherwise, return the value as is
|
485
472
|
return self._return_value
|
@@ -660,7 +647,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
660
647
|
def initialize_run(
|
661
648
|
self,
|
662
649
|
task_run_id: Optional[UUID] = None,
|
663
|
-
dependencies: Optional[
|
650
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
664
651
|
) -> Generator["SyncTaskRunEngine", Any, Any]:
|
665
652
|
"""
|
666
653
|
Enters a client context and creates a task run if needed.
|
@@ -700,7 +687,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
700
687
|
|
701
688
|
self._telemetry.start_span(
|
702
689
|
run=self.task_run,
|
703
|
-
name=self.task.name,
|
704
690
|
client=self.client,
|
705
691
|
parameters=self.parameters,
|
706
692
|
)
|
@@ -753,12 +739,14 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
753
739
|
def start(
|
754
740
|
self,
|
755
741
|
task_run_id: Optional[UUID] = None,
|
756
|
-
dependencies: Optional[
|
742
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
757
743
|
) -> Generator[None, None, None]:
|
758
744
|
with self.initialize_run(task_run_id=task_run_id, dependencies=dependencies):
|
759
|
-
with
|
760
|
-
self._telemetry.span
|
761
|
-
|
745
|
+
with (
|
746
|
+
trace.use_span(self._telemetry.span)
|
747
|
+
if self._telemetry.span
|
748
|
+
else nullcontext()
|
749
|
+
):
|
762
750
|
self.begin_run()
|
763
751
|
try:
|
764
752
|
yield
|
@@ -847,7 +835,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
847
835
|
|
848
836
|
async def can_retry(self, exc: Exception) -> bool:
|
849
837
|
retry_condition: Optional[
|
850
|
-
Callable[[Task[P, Coroutine[Any, Any, R]], TaskRun, State], bool]
|
838
|
+
Callable[["Task[P, Coroutine[Any, Any, R]]", TaskRun, State], bool]
|
851
839
|
] = self.task.retry_condition_fn
|
852
840
|
if not self.task_run:
|
853
841
|
raise ValueError("Task run is not set")
|
@@ -989,14 +977,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
989
977
|
self.task_run.run_count += 1
|
990
978
|
|
991
979
|
if new_state.is_final():
|
992
|
-
if (
|
993
|
-
isinstance(new_state.data, BaseResult)
|
994
|
-
and new_state.data.has_cached_object()
|
995
|
-
):
|
996
|
-
# Avoid fetching the result unless it is cached, otherwise we defeat
|
997
|
-
# the purpose of disabling `cache_result_in_memory`
|
998
|
-
result = await new_state.result(raise_on_failure=False, fetch=True)
|
999
|
-
elif isinstance(new_state.data, ResultRecord):
|
980
|
+
if isinstance(new_state.data, ResultRecord):
|
1000
981
|
result = new_state.data.result
|
1001
982
|
else:
|
1002
983
|
result = new_state.data
|
@@ -1016,10 +997,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1016
997
|
|
1017
998
|
async def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
|
1018
999
|
if self._return_value is not NotSet:
|
1019
|
-
|
1020
|
-
if isinstance(self._return_value, BaseResult):
|
1021
|
-
return await self._return_value.get()
|
1022
|
-
elif isinstance(self._return_value, ResultRecord):
|
1000
|
+
if isinstance(self._return_value, ResultRecord):
|
1023
1001
|
return self._return_value.result
|
1024
1002
|
# otherwise, return the value as is
|
1025
1003
|
return self._return_value
|
@@ -1199,7 +1177,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1199
1177
|
async def initialize_run(
|
1200
1178
|
self,
|
1201
1179
|
task_run_id: Optional[UUID] = None,
|
1202
|
-
dependencies: Optional[
|
1180
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1203
1181
|
) -> AsyncGenerator["AsyncTaskRunEngine", Any]:
|
1204
1182
|
"""
|
1205
1183
|
Enters a client context and creates a task run if needed.
|
@@ -1237,7 +1215,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1237
1215
|
|
1238
1216
|
await self._telemetry.async_start_span(
|
1239
1217
|
run=self.task_run,
|
1240
|
-
name=self.task.name,
|
1241
1218
|
client=self.client,
|
1242
1219
|
parameters=self.parameters,
|
1243
1220
|
)
|
@@ -1290,14 +1267,16 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1290
1267
|
async def start(
|
1291
1268
|
self,
|
1292
1269
|
task_run_id: Optional[UUID] = None,
|
1293
|
-
dependencies: Optional[
|
1270
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1294
1271
|
) -> AsyncGenerator[None, None]:
|
1295
1272
|
async with self.initialize_run(
|
1296
1273
|
task_run_id=task_run_id, dependencies=dependencies
|
1297
1274
|
):
|
1298
|
-
with
|
1299
|
-
self._telemetry.span
|
1300
|
-
|
1275
|
+
with (
|
1276
|
+
trace.use_span(self._telemetry.span)
|
1277
|
+
if self._telemetry.span
|
1278
|
+
else nullcontext()
|
1279
|
+
):
|
1301
1280
|
await self.begin_run()
|
1302
1281
|
try:
|
1303
1282
|
yield
|
@@ -1374,14 +1353,14 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1374
1353
|
|
1375
1354
|
|
1376
1355
|
def run_task_sync(
|
1377
|
-
task: Task[P, R],
|
1356
|
+
task: "Task[P, R]",
|
1378
1357
|
task_run_id: Optional[UUID] = None,
|
1379
1358
|
task_run: Optional[TaskRun] = None,
|
1380
|
-
parameters: Optional[
|
1381
|
-
wait_for: Optional[
|
1359
|
+
parameters: Optional[dict[str, Any]] = None,
|
1360
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1382
1361
|
return_type: Literal["state", "result"] = "result",
|
1383
|
-
dependencies: Optional[
|
1384
|
-
context: Optional[
|
1362
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1363
|
+
context: Optional[dict[str, Any]] = None,
|
1385
1364
|
) -> Union[R, State, None]:
|
1386
1365
|
engine = SyncTaskRunEngine[P, R](
|
1387
1366
|
task=task,
|
@@ -1401,14 +1380,14 @@ def run_task_sync(
|
|
1401
1380
|
|
1402
1381
|
|
1403
1382
|
async def run_task_async(
|
1404
|
-
task: Task[P, R],
|
1383
|
+
task: "Task[P, R]",
|
1405
1384
|
task_run_id: Optional[UUID] = None,
|
1406
1385
|
task_run: Optional[TaskRun] = None,
|
1407
|
-
parameters: Optional[
|
1408
|
-
wait_for: Optional[
|
1386
|
+
parameters: Optional[dict[str, Any]] = None,
|
1387
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1409
1388
|
return_type: Literal["state", "result"] = "result",
|
1410
|
-
dependencies: Optional[
|
1411
|
-
context: Optional[
|
1389
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1390
|
+
context: Optional[dict[str, Any]] = None,
|
1412
1391
|
) -> Union[R, State, None]:
|
1413
1392
|
engine = AsyncTaskRunEngine[P, R](
|
1414
1393
|
task=task,
|
@@ -1428,14 +1407,14 @@ async def run_task_async(
|
|
1428
1407
|
|
1429
1408
|
|
1430
1409
|
def run_generator_task_sync(
|
1431
|
-
task: Task[P, R],
|
1410
|
+
task: "Task[P, R]",
|
1432
1411
|
task_run_id: Optional[UUID] = None,
|
1433
1412
|
task_run: Optional[TaskRun] = None,
|
1434
|
-
parameters: Optional[
|
1435
|
-
wait_for: Optional[
|
1413
|
+
parameters: Optional[dict[str, Any]] = None,
|
1414
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1436
1415
|
return_type: Literal["state", "result"] = "result",
|
1437
|
-
dependencies: Optional[
|
1438
|
-
context: Optional[
|
1416
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1417
|
+
context: Optional[dict[str, Any]] = None,
|
1439
1418
|
) -> Generator[R, None, None]:
|
1440
1419
|
if return_type != "result":
|
1441
1420
|
raise ValueError("The return_type for a generator task must be 'result'")
|
@@ -1483,14 +1462,14 @@ def run_generator_task_sync(
|
|
1483
1462
|
|
1484
1463
|
|
1485
1464
|
async def run_generator_task_async(
|
1486
|
-
task: Task[P, R],
|
1465
|
+
task: "Task[P, R]",
|
1487
1466
|
task_run_id: Optional[UUID] = None,
|
1488
1467
|
task_run: Optional[TaskRun] = None,
|
1489
|
-
parameters: Optional[
|
1490
|
-
wait_for: Optional[
|
1468
|
+
parameters: Optional[dict[str, Any]] = None,
|
1469
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1491
1470
|
return_type: Literal["state", "result"] = "result",
|
1492
|
-
dependencies: Optional[
|
1493
|
-
context: Optional[
|
1471
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1472
|
+
context: Optional[dict[str, Any]] = None,
|
1494
1473
|
) -> AsyncGenerator[R, None]:
|
1495
1474
|
if return_type != "result":
|
1496
1475
|
raise ValueError("The return_type for a generator task must be 'result'")
|
@@ -1539,14 +1518,14 @@ async def run_generator_task_async(
|
|
1539
1518
|
|
1540
1519
|
|
1541
1520
|
def run_task(
|
1542
|
-
task: Task[P, Union[R, Coroutine[Any, Any, R]]],
|
1521
|
+
task: "Task[P, Union[R, Coroutine[Any, Any, R]]]",
|
1543
1522
|
task_run_id: Optional[UUID] = None,
|
1544
1523
|
task_run: Optional[TaskRun] = None,
|
1545
|
-
parameters: Optional[
|
1546
|
-
wait_for: Optional[
|
1524
|
+
parameters: Optional[dict[str, Any]] = None,
|
1525
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1547
1526
|
return_type: Literal["state", "result"] = "result",
|
1548
|
-
dependencies: Optional[
|
1549
|
-
context: Optional[
|
1527
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1528
|
+
context: Optional[dict[str, Any]] = None,
|
1550
1529
|
) -> Union[R, State, None, Coroutine[Any, Any, Union[R, State, None]]]:
|
1551
1530
|
"""
|
1552
1531
|
Runs the provided task.
|
prefect/task_runners.py
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
1
3
|
import abc
|
2
4
|
import asyncio
|
3
5
|
import sys
|
@@ -14,7 +16,6 @@ from typing import (
|
|
14
16
|
Iterable,
|
15
17
|
List,
|
16
18
|
Optional,
|
17
|
-
Set,
|
18
19
|
overload,
|
19
20
|
)
|
20
21
|
|
@@ -44,7 +45,7 @@ if TYPE_CHECKING:
|
|
44
45
|
P = ParamSpec("P")
|
45
46
|
T = TypeVar("T")
|
46
47
|
R = TypeVar("R")
|
47
|
-
F = TypeVar("F", bound=PrefectFuture, default=PrefectConcurrentFuture)
|
48
|
+
F = TypeVar("F", bound=PrefectFuture[Any], default=PrefectConcurrentFuture[Any])
|
48
49
|
|
49
50
|
|
50
51
|
class TaskRunner(abc.ABC, Generic[F]):
|
@@ -76,10 +77,10 @@ class TaskRunner(abc.ABC, Generic[F]):
|
|
76
77
|
@abc.abstractmethod
|
77
78
|
def submit(
|
78
79
|
self,
|
79
|
-
task: "Task",
|
80
|
-
parameters:
|
81
|
-
wait_for:
|
82
|
-
dependencies:
|
80
|
+
task: "Task[P, R]",
|
81
|
+
parameters: dict[str, Any],
|
82
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
83
|
+
dependencies: dict[str, set[TaskRunInput]] | None = None,
|
83
84
|
) -> F:
|
84
85
|
"""
|
85
86
|
Submit a task to the task run engine.
|
@@ -98,7 +99,7 @@ class TaskRunner(abc.ABC, Generic[F]):
|
|
98
99
|
def map(
|
99
100
|
self,
|
100
101
|
task: "Task[P, R]",
|
101
|
-
parameters:
|
102
|
+
parameters: dict[str, Any],
|
102
103
|
wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
|
103
104
|
) -> PrefectFutureList[F]:
|
104
105
|
"""
|
@@ -138,9 +139,9 @@ class TaskRunner(abc.ABC, Generic[F]):
|
|
138
139
|
# Ensure that any parameters in kwargs are expanded before this check
|
139
140
|
parameters = explode_variadic_parameter(task.fn, parameters)
|
140
141
|
|
141
|
-
iterable_parameters = {}
|
142
|
-
static_parameters = {}
|
143
|
-
annotated_parameters = {}
|
142
|
+
iterable_parameters: dict[str, Any] = {}
|
143
|
+
static_parameters: dict[str, Any] = {}
|
144
|
+
annotated_parameters: dict[str, Any] = {}
|
144
145
|
for key, val in parameters.items():
|
145
146
|
if isinstance(val, (allow_failure, quote)):
|
146
147
|
# Unwrap annotated parameters to determine if they are iterable
|
@@ -172,9 +173,9 @@ class TaskRunner(abc.ABC, Generic[F]):
|
|
172
173
|
|
173
174
|
map_length = list(lengths)[0]
|
174
175
|
|
175
|
-
futures: List[PrefectFuture] = []
|
176
|
+
futures: List[PrefectFuture[Any]] = []
|
176
177
|
for i in range(map_length):
|
177
|
-
call_parameters = {
|
178
|
+
call_parameters: dict[str, Any] = {
|
178
179
|
key: value[i] for key, value in iterable_parameters.items()
|
179
180
|
}
|
180
181
|
call_parameters.update(
|
@@ -212,12 +213,12 @@ class TaskRunner(abc.ABC, Generic[F]):
|
|
212
213
|
self._started = True
|
213
214
|
return self
|
214
215
|
|
215
|
-
def __exit__(self, exc_type, exc_value, traceback):
|
216
|
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any):
|
216
217
|
self.logger.debug("Stopping task runner")
|
217
218
|
self._started = False
|
218
219
|
|
219
220
|
|
220
|
-
class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
|
221
|
+
class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture[Any]]):
|
221
222
|
def __init__(self, max_workers: Optional[int] = None):
|
222
223
|
super().__init__()
|
223
224
|
self._executor: Optional[ThreadPoolExecutor] = None
|
@@ -235,9 +236,9 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
|
|
235
236
|
def submit(
|
236
237
|
self,
|
237
238
|
task: "Task[P, Coroutine[Any, Any, R]]",
|
238
|
-
parameters:
|
239
|
-
wait_for:
|
240
|
-
dependencies:
|
239
|
+
parameters: dict[str, Any],
|
240
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
241
|
+
dependencies: dict[str, set[TaskRunInput]] | None = None,
|
241
242
|
) -> PrefectConcurrentFuture[R]:
|
242
243
|
...
|
243
244
|
|
@@ -245,19 +246,19 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
|
|
245
246
|
def submit(
|
246
247
|
self,
|
247
248
|
task: "Task[Any, R]",
|
248
|
-
parameters:
|
249
|
-
wait_for:
|
250
|
-
dependencies:
|
249
|
+
parameters: dict[str, Any],
|
250
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
251
|
+
dependencies: dict[str, set[TaskRunInput]] | None = None,
|
251
252
|
) -> PrefectConcurrentFuture[R]:
|
252
253
|
...
|
253
254
|
|
254
255
|
def submit(
|
255
256
|
self,
|
256
|
-
task: "Task",
|
257
|
-
parameters:
|
258
|
-
wait_for:
|
259
|
-
dependencies:
|
260
|
-
):
|
257
|
+
task: "Task[P, R]",
|
258
|
+
parameters: dict[str, Any],
|
259
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
260
|
+
dependencies: dict[str, set[TaskRunInput]] | None = None,
|
261
|
+
) -> PrefectConcurrentFuture[R]:
|
261
262
|
"""
|
262
263
|
Submit a task to the task run engine running in a separate thread.
|
263
264
|
|
@@ -289,7 +290,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
|
|
289
290
|
else:
|
290
291
|
self.logger.debug(f"Submitting task {task.name} to thread pool executor...")
|
291
292
|
|
292
|
-
submit_kwargs = dict(
|
293
|
+
submit_kwargs: dict[str, Any] = dict(
|
293
294
|
task=task,
|
294
295
|
task_run_id=task_run_id,
|
295
296
|
parameters=parameters,
|
@@ -322,8 +323,8 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
|
|
322
323
|
def map(
|
323
324
|
self,
|
324
325
|
task: "Task[P, Coroutine[Any, Any, R]]",
|
325
|
-
parameters:
|
326
|
-
wait_for:
|
326
|
+
parameters: dict[str, Any],
|
327
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
327
328
|
) -> PrefectFutureList[PrefectConcurrentFuture[R]]:
|
328
329
|
...
|
329
330
|
|
@@ -331,17 +332,17 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
|
|
331
332
|
def map(
|
332
333
|
self,
|
333
334
|
task: "Task[Any, R]",
|
334
|
-
parameters:
|
335
|
-
wait_for:
|
335
|
+
parameters: dict[str, Any],
|
336
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
336
337
|
) -> PrefectFutureList[PrefectConcurrentFuture[R]]:
|
337
338
|
...
|
338
339
|
|
339
340
|
def map(
|
340
341
|
self,
|
341
|
-
task: "Task",
|
342
|
-
parameters:
|
343
|
-
wait_for:
|
344
|
-
):
|
342
|
+
task: "Task[P, R]",
|
343
|
+
parameters: dict[str, Any],
|
344
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
345
|
+
) -> PrefectFutureList[PrefectConcurrentFuture[R]]:
|
345
346
|
return super().map(task, parameters, wait_for)
|
346
347
|
|
347
348
|
def cancel_all(self):
|
@@ -358,7 +359,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
|
|
358
359
|
self._executor = ThreadPoolExecutor(max_workers=self._max_workers)
|
359
360
|
return self
|
360
361
|
|
361
|
-
def __exit__(self, exc_type, exc_value, traceback):
|
362
|
+
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any):
|
362
363
|
self.cancel_all()
|
363
364
|
if self._executor is not None:
|
364
365
|
self._executor.shutdown(cancel_futures=True)
|
@@ -375,7 +376,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
|
|
375
376
|
ConcurrentTaskRunner = ThreadPoolTaskRunner
|
376
377
|
|
377
378
|
|
378
|
-
class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
|
379
|
+
class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture[R]]):
|
379
380
|
def __init__(self):
|
380
381
|
super().__init__()
|
381
382
|
|
@@ -386,9 +387,9 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
|
|
386
387
|
def submit(
|
387
388
|
self,
|
388
389
|
task: "Task[P, Coroutine[Any, Any, R]]",
|
389
|
-
parameters:
|
390
|
-
wait_for:
|
391
|
-
dependencies:
|
390
|
+
parameters: dict[str, Any],
|
391
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
392
|
+
dependencies: dict[str, set[TaskRunInput]] | None = None,
|
392
393
|
) -> PrefectDistributedFuture[R]:
|
393
394
|
...
|
394
395
|
|
@@ -396,19 +397,19 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
|
|
396
397
|
def submit(
|
397
398
|
self,
|
398
399
|
task: "Task[Any, R]",
|
399
|
-
parameters:
|
400
|
-
wait_for:
|
401
|
-
dependencies:
|
400
|
+
parameters: dict[str, Any],
|
401
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
402
|
+
dependencies: dict[str, set[TaskRunInput]] | None = None,
|
402
403
|
) -> PrefectDistributedFuture[R]:
|
403
404
|
...
|
404
405
|
|
405
406
|
def submit(
|
406
407
|
self,
|
407
|
-
task: "Task",
|
408
|
-
parameters:
|
409
|
-
wait_for:
|
410
|
-
dependencies:
|
411
|
-
):
|
408
|
+
task: "Task[P, R]",
|
409
|
+
parameters: dict[str, Any],
|
410
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
411
|
+
dependencies: dict[str, set[TaskRunInput]] | None = None,
|
412
|
+
) -> PrefectDistributedFuture[R]:
|
412
413
|
"""
|
413
414
|
Submit a task to the task run engine running in a separate thread.
|
414
415
|
|
@@ -443,8 +444,8 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
|
|
443
444
|
def map(
|
444
445
|
self,
|
445
446
|
task: "Task[P, Coroutine[Any, Any, R]]",
|
446
|
-
parameters:
|
447
|
-
wait_for:
|
447
|
+
parameters: dict[str, Any],
|
448
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
448
449
|
) -> PrefectFutureList[PrefectDistributedFuture[R]]:
|
449
450
|
...
|
450
451
|
|
@@ -452,15 +453,15 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
|
|
452
453
|
def map(
|
453
454
|
self,
|
454
455
|
task: "Task[Any, R]",
|
455
|
-
parameters:
|
456
|
-
wait_for:
|
456
|
+
parameters: dict[str, Any],
|
457
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
457
458
|
) -> PrefectFutureList[PrefectDistributedFuture[R]]:
|
458
459
|
...
|
459
460
|
|
460
461
|
def map(
|
461
462
|
self,
|
462
|
-
task: "Task",
|
463
|
-
parameters:
|
464
|
-
wait_for:
|
465
|
-
):
|
463
|
+
task: "Task[P, R]",
|
464
|
+
parameters: dict[str, Any],
|
465
|
+
wait_for: Iterable[PrefectFuture[Any]] | None = None,
|
466
|
+
) -> PrefectFutureList[PrefectDistributedFuture[R]]:
|
466
467
|
return super().map(task, parameters, wait_for)
|
prefect/task_worker.py
CHANGED
@@ -21,7 +21,7 @@ from websockets.exceptions import InvalidStatusCode
|
|
21
21
|
|
22
22
|
from prefect import Task
|
23
23
|
from prefect._internal.concurrency.api import create_call, from_sync
|
24
|
-
from prefect.cache_policies import DEFAULT,
|
24
|
+
from prefect.cache_policies import DEFAULT, NO_CACHE
|
25
25
|
from prefect.client.orchestration import get_client
|
26
26
|
from prefect.client.schemas.objects import TaskRun
|
27
27
|
from prefect.client.subscriptions import Subscription
|
@@ -93,7 +93,7 @@ class TaskWorker:
|
|
93
93
|
if not isinstance(t, Task):
|
94
94
|
continue
|
95
95
|
|
96
|
-
if t.cache_policy in [None,
|
96
|
+
if t.cache_policy in [None, NO_CACHE, NotSet]:
|
97
97
|
self.tasks.append(
|
98
98
|
t.with_options(persist_result=True, cache_policy=DEFAULT)
|
99
99
|
)
|