prefect-client 3.1.9__py3-none-any.whl → 3.1.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_experimental/lineage.py +7 -8
- prefect/_internal/_logging.py +15 -3
- prefect/_internal/compatibility/async_dispatch.py +22 -16
- prefect/_internal/compatibility/deprecated.py +42 -18
- prefect/_internal/compatibility/migration.py +2 -2
- prefect/_internal/concurrency/inspection.py +12 -14
- prefect/_internal/concurrency/primitives.py +2 -2
- prefect/_internal/concurrency/services.py +154 -80
- prefect/_internal/concurrency/waiters.py +13 -9
- prefect/_internal/pydantic/annotations/pendulum.py +7 -7
- prefect/_internal/pytz.py +4 -3
- prefect/_internal/retries.py +10 -5
- prefect/_internal/schemas/bases.py +19 -10
- prefect/_internal/schemas/validators.py +227 -388
- prefect/_version.py +3 -3
- prefect/artifacts.py +61 -74
- prefect/automations.py +27 -7
- prefect/blocks/core.py +3 -3
- prefect/client/{orchestration.py → orchestration/__init__.py} +38 -701
- prefect/client/orchestration/_artifacts/__init__.py +0 -0
- prefect/client/orchestration/_artifacts/client.py +239 -0
- prefect/client/orchestration/_concurrency_limits/__init__.py +0 -0
- prefect/client/orchestration/_concurrency_limits/client.py +762 -0
- prefect/client/orchestration/_logs/__init__.py +0 -0
- prefect/client/orchestration/_logs/client.py +95 -0
- prefect/client/orchestration/_variables/__init__.py +0 -0
- prefect/client/orchestration/_variables/client.py +157 -0
- prefect/client/orchestration/base.py +46 -0
- prefect/client/orchestration/routes.py +145 -0
- prefect/client/schemas/actions.py +2 -2
- prefect/client/schemas/filters.py +5 -0
- prefect/client/schemas/objects.py +3 -10
- prefect/client/schemas/schedules.py +22 -10
- prefect/concurrency/_asyncio.py +87 -0
- prefect/concurrency/{events.py → _events.py} +10 -10
- prefect/concurrency/asyncio.py +20 -104
- prefect/concurrency/context.py +6 -4
- prefect/concurrency/services.py +26 -74
- prefect/concurrency/sync.py +23 -44
- prefect/concurrency/v1/_asyncio.py +63 -0
- prefect/concurrency/v1/{events.py → _events.py} +13 -15
- prefect/concurrency/v1/asyncio.py +27 -80
- prefect/concurrency/v1/context.py +6 -4
- prefect/concurrency/v1/services.py +33 -79
- prefect/concurrency/v1/sync.py +18 -37
- prefect/context.py +66 -70
- prefect/deployments/base.py +4 -144
- prefect/deployments/flow_runs.py +12 -2
- prefect/deployments/runner.py +11 -3
- prefect/deployments/steps/pull.py +13 -0
- prefect/events/clients.py +7 -1
- prefect/events/schemas/events.py +3 -2
- prefect/flow_engine.py +54 -47
- prefect/flows.py +2 -1
- prefect/futures.py +42 -27
- prefect/input/run_input.py +2 -1
- prefect/locking/filesystem.py +8 -7
- prefect/locking/memory.py +5 -3
- prefect/locking/protocol.py +1 -1
- prefect/main.py +1 -3
- prefect/plugins.py +12 -10
- prefect/results.py +3 -308
- prefect/runner/storage.py +87 -21
- prefect/serializers.py +32 -25
- prefect/settings/legacy.py +4 -4
- prefect/settings/models/api.py +3 -3
- prefect/settings/models/cli.py +3 -3
- prefect/settings/models/client.py +5 -3
- prefect/settings/models/cloud.py +3 -3
- prefect/settings/models/deployments.py +3 -3
- prefect/settings/models/experiments.py +4 -2
- prefect/settings/models/flows.py +3 -3
- prefect/settings/models/internal.py +4 -2
- prefect/settings/models/logging.py +4 -3
- prefect/settings/models/results.py +3 -3
- prefect/settings/models/root.py +3 -2
- prefect/settings/models/runner.py +4 -4
- prefect/settings/models/server/api.py +3 -3
- prefect/settings/models/server/database.py +11 -4
- prefect/settings/models/server/deployments.py +6 -2
- prefect/settings/models/server/ephemeral.py +4 -2
- prefect/settings/models/server/events.py +3 -2
- prefect/settings/models/server/flow_run_graph.py +6 -2
- prefect/settings/models/server/root.py +3 -3
- prefect/settings/models/server/services.py +26 -11
- prefect/settings/models/server/tasks.py +6 -3
- prefect/settings/models/server/ui.py +3 -3
- prefect/settings/models/tasks.py +5 -5
- prefect/settings/models/testing.py +3 -3
- prefect/settings/models/worker.py +5 -3
- prefect/settings/profiles.py +15 -2
- prefect/states.py +4 -7
- prefect/task_engine.py +54 -75
- prefect/tasks.py +84 -32
- prefect/telemetry/processors.py +6 -6
- prefect/telemetry/run_telemetry.py +13 -8
- prefect/telemetry/services.py +32 -31
- prefect/transactions.py +4 -15
- prefect/utilities/_git.py +34 -0
- prefect/utilities/asyncutils.py +1 -1
- prefect/utilities/engine.py +3 -19
- prefect/utilities/generics.py +18 -0
- prefect/workers/__init__.py +2 -0
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/METADATA +1 -1
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/RECORD +108 -99
- prefect/records/__init__.py +0 -1
- prefect/records/base.py +0 -235
- prefect/records/filesystem.py +0 -213
- prefect/records/memory.py +0 -184
- prefect/records/result_store.py +0 -70
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.9.dist-info → prefect_client-3.1.11.dist-info}/top_level.txt +0 -0
@@ -1,6 +1,7 @@
|
|
1
1
|
from datetime import timedelta
|
2
|
+
from typing import ClassVar
|
2
3
|
|
3
|
-
from pydantic import AliasChoices, AliasPath, Field
|
4
|
+
from pydantic import AliasChoices, AliasPath, ConfigDict, Field
|
4
5
|
|
5
6
|
from prefect.settings.base import PrefectBaseSettings, _build_settings_config
|
6
7
|
|
@@ -10,7 +11,7 @@ class ServerServicesCancellationCleanupSettings(PrefectBaseSettings):
|
|
10
11
|
Settings for controlling the cancellation cleanup service
|
11
12
|
"""
|
12
13
|
|
13
|
-
model_config = _build_settings_config(
|
14
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
14
15
|
("server", "services", "cancellation_cleanup")
|
15
16
|
)
|
16
17
|
|
@@ -40,7 +41,9 @@ class ServerServicesEventPersisterSettings(PrefectBaseSettings):
|
|
40
41
|
Settings for controlling the event persister service
|
41
42
|
"""
|
42
43
|
|
43
|
-
model_config = _build_settings_config(
|
44
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
45
|
+
("server", "services", "event_persister")
|
46
|
+
)
|
44
47
|
|
45
48
|
enabled: bool = Field(
|
46
49
|
default=True,
|
@@ -80,7 +83,7 @@ class ServerServicesFlowRunNotificationsSettings(PrefectBaseSettings):
|
|
80
83
|
Settings for controlling the flow run notifications service
|
81
84
|
"""
|
82
85
|
|
83
|
-
model_config = _build_settings_config(
|
86
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
84
87
|
("server", "services", "flow_run_notifications")
|
85
88
|
)
|
86
89
|
|
@@ -100,7 +103,9 @@ class ServerServicesForemanSettings(PrefectBaseSettings):
|
|
100
103
|
Settings for controlling the foreman service
|
101
104
|
"""
|
102
105
|
|
103
|
-
model_config = _build_settings_config(
|
106
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
107
|
+
("server", "services", "foreman")
|
108
|
+
)
|
104
109
|
|
105
110
|
enabled: bool = Field(
|
106
111
|
default=True,
|
@@ -179,7 +184,9 @@ class ServerServicesLateRunsSettings(PrefectBaseSettings):
|
|
179
184
|
Settings for controlling the late runs service
|
180
185
|
"""
|
181
186
|
|
182
|
-
model_config = _build_settings_config(
|
187
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
188
|
+
("server", "services", "late_runs")
|
189
|
+
)
|
183
190
|
|
184
191
|
enabled: bool = Field(
|
185
192
|
default=True,
|
@@ -221,7 +228,9 @@ class ServerServicesSchedulerSettings(PrefectBaseSettings):
|
|
221
228
|
Settings for controlling the scheduler service
|
222
229
|
"""
|
223
230
|
|
224
|
-
model_config = _build_settings_config(
|
231
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
232
|
+
("server", "services", "scheduler")
|
233
|
+
)
|
225
234
|
|
226
235
|
enabled: bool = Field(
|
227
236
|
default=True,
|
@@ -344,7 +353,9 @@ class ServerServicesPauseExpirationsSettings(PrefectBaseSettings):
|
|
344
353
|
Settings for controlling the pause expiration service
|
345
354
|
"""
|
346
355
|
|
347
|
-
model_config = _build_settings_config(
|
356
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
357
|
+
("server", "services", "pause_expirations")
|
358
|
+
)
|
348
359
|
|
349
360
|
enabled: bool = Field(
|
350
361
|
default=True,
|
@@ -378,7 +389,9 @@ class ServerServicesTaskRunRecorderSettings(PrefectBaseSettings):
|
|
378
389
|
Settings for controlling the task run recorder service
|
379
390
|
"""
|
380
391
|
|
381
|
-
model_config = _build_settings_config(
|
392
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
393
|
+
("server", "services", "task_run_recorder")
|
394
|
+
)
|
382
395
|
|
383
396
|
enabled: bool = Field(
|
384
397
|
default=True,
|
@@ -396,7 +409,9 @@ class ServerServicesTriggersSettings(PrefectBaseSettings):
|
|
396
409
|
Settings for controlling the triggers service
|
397
410
|
"""
|
398
411
|
|
399
|
-
model_config = _build_settings_config(
|
412
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
413
|
+
("server", "services", "triggers")
|
414
|
+
)
|
400
415
|
|
401
416
|
enabled: bool = Field(
|
402
417
|
default=True,
|
@@ -414,7 +429,7 @@ class ServerServicesSettings(PrefectBaseSettings):
|
|
414
429
|
Settings for controlling server services
|
415
430
|
"""
|
416
431
|
|
417
|
-
model_config = _build_settings_config(("server", "services"))
|
432
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "services"))
|
418
433
|
|
419
434
|
cancellation_cleanup: ServerServicesCancellationCleanupSettings = Field(
|
420
435
|
default_factory=ServerServicesCancellationCleanupSettings,
|
@@ -1,6 +1,7 @@
|
|
1
1
|
from datetime import timedelta
|
2
|
+
from typing import ClassVar
|
2
3
|
|
3
|
-
from pydantic import AliasChoices, AliasPath, Field
|
4
|
+
from pydantic import AliasChoices, AliasPath, ConfigDict, Field
|
4
5
|
|
5
6
|
from prefect.settings.base import PrefectBaseSettings, _build_settings_config
|
6
7
|
|
@@ -10,7 +11,9 @@ class ServerTasksSchedulingSettings(PrefectBaseSettings):
|
|
10
11
|
Settings for controlling server-side behavior related to task scheduling
|
11
12
|
"""
|
12
13
|
|
13
|
-
model_config = _build_settings_config(
|
14
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(
|
15
|
+
("server", "tasks", "scheduling")
|
16
|
+
)
|
14
17
|
|
15
18
|
max_scheduled_queue_size: int = Field(
|
16
19
|
default=1000,
|
@@ -48,7 +51,7 @@ class ServerTasksSettings(PrefectBaseSettings):
|
|
48
51
|
Settings for controlling server-side behavior related to tasks
|
49
52
|
"""
|
50
53
|
|
51
|
-
model_config = _build_settings_config(("server", "tasks"))
|
54
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "tasks"))
|
52
55
|
|
53
56
|
tag_concurrency_slot_wait_seconds: float = Field(
|
54
57
|
default=30,
|
@@ -1,12 +1,12 @@
|
|
1
|
-
from typing import Optional
|
1
|
+
from typing import ClassVar, Optional
|
2
2
|
|
3
|
-
from pydantic import AliasChoices, AliasPath, Field
|
3
|
+
from pydantic import AliasChoices, AliasPath, ConfigDict, Field
|
4
4
|
|
5
5
|
from prefect.settings.base import PrefectBaseSettings, _build_settings_config
|
6
6
|
|
7
7
|
|
8
8
|
class ServerUISettings(PrefectBaseSettings):
|
9
|
-
model_config = _build_settings_config(("server", "ui"))
|
9
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "ui"))
|
10
10
|
|
11
11
|
enabled: bool = Field(
|
12
12
|
default=True,
|
prefect/settings/models/tasks.py
CHANGED
@@ -1,12 +1,12 @@
|
|
1
|
-
from typing import Optional, Union
|
1
|
+
from typing import ClassVar, Optional, Union
|
2
2
|
|
3
|
-
from pydantic import AliasChoices, AliasPath, Field
|
3
|
+
from pydantic import AliasChoices, AliasPath, ConfigDict, Field
|
4
4
|
|
5
5
|
from prefect.settings.base import PrefectBaseSettings, _build_settings_config
|
6
6
|
|
7
7
|
|
8
8
|
class TasksRunnerSettings(PrefectBaseSettings):
|
9
|
-
model_config = _build_settings_config(("tasks", "runner"))
|
9
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(("tasks", "runner"))
|
10
10
|
|
11
11
|
thread_pool_max_workers: Optional[int] = Field(
|
12
12
|
default=None,
|
@@ -21,7 +21,7 @@ class TasksRunnerSettings(PrefectBaseSettings):
|
|
21
21
|
|
22
22
|
|
23
23
|
class TasksSchedulingSettings(PrefectBaseSettings):
|
24
|
-
model_config = _build_settings_config(("tasks", "scheduling"))
|
24
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(("tasks", "scheduling"))
|
25
25
|
|
26
26
|
default_storage_block: Optional[str] = Field(
|
27
27
|
default=None,
|
@@ -45,7 +45,7 @@ class TasksSchedulingSettings(PrefectBaseSettings):
|
|
45
45
|
|
46
46
|
|
47
47
|
class TasksSettings(PrefectBaseSettings):
|
48
|
-
model_config = _build_settings_config(("tasks",))
|
48
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(("tasks",))
|
49
49
|
|
50
50
|
refresh_cache: bool = Field(
|
51
51
|
default=False,
|
@@ -1,12 +1,12 @@
|
|
1
|
-
from typing import Any, Optional
|
1
|
+
from typing import Any, ClassVar, Optional
|
2
2
|
|
3
|
-
from pydantic import AliasChoices, AliasPath, Field
|
3
|
+
from pydantic import AliasChoices, AliasPath, ConfigDict, Field
|
4
4
|
|
5
5
|
from prefect.settings.base import PrefectBaseSettings, _build_settings_config
|
6
6
|
|
7
7
|
|
8
8
|
class TestingSettings(PrefectBaseSettings):
|
9
|
-
model_config = _build_settings_config(("testing",))
|
9
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(("testing",))
|
10
10
|
|
11
11
|
test_mode: bool = Field(
|
12
12
|
default=False,
|
@@ -1,10 +1,12 @@
|
|
1
|
-
from
|
1
|
+
from typing import ClassVar
|
2
|
+
|
3
|
+
from pydantic import ConfigDict, Field
|
2
4
|
|
3
5
|
from prefect.settings.base import PrefectBaseSettings, _build_settings_config
|
4
6
|
|
5
7
|
|
6
8
|
class WorkerWebserverSettings(PrefectBaseSettings):
|
7
|
-
model_config = _build_settings_config(("worker", "webserver"))
|
9
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(("worker", "webserver"))
|
8
10
|
|
9
11
|
host: str = Field(
|
10
12
|
default="0.0.0.0",
|
@@ -18,7 +20,7 @@ class WorkerWebserverSettings(PrefectBaseSettings):
|
|
18
20
|
|
19
21
|
|
20
22
|
class WorkerSettings(PrefectBaseSettings):
|
21
|
-
model_config = _build_settings_config(("worker",))
|
23
|
+
model_config: ClassVar[ConfigDict] = _build_settings_config(("worker",))
|
22
24
|
|
23
25
|
heartbeat_seconds: float = Field(
|
24
26
|
default=30,
|
prefect/settings/profiles.py
CHANGED
@@ -1,7 +1,18 @@
|
|
1
1
|
import inspect
|
2
2
|
import warnings
|
3
3
|
from pathlib import Path
|
4
|
-
from typing import
|
4
|
+
from typing import (
|
5
|
+
Annotated,
|
6
|
+
Any,
|
7
|
+
ClassVar,
|
8
|
+
Dict,
|
9
|
+
Iterable,
|
10
|
+
List,
|
11
|
+
Optional,
|
12
|
+
Set,
|
13
|
+
Tuple,
|
14
|
+
Union,
|
15
|
+
)
|
5
16
|
|
6
17
|
import toml
|
7
18
|
from pydantic import (
|
@@ -48,7 +59,9 @@ def _cast_settings(
|
|
48
59
|
class Profile(BaseModel):
|
49
60
|
"""A user profile containing settings."""
|
50
61
|
|
51
|
-
model_config = ConfigDict(
|
62
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(
|
63
|
+
extra="ignore", arbitrary_types_allowed=True
|
64
|
+
)
|
52
65
|
|
53
66
|
name: str
|
54
67
|
settings: Annotated[Dict[Setting, Any], BeforeValidator(_cast_settings)] = Field(
|
prefect/states.py
CHANGED
@@ -29,7 +29,6 @@ from prefect.exceptions import (
|
|
29
29
|
)
|
30
30
|
from prefect.logging.loggers import get_logger, get_run_logger
|
31
31
|
from prefect.results import (
|
32
|
-
BaseResult,
|
33
32
|
R,
|
34
33
|
ResultRecord,
|
35
34
|
ResultRecordMetadata,
|
@@ -140,7 +139,7 @@ async def _get_state_result(
|
|
140
139
|
):
|
141
140
|
raise await get_state_exception(state)
|
142
141
|
|
143
|
-
if isinstance(state.data,
|
142
|
+
if isinstance(state.data, ResultRecordMetadata):
|
144
143
|
result = await _get_state_result_data_with_retries(
|
145
144
|
state, retry_result_failure=retry_result_failure
|
146
145
|
)
|
@@ -326,7 +325,7 @@ async def return_value_to_state(
|
|
326
325
|
state = retval
|
327
326
|
# Unless the user has already constructed a result explicitly, use the store
|
328
327
|
# to update the data to the correct type
|
329
|
-
if not isinstance(state.data, (
|
328
|
+
if not isinstance(state.data, (ResultRecord, ResultRecordMetadata)):
|
330
329
|
result_record = result_store.create_result_record(
|
331
330
|
state.data,
|
332
331
|
key=key,
|
@@ -402,7 +401,7 @@ async def return_value_to_state(
|
|
402
401
|
data = retval
|
403
402
|
|
404
403
|
# Otherwise, they just gave data and this is a completed retval
|
405
|
-
if isinstance(data,
|
404
|
+
if isinstance(data, ResultRecord):
|
406
405
|
return Completed(data=data)
|
407
406
|
else:
|
408
407
|
result_record = result_store.create_result_record(
|
@@ -457,9 +456,7 @@ async def get_state_exception(state: State) -> BaseException:
|
|
457
456
|
else:
|
458
457
|
raise ValueError(f"Expected failed or crashed state got {state!r}.")
|
459
458
|
|
460
|
-
if isinstance(state.data,
|
461
|
-
result = await _get_state_result_data_with_retries(state)
|
462
|
-
elif isinstance(state.data, ResultRecord):
|
459
|
+
if isinstance(state.data, ResultRecord):
|
463
460
|
result = state.data.result
|
464
461
|
elif isinstance(state.data, ResultRecordMetadata):
|
465
462
|
record = await ResultRecord._from_metadata(state.data)
|
prefect/task_engine.py
CHANGED
@@ -9,18 +9,16 @@ from dataclasses import dataclass, field
|
|
9
9
|
from functools import partial
|
10
10
|
from textwrap import dedent
|
11
11
|
from typing import (
|
12
|
+
TYPE_CHECKING,
|
12
13
|
Any,
|
13
14
|
AsyncGenerator,
|
14
15
|
Callable,
|
15
16
|
Coroutine,
|
16
|
-
Dict,
|
17
17
|
Generator,
|
18
18
|
Generic,
|
19
|
-
Iterable,
|
20
19
|
Literal,
|
21
20
|
Optional,
|
22
21
|
Sequence,
|
23
|
-
Set,
|
24
22
|
Type,
|
25
23
|
TypeVar,
|
26
24
|
Union,
|
@@ -32,7 +30,6 @@ import pendulum
|
|
32
30
|
from opentelemetry import trace
|
33
31
|
from typing_extensions import ParamSpec
|
34
32
|
|
35
|
-
from prefect import Task
|
36
33
|
from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_client
|
37
34
|
from prefect.client.schemas import TaskRun
|
38
35
|
from prefect.client.schemas.objects import State, TaskRunInput
|
@@ -55,10 +52,8 @@ from prefect.exceptions import (
|
|
55
52
|
TerminationSignal,
|
56
53
|
UpstreamTaskError,
|
57
54
|
)
|
58
|
-
from prefect.futures import PrefectFuture
|
59
55
|
from prefect.logging.loggers import get_logger, patch_print, task_run_logger
|
60
56
|
from prefect.results import (
|
61
|
-
BaseResult,
|
62
57
|
ResultRecord,
|
63
58
|
_format_user_supplied_storage_key,
|
64
59
|
get_result_store,
|
@@ -95,6 +90,9 @@ from prefect.utilities.engine import (
|
|
95
90
|
from prefect.utilities.math import clamped_poisson_interval
|
96
91
|
from prefect.utilities.timeout import timeout, timeout_async
|
97
92
|
|
93
|
+
if TYPE_CHECKING:
|
94
|
+
from prefect.tasks import OneOrManyFutureOrResult, Task
|
95
|
+
|
98
96
|
P = ParamSpec("P")
|
99
97
|
R = TypeVar("R")
|
100
98
|
|
@@ -107,13 +105,13 @@ class TaskRunTimeoutError(TimeoutError):
|
|
107
105
|
|
108
106
|
@dataclass
|
109
107
|
class BaseTaskRunEngine(Generic[P, R]):
|
110
|
-
task: Union[Task[P, R], Task[P, Coroutine[Any, Any, R]]]
|
108
|
+
task: Union["Task[P, R]", "Task[P, Coroutine[Any, Any, R]]"]
|
111
109
|
logger: logging.Logger = field(default_factory=lambda: get_logger("engine"))
|
112
|
-
parameters: Optional[
|
110
|
+
parameters: Optional[dict[str, Any]] = None
|
113
111
|
task_run: Optional[TaskRun] = None
|
114
112
|
retries: int = 0
|
115
|
-
wait_for: Optional[
|
116
|
-
context: Optional[
|
113
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None
|
114
|
+
context: Optional[dict[str, Any]] = None
|
117
115
|
# holds the return value from the user code
|
118
116
|
_return_value: Union[R, Type[NotSet]] = NotSet
|
119
117
|
# holds the exception raised by the user code, if any
|
@@ -210,6 +208,7 @@ class BaseTaskRunEngine(Generic[P, R]):
|
|
210
208
|
)
|
211
209
|
self.task_run.name = task_run_name
|
212
210
|
self._task_name_set = True
|
211
|
+
self._telemetry.update_run_name(name=task_run_name)
|
213
212
|
|
214
213
|
def _wait_for_dependencies(self):
|
215
214
|
if not self.wait_for:
|
@@ -305,7 +304,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
305
304
|
|
306
305
|
def can_retry(self, exc: Exception) -> bool:
|
307
306
|
retry_condition: Optional[
|
308
|
-
Callable[[Task[P, Coroutine[Any, Any, R]], TaskRun, State], bool]
|
307
|
+
Callable[["Task[P, Coroutine[Any, Any, R]]", TaskRun, State], bool]
|
309
308
|
] = self.task.retry_condition_fn
|
310
309
|
if not self.task_run:
|
311
310
|
raise ValueError("Task run is not set")
|
@@ -448,13 +447,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
448
447
|
self.task_run.run_count += 1
|
449
448
|
|
450
449
|
if new_state.is_final():
|
451
|
-
if isinstance(state.data,
|
452
|
-
# Avoid fetching the result unless it is cached, otherwise we defeat
|
453
|
-
# the purpose of disabling `cache_result_in_memory`
|
454
|
-
result = state.result(raise_on_failure=False, fetch=True)
|
455
|
-
if asyncio.iscoroutine(result):
|
456
|
-
result = run_coro_as_sync(result)
|
457
|
-
elif isinstance(state.data, ResultRecord):
|
450
|
+
if isinstance(state.data, ResultRecord):
|
458
451
|
result = state.data.result
|
459
452
|
else:
|
460
453
|
result = state.data
|
@@ -473,13 +466,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
473
466
|
|
474
467
|
def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
|
475
468
|
if self._return_value is not NotSet:
|
476
|
-
|
477
|
-
if isinstance(self._return_value, BaseResult):
|
478
|
-
_result = self._return_value.get()
|
479
|
-
if asyncio.iscoroutine(_result):
|
480
|
-
_result = run_coro_as_sync(_result)
|
481
|
-
return _result
|
482
|
-
elif isinstance(self._return_value, ResultRecord):
|
469
|
+
if isinstance(self._return_value, ResultRecord):
|
483
470
|
return self._return_value.result
|
484
471
|
# otherwise, return the value as is
|
485
472
|
return self._return_value
|
@@ -660,7 +647,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
660
647
|
def initialize_run(
|
661
648
|
self,
|
662
649
|
task_run_id: Optional[UUID] = None,
|
663
|
-
dependencies: Optional[
|
650
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
664
651
|
) -> Generator["SyncTaskRunEngine", Any, Any]:
|
665
652
|
"""
|
666
653
|
Enters a client context and creates a task run if needed.
|
@@ -700,7 +687,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
700
687
|
|
701
688
|
self._telemetry.start_span(
|
702
689
|
run=self.task_run,
|
703
|
-
name=self.task.name,
|
704
690
|
client=self.client,
|
705
691
|
parameters=self.parameters,
|
706
692
|
)
|
@@ -753,12 +739,14 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
753
739
|
def start(
|
754
740
|
self,
|
755
741
|
task_run_id: Optional[UUID] = None,
|
756
|
-
dependencies: Optional[
|
742
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
757
743
|
) -> Generator[None, None, None]:
|
758
744
|
with self.initialize_run(task_run_id=task_run_id, dependencies=dependencies):
|
759
|
-
with
|
760
|
-
self._telemetry.span
|
761
|
-
|
745
|
+
with (
|
746
|
+
trace.use_span(self._telemetry.span)
|
747
|
+
if self._telemetry.span
|
748
|
+
else nullcontext()
|
749
|
+
):
|
762
750
|
self.begin_run()
|
763
751
|
try:
|
764
752
|
yield
|
@@ -847,7 +835,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
847
835
|
|
848
836
|
async def can_retry(self, exc: Exception) -> bool:
|
849
837
|
retry_condition: Optional[
|
850
|
-
Callable[[Task[P, Coroutine[Any, Any, R]], TaskRun, State], bool]
|
838
|
+
Callable[["Task[P, Coroutine[Any, Any, R]]", TaskRun, State], bool]
|
851
839
|
] = self.task.retry_condition_fn
|
852
840
|
if not self.task_run:
|
853
841
|
raise ValueError("Task run is not set")
|
@@ -989,14 +977,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
989
977
|
self.task_run.run_count += 1
|
990
978
|
|
991
979
|
if new_state.is_final():
|
992
|
-
if (
|
993
|
-
isinstance(new_state.data, BaseResult)
|
994
|
-
and new_state.data.has_cached_object()
|
995
|
-
):
|
996
|
-
# Avoid fetching the result unless it is cached, otherwise we defeat
|
997
|
-
# the purpose of disabling `cache_result_in_memory`
|
998
|
-
result = await new_state.result(raise_on_failure=False, fetch=True)
|
999
|
-
elif isinstance(new_state.data, ResultRecord):
|
980
|
+
if isinstance(new_state.data, ResultRecord):
|
1000
981
|
result = new_state.data.result
|
1001
982
|
else:
|
1002
983
|
result = new_state.data
|
@@ -1016,10 +997,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1016
997
|
|
1017
998
|
async def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
|
1018
999
|
if self._return_value is not NotSet:
|
1019
|
-
|
1020
|
-
if isinstance(self._return_value, BaseResult):
|
1021
|
-
return await self._return_value.get()
|
1022
|
-
elif isinstance(self._return_value, ResultRecord):
|
1000
|
+
if isinstance(self._return_value, ResultRecord):
|
1023
1001
|
return self._return_value.result
|
1024
1002
|
# otherwise, return the value as is
|
1025
1003
|
return self._return_value
|
@@ -1199,7 +1177,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1199
1177
|
async def initialize_run(
|
1200
1178
|
self,
|
1201
1179
|
task_run_id: Optional[UUID] = None,
|
1202
|
-
dependencies: Optional[
|
1180
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1203
1181
|
) -> AsyncGenerator["AsyncTaskRunEngine", Any]:
|
1204
1182
|
"""
|
1205
1183
|
Enters a client context and creates a task run if needed.
|
@@ -1237,7 +1215,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1237
1215
|
|
1238
1216
|
await self._telemetry.async_start_span(
|
1239
1217
|
run=self.task_run,
|
1240
|
-
name=self.task.name,
|
1241
1218
|
client=self.client,
|
1242
1219
|
parameters=self.parameters,
|
1243
1220
|
)
|
@@ -1290,14 +1267,16 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1290
1267
|
async def start(
|
1291
1268
|
self,
|
1292
1269
|
task_run_id: Optional[UUID] = None,
|
1293
|
-
dependencies: Optional[
|
1270
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1294
1271
|
) -> AsyncGenerator[None, None]:
|
1295
1272
|
async with self.initialize_run(
|
1296
1273
|
task_run_id=task_run_id, dependencies=dependencies
|
1297
1274
|
):
|
1298
|
-
with
|
1299
|
-
self._telemetry.span
|
1300
|
-
|
1275
|
+
with (
|
1276
|
+
trace.use_span(self._telemetry.span)
|
1277
|
+
if self._telemetry.span
|
1278
|
+
else nullcontext()
|
1279
|
+
):
|
1301
1280
|
await self.begin_run()
|
1302
1281
|
try:
|
1303
1282
|
yield
|
@@ -1374,14 +1353,14 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1374
1353
|
|
1375
1354
|
|
1376
1355
|
def run_task_sync(
|
1377
|
-
task: Task[P, R],
|
1356
|
+
task: "Task[P, R]",
|
1378
1357
|
task_run_id: Optional[UUID] = None,
|
1379
1358
|
task_run: Optional[TaskRun] = None,
|
1380
|
-
parameters: Optional[
|
1381
|
-
wait_for: Optional[
|
1359
|
+
parameters: Optional[dict[str, Any]] = None,
|
1360
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1382
1361
|
return_type: Literal["state", "result"] = "result",
|
1383
|
-
dependencies: Optional[
|
1384
|
-
context: Optional[
|
1362
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1363
|
+
context: Optional[dict[str, Any]] = None,
|
1385
1364
|
) -> Union[R, State, None]:
|
1386
1365
|
engine = SyncTaskRunEngine[P, R](
|
1387
1366
|
task=task,
|
@@ -1401,14 +1380,14 @@ def run_task_sync(
|
|
1401
1380
|
|
1402
1381
|
|
1403
1382
|
async def run_task_async(
|
1404
|
-
task: Task[P, R],
|
1383
|
+
task: "Task[P, R]",
|
1405
1384
|
task_run_id: Optional[UUID] = None,
|
1406
1385
|
task_run: Optional[TaskRun] = None,
|
1407
|
-
parameters: Optional[
|
1408
|
-
wait_for: Optional[
|
1386
|
+
parameters: Optional[dict[str, Any]] = None,
|
1387
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1409
1388
|
return_type: Literal["state", "result"] = "result",
|
1410
|
-
dependencies: Optional[
|
1411
|
-
context: Optional[
|
1389
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1390
|
+
context: Optional[dict[str, Any]] = None,
|
1412
1391
|
) -> Union[R, State, None]:
|
1413
1392
|
engine = AsyncTaskRunEngine[P, R](
|
1414
1393
|
task=task,
|
@@ -1428,14 +1407,14 @@ async def run_task_async(
|
|
1428
1407
|
|
1429
1408
|
|
1430
1409
|
def run_generator_task_sync(
|
1431
|
-
task: Task[P, R],
|
1410
|
+
task: "Task[P, R]",
|
1432
1411
|
task_run_id: Optional[UUID] = None,
|
1433
1412
|
task_run: Optional[TaskRun] = None,
|
1434
|
-
parameters: Optional[
|
1435
|
-
wait_for: Optional[
|
1413
|
+
parameters: Optional[dict[str, Any]] = None,
|
1414
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1436
1415
|
return_type: Literal["state", "result"] = "result",
|
1437
|
-
dependencies: Optional[
|
1438
|
-
context: Optional[
|
1416
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1417
|
+
context: Optional[dict[str, Any]] = None,
|
1439
1418
|
) -> Generator[R, None, None]:
|
1440
1419
|
if return_type != "result":
|
1441
1420
|
raise ValueError("The return_type for a generator task must be 'result'")
|
@@ -1483,14 +1462,14 @@ def run_generator_task_sync(
|
|
1483
1462
|
|
1484
1463
|
|
1485
1464
|
async def run_generator_task_async(
|
1486
|
-
task: Task[P, R],
|
1465
|
+
task: "Task[P, R]",
|
1487
1466
|
task_run_id: Optional[UUID] = None,
|
1488
1467
|
task_run: Optional[TaskRun] = None,
|
1489
|
-
parameters: Optional[
|
1490
|
-
wait_for: Optional[
|
1468
|
+
parameters: Optional[dict[str, Any]] = None,
|
1469
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1491
1470
|
return_type: Literal["state", "result"] = "result",
|
1492
|
-
dependencies: Optional[
|
1493
|
-
context: Optional[
|
1471
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1472
|
+
context: Optional[dict[str, Any]] = None,
|
1494
1473
|
) -> AsyncGenerator[R, None]:
|
1495
1474
|
if return_type != "result":
|
1496
1475
|
raise ValueError("The return_type for a generator task must be 'result'")
|
@@ -1539,14 +1518,14 @@ async def run_generator_task_async(
|
|
1539
1518
|
|
1540
1519
|
|
1541
1520
|
def run_task(
|
1542
|
-
task: Task[P, Union[R, Coroutine[Any, Any, R]]],
|
1521
|
+
task: "Task[P, Union[R, Coroutine[Any, Any, R]]]",
|
1543
1522
|
task_run_id: Optional[UUID] = None,
|
1544
1523
|
task_run: Optional[TaskRun] = None,
|
1545
|
-
parameters: Optional[
|
1546
|
-
wait_for: Optional[
|
1524
|
+
parameters: Optional[dict[str, Any]] = None,
|
1525
|
+
wait_for: Optional["OneOrManyFutureOrResult[Any]"] = None,
|
1547
1526
|
return_type: Literal["state", "result"] = "result",
|
1548
|
-
dependencies: Optional[
|
1549
|
-
context: Optional[
|
1527
|
+
dependencies: Optional[dict[str, set[TaskRunInput]]] = None,
|
1528
|
+
context: Optional[dict[str, Any]] = None,
|
1550
1529
|
) -> Union[R, State, None, Coroutine[Any, Any, Union[R, State, None]]]:
|
1551
1530
|
"""
|
1552
1531
|
Runs the provided task.
|