prefect-client 3.1.15__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_experimental/sla/objects.py +29 -1
- prefect/_internal/compatibility/deprecated.py +4 -4
- prefect/_internal/compatibility/migration.py +1 -1
- prefect/_internal/concurrency/calls.py +1 -2
- prefect/_internal/concurrency/cancellation.py +2 -4
- prefect/_internal/concurrency/threads.py +3 -3
- prefect/_internal/schemas/bases.py +3 -11
- prefect/_internal/schemas/validators.py +36 -60
- prefect/_result_records.py +235 -0
- prefect/_version.py +3 -3
- prefect/agent.py +1 -0
- prefect/automations.py +4 -8
- prefect/blocks/notifications.py +8 -8
- prefect/cache_policies.py +2 -0
- prefect/client/base.py +7 -8
- prefect/client/collections.py +3 -6
- prefect/client/orchestration/__init__.py +15 -263
- prefect/client/orchestration/_deployments/client.py +14 -6
- prefect/client/orchestration/_flow_runs/client.py +10 -6
- prefect/client/orchestration/_work_pools/__init__.py +0 -0
- prefect/client/orchestration/_work_pools/client.py +598 -0
- prefect/client/orchestration/base.py +9 -2
- prefect/client/schemas/actions.py +66 -2
- prefect/client/schemas/objects.py +22 -50
- prefect/client/schemas/schedules.py +7 -18
- prefect/client/types/flexible_schedule_list.py +2 -1
- prefect/context.py +2 -3
- prefect/deployments/flow_runs.py +1 -1
- prefect/deployments/runner.py +119 -43
- prefect/deployments/schedules.py +7 -1
- prefect/engine.py +4 -9
- prefect/events/schemas/automations.py +4 -2
- prefect/events/utilities.py +15 -13
- prefect/exceptions.py +1 -1
- prefect/flow_engine.py +8 -8
- prefect/flow_runs.py +4 -8
- prefect/flows.py +30 -22
- prefect/infrastructure/__init__.py +1 -0
- prefect/infrastructure/base.py +1 -0
- prefect/infrastructure/provisioners/__init__.py +3 -6
- prefect/infrastructure/provisioners/coiled.py +3 -3
- prefect/infrastructure/provisioners/container_instance.py +1 -0
- prefect/infrastructure/provisioners/ecs.py +6 -6
- prefect/infrastructure/provisioners/modal.py +3 -3
- prefect/input/run_input.py +5 -7
- prefect/locking/filesystem.py +4 -3
- prefect/main.py +1 -1
- prefect/results.py +42 -249
- prefect/runner/runner.py +9 -4
- prefect/runner/server.py +5 -5
- prefect/runner/storage.py +12 -10
- prefect/runner/submit.py +2 -4
- prefect/schedules.py +231 -0
- prefect/serializers.py +5 -5
- prefect/settings/__init__.py +2 -1
- prefect/settings/base.py +3 -3
- prefect/settings/models/root.py +4 -0
- prefect/settings/models/server/services.py +50 -9
- prefect/settings/sources.py +4 -4
- prefect/states.py +42 -11
- prefect/task_engine.py +10 -10
- prefect/task_runners.py +11 -22
- prefect/task_worker.py +9 -9
- prefect/tasks.py +22 -41
- prefect/telemetry/bootstrap.py +4 -6
- prefect/telemetry/services.py +2 -4
- prefect/types/__init__.py +2 -1
- prefect/types/_datetime.py +28 -1
- prefect/utilities/_engine.py +0 -1
- prefect/utilities/asyncutils.py +4 -8
- prefect/utilities/collections.py +13 -22
- prefect/utilities/dispatch.py +2 -4
- prefect/utilities/dockerutils.py +6 -6
- prefect/utilities/importtools.py +1 -68
- prefect/utilities/names.py +1 -1
- prefect/utilities/processutils.py +3 -6
- prefect/utilities/pydantic.py +4 -6
- prefect/utilities/schema_tools/hydration.py +6 -5
- prefect/utilities/templating.py +16 -10
- prefect/utilities/visualization.py +2 -4
- prefect/workers/base.py +3 -3
- prefect/workers/block.py +1 -0
- prefect/workers/cloud.py +1 -0
- prefect/workers/process.py +1 -0
- {prefect_client-3.1.15.dist-info → prefect_client-3.2.0.dist-info}/METADATA +1 -1
- {prefect_client-3.1.15.dist-info → prefect_client-3.2.0.dist-info}/RECORD +89 -85
- {prefect_client-3.1.15.dist-info → prefect_client-3.2.0.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.15.dist-info → prefect_client-3.2.0.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.15.dist-info → prefect_client-3.2.0.dist-info}/top_level.txt +0 -0
prefect/schedules.py
ADDED
@@ -0,0 +1,231 @@
|
|
1
|
+
"""
|
2
|
+
This module contains functionality for creating schedules for deployments.
|
3
|
+
"""
|
4
|
+
|
5
|
+
from __future__ import annotations
|
6
|
+
|
7
|
+
import dataclasses
|
8
|
+
import datetime
|
9
|
+
from functools import partial
|
10
|
+
from typing import Any
|
11
|
+
|
12
|
+
from prefect._internal.schemas.validators import (
|
13
|
+
validate_cron_string,
|
14
|
+
validate_rrule_string,
|
15
|
+
)
|
16
|
+
|
17
|
+
|
18
|
+
@dataclasses.dataclass(frozen=True)
|
19
|
+
class Schedule:
|
20
|
+
"""
|
21
|
+
A dataclass representing a schedule.
|
22
|
+
|
23
|
+
Note that only one of `interval`, `cron`, or `rrule` can be defined at a time.
|
24
|
+
|
25
|
+
Attributes:
|
26
|
+
interval: A timedelta representing the frequency of the schedule.
|
27
|
+
cron: A valid cron string (e.g. "0 0 * * *").
|
28
|
+
rrule: A valid RRule string (e.g. "RRULE:FREQ=DAILY;INTERVAL=1").
|
29
|
+
timezone: A valid timezone string in IANA tzdata format (e.g. America/New_York).
|
30
|
+
anchor_date: An anchor date to schedule increments against; if not provided,
|
31
|
+
the current timestamp will be used.
|
32
|
+
day_or: Control how `day` and `day_of_week` entries are handled.
|
33
|
+
Defaults to True, matching cron which connects those values using
|
34
|
+
OR. If the switch is set to False, the values are connected using AND.
|
35
|
+
This behaves like fcron and enables you to e.g. define a job that
|
36
|
+
executes each 2nd friday of a month by setting the days of month and
|
37
|
+
the weekday.
|
38
|
+
active: Whether or not the schedule is active.
|
39
|
+
parameters: A dictionary containing parameter overrides for the schedule.
|
40
|
+
slug: A unique identifier for the schedule.
|
41
|
+
"""
|
42
|
+
|
43
|
+
interval: datetime.timedelta | None = None
|
44
|
+
cron: str | None = None
|
45
|
+
rrule: str | None = None
|
46
|
+
timezone: str | None = None
|
47
|
+
anchor_date: datetime.datetime = dataclasses.field(
|
48
|
+
default_factory=partial(datetime.datetime.now, tz=datetime.timezone.utc)
|
49
|
+
)
|
50
|
+
day_or: bool = False
|
51
|
+
active: bool = True
|
52
|
+
parameters: dict[str, Any] = dataclasses.field(default_factory=dict)
|
53
|
+
slug: str | None = None
|
54
|
+
|
55
|
+
def __post_init__(self) -> None:
|
56
|
+
defined_fields = [
|
57
|
+
field
|
58
|
+
for field in ["interval", "cron", "rrule"]
|
59
|
+
if getattr(self, field) is not None
|
60
|
+
]
|
61
|
+
if len(defined_fields) > 1:
|
62
|
+
raise ValueError(
|
63
|
+
f"Only one schedule type can be defined at a time. Found: {', '.join(defined_fields)}"
|
64
|
+
)
|
65
|
+
|
66
|
+
if self.cron is not None:
|
67
|
+
validate_cron_string(self.cron)
|
68
|
+
if self.rrule is not None:
|
69
|
+
validate_rrule_string(self.rrule)
|
70
|
+
|
71
|
+
|
72
|
+
def Cron(
|
73
|
+
cron: str,
|
74
|
+
/,
|
75
|
+
timezone: str | None = None,
|
76
|
+
day_or: bool = False,
|
77
|
+
active: bool = True,
|
78
|
+
parameters: dict[str, Any] | None = None,
|
79
|
+
slug: str | None = None,
|
80
|
+
) -> Schedule:
|
81
|
+
"""
|
82
|
+
Creates a cron schedule.
|
83
|
+
|
84
|
+
Args:
|
85
|
+
cron: A valid cron string (e.g. "0 0 * * *").
|
86
|
+
timezone: A valid timezone string in IANA tzdata format (e.g. America/New_York).
|
87
|
+
day_or: Control how `day` and `day_of_week` entries are handled.
|
88
|
+
Defaults to True, matching cron which connects those values using
|
89
|
+
OR. If the switch is set to False, the values are connected using AND.
|
90
|
+
This behaves like fcron and enables you to e.g. define a job that
|
91
|
+
executes each 2nd friday of a month by setting the days of month and
|
92
|
+
the weekday.
|
93
|
+
active: Whether or not the schedule is active.
|
94
|
+
parameters: A dictionary containing parameter overrides for the schedule.
|
95
|
+
slug: A unique identifier for the schedule.
|
96
|
+
|
97
|
+
Returns:
|
98
|
+
A cron schedule.
|
99
|
+
|
100
|
+
Examples:
|
101
|
+
Create a cron schedule that runs every day at 12:00 AM UTC:
|
102
|
+
```python
|
103
|
+
from prefect.schedules import Cron
|
104
|
+
|
105
|
+
Cron("0 0 * * *")
|
106
|
+
```
|
107
|
+
|
108
|
+
Create a cron schedule that runs every Monday at 8:00 AM in the America/New_York timezone:
|
109
|
+
```python
|
110
|
+
from prefect.schedules import Cron
|
111
|
+
|
112
|
+
Cron("0 8 * * 1", timezone="America/New_York")
|
113
|
+
```
|
114
|
+
|
115
|
+
"""
|
116
|
+
if parameters is None:
|
117
|
+
parameters = {}
|
118
|
+
return Schedule(
|
119
|
+
cron=cron,
|
120
|
+
timezone=timezone,
|
121
|
+
day_or=day_or,
|
122
|
+
active=active,
|
123
|
+
parameters=parameters,
|
124
|
+
slug=slug,
|
125
|
+
)
|
126
|
+
|
127
|
+
|
128
|
+
def Interval(
|
129
|
+
interval: datetime.timedelta | float | int,
|
130
|
+
/,
|
131
|
+
anchor_date: datetime.datetime | None = None,
|
132
|
+
timezone: str | None = None,
|
133
|
+
active: bool = True,
|
134
|
+
parameters: dict[str, Any] | None = None,
|
135
|
+
slug: str | None = None,
|
136
|
+
) -> Schedule:
|
137
|
+
"""
|
138
|
+
Creates an interval schedule.
|
139
|
+
|
140
|
+
Args:
|
141
|
+
interval: The interval to use for the schedule. If an integer is provided,
|
142
|
+
it will be interpreted as seconds.
|
143
|
+
anchor_date: The anchor date to use for the schedule.
|
144
|
+
timezone: A valid timezone string in IANA tzdata format (e.g. America/New_York).
|
145
|
+
active: Whether or not the schedule is active.
|
146
|
+
parameters: A dictionary containing parameter overrides for the schedule.
|
147
|
+
slug: A unique identifier for the schedule.
|
148
|
+
|
149
|
+
Returns:
|
150
|
+
An interval schedule.
|
151
|
+
|
152
|
+
Examples:
|
153
|
+
Create an interval schedule that runs every hour:
|
154
|
+
```python
|
155
|
+
from datetime import timedelta
|
156
|
+
|
157
|
+
from prefect.schedules import Interval
|
158
|
+
|
159
|
+
Interval(timedelta(hours=1))
|
160
|
+
```
|
161
|
+
|
162
|
+
Create an interval schedule that runs every 60 seconds starting at a specific date:
|
163
|
+
```python
|
164
|
+
from datetime import datetime
|
165
|
+
|
166
|
+
from prefect.schedules import Interval
|
167
|
+
|
168
|
+
Interval(60, anchor_date=datetime(2024, 1, 1))
|
169
|
+
```
|
170
|
+
"""
|
171
|
+
if isinstance(interval, (float, int)):
|
172
|
+
interval = datetime.timedelta(seconds=interval)
|
173
|
+
if anchor_date is None:
|
174
|
+
anchor_date = datetime.datetime.now(tz=datetime.timezone.utc)
|
175
|
+
if parameters is None:
|
176
|
+
parameters = {}
|
177
|
+
return Schedule(
|
178
|
+
interval=interval,
|
179
|
+
anchor_date=anchor_date,
|
180
|
+
timezone=timezone,
|
181
|
+
active=active,
|
182
|
+
parameters=parameters,
|
183
|
+
slug=slug,
|
184
|
+
)
|
185
|
+
|
186
|
+
|
187
|
+
def RRule(
|
188
|
+
rrule: str,
|
189
|
+
/,
|
190
|
+
timezone: str | None = None,
|
191
|
+
active: bool = True,
|
192
|
+
parameters: dict[str, Any] | None = None,
|
193
|
+
slug: str | None = None,
|
194
|
+
) -> Schedule:
|
195
|
+
"""
|
196
|
+
Creates an RRule schedule.
|
197
|
+
|
198
|
+
Args:
|
199
|
+
rrule: A valid RRule string (e.g. "RRULE:FREQ=DAILY;INTERVAL=1").
|
200
|
+
timezone: A valid timezone string in IANA tzdata format (e.g. America/New_York).
|
201
|
+
active: Whether or not the schedule is active.
|
202
|
+
parameters: A dictionary containing parameter overrides for the schedule.
|
203
|
+
slug: A unique identifier for the schedule.
|
204
|
+
|
205
|
+
Returns:
|
206
|
+
An RRule schedule.
|
207
|
+
|
208
|
+
Examples:
|
209
|
+
Create an RRule schedule that runs every day at 12:00 AM UTC:
|
210
|
+
```python
|
211
|
+
from prefect.schedules import RRule
|
212
|
+
|
213
|
+
RRule("RRULE:FREQ=DAILY;INTERVAL=1")
|
214
|
+
```
|
215
|
+
|
216
|
+
Create an RRule schedule that runs every 2nd friday of the month in the America/Chicago timezone:
|
217
|
+
```python
|
218
|
+
from prefect.schedules import RRule
|
219
|
+
|
220
|
+
RRule("RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=2FR", timezone="America/Chicago")
|
221
|
+
```
|
222
|
+
"""
|
223
|
+
if parameters is None:
|
224
|
+
parameters = {}
|
225
|
+
return Schedule(
|
226
|
+
rrule=rrule,
|
227
|
+
timezone=timezone,
|
228
|
+
active=active,
|
229
|
+
parameters=parameters,
|
230
|
+
slug=slug,
|
231
|
+
)
|
prefect/serializers.py
CHANGED
@@ -75,17 +75,17 @@ class Serializer(BaseModel, Generic[D]):
|
|
75
75
|
"""
|
76
76
|
|
77
77
|
def __init__(self, **data: Any) -> None:
|
78
|
-
type_string =
|
78
|
+
type_string = (
|
79
|
+
get_dispatch_key(self) if type(self) is not Serializer else "__base__"
|
80
|
+
)
|
79
81
|
data.setdefault("type", type_string)
|
80
82
|
super().__init__(**data)
|
81
83
|
|
82
84
|
@overload
|
83
|
-
def __new__(cls, *, type: str, **kwargs: Any) -> "Serializer[Any]":
|
84
|
-
...
|
85
|
+
def __new__(cls, *, type: str, **kwargs: Any) -> "Serializer[Any]": ...
|
85
86
|
|
86
87
|
@overload
|
87
|
-
def __new__(cls, *, type: None = ..., **kwargs: Any) -> Self:
|
88
|
-
...
|
88
|
+
def __new__(cls, *, type: None = ..., **kwargs: Any) -> Self: ...
|
89
89
|
|
90
90
|
def __new__(cls, **kwargs: Any) -> Union[Self, "Serializer[Any]"]:
|
91
91
|
if type_ := kwargs.get("type"):
|
prefect/settings/__init__.py
CHANGED
@@ -15,7 +15,7 @@ from prefect.settings.legacy import (
|
|
15
15
|
_get_settings_fields,
|
16
16
|
_get_valid_setting_names,
|
17
17
|
)
|
18
|
-
from prefect.settings.models.root import Settings
|
18
|
+
from prefect.settings.models.root import Settings, canonical_environment_prefix
|
19
19
|
|
20
20
|
from prefect.settings.profiles import (
|
21
21
|
Profile,
|
@@ -51,6 +51,7 @@ __all__ = [ # noqa: F822
|
|
51
51
|
"load_profiles",
|
52
52
|
"get_current_settings",
|
53
53
|
"temporary_settings",
|
54
|
+
"canonical_environment_prefix",
|
54
55
|
"DEFAULT_PROFILES_PATH",
|
55
56
|
# add public settings here for auto-completion
|
56
57
|
"PREFECT_API_AUTH_STRING", # type: ignore
|
prefect/settings/base.py
CHANGED
@@ -108,9 +108,9 @@ class PrefectBaseSettings(BaseSettings):
|
|
108
108
|
)
|
109
109
|
env_variables.update(child_env)
|
110
110
|
elif (value := env.get(key)) is not None:
|
111
|
-
env_variables[
|
112
|
-
|
113
|
-
|
111
|
+
env_variables[f"{self.model_config.get('env_prefix')}{key.upper()}"] = (
|
112
|
+
_to_environment_variable_value(value)
|
113
|
+
)
|
114
114
|
return env_variables
|
115
115
|
|
116
116
|
@model_serializer(
|
prefect/settings/models/root.py
CHANGED
@@ -441,3 +441,7 @@ def _default_database_connection_url(settings: "Settings") -> SecretStr:
|
|
441
441
|
f"Unsupported database driver: {settings.server.database.driver}"
|
442
442
|
)
|
443
443
|
return SecretStr(value)
|
444
|
+
|
445
|
+
|
446
|
+
def canonical_environment_prefix(settings: "Settings") -> str:
|
447
|
+
return settings.model_config.get("env_prefix") or ""
|
@@ -7,7 +7,14 @@ from pydantic_settings import SettingsConfigDict
|
|
7
7
|
from prefect.settings.base import PrefectBaseSettings, build_settings_config
|
8
8
|
|
9
9
|
|
10
|
-
class
|
10
|
+
class ServicesBaseSetting(PrefectBaseSettings):
|
11
|
+
enabled: bool = Field(
|
12
|
+
default=True,
|
13
|
+
description="Whether or not to start the service in the server application.",
|
14
|
+
)
|
15
|
+
|
16
|
+
|
17
|
+
class ServerServicesCancellationCleanupSettings(ServicesBaseSetting):
|
11
18
|
"""
|
12
19
|
Settings for controlling the cancellation cleanup service
|
13
20
|
"""
|
@@ -37,7 +44,7 @@ class ServerServicesCancellationCleanupSettings(PrefectBaseSettings):
|
|
37
44
|
)
|
38
45
|
|
39
46
|
|
40
|
-
class ServerServicesEventPersisterSettings(
|
47
|
+
class ServerServicesEventPersisterSettings(ServicesBaseSetting):
|
41
48
|
"""
|
42
49
|
Settings for controlling the event persister service
|
43
50
|
"""
|
@@ -78,8 +85,38 @@ class ServerServicesEventPersisterSettings(PrefectBaseSettings):
|
|
78
85
|
),
|
79
86
|
)
|
80
87
|
|
88
|
+
batch_size_delete: int = Field(
|
89
|
+
default=10_000,
|
90
|
+
gt=0,
|
91
|
+
description="The number of expired events and event resources the event persister will attempt to delete in one batch.",
|
92
|
+
validation_alias=AliasChoices(
|
93
|
+
AliasPath("batch_size_delete"),
|
94
|
+
"prefect_server_services_event_persister_batch_size_delete",
|
95
|
+
),
|
96
|
+
)
|
97
|
+
|
98
|
+
|
99
|
+
class ServerServicesEventLoggerSettings(ServicesBaseSetting):
|
100
|
+
"""
|
101
|
+
Settings for controlling the event logger service
|
102
|
+
"""
|
103
|
+
|
104
|
+
model_config: ClassVar[SettingsConfigDict] = build_settings_config(
|
105
|
+
("server", "services", "event_logger")
|
106
|
+
)
|
107
|
+
|
108
|
+
enabled: bool = Field(
|
109
|
+
default=False,
|
110
|
+
description="Whether or not to start the event logger service in the server application.",
|
111
|
+
validation_alias=AliasChoices(
|
112
|
+
AliasPath("enabled"),
|
113
|
+
"prefect_server_services_event_logger_enabled",
|
114
|
+
"prefect_api_services_event_logger_enabled",
|
115
|
+
),
|
116
|
+
)
|
117
|
+
|
81
118
|
|
82
|
-
class ServerServicesFlowRunNotificationsSettings(
|
119
|
+
class ServerServicesFlowRunNotificationsSettings(ServicesBaseSetting):
|
83
120
|
"""
|
84
121
|
Settings for controlling the flow run notifications service
|
85
122
|
"""
|
@@ -99,7 +136,7 @@ class ServerServicesFlowRunNotificationsSettings(PrefectBaseSettings):
|
|
99
136
|
)
|
100
137
|
|
101
138
|
|
102
|
-
class ServerServicesForemanSettings(
|
139
|
+
class ServerServicesForemanSettings(ServicesBaseSetting):
|
103
140
|
"""
|
104
141
|
Settings for controlling the foreman service
|
105
142
|
"""
|
@@ -180,7 +217,7 @@ class ServerServicesForemanSettings(PrefectBaseSettings):
|
|
180
217
|
)
|
181
218
|
|
182
219
|
|
183
|
-
class ServerServicesLateRunsSettings(
|
220
|
+
class ServerServicesLateRunsSettings(ServicesBaseSetting):
|
184
221
|
"""
|
185
222
|
Settings for controlling the late runs service
|
186
223
|
"""
|
@@ -224,7 +261,7 @@ class ServerServicesLateRunsSettings(PrefectBaseSettings):
|
|
224
261
|
)
|
225
262
|
|
226
263
|
|
227
|
-
class ServerServicesSchedulerSettings(
|
264
|
+
class ServerServicesSchedulerSettings(ServicesBaseSetting):
|
228
265
|
"""
|
229
266
|
Settings for controlling the scheduler service
|
230
267
|
"""
|
@@ -349,7 +386,7 @@ class ServerServicesSchedulerSettings(PrefectBaseSettings):
|
|
349
386
|
)
|
350
387
|
|
351
388
|
|
352
|
-
class ServerServicesPauseExpirationsSettings(
|
389
|
+
class ServerServicesPauseExpirationsSettings(ServicesBaseSetting):
|
353
390
|
"""
|
354
391
|
Settings for controlling the pause expiration service
|
355
392
|
"""
|
@@ -385,7 +422,7 @@ class ServerServicesPauseExpirationsSettings(PrefectBaseSettings):
|
|
385
422
|
)
|
386
423
|
|
387
424
|
|
388
|
-
class ServerServicesTaskRunRecorderSettings(
|
425
|
+
class ServerServicesTaskRunRecorderSettings(ServicesBaseSetting):
|
389
426
|
"""
|
390
427
|
Settings for controlling the task run recorder service
|
391
428
|
"""
|
@@ -405,7 +442,7 @@ class ServerServicesTaskRunRecorderSettings(PrefectBaseSettings):
|
|
405
442
|
)
|
406
443
|
|
407
444
|
|
408
|
-
class ServerServicesTriggersSettings(
|
445
|
+
class ServerServicesTriggersSettings(ServicesBaseSetting):
|
409
446
|
"""
|
410
447
|
Settings for controlling the triggers service
|
411
448
|
"""
|
@@ -442,6 +479,10 @@ class ServerServicesSettings(PrefectBaseSettings):
|
|
442
479
|
default_factory=ServerServicesEventPersisterSettings,
|
443
480
|
description="Settings for controlling the event persister service",
|
444
481
|
)
|
482
|
+
event_logger: ServerServicesEventLoggerSettings = Field(
|
483
|
+
default_factory=ServerServicesEventLoggerSettings,
|
484
|
+
description="Settings for controlling the event logger service",
|
485
|
+
)
|
445
486
|
flow_run_notifications: ServerServicesFlowRunNotificationsSettings = Field(
|
446
487
|
default_factory=ServerServicesFlowRunNotificationsSettings,
|
447
488
|
description="Settings for controlling the flow run notifications service",
|
prefect/settings/sources.py
CHANGED
@@ -188,7 +188,7 @@ class ProfileSettingsTomlLoader(PydanticBaseSettingsSource):
|
|
188
188
|
self.field_is_complex(field),
|
189
189
|
)
|
190
190
|
|
191
|
-
name = f"{self.config.get('env_prefix','')}{field_name.upper()}"
|
191
|
+
name = f"{self.config.get('env_prefix', '')}{field_name.upper()}"
|
192
192
|
value = self.profile_settings.get(name)
|
193
193
|
return value, field_name, self.field_is_complex(field)
|
194
194
|
|
@@ -266,9 +266,9 @@ class PrefectTomlConfigSettingsSource(TomlConfigSettingsSourceBase):
|
|
266
266
|
settings_cls: Type[BaseSettings],
|
267
267
|
):
|
268
268
|
super().__init__(settings_cls)
|
269
|
-
self.toml_file_path: Path | str | Sequence[
|
270
|
-
|
271
|
-
|
269
|
+
self.toml_file_path: Path | str | Sequence[Path | str] | None = (
|
270
|
+
settings_cls.model_config.get("toml_file", DEFAULT_PREFECT_TOML_PATH)
|
271
|
+
)
|
272
272
|
self.toml_data: dict[str, Any] = self._read_files(self.toml_file_path)
|
273
273
|
self.toml_table_header: tuple[str, ...] = settings_cls.model_config.get(
|
274
274
|
"prefect_toml_table_header", tuple()
|
prefect/states.py
CHANGED
@@ -12,13 +12,11 @@ from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Type
|
|
12
12
|
|
13
13
|
import anyio
|
14
14
|
import httpx
|
15
|
-
import pendulum
|
16
15
|
from opentelemetry import propagate
|
17
16
|
from typing_extensions import TypeGuard
|
18
17
|
|
19
18
|
from prefect._internal.compatibility import deprecated
|
20
|
-
from prefect.client.schemas import State
|
21
|
-
from prefect.client.schemas import StateDetails, StateType
|
19
|
+
from prefect.client.schemas.objects import State, StateDetails, StateType
|
22
20
|
from prefect.exceptions import (
|
23
21
|
CancelledRun,
|
24
22
|
CrashedRun,
|
@@ -30,6 +28,7 @@ from prefect.exceptions import (
|
|
30
28
|
UnfinishedRun,
|
31
29
|
)
|
32
30
|
from prefect.logging.loggers import get_logger, get_run_logger
|
31
|
+
from prefect.types._datetime import DateTime, PendulumDuration
|
33
32
|
from prefect.utilities.annotations import BaseAnnotation
|
34
33
|
from prefect.utilities.asyncutils import in_async_main_thread, sync_compatible
|
35
34
|
from prefect.utilities.collections import ensure_iterable
|
@@ -37,6 +36,7 @@ from prefect.utilities.collections import ensure_iterable
|
|
37
36
|
if TYPE_CHECKING:
|
38
37
|
import logging
|
39
38
|
|
39
|
+
from prefect.client.schemas.actions import StateCreate
|
40
40
|
from prefect.results import (
|
41
41
|
R,
|
42
42
|
ResultStore,
|
@@ -45,6 +45,34 @@ if TYPE_CHECKING:
|
|
45
45
|
logger: "logging.Logger" = get_logger("states")
|
46
46
|
|
47
47
|
|
48
|
+
def to_state_create(state: State) -> "StateCreate":
|
49
|
+
"""
|
50
|
+
Convert the state to a `StateCreate` type which can be used to set the state of
|
51
|
+
a run in the API.
|
52
|
+
|
53
|
+
This method will drop this state's `data` if it is not a result type. Only
|
54
|
+
results should be sent to the API. Other data is only available locally.
|
55
|
+
"""
|
56
|
+
from prefect.client.schemas.actions import StateCreate
|
57
|
+
from prefect.results import (
|
58
|
+
ResultRecord,
|
59
|
+
should_persist_result,
|
60
|
+
)
|
61
|
+
|
62
|
+
if isinstance(state.data, ResultRecord) and should_persist_result():
|
63
|
+
data = state.data.metadata # pyright: ignore[reportUnknownMemberType] unable to narrow ResultRecord type
|
64
|
+
else:
|
65
|
+
data = None
|
66
|
+
|
67
|
+
return StateCreate(
|
68
|
+
type=state.type,
|
69
|
+
name=state.name,
|
70
|
+
message=state.message,
|
71
|
+
data=data,
|
72
|
+
state_details=state.state_details,
|
73
|
+
)
|
74
|
+
|
75
|
+
|
48
76
|
@deprecated.deprecated_parameter(
|
49
77
|
"fetch",
|
50
78
|
when=lambda fetch: fetch is not True,
|
@@ -97,10 +125,10 @@ async def _get_state_result_data_with_retries(
|
|
97
125
|
# grace here about missing results. The exception below could come in the form
|
98
126
|
# of a missing file, a short read, or other types of errors depending on the
|
99
127
|
# result storage backend.
|
100
|
-
from prefect.
|
101
|
-
ResultRecord,
|
128
|
+
from prefect._result_records import (
|
102
129
|
ResultRecordMetadata,
|
103
130
|
)
|
131
|
+
from prefect.results import ResultStore
|
104
132
|
|
105
133
|
if retry_result_failure is False:
|
106
134
|
max_attempts = 1
|
@@ -110,7 +138,7 @@ async def _get_state_result_data_with_retries(
|
|
110
138
|
for i in range(1, max_attempts + 1):
|
111
139
|
try:
|
112
140
|
if isinstance(state.data, ResultRecordMetadata):
|
113
|
-
record = await
|
141
|
+
record = await ResultStore._from_metadata(state.data)
|
114
142
|
return record.result
|
115
143
|
else:
|
116
144
|
return await state.data.get()
|
@@ -462,10 +490,11 @@ async def get_state_exception(state: State) -> BaseException:
|
|
462
490
|
- `CrashedRun` if the state type is CRASHED.
|
463
491
|
- `CancelledRun` if the state type is CANCELLED.
|
464
492
|
"""
|
465
|
-
from prefect.
|
493
|
+
from prefect._result_records import (
|
466
494
|
ResultRecord,
|
467
495
|
ResultRecordMetadata,
|
468
496
|
)
|
497
|
+
from prefect.results import ResultStore
|
469
498
|
|
470
499
|
if state.is_failed():
|
471
500
|
wrapper = FailedRun
|
@@ -482,7 +511,7 @@ async def get_state_exception(state: State) -> BaseException:
|
|
482
511
|
if isinstance(state.data, ResultRecord):
|
483
512
|
result = state.data.result
|
484
513
|
elif isinstance(state.data, ResultRecordMetadata):
|
485
|
-
record = await
|
514
|
+
record = await ResultStore._from_metadata(state.data)
|
486
515
|
result = record.result
|
487
516
|
elif state.data is None:
|
488
517
|
result = None
|
@@ -631,7 +660,7 @@ def Scheduled(
|
|
631
660
|
"""
|
632
661
|
state_details = StateDetails.model_validate(kwargs.pop("state_details", {}))
|
633
662
|
if scheduled_time is None:
|
634
|
-
scheduled_time =
|
663
|
+
scheduled_time = DateTime.now("UTC")
|
635
664
|
elif state_details.scheduled_time:
|
636
665
|
raise ValueError("An extra scheduled_time was provided in state_details")
|
637
666
|
state_details.scheduled_time = scheduled_time
|
@@ -729,8 +758,10 @@ def Paused(
|
|
729
758
|
if pause_expiration_time is None and timeout_seconds is None:
|
730
759
|
pass
|
731
760
|
else:
|
732
|
-
state_details.pause_timeout =
|
733
|
-
|
761
|
+
state_details.pause_timeout = (
|
762
|
+
DateTime.instance(pause_expiration_time)
|
763
|
+
if pause_expiration_time
|
764
|
+
else DateTime.now("UTC") + PendulumDuration(seconds=timeout_seconds or 0)
|
734
765
|
)
|
735
766
|
|
736
767
|
state_details.pause_reschedule = reschedule
|
prefect/task_engine.py
CHANGED
@@ -28,7 +28,6 @@ from typing import (
|
|
28
28
|
from uuid import UUID
|
29
29
|
|
30
30
|
import anyio
|
31
|
-
import pendulum
|
32
31
|
from opentelemetry import trace
|
33
32
|
from typing_extensions import ParamSpec, Self
|
34
33
|
|
@@ -80,6 +79,7 @@ from prefect.states import (
|
|
80
79
|
)
|
81
80
|
from prefect.telemetry.run_telemetry import RunTelemetry
|
82
81
|
from prefect.transactions import IsolationLevel, Transaction, transaction
|
82
|
+
from prefect.types._datetime import DateTime, PendulumDuration
|
83
83
|
from prefect.utilities._engine import get_hook_name
|
84
84
|
from prefect.utilities.annotations import NotSet
|
85
85
|
from prefect.utilities.asyncutils import run_coro_as_sync
|
@@ -249,7 +249,7 @@ class BaseTaskRunEngine(Generic[P, R]):
|
|
249
249
|
display_state = repr(self.state) if PREFECT_DEBUG_MODE else str(self.state)
|
250
250
|
level = logging.INFO if self.state.is_completed() else logging.ERROR
|
251
251
|
msg = f"Finished in state {display_state}"
|
252
|
-
if self.state.is_pending():
|
252
|
+
if self.state.is_pending() and self.state.name != "NotReady":
|
253
253
|
msg += (
|
254
254
|
"\nPlease wait for all submitted tasks to complete"
|
255
255
|
" before exiting your flow by calling `.wait()` on the "
|
@@ -437,7 +437,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
437
437
|
if last_state.timestamp == new_state.timestamp:
|
438
438
|
# Ensure that the state timestamp is unique, or at least not equal to the last state.
|
439
439
|
# This might occur especially on Windows where the timestamp resolution is limited.
|
440
|
-
new_state.timestamp +=
|
440
|
+
new_state.timestamp += PendulumDuration(microseconds=1)
|
441
441
|
|
442
442
|
# Ensure that the state_details are populated with the current run IDs
|
443
443
|
new_state.state_details.task_run_id = self.task_run.id
|
@@ -486,7 +486,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
486
486
|
|
487
487
|
def handle_success(self, result: R, transaction: Transaction) -> R:
|
488
488
|
if self.task.cache_expiration is not None:
|
489
|
-
expiration =
|
489
|
+
expiration = DateTime.now("utc") + self.task.cache_expiration
|
490
490
|
else:
|
491
491
|
expiration = None
|
492
492
|
|
@@ -535,7 +535,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
535
535
|
else self.task.retry_delay_seconds
|
536
536
|
)
|
537
537
|
new_state = AwaitingRetry(
|
538
|
-
scheduled_time=
|
538
|
+
scheduled_time=DateTime.now("utc").add(seconds=delay)
|
539
539
|
)
|
540
540
|
else:
|
541
541
|
delay = None
|
@@ -728,7 +728,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
728
728
|
async def wait_until_ready(self) -> None:
|
729
729
|
"""Waits until the scheduled time (if its the future), then enters Running."""
|
730
730
|
if scheduled_time := self.state.state_details.scheduled_time:
|
731
|
-
sleep_time = (scheduled_time -
|
731
|
+
sleep_time = (scheduled_time - DateTime.now("utc")).total_seconds()
|
732
732
|
await anyio.sleep(sleep_time if sleep_time > 0 else 0)
|
733
733
|
new_state = Retrying() if self.state.name == "AwaitingRetry" else Running()
|
734
734
|
self.set_state(
|
@@ -970,7 +970,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
970
970
|
if last_state.timestamp == new_state.timestamp:
|
971
971
|
# Ensure that the state timestamp is unique, or at least not equal to the last state.
|
972
972
|
# This might occur especially on Windows where the timestamp resolution is limited.
|
973
|
-
new_state.timestamp +=
|
973
|
+
new_state.timestamp += PendulumDuration(microseconds=1)
|
974
974
|
|
975
975
|
# Ensure that the state_details are populated with the current run IDs
|
976
976
|
new_state.state_details.task_run_id = self.task_run.id
|
@@ -1020,7 +1020,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1020
1020
|
|
1021
1021
|
async def handle_success(self, result: R, transaction: Transaction) -> R:
|
1022
1022
|
if self.task.cache_expiration is not None:
|
1023
|
-
expiration =
|
1023
|
+
expiration = DateTime.now("utc") + self.task.cache_expiration
|
1024
1024
|
else:
|
1025
1025
|
expiration = None
|
1026
1026
|
|
@@ -1068,7 +1068,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1068
1068
|
else self.task.retry_delay_seconds
|
1069
1069
|
)
|
1070
1070
|
new_state = AwaitingRetry(
|
1071
|
-
scheduled_time=
|
1071
|
+
scheduled_time=DateTime.now("utc").add(seconds=delay)
|
1072
1072
|
)
|
1073
1073
|
else:
|
1074
1074
|
delay = None
|
@@ -1259,7 +1259,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
|
|
1259
1259
|
async def wait_until_ready(self) -> None:
|
1260
1260
|
"""Waits until the scheduled time (if its the future), then enters Running."""
|
1261
1261
|
if scheduled_time := self.state.state_details.scheduled_time:
|
1262
|
-
sleep_time = (scheduled_time -
|
1262
|
+
sleep_time = (scheduled_time - DateTime.now("utc")).total_seconds()
|
1263
1263
|
await anyio.sleep(sleep_time if sleep_time > 0 else 0)
|
1264
1264
|
new_state = Retrying() if self.state.name == "AwaitingRetry" else Running()
|
1265
1265
|
await self.set_state(
|