prefect-client 3.0.1__py3-none-any.whl → 3.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/compatibility/deprecated.py +1 -1
- prefect/blocks/notifications.py +21 -0
- prefect/blocks/webhook.py +8 -0
- prefect/client/orchestration.py +39 -20
- prefect/client/schemas/actions.py +2 -2
- prefect/client/schemas/objects.py +24 -6
- prefect/client/types/flexible_schedule_list.py +1 -1
- prefect/concurrency/asyncio.py +45 -6
- prefect/concurrency/services.py +1 -1
- prefect/concurrency/sync.py +21 -27
- prefect/concurrency/v1/asyncio.py +3 -0
- prefect/concurrency/v1/sync.py +4 -5
- prefect/context.py +5 -1
- prefect/deployments/runner.py +1 -0
- prefect/events/actions.py +6 -0
- prefect/flow_engine.py +12 -4
- prefect/locking/filesystem.py +243 -0
- prefect/logging/handlers.py +0 -2
- prefect/logging/loggers.py +0 -18
- prefect/logging/logging.yml +1 -0
- prefect/main.py +19 -5
- prefect/records/base.py +12 -0
- prefect/records/filesystem.py +6 -2
- prefect/records/memory.py +6 -0
- prefect/records/result_store.py +6 -0
- prefect/results.py +169 -25
- prefect/runner/runner.py +74 -5
- prefect/settings.py +1 -1
- prefect/states.py +34 -17
- prefect/task_engine.py +31 -37
- prefect/transactions.py +105 -50
- prefect/utilities/engine.py +16 -8
- prefect/utilities/importtools.py +1 -0
- prefect/utilities/urls.py +70 -12
- prefect/workers/base.py +14 -6
- {prefect_client-3.0.1.dist-info → prefect_client-3.0.2.dist-info}/METADATA +1 -1
- {prefect_client-3.0.1.dist-info → prefect_client-3.0.2.dist-info}/RECORD +40 -39
- {prefect_client-3.0.1.dist-info → prefect_client-3.0.2.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.1.dist-info → prefect_client-3.0.2.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.1.dist-info → prefect_client-3.0.2.dist-info}/top_level.txt +0 -0
@@ -30,7 +30,7 @@ M = TypeVar("M", bound=BaseModel)
|
|
30
30
|
|
31
31
|
|
32
32
|
DEPRECATED_WARNING = (
|
33
|
-
"{name} has been deprecated{when}. It will not be available after {end_date}."
|
33
|
+
"{name} has been deprecated{when}. It will not be available in new releases after {end_date}."
|
34
34
|
" {help}"
|
35
35
|
)
|
36
36
|
DEPRECATED_MOVED_WARNING = (
|
prefect/blocks/notifications.py
CHANGED
@@ -10,6 +10,7 @@ from prefect.logging import LogEavesdropper
|
|
10
10
|
from prefect.types import SecretDict
|
11
11
|
from prefect.utilities.asyncutils import sync_compatible
|
12
12
|
from prefect.utilities.templating import apply_values, find_placeholders
|
13
|
+
from prefect.utilities.urls import validate_restricted_url
|
13
14
|
|
14
15
|
PREFECT_NOTIFY_TYPE_DEFAULT = "prefect_default"
|
15
16
|
|
@@ -80,6 +81,26 @@ class AppriseNotificationBlock(AbstractAppriseNotificationBlock, ABC):
|
|
80
81
|
description="Incoming webhook URL used to send notifications.",
|
81
82
|
examples=["https://hooks.example.com/XXX"],
|
82
83
|
)
|
84
|
+
allow_private_urls: bool = Field(
|
85
|
+
default=True,
|
86
|
+
description="Whether to allow notifications to private URLs. Defaults to True.",
|
87
|
+
)
|
88
|
+
|
89
|
+
@sync_compatible
|
90
|
+
async def notify(
|
91
|
+
self,
|
92
|
+
body: str,
|
93
|
+
subject: Optional[str] = None,
|
94
|
+
):
|
95
|
+
if not self.allow_private_urls:
|
96
|
+
try:
|
97
|
+
validate_restricted_url(self.url.get_secret_value())
|
98
|
+
except ValueError as exc:
|
99
|
+
if self._raise_on_failure:
|
100
|
+
raise NotificationError(str(exc))
|
101
|
+
raise
|
102
|
+
|
103
|
+
await super().notify(body, subject)
|
83
104
|
|
84
105
|
|
85
106
|
# TODO: Move to prefect-slack once collection block auto-registration is
|
prefect/blocks/webhook.py
CHANGED
@@ -6,6 +6,7 @@ from typing_extensions import Literal
|
|
6
6
|
|
7
7
|
from prefect.blocks.core import Block
|
8
8
|
from prefect.types import SecretDict
|
9
|
+
from prefect.utilities.urls import validate_restricted_url
|
9
10
|
|
10
11
|
# Use a global HTTP transport to maintain a process-wide connection pool for
|
11
12
|
# interservice requests
|
@@ -39,6 +40,10 @@ class Webhook(Block):
|
|
39
40
|
title="Webhook Headers",
|
40
41
|
description="A dictionary of headers to send with the webhook request.",
|
41
42
|
)
|
43
|
+
allow_private_urls: bool = Field(
|
44
|
+
default=True,
|
45
|
+
description="Whether to allow notifications to private URLs. Defaults to True.",
|
46
|
+
)
|
42
47
|
|
43
48
|
def block_initialization(self):
|
44
49
|
self._client = AsyncClient(transport=_http_transport)
|
@@ -50,6 +55,9 @@ class Webhook(Block):
|
|
50
55
|
Args:
|
51
56
|
payload: an optional payload to send when calling the webhook.
|
52
57
|
"""
|
58
|
+
if not self.allow_private_urls:
|
59
|
+
validate_restricted_url(self.url.get_secret_value())
|
60
|
+
|
53
61
|
async with self._client:
|
54
62
|
return await self._client.request(
|
55
63
|
method=self.method,
|
prefect/client/orchestration.py
CHANGED
@@ -94,7 +94,6 @@ from prefect.client.schemas.objects import (
|
|
94
94
|
FlowRunPolicy,
|
95
95
|
Log,
|
96
96
|
Parameter,
|
97
|
-
QueueFilter,
|
98
97
|
TaskRunPolicy,
|
99
98
|
TaskRunResult,
|
100
99
|
Variable,
|
@@ -994,7 +993,6 @@ class PrefectClient:
|
|
994
993
|
async def create_work_queue(
|
995
994
|
self,
|
996
995
|
name: str,
|
997
|
-
tags: Optional[List[str]] = None,
|
998
996
|
description: Optional[str] = None,
|
999
997
|
is_paused: Optional[bool] = None,
|
1000
998
|
concurrency_limit: Optional[int] = None,
|
@@ -1006,8 +1004,6 @@ class PrefectClient:
|
|
1006
1004
|
|
1007
1005
|
Args:
|
1008
1006
|
name: a unique name for the work queue
|
1009
|
-
tags: DEPRECATED: an optional list of tags to filter on; only work scheduled with these tags
|
1010
|
-
will be included in the queue. This option will be removed on 2023-02-23.
|
1011
1007
|
description: An optional description for the work queue.
|
1012
1008
|
is_paused: Whether or not the work queue is paused.
|
1013
1009
|
concurrency_limit: An optional concurrency limit for the work queue.
|
@@ -1021,18 +1017,7 @@ class PrefectClient:
|
|
1021
1017
|
Returns:
|
1022
1018
|
The created work queue
|
1023
1019
|
"""
|
1024
|
-
|
1025
|
-
warnings.warn(
|
1026
|
-
(
|
1027
|
-
"The use of tags for creating work queue filters is deprecated."
|
1028
|
-
" This option will be removed on 2023-02-23."
|
1029
|
-
),
|
1030
|
-
DeprecationWarning,
|
1031
|
-
)
|
1032
|
-
filter = QueueFilter(tags=tags)
|
1033
|
-
else:
|
1034
|
-
filter = None
|
1035
|
-
create_model = WorkQueueCreate(name=name, filter=filter)
|
1020
|
+
create_model = WorkQueueCreate(name=name, filter=None)
|
1036
1021
|
if description is not None:
|
1037
1022
|
create_model.description = description
|
1038
1023
|
if is_paused is not None:
|
@@ -2158,7 +2143,10 @@ class PrefectClient:
|
|
2158
2143
|
try:
|
2159
2144
|
response = await self._client.post(
|
2160
2145
|
f"/flow_runs/{flow_run_id}/set_state",
|
2161
|
-
json=dict(
|
2146
|
+
json=dict(
|
2147
|
+
state=state_create.model_dump(mode="json", serialize_as_any=True),
|
2148
|
+
force=force,
|
2149
|
+
),
|
2162
2150
|
)
|
2163
2151
|
except httpx.HTTPStatusError as e:
|
2164
2152
|
if e.response.status_code == status.HTTP_404_NOT_FOUND:
|
@@ -3055,7 +3043,11 @@ class PrefectClient:
|
|
3055
3043
|
return response.json()
|
3056
3044
|
|
3057
3045
|
async def increment_concurrency_slots(
|
3058
|
-
self,
|
3046
|
+
self,
|
3047
|
+
names: List[str],
|
3048
|
+
slots: int,
|
3049
|
+
mode: str,
|
3050
|
+
create_if_missing: Optional[bool] = None,
|
3059
3051
|
) -> httpx.Response:
|
3060
3052
|
return await self._client.post(
|
3061
3053
|
"/v2/concurrency_limits/increment",
|
@@ -3063,7 +3055,7 @@ class PrefectClient:
|
|
3063
3055
|
"names": names,
|
3064
3056
|
"slots": slots,
|
3065
3057
|
"mode": mode,
|
3066
|
-
"create_if_missing": create_if_missing,
|
3058
|
+
"create_if_missing": create_if_missing if create_if_missing else False,
|
3067
3059
|
},
|
3068
3060
|
)
|
3069
3061
|
|
@@ -3140,6 +3132,30 @@ class PrefectClient:
|
|
3140
3132
|
else:
|
3141
3133
|
raise
|
3142
3134
|
|
3135
|
+
async def upsert_global_concurrency_limit_by_name(self, name: str, limit: int):
|
3136
|
+
"""Creates a global concurrency limit with the given name and limit if one does not already exist.
|
3137
|
+
|
3138
|
+
If one does already exist matching the name then update it's limit if it is different.
|
3139
|
+
|
3140
|
+
Note: This is not done atomically.
|
3141
|
+
"""
|
3142
|
+
try:
|
3143
|
+
existing_limit = await self.read_global_concurrency_limit_by_name(name)
|
3144
|
+
except prefect.exceptions.ObjectNotFound:
|
3145
|
+
existing_limit = None
|
3146
|
+
|
3147
|
+
if not existing_limit:
|
3148
|
+
await self.create_global_concurrency_limit(
|
3149
|
+
GlobalConcurrencyLimitCreate(
|
3150
|
+
name=name,
|
3151
|
+
limit=limit,
|
3152
|
+
)
|
3153
|
+
)
|
3154
|
+
elif existing_limit.limit != limit:
|
3155
|
+
await self.update_global_concurrency_limit(
|
3156
|
+
name, GlobalConcurrencyLimitUpdate(limit=limit)
|
3157
|
+
)
|
3158
|
+
|
3143
3159
|
async def read_global_concurrency_limits(
|
3144
3160
|
self, limit: int = 10, offset: int = 0
|
3145
3161
|
) -> List[GlobalConcurrencyLimitResponse]:
|
@@ -3934,7 +3950,10 @@ class SyncPrefectClient:
|
|
3934
3950
|
try:
|
3935
3951
|
response = self._client.post(
|
3936
3952
|
f"/flow_runs/{flow_run_id}/set_state",
|
3937
|
-
json=dict(
|
3953
|
+
json=dict(
|
3954
|
+
state=state_create.model_dump(mode="json", serialize_as_any=True),
|
3955
|
+
force=force,
|
3956
|
+
),
|
3938
3957
|
)
|
3939
3958
|
except httpx.HTTPStatusError as e:
|
3940
3959
|
if e.response.status_code == status.HTTP_404_NOT_FOUND:
|
@@ -38,7 +38,7 @@ from prefect.utilities.collections import listrepr
|
|
38
38
|
from prefect.utilities.pydantic import get_class_fields_only
|
39
39
|
|
40
40
|
if TYPE_CHECKING:
|
41
|
-
from prefect.results import BaseResult
|
41
|
+
from prefect.results import BaseResult, ResultRecordMetadata
|
42
42
|
|
43
43
|
R = TypeVar("R")
|
44
44
|
|
@@ -50,7 +50,7 @@ class StateCreate(ActionBaseModel):
|
|
50
50
|
name: Optional[str] = Field(default=None)
|
51
51
|
message: Optional[str] = Field(default=None, examples=["Run started"])
|
52
52
|
state_details: StateDetails = Field(default_factory=StateDetails)
|
53
|
-
data: Union["BaseResult[R]", Any] = Field(
|
53
|
+
data: Union["BaseResult[R]", "ResultRecordMetadata", Any] = Field(
|
54
54
|
default=None,
|
55
55
|
)
|
56
56
|
|
@@ -3,6 +3,7 @@ import warnings
|
|
3
3
|
from functools import partial
|
4
4
|
from typing import (
|
5
5
|
TYPE_CHECKING,
|
6
|
+
Annotated,
|
6
7
|
Any,
|
7
8
|
Dict,
|
8
9
|
Generic,
|
@@ -17,10 +18,12 @@ import orjson
|
|
17
18
|
import pendulum
|
18
19
|
from pydantic import (
|
19
20
|
ConfigDict,
|
21
|
+
Discriminator,
|
20
22
|
Field,
|
21
23
|
HttpUrl,
|
22
24
|
IPvAnyNetwork,
|
23
25
|
SerializationInfo,
|
26
|
+
Tag,
|
24
27
|
field_validator,
|
25
28
|
model_serializer,
|
26
29
|
model_validator,
|
@@ -59,7 +62,7 @@ from prefect.utilities.names import generate_slug
|
|
59
62
|
from prefect.utilities.pydantic import handle_secret_render
|
60
63
|
|
61
64
|
if TYPE_CHECKING:
|
62
|
-
from prefect.results import BaseResult
|
65
|
+
from prefect.results import BaseResult, ResultRecordMetadata
|
63
66
|
|
64
67
|
|
65
68
|
R = TypeVar("R", default=Any)
|
@@ -158,6 +161,14 @@ class StateDetails(PrefectBaseModel):
|
|
158
161
|
task_parameters_id: Optional[UUID] = None
|
159
162
|
|
160
163
|
|
164
|
+
def data_discriminator(x: Any) -> str:
|
165
|
+
if isinstance(x, dict) and "type" in x:
|
166
|
+
return "BaseResult"
|
167
|
+
elif isinstance(x, dict) and "storage_key" in x:
|
168
|
+
return "ResultRecordMetadata"
|
169
|
+
return "Any"
|
170
|
+
|
171
|
+
|
161
172
|
class State(ObjectBaseModel, Generic[R]):
|
162
173
|
"""
|
163
174
|
The state of a run.
|
@@ -168,9 +179,14 @@ class State(ObjectBaseModel, Generic[R]):
|
|
168
179
|
timestamp: DateTime = Field(default_factory=lambda: pendulum.now("UTC"))
|
169
180
|
message: Optional[str] = Field(default=None, examples=["Run started"])
|
170
181
|
state_details: StateDetails = Field(default_factory=StateDetails)
|
171
|
-
data:
|
172
|
-
|
173
|
-
|
182
|
+
data: Annotated[
|
183
|
+
Union[
|
184
|
+
Annotated["BaseResult[R]", Tag("BaseResult")],
|
185
|
+
Annotated["ResultRecordMetadata", Tag("ResultRecordMetadata")],
|
186
|
+
Annotated[Any, Tag("Any")],
|
187
|
+
],
|
188
|
+
Discriminator(data_discriminator),
|
189
|
+
] = Field(default=None)
|
174
190
|
|
175
191
|
@overload
|
176
192
|
def result(self: "State[R]", raise_on_failure: bool = True) -> R:
|
@@ -276,10 +292,12 @@ class State(ObjectBaseModel, Generic[R]):
|
|
276
292
|
results should be sent to the API. Other data is only available locally.
|
277
293
|
"""
|
278
294
|
from prefect.client.schemas.actions import StateCreate
|
279
|
-
from prefect.results import BaseResult
|
295
|
+
from prefect.results import BaseResult, ResultRecord, should_persist_result
|
280
296
|
|
281
|
-
if isinstance(self.data, BaseResult)
|
297
|
+
if isinstance(self.data, BaseResult):
|
282
298
|
data = self.data
|
299
|
+
elif isinstance(self.data, ResultRecord) and should_persist_result():
|
300
|
+
data = self.data.metadata
|
283
301
|
else:
|
284
302
|
data = None
|
285
303
|
|
@@ -7,5 +7,5 @@ if TYPE_CHECKING:
|
|
7
7
|
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
8
8
|
|
9
9
|
FlexibleScheduleList: TypeAlias = Sequence[
|
10
|
-
Union[DeploymentScheduleCreate, dict[str, Any], "SCHEDULE_TYPES"]
|
10
|
+
Union["DeploymentScheduleCreate", dict[str, Any], "SCHEDULE_TYPES"]
|
11
11
|
]
|
prefect/concurrency/asyncio.py
CHANGED
@@ -6,6 +6,8 @@ import anyio
|
|
6
6
|
import httpx
|
7
7
|
import pendulum
|
8
8
|
|
9
|
+
from prefect._internal.compatibility.deprecated import deprecated_parameter
|
10
|
+
|
9
11
|
try:
|
10
12
|
from pendulum import Interval
|
11
13
|
except ImportError:
|
@@ -14,6 +16,8 @@ except ImportError:
|
|
14
16
|
|
15
17
|
from prefect.client.orchestration import get_client
|
16
18
|
from prefect.client.schemas.responses import MinimalConcurrencyLimitResponse
|
19
|
+
from prefect.logging.loggers import get_run_logger
|
20
|
+
from prefect.utilities.asyncutils import sync_compatible
|
17
21
|
|
18
22
|
from .context import ConcurrencyContext
|
19
23
|
from .events import (
|
@@ -36,8 +40,9 @@ async def concurrency(
|
|
36
40
|
names: Union[str, List[str]],
|
37
41
|
occupy: int = 1,
|
38
42
|
timeout_seconds: Optional[float] = None,
|
39
|
-
create_if_missing: bool = True,
|
40
43
|
max_retries: Optional[int] = None,
|
44
|
+
create_if_missing: Optional[bool] = None,
|
45
|
+
strict: bool = False,
|
41
46
|
) -> AsyncGenerator[None, None]:
|
42
47
|
"""A context manager that acquires and releases concurrency slots from the
|
43
48
|
given concurrency limits.
|
@@ -47,11 +52,13 @@ async def concurrency(
|
|
47
52
|
occupy: The number of slots to acquire and hold from each limit.
|
48
53
|
timeout_seconds: The number of seconds to wait for the slots to be acquired before
|
49
54
|
raising a `TimeoutError`. A timeout of `None` will wait indefinitely.
|
50
|
-
create_if_missing: Whether to create the concurrency limits if they do not exist.
|
51
55
|
max_retries: The maximum number of retries to acquire the concurrency slots.
|
56
|
+
strict: A boolean specifying whether to raise an error if the concurrency limit does not exist.
|
57
|
+
Defaults to `False`.
|
52
58
|
|
53
59
|
Raises:
|
54
60
|
TimeoutError: If the slots are not acquired within the given timeout.
|
61
|
+
ConcurrencySlotAcquisitionError: If the concurrency limit does not exist and `strict` is `True`.
|
55
62
|
|
56
63
|
Example:
|
57
64
|
A simple example of using the async `concurrency` context manager:
|
@@ -78,6 +85,7 @@ async def concurrency(
|
|
78
85
|
timeout_seconds=timeout_seconds,
|
79
86
|
create_if_missing=create_if_missing,
|
80
87
|
max_retries=max_retries,
|
88
|
+
strict=strict,
|
81
89
|
)
|
82
90
|
acquisition_time = pendulum.now("UTC")
|
83
91
|
emitted_events = _emit_concurrency_acquisition_events(limits, occupy)
|
@@ -106,7 +114,8 @@ async def rate_limit(
|
|
106
114
|
names: Union[str, List[str]],
|
107
115
|
occupy: int = 1,
|
108
116
|
timeout_seconds: Optional[float] = None,
|
109
|
-
create_if_missing: Optional[bool] =
|
117
|
+
create_if_missing: Optional[bool] = None,
|
118
|
+
strict: bool = False,
|
110
119
|
) -> None:
|
111
120
|
"""Block execution until an `occupy` number of slots of the concurrency
|
112
121
|
limits given in `names` are acquired. Requires that all given concurrency
|
@@ -117,7 +126,12 @@ async def rate_limit(
|
|
117
126
|
occupy: The number of slots to acquire and hold from each limit.
|
118
127
|
timeout_seconds: The number of seconds to wait for the slots to be acquired before
|
119
128
|
raising a `TimeoutError`. A timeout of `None` will wait indefinitely.
|
120
|
-
|
129
|
+
strict: A boolean specifying whether to raise an error if the concurrency limit does not exist.
|
130
|
+
Defaults to `False`.
|
131
|
+
|
132
|
+
Raises:
|
133
|
+
TimeoutError: If the slots are not acquired within the given timeout.
|
134
|
+
ConcurrencySlotAcquisitionError: If the concurrency limit does not exist and `strict` is `True`.
|
121
135
|
"""
|
122
136
|
if not names:
|
123
137
|
return
|
@@ -130,17 +144,27 @@ async def rate_limit(
|
|
130
144
|
mode="rate_limit",
|
131
145
|
timeout_seconds=timeout_seconds,
|
132
146
|
create_if_missing=create_if_missing,
|
147
|
+
strict=strict,
|
133
148
|
)
|
134
149
|
_emit_concurrency_acquisition_events(limits, occupy)
|
135
150
|
|
136
151
|
|
152
|
+
@sync_compatible
|
153
|
+
@deprecated_parameter(
|
154
|
+
name="create_if_missing",
|
155
|
+
start_date="Sep 2024",
|
156
|
+
end_date="Oct 2024",
|
157
|
+
when=lambda x: x is not None,
|
158
|
+
help="Limits must be explicitly created before acquiring concurrency slots; see `strict` if you want to enforce this behavior.",
|
159
|
+
)
|
137
160
|
async def _acquire_concurrency_slots(
|
138
161
|
names: List[str],
|
139
162
|
slots: int,
|
140
163
|
mode: Union[Literal["concurrency"], Literal["rate_limit"]] = "concurrency",
|
141
164
|
timeout_seconds: Optional[float] = None,
|
142
|
-
create_if_missing: Optional[bool] =
|
165
|
+
create_if_missing: Optional[bool] = None,
|
143
166
|
max_retries: Optional[int] = None,
|
167
|
+
strict: bool = False,
|
144
168
|
) -> List[MinimalConcurrencyLimitResponse]:
|
145
169
|
service = ConcurrencySlotAcquisitionService.instance(frozenset(names))
|
146
170
|
future = service.send(
|
@@ -158,9 +182,24 @@ async def _acquire_concurrency_slots(
|
|
158
182
|
f"Unable to acquire concurrency slots on {names!r}"
|
159
183
|
) from response_or_exception
|
160
184
|
|
161
|
-
|
185
|
+
retval = _response_to_minimal_concurrency_limit_response(response_or_exception)
|
186
|
+
|
187
|
+
if strict and not retval:
|
188
|
+
raise ConcurrencySlotAcquisitionError(
|
189
|
+
f"Concurrency limits {names!r} must be created before acquiring slots"
|
190
|
+
)
|
191
|
+
elif not retval:
|
192
|
+
try:
|
193
|
+
logger = get_run_logger()
|
194
|
+
logger.warning(
|
195
|
+
f"Concurrency limits {names!r} do not exist - skipping acquisition."
|
196
|
+
)
|
197
|
+
except Exception:
|
198
|
+
pass
|
199
|
+
return retval
|
162
200
|
|
163
201
|
|
202
|
+
@sync_compatible
|
164
203
|
async def _release_concurrency_slots(
|
165
204
|
names: List[str], slots: int, occupancy_seconds: float
|
166
205
|
) -> List[MinimalConcurrencyLimitResponse]:
|
prefect/concurrency/services.py
CHANGED
@@ -63,7 +63,7 @@ class ConcurrencySlotAcquisitionService(QueueService):
|
|
63
63
|
slots: int,
|
64
64
|
mode: str,
|
65
65
|
timeout_seconds: Optional[float] = None,
|
66
|
-
create_if_missing: Optional[bool] =
|
66
|
+
create_if_missing: Optional[bool] = None,
|
67
67
|
max_retries: Optional[int] = None,
|
68
68
|
) -> httpx.Response:
|
69
69
|
with timeout_async(seconds=timeout_seconds):
|
prefect/concurrency/sync.py
CHANGED
@@ -1,8 +1,5 @@
|
|
1
1
|
from contextlib import contextmanager
|
2
2
|
from typing import (
|
3
|
-
Any,
|
4
|
-
Awaitable,
|
5
|
-
Callable,
|
6
3
|
Generator,
|
7
4
|
List,
|
8
5
|
Optional,
|
@@ -19,8 +16,6 @@ except ImportError:
|
|
19
16
|
# pendulum < 3
|
20
17
|
from pendulum.period import Period as Interval # type: ignore
|
21
18
|
|
22
|
-
from prefect._internal.concurrency.api import create_call, from_sync
|
23
|
-
from prefect._internal.concurrency.event_loop import get_running_loop
|
24
19
|
from prefect.client.schemas.responses import MinimalConcurrencyLimitResponse
|
25
20
|
|
26
21
|
from .asyncio import (
|
@@ -40,8 +35,9 @@ def concurrency(
|
|
40
35
|
names: Union[str, List[str]],
|
41
36
|
occupy: int = 1,
|
42
37
|
timeout_seconds: Optional[float] = None,
|
43
|
-
create_if_missing: bool = True,
|
44
38
|
max_retries: Optional[int] = None,
|
39
|
+
strict: bool = False,
|
40
|
+
create_if_missing: Optional[bool] = None,
|
45
41
|
) -> Generator[None, None, None]:
|
46
42
|
"""A context manager that acquires and releases concurrency slots from the
|
47
43
|
given concurrency limits.
|
@@ -51,11 +47,13 @@ def concurrency(
|
|
51
47
|
occupy: The number of slots to acquire and hold from each limit.
|
52
48
|
timeout_seconds: The number of seconds to wait for the slots to be acquired before
|
53
49
|
raising a `TimeoutError`. A timeout of `None` will wait indefinitely.
|
54
|
-
create_if_missing: Whether to create the concurrency limits if they do not exist.
|
55
50
|
max_retries: The maximum number of retries to acquire the concurrency slots.
|
51
|
+
strict: A boolean specifying whether to raise an error if the concurrency limit does not exist.
|
52
|
+
Defaults to `False`.
|
56
53
|
|
57
54
|
Raises:
|
58
55
|
TimeoutError: If the slots are not acquired within the given timeout.
|
56
|
+
ConcurrencySlotAcquisitionError: If the concurrency limit does not exist and `strict` is `True`.
|
59
57
|
|
60
58
|
Example:
|
61
59
|
A simple example of using the sync `concurrency` context manager:
|
@@ -76,13 +74,14 @@ def concurrency(
|
|
76
74
|
|
77
75
|
names = names if isinstance(names, list) else [names]
|
78
76
|
|
79
|
-
limits: List[MinimalConcurrencyLimitResponse] =
|
80
|
-
_acquire_concurrency_slots,
|
77
|
+
limits: List[MinimalConcurrencyLimitResponse] = _acquire_concurrency_slots(
|
81
78
|
names,
|
82
79
|
occupy,
|
83
80
|
timeout_seconds=timeout_seconds,
|
84
81
|
create_if_missing=create_if_missing,
|
82
|
+
strict=strict,
|
85
83
|
max_retries=max_retries,
|
84
|
+
_sync=True,
|
86
85
|
)
|
87
86
|
acquisition_time = pendulum.now("UTC")
|
88
87
|
emitted_events = _emit_concurrency_acquisition_events(limits, occupy)
|
@@ -91,11 +90,11 @@ def concurrency(
|
|
91
90
|
yield
|
92
91
|
finally:
|
93
92
|
occupancy_period = cast(Interval, pendulum.now("UTC") - acquisition_time)
|
94
|
-
|
95
|
-
_release_concurrency_slots,
|
93
|
+
_release_concurrency_slots(
|
96
94
|
names,
|
97
95
|
occupy,
|
98
96
|
occupancy_period.total_seconds(),
|
97
|
+
_sync=True,
|
99
98
|
)
|
100
99
|
_emit_concurrency_release_events(limits, occupy, emitted_events)
|
101
100
|
|
@@ -104,7 +103,8 @@ def rate_limit(
|
|
104
103
|
names: Union[str, List[str]],
|
105
104
|
occupy: int = 1,
|
106
105
|
timeout_seconds: Optional[float] = None,
|
107
|
-
create_if_missing: Optional[bool] =
|
106
|
+
create_if_missing: Optional[bool] = None,
|
107
|
+
strict: bool = False,
|
108
108
|
) -> None:
|
109
109
|
"""Block execution until an `occupy` number of slots of the concurrency
|
110
110
|
limits given in `names` are acquired. Requires that all given concurrency
|
@@ -115,31 +115,25 @@ def rate_limit(
|
|
115
115
|
occupy: The number of slots to acquire and hold from each limit.
|
116
116
|
timeout_seconds: The number of seconds to wait for the slots to be acquired before
|
117
117
|
raising a `TimeoutError`. A timeout of `None` will wait indefinitely.
|
118
|
-
|
118
|
+
strict: A boolean specifying whether to raise an error if the concurrency limit does not exist.
|
119
|
+
Defaults to `False`.
|
120
|
+
|
121
|
+
Raises:
|
122
|
+
TimeoutError: If the slots are not acquired within the given timeout.
|
123
|
+
ConcurrencySlotAcquisitionError: If the concurrency limit does not exist and `strict` is `True`.
|
119
124
|
"""
|
120
125
|
if not names:
|
121
126
|
return
|
122
127
|
|
123
128
|
names = names if isinstance(names, list) else [names]
|
124
129
|
|
125
|
-
limits =
|
126
|
-
_acquire_concurrency_slots,
|
130
|
+
limits = _acquire_concurrency_slots(
|
127
131
|
names,
|
128
132
|
occupy,
|
129
133
|
mode="rate_limit",
|
130
134
|
timeout_seconds=timeout_seconds,
|
131
135
|
create_if_missing=create_if_missing,
|
136
|
+
strict=strict,
|
137
|
+
_sync=True,
|
132
138
|
)
|
133
139
|
_emit_concurrency_acquisition_events(limits, occupy)
|
134
|
-
|
135
|
-
|
136
|
-
def _call_async_function_from_sync(
|
137
|
-
fn: Callable[..., Awaitable[T]], *args: Any, **kwargs: Any
|
138
|
-
) -> T:
|
139
|
-
loop = get_running_loop()
|
140
|
-
call = create_call(fn, *args, **kwargs)
|
141
|
-
|
142
|
-
if loop is not None:
|
143
|
-
return from_sync.call_soon_in_loop_thread(call).result()
|
144
|
-
else:
|
145
|
-
return call() # type: ignore [return-value]
|
@@ -16,6 +16,7 @@ except ImportError:
|
|
16
16
|
from pendulum.period import Period as Interval # type: ignore
|
17
17
|
|
18
18
|
from prefect.client.orchestration import get_client
|
19
|
+
from prefect.utilities.asyncutils import sync_compatible
|
19
20
|
|
20
21
|
from .context import ConcurrencyContext
|
21
22
|
from .events import (
|
@@ -98,6 +99,7 @@ async def concurrency(
|
|
98
99
|
_emit_concurrency_release_events(limits, emitted_events, task_run_id)
|
99
100
|
|
100
101
|
|
102
|
+
@sync_compatible
|
101
103
|
async def _acquire_concurrency_slots(
|
102
104
|
names: List[str],
|
103
105
|
task_run_id: UUID,
|
@@ -120,6 +122,7 @@ async def _acquire_concurrency_slots(
|
|
120
122
|
return _response_to_concurrency_limit_response(response_or_exception)
|
121
123
|
|
122
124
|
|
125
|
+
@sync_compatible
|
123
126
|
async def _release_concurrency_slots(
|
124
127
|
names: List[str],
|
125
128
|
task_run_id: UUID,
|
prefect/concurrency/v1/sync.py
CHANGED
@@ -12,7 +12,6 @@ from uuid import UUID
|
|
12
12
|
import pendulum
|
13
13
|
|
14
14
|
from ...client.schemas.responses import MinimalConcurrencyLimitResponse
|
15
|
-
from ..sync import _call_async_function_from_sync
|
16
15
|
|
17
16
|
try:
|
18
17
|
from pendulum import Interval
|
@@ -70,11 +69,11 @@ def concurrency(
|
|
70
69
|
|
71
70
|
names = names if isinstance(names, list) else [names]
|
72
71
|
|
73
|
-
limits: List[MinimalConcurrencyLimitResponse] =
|
74
|
-
_acquire_concurrency_slots,
|
72
|
+
limits: List[MinimalConcurrencyLimitResponse] = _acquire_concurrency_slots(
|
75
73
|
names,
|
76
74
|
timeout_seconds=timeout_seconds,
|
77
75
|
task_run_id=task_run_id,
|
76
|
+
_sync=True,
|
78
77
|
)
|
79
78
|
acquisition_time = pendulum.now("UTC")
|
80
79
|
emitted_events = _emit_concurrency_acquisition_events(limits, task_run_id)
|
@@ -83,10 +82,10 @@ def concurrency(
|
|
83
82
|
yield
|
84
83
|
finally:
|
85
84
|
occupancy_period = cast(Interval, pendulum.now("UTC") - acquisition_time)
|
86
|
-
|
87
|
-
_release_concurrency_slots,
|
85
|
+
_release_concurrency_slots(
|
88
86
|
names,
|
89
87
|
task_run_id,
|
90
88
|
occupancy_period.total_seconds(),
|
89
|
+
_sync=True,
|
91
90
|
)
|
92
91
|
_emit_concurrency_release_events(limits, emitted_events, task_run_id)
|
prefect/context.py
CHANGED
@@ -40,7 +40,7 @@ from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_c
|
|
40
40
|
from prefect.client.schemas import FlowRun, TaskRun
|
41
41
|
from prefect.events.worker import EventsWorker
|
42
42
|
from prefect.exceptions import MissingContextError
|
43
|
-
from prefect.results import ResultStore
|
43
|
+
from prefect.results import ResultStore, get_default_persist_setting
|
44
44
|
from prefect.settings import PREFECT_HOME, Profile, Settings
|
45
45
|
from prefect.states import State
|
46
46
|
from prefect.task_runners import TaskRunner
|
@@ -343,6 +343,7 @@ class EngineContext(RunContext):
|
|
343
343
|
|
344
344
|
# Result handling
|
345
345
|
result_store: ResultStore
|
346
|
+
persist_result: bool = Field(default_factory=get_default_persist_setting)
|
346
347
|
|
347
348
|
# Counter for task calls allowing unique
|
348
349
|
task_run_dynamic_keys: Dict[str, int] = Field(default_factory=dict)
|
@@ -372,6 +373,7 @@ class EngineContext(RunContext):
|
|
372
373
|
"start_time",
|
373
374
|
"input_keyset",
|
374
375
|
"result_store",
|
376
|
+
"persist_result",
|
375
377
|
},
|
376
378
|
exclude_unset=True,
|
377
379
|
)
|
@@ -397,6 +399,7 @@ class TaskRunContext(RunContext):
|
|
397
399
|
|
398
400
|
# Result handling
|
399
401
|
result_store: ResultStore
|
402
|
+
persist_result: bool = Field(default_factory=get_default_persist_setting)
|
400
403
|
|
401
404
|
__var__ = ContextVar("task_run")
|
402
405
|
|
@@ -410,6 +413,7 @@ class TaskRunContext(RunContext):
|
|
410
413
|
"start_time",
|
411
414
|
"input_keyset",
|
412
415
|
"result_store",
|
416
|
+
"persist_result",
|
413
417
|
},
|
414
418
|
exclude_unset=True,
|
415
419
|
)
|
prefect/deployments/runner.py
CHANGED
@@ -462,6 +462,7 @@ class RunnerDeployment(BaseModel):
|
|
462
462
|
paused: Whether or not to set this deployment as paused.
|
463
463
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
464
464
|
Used to define multiple schedules or additional scheduling options like `timezone`.
|
465
|
+
concurrency_limit: The maximum number of concurrent runs this deployment will allow.
|
465
466
|
triggers: A list of triggers that should kick of a run of this flow.
|
466
467
|
parameters: A dictionary of default parameter values to pass to runs of this flow.
|
467
468
|
description: A description for the created deployment. Defaults to the flow's
|
prefect/events/actions.py
CHANGED
@@ -113,6 +113,12 @@ class CancelFlowRun(Action):
|
|
113
113
|
type: Literal["cancel-flow-run"] = "cancel-flow-run"
|
114
114
|
|
115
115
|
|
116
|
+
class ResumeFlowRun(Action):
|
117
|
+
"""Resumes a flow run associated with the trigger"""
|
118
|
+
|
119
|
+
type: Literal["resume-flow-run"] = "resume-flow-run"
|
120
|
+
|
121
|
+
|
116
122
|
class SuspendFlowRun(Action):
|
117
123
|
"""Suspends a flow run associated with the trigger"""
|
118
124
|
|