prefect-client 3.0.2__py3-none-any.whl → 3.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/retries.py +1 -3
- prefect/_internal/schemas/validators.py +1 -1
- prefect/blocks/core.py +5 -4
- prefect/blocks/webhook.py +9 -1
- prefect/cache_policies.py +98 -28
- prefect/client/cloud.py +9 -0
- prefect/client/orchestration.py +3 -0
- prefect/client/schemas/actions.py +8 -0
- prefect/client/schemas/filters.py +4 -2
- prefect/client/schemas/objects.py +36 -1
- prefect/client/schemas/responses.py +15 -1
- prefect/client/subscriptions.py +3 -3
- prefect/context.py +1 -5
- prefect/deployments/base.py +12 -0
- prefect/deployments/runner.py +42 -5
- prefect/events/clients.py +40 -22
- prefect/filesystems.py +26 -1
- prefect/flows.py +25 -12
- prefect/locking/filesystem.py +3 -3
- prefect/plugins.py +9 -1
- prefect/results.py +96 -11
- prefect/runner/runner.py +12 -45
- prefect/settings.py +36 -0
- prefect/task_engine.py +28 -7
- prefect/transactions.py +11 -8
- prefect/utilities/asyncutils.py +7 -0
- prefect/utilities/collections.py +3 -2
- prefect/utilities/engine.py +4 -1
- prefect/workers/base.py +7 -62
- {prefect_client-3.0.2.dist-info → prefect_client-3.0.4.dist-info}/METADATA +1 -1
- {prefect_client-3.0.2.dist-info → prefect_client-3.0.4.dist-info}/RECORD +34 -34
- {prefect_client-3.0.2.dist-info → prefect_client-3.0.4.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.2.dist-info → prefect_client-3.0.4.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.2.dist-info → prefect_client-3.0.4.dist-info}/top_level.txt +0 -0
prefect/_internal/retries.py
CHANGED
@@ -2,11 +2,9 @@ import asyncio
|
|
2
2
|
from functools import wraps
|
3
3
|
from typing import Any, Callable, Tuple, Type
|
4
4
|
|
5
|
-
from prefect.
|
5
|
+
from prefect._internal._logging import logger
|
6
6
|
from prefect.utilities.math import clamped_poisson_interval
|
7
7
|
|
8
|
-
logger = get_logger("retries")
|
9
|
-
|
10
8
|
|
11
9
|
def exponential_backoff_with_jitter(
|
12
10
|
attempt: int, base_delay: float, max_delay: float
|
@@ -70,7 +70,7 @@ def validate_schema(schema: dict):
|
|
70
70
|
try:
|
71
71
|
if schema is not None:
|
72
72
|
# Most closely matches the schemas generated by pydantic
|
73
|
-
jsonschema.
|
73
|
+
jsonschema.Draft202012Validator.check_schema(schema)
|
74
74
|
except jsonschema.SchemaError as exc:
|
75
75
|
raise ValueError(
|
76
76
|
"The provided schema is not a valid json schema. Schema error:"
|
prefect/blocks/core.py
CHANGED
@@ -40,7 +40,6 @@ from pydantic import (
|
|
40
40
|
from pydantic.json_schema import GenerateJsonSchema
|
41
41
|
from typing_extensions import Literal, ParamSpec, Self, get_args
|
42
42
|
|
43
|
-
import prefect
|
44
43
|
import prefect.exceptions
|
45
44
|
from prefect.client.schemas import (
|
46
45
|
DEFAULT_BLOCK_SCHEMA_VERSION,
|
@@ -52,6 +51,7 @@ from prefect.client.schemas import (
|
|
52
51
|
from prefect.client.utilities import inject_client
|
53
52
|
from prefect.events import emit_event
|
54
53
|
from prefect.logging.loggers import disable_logger
|
54
|
+
from prefect.plugins import load_prefect_collections
|
55
55
|
from prefect.types import SecretDict
|
56
56
|
from prefect.utilities.asyncutils import sync_compatible
|
57
57
|
from prefect.utilities.collections import listrepr, remove_nested_keys, visit_collection
|
@@ -86,7 +86,7 @@ class InvalidBlockRegistration(Exception):
|
|
86
86
|
"""
|
87
87
|
|
88
88
|
|
89
|
-
def _collect_nested_reference_strings(obj: Dict):
|
89
|
+
def _collect_nested_reference_strings(obj: Dict) -> List[str]:
|
90
90
|
"""
|
91
91
|
Collects all nested reference strings (e.g. #/definitions/Model) from a given object.
|
92
92
|
"""
|
@@ -739,9 +739,10 @@ class Block(BaseModel, ABC):
|
|
739
739
|
"""
|
740
740
|
Retrieve the block class implementation given a key.
|
741
741
|
"""
|
742
|
+
|
742
743
|
# Ensure collections are imported and have the opportunity to register types
|
743
|
-
# before looking up the block class
|
744
|
-
|
744
|
+
# before looking up the block class, but only do this once
|
745
|
+
load_prefect_collections()
|
745
746
|
|
746
747
|
return lookup_type(cls, key)
|
747
748
|
|
prefect/blocks/webhook.py
CHANGED
@@ -11,6 +11,7 @@ from prefect.utilities.urls import validate_restricted_url
|
|
11
11
|
# Use a global HTTP transport to maintain a process-wide connection pool for
|
12
12
|
# interservice requests
|
13
13
|
_http_transport = AsyncHTTPTransport()
|
14
|
+
_insecure_http_transport = AsyncHTTPTransport(verify=False)
|
14
15
|
|
15
16
|
|
16
17
|
class Webhook(Block):
|
@@ -44,9 +45,16 @@ class Webhook(Block):
|
|
44
45
|
default=True,
|
45
46
|
description="Whether to allow notifications to private URLs. Defaults to True.",
|
46
47
|
)
|
48
|
+
verify: bool = Field(
|
49
|
+
default=True,
|
50
|
+
description="Whether or not to enforce a secure connection to the webhook.",
|
51
|
+
)
|
47
52
|
|
48
53
|
def block_initialization(self):
|
49
|
-
self.
|
54
|
+
if self.verify:
|
55
|
+
self._client = AsyncClient(transport=_http_transport)
|
56
|
+
else:
|
57
|
+
self._client = AsyncClient(transport=_insecure_http_transport)
|
50
58
|
|
51
59
|
async def call(self, payload: Optional[dict] = None) -> Response:
|
52
60
|
"""
|
prefect/cache_policies.py
CHANGED
@@ -1,10 +1,19 @@
|
|
1
1
|
import inspect
|
2
|
-
from
|
3
|
-
from
|
2
|
+
from copy import deepcopy
|
3
|
+
from dataclasses import dataclass, field
|
4
|
+
from pathlib import Path
|
5
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Union
|
6
|
+
|
7
|
+
from typing_extensions import Self
|
4
8
|
|
5
9
|
from prefect.context import TaskRunContext
|
6
10
|
from prefect.utilities.hashing import hash_objects
|
7
11
|
|
12
|
+
if TYPE_CHECKING:
|
13
|
+
from prefect.filesystems import WritableFileSystem
|
14
|
+
from prefect.locking.protocol import LockManager
|
15
|
+
from prefect.transactions import IsolationLevel
|
16
|
+
|
8
17
|
|
9
18
|
@dataclass
|
10
19
|
class CachePolicy:
|
@@ -12,6 +21,14 @@ class CachePolicy:
|
|
12
21
|
Base class for all cache policies.
|
13
22
|
"""
|
14
23
|
|
24
|
+
key_storage: Union["WritableFileSystem", str, Path, None] = None
|
25
|
+
isolation_level: Union[
|
26
|
+
Literal["READ_COMMITTED", "SERIALIZABLE"],
|
27
|
+
"IsolationLevel",
|
28
|
+
None,
|
29
|
+
] = None
|
30
|
+
lock_manager: Optional["LockManager"] = None
|
31
|
+
|
15
32
|
@classmethod
|
16
33
|
def from_cache_key_fn(
|
17
34
|
cls, cache_key_fn: Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
|
@@ -21,6 +38,37 @@ class CachePolicy:
|
|
21
38
|
"""
|
22
39
|
return CacheKeyFnPolicy(cache_key_fn=cache_key_fn)
|
23
40
|
|
41
|
+
def configure(
|
42
|
+
self,
|
43
|
+
key_storage: Union["WritableFileSystem", str, Path, None] = None,
|
44
|
+
lock_manager: Optional["LockManager"] = None,
|
45
|
+
isolation_level: Union[
|
46
|
+
Literal["READ_COMMITTED", "SERIALIZABLE"], "IsolationLevel", None
|
47
|
+
] = None,
|
48
|
+
) -> Self:
|
49
|
+
"""
|
50
|
+
Configure the cache policy with the given key storage, lock manager, and isolation level.
|
51
|
+
|
52
|
+
Args:
|
53
|
+
key_storage: The storage to use for cache keys. If not provided,
|
54
|
+
the current key storage will be used.
|
55
|
+
lock_manager: The lock manager to use for the cache policy. If not provided,
|
56
|
+
the current lock manager will be used.
|
57
|
+
isolation_level: The isolation level to use for the cache policy. If not provided,
|
58
|
+
the current isolation level will be used.
|
59
|
+
|
60
|
+
Returns:
|
61
|
+
A new cache policy with the given key storage, lock manager, and isolation level.
|
62
|
+
"""
|
63
|
+
new = deepcopy(self)
|
64
|
+
if key_storage is not None:
|
65
|
+
new.key_storage = key_storage
|
66
|
+
if lock_manager is not None:
|
67
|
+
new.lock_manager = lock_manager
|
68
|
+
if isolation_level is not None:
|
69
|
+
new.isolation_level = isolation_level
|
70
|
+
return new
|
71
|
+
|
24
72
|
def compute_key(
|
25
73
|
self,
|
26
74
|
task_ctx: TaskRunContext,
|
@@ -30,35 +78,48 @@ class CachePolicy:
|
|
30
78
|
) -> Optional[str]:
|
31
79
|
raise NotImplementedError
|
32
80
|
|
33
|
-
def __sub__(self, other: str) -> "
|
81
|
+
def __sub__(self, other: str) -> "CachePolicy":
|
34
82
|
if not isinstance(other, str):
|
35
83
|
raise TypeError("Can only subtract strings from key policies.")
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
new = Inputs(exclude=[other])
|
41
|
-
policies = self.policies or []
|
42
|
-
return CompoundCachePolicy(policies=policies + [new])
|
43
|
-
else:
|
44
|
-
new = Inputs(exclude=[other])
|
45
|
-
return CompoundCachePolicy(policies=[self, new])
|
46
|
-
|
47
|
-
def __add__(self, other: "CachePolicy") -> "CompoundCachePolicy":
|
84
|
+
new = Inputs(exclude=[other])
|
85
|
+
return CompoundCachePolicy(policies=[self, new])
|
86
|
+
|
87
|
+
def __add__(self, other: "CachePolicy") -> "CachePolicy":
|
48
88
|
# adding _None is a no-op
|
49
89
|
if isinstance(other, _None):
|
50
90
|
return self
|
51
|
-
elif isinstance(self, _None):
|
52
|
-
return other
|
53
91
|
|
54
|
-
if
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
92
|
+
if (
|
93
|
+
other.key_storage is not None
|
94
|
+
and self.key_storage is not None
|
95
|
+
and other.key_storage != self.key_storage
|
96
|
+
):
|
97
|
+
raise ValueError(
|
98
|
+
"Cannot add CachePolicies with different storage locations."
|
99
|
+
)
|
100
|
+
if (
|
101
|
+
other.isolation_level is not None
|
102
|
+
and self.isolation_level is not None
|
103
|
+
and other.isolation_level != self.isolation_level
|
104
|
+
):
|
105
|
+
raise ValueError(
|
106
|
+
"Cannot add CachePolicies with different isolation levels."
|
107
|
+
)
|
108
|
+
if (
|
109
|
+
other.lock_manager is not None
|
110
|
+
and self.lock_manager is not None
|
111
|
+
and other.lock_manager != self.lock_manager
|
112
|
+
):
|
113
|
+
raise ValueError(
|
114
|
+
"Cannot add CachePolicies with different lock implementations."
|
115
|
+
)
|
116
|
+
|
117
|
+
return CompoundCachePolicy(
|
118
|
+
policies=[self, other],
|
119
|
+
key_storage=self.key_storage or other.key_storage,
|
120
|
+
isolation_level=self.isolation_level or other.isolation_level,
|
121
|
+
lock_manager=self.lock_manager or other.lock_manager,
|
122
|
+
)
|
62
123
|
|
63
124
|
|
64
125
|
@dataclass
|
@@ -93,7 +154,7 @@ class CompoundCachePolicy(CachePolicy):
|
|
93
154
|
Any keys that return `None` will be ignored.
|
94
155
|
"""
|
95
156
|
|
96
|
-
policies:
|
157
|
+
policies: List[CachePolicy] = field(default_factory=list)
|
97
158
|
|
98
159
|
def compute_key(
|
99
160
|
self,
|
@@ -103,7 +164,7 @@ class CompoundCachePolicy(CachePolicy):
|
|
103
164
|
**kwargs,
|
104
165
|
) -> Optional[str]:
|
105
166
|
keys = []
|
106
|
-
for policy in self.policies
|
167
|
+
for policy in self.policies:
|
107
168
|
policy_key = policy.compute_key(
|
108
169
|
task_ctx=task_ctx,
|
109
170
|
inputs=inputs,
|
@@ -133,6 +194,10 @@ class _None(CachePolicy):
|
|
133
194
|
) -> Optional[str]:
|
134
195
|
return None
|
135
196
|
|
197
|
+
def __add__(self, other: "CachePolicy") -> "CachePolicy":
|
198
|
+
# adding _None is a no-op
|
199
|
+
return other
|
200
|
+
|
136
201
|
|
137
202
|
@dataclass
|
138
203
|
class TaskSource(CachePolicy):
|
@@ -208,7 +273,7 @@ class Inputs(CachePolicy):
|
|
208
273
|
Policy that computes a cache key based on a hash of the runtime inputs provided to the task..
|
209
274
|
"""
|
210
275
|
|
211
|
-
exclude:
|
276
|
+
exclude: List[str] = field(default_factory=list)
|
212
277
|
|
213
278
|
def compute_key(
|
214
279
|
self,
|
@@ -230,6 +295,11 @@ class Inputs(CachePolicy):
|
|
230
295
|
|
231
296
|
return hash_objects(hashed_inputs)
|
232
297
|
|
298
|
+
def __sub__(self, other: str) -> "CachePolicy":
|
299
|
+
if not isinstance(other, str):
|
300
|
+
raise TypeError("Can only subtract strings from key policies.")
|
301
|
+
return Inputs(exclude=self.exclude + [other])
|
302
|
+
|
233
303
|
|
234
304
|
INPUTS = Inputs()
|
235
305
|
NONE = _None()
|
prefect/client/cloud.py
CHANGED
@@ -17,6 +17,7 @@ from prefect.client.schemas.objects import (
|
|
17
17
|
from prefect.exceptions import ObjectNotFound, PrefectException
|
18
18
|
from prefect.settings import (
|
19
19
|
PREFECT_API_KEY,
|
20
|
+
PREFECT_API_URL,
|
20
21
|
PREFECT_CLOUD_API_URL,
|
21
22
|
PREFECT_UNIT_TEST_MODE,
|
22
23
|
)
|
@@ -110,6 +111,14 @@ class CloudClient:
|
|
110
111
|
)
|
111
112
|
return workspaces
|
112
113
|
|
114
|
+
async def read_current_workspace(self) -> Workspace:
|
115
|
+
workspaces = await self.read_workspaces()
|
116
|
+
current_api_url = PREFECT_API_URL.value()
|
117
|
+
for workspace in workspaces:
|
118
|
+
if workspace.api_url() == current_api_url.rstrip("/"):
|
119
|
+
return workspace
|
120
|
+
raise ValueError("Current workspace not found")
|
121
|
+
|
113
122
|
async def read_worker_metadata(self) -> Dict[str, Any]:
|
114
123
|
response = await self.get(
|
115
124
|
f"{self.workspace_base_url}/collections/work_pool_types"
|
prefect/client/orchestration.py
CHANGED
@@ -86,6 +86,7 @@ from prefect.client.schemas.objects import (
|
|
86
86
|
BlockSchema,
|
87
87
|
BlockType,
|
88
88
|
ConcurrencyLimit,
|
89
|
+
ConcurrencyOptions,
|
89
90
|
Constant,
|
90
91
|
DeploymentSchedule,
|
91
92
|
Flow,
|
@@ -1639,6 +1640,7 @@ class PrefectClient:
|
|
1639
1640
|
version: Optional[str] = None,
|
1640
1641
|
schedules: Optional[List[DeploymentScheduleCreate]] = None,
|
1641
1642
|
concurrency_limit: Optional[int] = None,
|
1643
|
+
concurrency_options: Optional[ConcurrencyOptions] = None,
|
1642
1644
|
parameters: Optional[Dict[str, Any]] = None,
|
1643
1645
|
description: Optional[str] = None,
|
1644
1646
|
work_queue_name: Optional[str] = None,
|
@@ -1697,6 +1699,7 @@ class PrefectClient:
|
|
1697
1699
|
paused=paused,
|
1698
1700
|
schedules=schedules or [],
|
1699
1701
|
concurrency_limit=concurrency_limit,
|
1702
|
+
concurrency_options=concurrency_options,
|
1700
1703
|
pull_steps=pull_steps,
|
1701
1704
|
enforce_parameter_schema=enforce_parameter_schema,
|
1702
1705
|
)
|
@@ -161,6 +161,10 @@ class DeploymentCreate(ActionBaseModel):
|
|
161
161
|
default=None,
|
162
162
|
description="The concurrency limit for the deployment.",
|
163
163
|
)
|
164
|
+
concurrency_options: Optional[objects.ConcurrencyOptions] = Field(
|
165
|
+
default=None,
|
166
|
+
description="The concurrency options for the deployment.",
|
167
|
+
)
|
164
168
|
enforce_parameter_schema: Optional[bool] = Field(
|
165
169
|
default=None,
|
166
170
|
description=(
|
@@ -237,6 +241,10 @@ class DeploymentUpdate(ActionBaseModel):
|
|
237
241
|
default=None,
|
238
242
|
description="The concurrency limit for the deployment.",
|
239
243
|
)
|
244
|
+
concurrency_options: Optional[objects.ConcurrencyOptions] = Field(
|
245
|
+
default=None,
|
246
|
+
description="The concurrency options for the deployment.",
|
247
|
+
)
|
240
248
|
tags: List[str] = Field(default_factory=list)
|
241
249
|
work_queue_name: Optional[str] = Field(None)
|
242
250
|
work_pool_name: Optional[str] = Field(
|
@@ -506,7 +506,7 @@ class DeploymentFilterTags(PrefectBaseModel, OperatorMixin):
|
|
506
506
|
|
507
507
|
|
508
508
|
class DeploymentFilterConcurrencyLimit(PrefectBaseModel):
|
509
|
-
"""
|
509
|
+
"""DEPRECATED: Prefer `Deployment.concurrency_limit_id` over `Deployment.concurrency_limit`."""
|
510
510
|
|
511
511
|
ge_: Optional[int] = Field(
|
512
512
|
default=None,
|
@@ -538,7 +538,9 @@ class DeploymentFilter(PrefectBaseModel, OperatorMixin):
|
|
538
538
|
default=None, description="Filter criteria for `Deployment.work_queue_name`"
|
539
539
|
)
|
540
540
|
concurrency_limit: Optional[DeploymentFilterConcurrencyLimit] = Field(
|
541
|
-
default=None,
|
541
|
+
default=None,
|
542
|
+
description="DEPRECATED: Prefer `Deployment.concurrency_limit_id` over `Deployment.concurrency_limit`. If provided, will be ignored for backwards-compatibility. Will be removed after December 2024.",
|
543
|
+
deprecated=True,
|
542
544
|
)
|
543
545
|
|
544
546
|
|
@@ -141,6 +141,30 @@ class WorkQueueStatus(AutoEnum):
|
|
141
141
|
PAUSED = AutoEnum.auto()
|
142
142
|
|
143
143
|
|
144
|
+
class ConcurrencyLimitStrategy(AutoEnum):
|
145
|
+
"""Enumeration of concurrency limit strategies."""
|
146
|
+
|
147
|
+
ENQUEUE = AutoEnum.auto()
|
148
|
+
CANCEL_NEW = AutoEnum.auto()
|
149
|
+
|
150
|
+
|
151
|
+
class ConcurrencyOptions(PrefectBaseModel):
|
152
|
+
"""
|
153
|
+
Class for storing the concurrency config in database.
|
154
|
+
"""
|
155
|
+
|
156
|
+
collision_strategy: ConcurrencyLimitStrategy
|
157
|
+
|
158
|
+
|
159
|
+
class ConcurrencyLimitConfig(PrefectBaseModel):
|
160
|
+
"""
|
161
|
+
Class for storing the concurrency limit config in database.
|
162
|
+
"""
|
163
|
+
|
164
|
+
limit: int
|
165
|
+
collision_strategy: ConcurrencyLimitStrategy = ConcurrencyLimitStrategy.ENQUEUE
|
166
|
+
|
167
|
+
|
144
168
|
class StateDetails(PrefectBaseModel):
|
145
169
|
flow_run_id: Optional[UUID] = None
|
146
170
|
task_run_id: Optional[UUID] = None
|
@@ -207,7 +231,9 @@ class State(ObjectBaseModel, Generic[R]):
|
|
207
231
|
|
208
232
|
Args:
|
209
233
|
raise_on_failure: a boolean specifying whether to raise an exception
|
210
|
-
if the state is of type `FAILED` and the underlying data is an exception
|
234
|
+
if the state is of type `FAILED` and the underlying data is an exception. When flow
|
235
|
+
was run in a different memory space (using `run_deployment`), this will only raise
|
236
|
+
if `fetch` is `True`.
|
211
237
|
fetch: a boolean specifying whether to resolve references to persisted
|
212
238
|
results into data. For synchronous users, this defaults to `True`.
|
213
239
|
For asynchronous users, this defaults to `False` for backwards
|
@@ -273,6 +299,15 @@ class State(ObjectBaseModel, Generic[R]):
|
|
273
299
|
>>> state = await my_flow(return_state=True)
|
274
300
|
>>> await state.result()
|
275
301
|
hello
|
302
|
+
|
303
|
+
Get the result with `raise_on_failure` from a flow run in a different memory space
|
304
|
+
|
305
|
+
>>> @flow
|
306
|
+
>>> async def my_flow():
|
307
|
+
>>> raise ValueError("oh no!")
|
308
|
+
>>> my_flow.deploy("my_deployment/my_flow")
|
309
|
+
>>> flow_run = run_deployment("my_deployment/my_flow")
|
310
|
+
>>> await flow_run.state.result(raise_on_failure=True, fetch=True) # Raises `ValueError("oh no!")`
|
276
311
|
"""
|
277
312
|
from prefect.states import get_state_result
|
278
313
|
|
@@ -314,11 +314,25 @@ class DeploymentResponse(ObjectBaseModel):
|
|
314
314
|
default=..., description="The flow id associated with the deployment."
|
315
315
|
)
|
316
316
|
concurrency_limit: Optional[int] = Field(
|
317
|
-
default=None,
|
317
|
+
default=None,
|
318
|
+
description="DEPRECATED: Prefer `global_concurrency_limit`. Will always be None for backwards compatibility. Will be removed after December 2024.",
|
319
|
+
deprecated=True,
|
320
|
+
)
|
321
|
+
global_concurrency_limit: Optional["GlobalConcurrencyLimitResponse"] = Field(
|
322
|
+
default=None,
|
323
|
+
description="The global concurrency limit object for enforcing the maximum number of flow runs that can be active at once.",
|
324
|
+
)
|
325
|
+
concurrency_options: Optional[objects.ConcurrencyOptions] = Field(
|
326
|
+
default=None,
|
327
|
+
description="The concurrency options for the deployment.",
|
318
328
|
)
|
319
329
|
paused: bool = Field(
|
320
330
|
default=False, description="Whether or not the deployment is paused."
|
321
331
|
)
|
332
|
+
concurrency_options: Optional[objects.ConcurrencyOptions] = Field(
|
333
|
+
default=None,
|
334
|
+
description="The concurrency options for the deployment.",
|
335
|
+
)
|
322
336
|
schedules: List[objects.DeploymentSchedule] = Field(
|
323
337
|
default_factory=list, description="A list of schedules for the deployment."
|
324
338
|
)
|
prefect/client/subscriptions.py
CHANGED
@@ -84,13 +84,13 @@ class Subscription(Generic[S]):
|
|
84
84
|
AssertionError,
|
85
85
|
websockets.exceptions.ConnectionClosedError,
|
86
86
|
) as e:
|
87
|
-
if isinstance(e, AssertionError) or e.code == WS_1008_POLICY_VIOLATION:
|
87
|
+
if isinstance(e, AssertionError) or e.rcvd.code == WS_1008_POLICY_VIOLATION:
|
88
88
|
if isinstance(e, AssertionError):
|
89
89
|
reason = e.args[0]
|
90
90
|
elif isinstance(e, websockets.exceptions.ConnectionClosedError):
|
91
|
-
reason = e.reason
|
91
|
+
reason = e.rcvd.reason
|
92
92
|
|
93
|
-
if isinstance(e, AssertionError) or e.code == WS_1008_POLICY_VIOLATION:
|
93
|
+
if isinstance(e, AssertionError) or e.rcvd.code == WS_1008_POLICY_VIOLATION:
|
94
94
|
raise Exception(
|
95
95
|
"Unable to authenticate to the subscription. Please "
|
96
96
|
"ensure the provided `PREFECT_API_KEY` you are using is "
|
prefect/context.py
CHANGED
@@ -9,7 +9,6 @@ For more user-accessible information about the current run, see [`prefect.runtim
|
|
9
9
|
import os
|
10
10
|
import sys
|
11
11
|
import warnings
|
12
|
-
import weakref
|
13
12
|
from contextlib import ExitStack, asynccontextmanager, contextmanager
|
14
13
|
from contextvars import ContextVar, Token
|
15
14
|
from pathlib import Path
|
@@ -353,10 +352,7 @@ class EngineContext(RunContext):
|
|
353
352
|
|
354
353
|
# Tracking for result from task runs in this flow run for dependency tracking
|
355
354
|
# Holds the ID of the object returned by the task run and task run state
|
356
|
-
|
357
|
-
task_run_results: Mapping[int, State] = Field(
|
358
|
-
default_factory=weakref.WeakValueDictionary
|
359
|
-
)
|
355
|
+
task_run_results: Mapping[int, State] = Field(default_factory=dict)
|
360
356
|
|
361
357
|
# Events worker to emit events
|
362
358
|
events: Optional[EventsWorker] = None
|
prefect/deployments/base.py
CHANGED
@@ -20,6 +20,7 @@ import yaml
|
|
20
20
|
from ruamel.yaml import YAML
|
21
21
|
|
22
22
|
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
23
|
+
from prefect.client.schemas.objects import ConcurrencyLimitStrategy
|
23
24
|
from prefect.client.schemas.schedules import IntervalSchedule
|
24
25
|
from prefect.logging import get_logger
|
25
26
|
from prefect.settings import PREFECT_DEBUG_MODE
|
@@ -277,6 +278,17 @@ def _format_deployment_for_saving_to_prefect_file(
|
|
277
278
|
|
278
279
|
deployment["schedules"] = schedules
|
279
280
|
|
281
|
+
if deployment.get("concurrency_limit"):
|
282
|
+
concurrency_limit = deployment["concurrency_limit"]
|
283
|
+
if isinstance(concurrency_limit, dict):
|
284
|
+
if isinstance(
|
285
|
+
concurrency_limit["collision_strategy"], ConcurrencyLimitStrategy
|
286
|
+
):
|
287
|
+
concurrency_limit["collision_strategy"] = str(
|
288
|
+
concurrency_limit["collision_strategy"].value
|
289
|
+
)
|
290
|
+
deployment["concurrency_limit"] = concurrency_limit
|
291
|
+
|
280
292
|
return deployment
|
281
293
|
|
282
294
|
|
prefect/deployments/runner.py
CHANGED
@@ -54,6 +54,7 @@ from prefect._internal.schemas.validators import (
|
|
54
54
|
)
|
55
55
|
from prefect.client.orchestration import get_client
|
56
56
|
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
57
|
+
from prefect.client.schemas.objects import ConcurrencyLimitConfig, ConcurrencyOptions
|
57
58
|
from prefect.client.schemas.schedules import (
|
58
59
|
SCHEDULE_TYPES,
|
59
60
|
construct_schedule,
|
@@ -147,6 +148,10 @@ class RunnerDeployment(BaseModel):
|
|
147
148
|
default=None,
|
148
149
|
description="The maximum number of concurrent runs of this deployment.",
|
149
150
|
)
|
151
|
+
concurrency_options: Optional[ConcurrencyOptions] = Field(
|
152
|
+
default=None,
|
153
|
+
description="The concurrency limit config for the deployment.",
|
154
|
+
)
|
150
155
|
paused: Optional[bool] = Field(
|
151
156
|
default=None, description="Whether or not the deployment is paused."
|
152
157
|
)
|
@@ -279,6 +284,7 @@ class RunnerDeployment(BaseModel):
|
|
279
284
|
paused=self.paused,
|
280
285
|
schedules=self.schedules,
|
281
286
|
concurrency_limit=self.concurrency_limit,
|
287
|
+
concurrency_options=self.concurrency_options,
|
282
288
|
parameters=self.parameters,
|
283
289
|
description=self.description,
|
284
290
|
tags=self.tags,
|
@@ -437,7 +443,7 @@ class RunnerDeployment(BaseModel):
|
|
437
443
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
438
444
|
paused: Optional[bool] = None,
|
439
445
|
schedules: Optional["FlexibleScheduleList"] = None,
|
440
|
-
concurrency_limit: Optional[int] = None,
|
446
|
+
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
441
447
|
parameters: Optional[dict] = None,
|
442
448
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
443
449
|
description: Optional[str] = None,
|
@@ -488,11 +494,20 @@ class RunnerDeployment(BaseModel):
|
|
488
494
|
|
489
495
|
job_variables = job_variables or {}
|
490
496
|
|
497
|
+
if isinstance(concurrency_limit, ConcurrencyLimitConfig):
|
498
|
+
concurrency_options = {
|
499
|
+
"collision_strategy": concurrency_limit.collision_strategy
|
500
|
+
}
|
501
|
+
concurrency_limit = concurrency_limit.limit
|
502
|
+
else:
|
503
|
+
concurrency_options = None
|
504
|
+
|
491
505
|
deployment = cls(
|
492
506
|
name=Path(name).stem,
|
493
507
|
flow_name=flow.name,
|
494
508
|
schedules=constructed_schedules,
|
495
509
|
concurrency_limit=concurrency_limit,
|
510
|
+
concurrency_options=concurrency_options,
|
496
511
|
paused=paused,
|
497
512
|
tags=tags or [],
|
498
513
|
triggers=triggers or [],
|
@@ -559,6 +574,7 @@ class RunnerDeployment(BaseModel):
|
|
559
574
|
cls,
|
560
575
|
entrypoint: str,
|
561
576
|
name: str,
|
577
|
+
flow_name: Optional[str] = None,
|
562
578
|
interval: Optional[
|
563
579
|
Union[Iterable[Union[int, float, timedelta]], int, float, timedelta]
|
564
580
|
] = None,
|
@@ -566,7 +582,7 @@ class RunnerDeployment(BaseModel):
|
|
566
582
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
567
583
|
paused: Optional[bool] = None,
|
568
584
|
schedules: Optional["FlexibleScheduleList"] = None,
|
569
|
-
concurrency_limit: Optional[int] = None,
|
585
|
+
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
570
586
|
parameters: Optional[dict] = None,
|
571
587
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
572
588
|
description: Optional[str] = None,
|
@@ -584,6 +600,7 @@ class RunnerDeployment(BaseModel):
|
|
584
600
|
entrypoint: The path to a file containing a flow and the name of the flow function in
|
585
601
|
the format `./path/to/file.py:flow_func_name`.
|
586
602
|
name: A name for the deployment
|
603
|
+
flow_name: The name of the flow to deploy
|
587
604
|
interval: An interval on which to execute the current flow. Accepts either a number
|
588
605
|
or a timedelta object. If a number is given, it will be interpreted as seconds.
|
589
606
|
cron: A cron schedule of when to execute runs of this flow.
|
@@ -619,11 +636,20 @@ class RunnerDeployment(BaseModel):
|
|
619
636
|
schedules=schedules,
|
620
637
|
)
|
621
638
|
|
639
|
+
if isinstance(concurrency_limit, ConcurrencyLimitConfig):
|
640
|
+
concurrency_options = {
|
641
|
+
"collision_strategy": concurrency_limit.collision_strategy
|
642
|
+
}
|
643
|
+
concurrency_limit = concurrency_limit.limit
|
644
|
+
else:
|
645
|
+
concurrency_options = None
|
646
|
+
|
622
647
|
deployment = cls(
|
623
648
|
name=Path(name).stem,
|
624
|
-
flow_name=flow.name,
|
649
|
+
flow_name=flow_name or flow.name,
|
625
650
|
schedules=constructed_schedules,
|
626
651
|
concurrency_limit=concurrency_limit,
|
652
|
+
concurrency_options=concurrency_options,
|
627
653
|
paused=paused,
|
628
654
|
tags=tags or [],
|
629
655
|
triggers=triggers or [],
|
@@ -649,6 +675,7 @@ class RunnerDeployment(BaseModel):
|
|
649
675
|
storage: RunnerStorage,
|
650
676
|
entrypoint: str,
|
651
677
|
name: str,
|
678
|
+
flow_name: Optional[str] = None,
|
652
679
|
interval: Optional[
|
653
680
|
Union[Iterable[Union[int, float, timedelta]], int, float, timedelta]
|
654
681
|
] = None,
|
@@ -656,7 +683,7 @@ class RunnerDeployment(BaseModel):
|
|
656
683
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
657
684
|
paused: Optional[bool] = None,
|
658
685
|
schedules: Optional["FlexibleScheduleList"] = None,
|
659
|
-
concurrency_limit: Optional[int] = None,
|
686
|
+
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
660
687
|
parameters: Optional[dict] = None,
|
661
688
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
662
689
|
description: Optional[str] = None,
|
@@ -675,6 +702,7 @@ class RunnerDeployment(BaseModel):
|
|
675
702
|
entrypoint: The path to a file containing a flow and the name of the flow function in
|
676
703
|
the format `./path/to/file.py:flow_func_name`.
|
677
704
|
name: A name for the deployment
|
705
|
+
flow_name: The name of the flow to deploy
|
678
706
|
storage: A storage object to use for retrieving flow code. If not provided, a
|
679
707
|
URL must be provided.
|
680
708
|
interval: An interval on which to execute the current flow. Accepts either a number
|
@@ -706,6 +734,14 @@ class RunnerDeployment(BaseModel):
|
|
706
734
|
schedules=schedules,
|
707
735
|
)
|
708
736
|
|
737
|
+
if isinstance(concurrency_limit, ConcurrencyLimitConfig):
|
738
|
+
concurrency_options = {
|
739
|
+
"collision_strategy": concurrency_limit.collision_strategy
|
740
|
+
}
|
741
|
+
concurrency_limit = concurrency_limit.limit
|
742
|
+
else:
|
743
|
+
concurrency_options = None
|
744
|
+
|
709
745
|
job_variables = job_variables or {}
|
710
746
|
|
711
747
|
with tempfile.TemporaryDirectory() as tmpdir:
|
@@ -719,9 +755,10 @@ class RunnerDeployment(BaseModel):
|
|
719
755
|
|
720
756
|
deployment = cls(
|
721
757
|
name=Path(name).stem,
|
722
|
-
flow_name=flow.name,
|
758
|
+
flow_name=flow_name or flow.name,
|
723
759
|
schedules=constructed_schedules,
|
724
760
|
concurrency_limit=concurrency_limit,
|
761
|
+
concurrency_options=concurrency_options,
|
725
762
|
paused=paused,
|
726
763
|
tags=tags or [],
|
727
764
|
triggers=triggers or [],
|