prefect-client 3.0.0rc18__py3-none-any.whl → 3.0.0rc20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +0 -3
- prefect/_internal/concurrency/services.py +14 -0
- prefect/_internal/schemas/bases.py +1 -0
- prefect/blocks/core.py +41 -30
- prefect/blocks/system.py +48 -12
- prefect/client/cloud.py +56 -7
- prefect/client/collections.py +1 -1
- prefect/client/orchestration.py +111 -8
- prefect/client/schemas/objects.py +40 -2
- prefect/concurrency/asyncio.py +8 -2
- prefect/concurrency/services.py +16 -6
- prefect/concurrency/sync.py +4 -1
- prefect/concurrency/v1/__init__.py +0 -0
- prefect/concurrency/v1/asyncio.py +143 -0
- prefect/concurrency/v1/context.py +27 -0
- prefect/concurrency/v1/events.py +61 -0
- prefect/concurrency/v1/services.py +116 -0
- prefect/concurrency/v1/sync.py +92 -0
- prefect/context.py +2 -2
- prefect/deployments/flow_runs.py +0 -7
- prefect/deployments/runner.py +11 -0
- prefect/events/clients.py +41 -0
- prefect/events/related.py +72 -73
- prefect/events/utilities.py +2 -0
- prefect/events/worker.py +12 -3
- prefect/exceptions.py +6 -0
- prefect/flow_engine.py +5 -0
- prefect/flows.py +9 -2
- prefect/logging/handlers.py +4 -1
- prefect/main.py +8 -6
- prefect/records/base.py +74 -18
- prefect/records/filesystem.py +207 -0
- prefect/records/memory.py +16 -3
- prefect/records/result_store.py +19 -14
- prefect/results.py +232 -169
- prefect/runner/runner.py +7 -4
- prefect/settings.py +14 -15
- prefect/states.py +73 -18
- prefect/task_engine.py +127 -221
- prefect/task_worker.py +7 -39
- prefect/tasks.py +0 -7
- prefect/transactions.py +89 -27
- prefect/utilities/annotations.py +4 -3
- prefect/utilities/asyncutils.py +4 -4
- prefect/utilities/callables.py +1 -3
- prefect/utilities/dispatch.py +16 -11
- prefect/utilities/engine.py +1 -4
- prefect/utilities/schema_tools/hydration.py +13 -0
- prefect/workers/base.py +78 -18
- {prefect_client-3.0.0rc18.dist-info → prefect_client-3.0.0rc20.dist-info}/METADATA +3 -4
- {prefect_client-3.0.0rc18.dist-info → prefect_client-3.0.0rc20.dist-info}/RECORD +54 -48
- prefect/manifests.py +0 -21
- {prefect_client-3.0.0rc18.dist-info → prefect_client-3.0.0rc20.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.0rc18.dist-info → prefect_client-3.0.0rc20.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.0rc18.dist-info → prefect_client-3.0.0rc20.dist-info}/top_level.txt +0 -0
prefect/__init__.py
CHANGED
@@ -31,7 +31,6 @@ if TYPE_CHECKING:
|
|
31
31
|
Flow,
|
32
32
|
get_client,
|
33
33
|
get_run_logger,
|
34
|
-
Manifest,
|
35
34
|
State,
|
36
35
|
tags,
|
37
36
|
task,
|
@@ -60,7 +59,6 @@ _public_api: dict[str, tuple[str, str]] = {
|
|
60
59
|
"Flow": (__spec__.parent, ".main"),
|
61
60
|
"get_client": (__spec__.parent, ".main"),
|
62
61
|
"get_run_logger": (__spec__.parent, ".main"),
|
63
|
-
"Manifest": (__spec__.parent, ".main"),
|
64
62
|
"State": (__spec__.parent, ".main"),
|
65
63
|
"tags": (__spec__.parent, ".main"),
|
66
64
|
"task": (__spec__.parent, ".main"),
|
@@ -81,7 +79,6 @@ __all__ = [
|
|
81
79
|
"Flow",
|
82
80
|
"get_client",
|
83
81
|
"get_run_logger",
|
84
|
-
"Manifest",
|
85
82
|
"State",
|
86
83
|
"tags",
|
87
84
|
"task",
|
@@ -39,6 +39,7 @@ class QueueService(abc.ABC, Generic[T]):
|
|
39
39
|
daemon=True,
|
40
40
|
name=f"{type(self).__name__}Thread",
|
41
41
|
)
|
42
|
+
self._logger = logging.getLogger(f"{type(self).__name__}")
|
42
43
|
|
43
44
|
def start(self):
|
44
45
|
logger.debug("Starting service %r", self)
|
@@ -144,11 +145,24 @@ class QueueService(abc.ABC, Generic[T]):
|
|
144
145
|
self._done_event.set()
|
145
146
|
|
146
147
|
async def _main_loop(self):
|
148
|
+
last_log_time = 0
|
149
|
+
log_interval = 4 # log every 4 seconds
|
150
|
+
|
147
151
|
while True:
|
148
152
|
item: T = await self._queue_get_thread.submit(
|
149
153
|
create_call(self._queue.get)
|
150
154
|
).aresult()
|
151
155
|
|
156
|
+
if self._stopped:
|
157
|
+
current_time = asyncio.get_event_loop().time()
|
158
|
+
queue_size = self._queue.qsize()
|
159
|
+
|
160
|
+
if current_time - last_log_time >= log_interval and queue_size > 0:
|
161
|
+
self._logger.warning(
|
162
|
+
f"Still processing items: {queue_size} items remaining..."
|
163
|
+
)
|
164
|
+
last_log_time = current_time
|
165
|
+
|
152
166
|
if item is None:
|
153
167
|
logger.debug("Exiting service %r", self)
|
154
168
|
self._queue.task_done()
|
prefect/blocks/core.py
CHANGED
@@ -24,9 +24,7 @@ from typing import (
|
|
24
24
|
)
|
25
25
|
from uuid import UUID, uuid4
|
26
26
|
|
27
|
-
from griffe
|
28
|
-
from griffe.docstrings.dataclasses import DocstringSection, DocstringSectionKind
|
29
|
-
from griffe.docstrings.parsers import Parser, parse
|
27
|
+
from griffe import Docstring, DocstringSection, DocstringSectionKind, Parser, parse
|
30
28
|
from packaging.version import InvalidVersion, Version
|
31
29
|
from pydantic import (
|
32
30
|
BaseModel,
|
@@ -130,7 +128,9 @@ def _is_subclass(cls, parent_cls) -> bool:
|
|
130
128
|
Checks if a given class is a subclass of another class. Unlike issubclass,
|
131
129
|
this will not throw an exception if cls is an instance instead of a type.
|
132
130
|
"""
|
133
|
-
|
131
|
+
# For python<=3.11 inspect.isclass() will return True for parametrized types (e.g. list[str])
|
132
|
+
# so we need to check for get_origin() to avoid TypeError for issubclass.
|
133
|
+
return inspect.isclass(cls) and not get_origin(cls) and issubclass(cls, parent_cls)
|
134
134
|
|
135
135
|
|
136
136
|
def _collect_secret_fields(
|
@@ -138,19 +138,23 @@ def _collect_secret_fields(
|
|
138
138
|
) -> None:
|
139
139
|
"""
|
140
140
|
Recursively collects all secret fields from a given type and adds them to the
|
141
|
-
secrets list, supporting nested Union /
|
142
|
-
mutates the input secrets list, thus does not return anything.
|
141
|
+
secrets list, supporting nested Union / Dict / Tuple / List / BaseModel fields.
|
142
|
+
Also, note, this function mutates the input secrets list, thus does not return anything.
|
143
143
|
"""
|
144
|
-
if get_origin(type_)
|
145
|
-
for
|
146
|
-
_collect_secret_fields(name,
|
144
|
+
if get_origin(type_) in (Union, dict, list, tuple):
|
145
|
+
for nested_type in get_args(type_):
|
146
|
+
_collect_secret_fields(name, nested_type, secrets)
|
147
147
|
return
|
148
148
|
elif _is_subclass(type_, BaseModel):
|
149
149
|
for field_name, field in type_.model_fields.items():
|
150
150
|
_collect_secret_fields(f"{name}.{field_name}", field.annotation, secrets)
|
151
151
|
return
|
152
152
|
|
153
|
-
if type_ in (SecretStr, SecretBytes)
|
153
|
+
if type_ in (SecretStr, SecretBytes) or (
|
154
|
+
isinstance(type_, type)
|
155
|
+
and getattr(type_, "__module__", None) == "pydantic.types"
|
156
|
+
and getattr(type_, "__name__", None) == "Secret"
|
157
|
+
):
|
154
158
|
secrets.append(name)
|
155
159
|
elif type_ == SecretDict:
|
156
160
|
# Append .* to field name to signify that all values under this
|
@@ -232,21 +236,25 @@ def schema_extra(schema: Dict[str, Any], model: Type["Block"]):
|
|
232
236
|
|
233
237
|
# create block schema references
|
234
238
|
refs = schema["block_schema_references"] = {}
|
239
|
+
|
240
|
+
def collect_block_schema_references(field_name: str, annotation: type) -> None:
|
241
|
+
"""Walk through the annotation and collect block schemas for any nested blocks."""
|
242
|
+
if Block.is_block_class(annotation):
|
243
|
+
if isinstance(refs.get(field_name), list):
|
244
|
+
refs[field_name].append(annotation._to_block_schema_reference_dict())
|
245
|
+
elif isinstance(refs.get(field_name), dict):
|
246
|
+
refs[field_name] = [
|
247
|
+
refs[field_name],
|
248
|
+
annotation._to_block_schema_reference_dict(),
|
249
|
+
]
|
250
|
+
else:
|
251
|
+
refs[field_name] = annotation._to_block_schema_reference_dict()
|
252
|
+
if get_origin(annotation) in (Union, list, tuple, dict):
|
253
|
+
for type_ in get_args(annotation):
|
254
|
+
collect_block_schema_references(field_name, type_)
|
255
|
+
|
235
256
|
for name, field in model.model_fields.items():
|
236
|
-
|
237
|
-
refs[name] = field.annotation._to_block_schema_reference_dict()
|
238
|
-
if get_origin(field.annotation) in [Union, list]:
|
239
|
-
for type_ in get_args(field.annotation):
|
240
|
-
if Block.is_block_class(type_):
|
241
|
-
if isinstance(refs.get(name), list):
|
242
|
-
refs[name].append(type_._to_block_schema_reference_dict())
|
243
|
-
elif isinstance(refs.get(name), dict):
|
244
|
-
refs[name] = [
|
245
|
-
refs[name],
|
246
|
-
type_._to_block_schema_reference_dict(),
|
247
|
-
]
|
248
|
-
else:
|
249
|
-
refs[name] = type_._to_block_schema_reference_dict()
|
257
|
+
collect_block_schema_references(name, field.annotation)
|
250
258
|
|
251
259
|
|
252
260
|
@register_base_type
|
@@ -1067,13 +1075,16 @@ class Block(BaseModel, ABC):
|
|
1067
1075
|
"subclass and not on a Block interface class directly."
|
1068
1076
|
)
|
1069
1077
|
|
1078
|
+
async def register_blocks_in_annotation(annotation: type) -> None:
|
1079
|
+
"""Walk through the annotation and register any nested blocks."""
|
1080
|
+
if Block.is_block_class(annotation):
|
1081
|
+
await annotation.register_type_and_schema(client=client)
|
1082
|
+
elif get_origin(annotation) in (Union, tuple, list, dict):
|
1083
|
+
for inner_annotation in get_args(annotation):
|
1084
|
+
await register_blocks_in_annotation(inner_annotation)
|
1085
|
+
|
1070
1086
|
for field in cls.model_fields.values():
|
1071
|
-
|
1072
|
-
await field.annotation.register_type_and_schema(client=client)
|
1073
|
-
if get_origin(field.annotation) is Union:
|
1074
|
-
for annotation in get_args(field.annotation):
|
1075
|
-
if Block.is_block_class(annotation):
|
1076
|
-
await annotation.register_type_and_schema(client=client)
|
1087
|
+
await register_blocks_in_annotation(field.annotation)
|
1077
1088
|
|
1078
1089
|
try:
|
1079
1090
|
block_type = await client.read_block_type_by_slug(
|
prefect/blocks/system.py
CHANGED
@@ -1,11 +1,26 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
from
|
1
|
+
import json
|
2
|
+
from typing import Annotated, Any, Generic, TypeVar, Union
|
3
|
+
|
4
|
+
from pydantic import (
|
5
|
+
Field,
|
6
|
+
JsonValue,
|
7
|
+
SecretStr,
|
8
|
+
StrictStr,
|
9
|
+
field_validator,
|
10
|
+
)
|
11
|
+
from pydantic import Secret as PydanticSecret
|
12
|
+
from pydantic_extra_types.pendulum_dt import DateTime as PydanticDateTime
|
5
13
|
|
6
14
|
from prefect._internal.compatibility.deprecated import deprecated_class
|
7
15
|
from prefect.blocks.core import Block
|
8
16
|
|
17
|
+
_SecretValueType = Union[
|
18
|
+
Annotated[StrictStr, Field(title="string")],
|
19
|
+
Annotated[JsonValue, Field(title="JSON")],
|
20
|
+
]
|
21
|
+
|
22
|
+
T = TypeVar("T", bound=_SecretValueType)
|
23
|
+
|
9
24
|
|
10
25
|
@deprecated_class(
|
11
26
|
start_date="Jun 2024",
|
@@ -86,24 +101,26 @@ class DateTime(Block):
|
|
86
101
|
_logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/8b3da9a6621e92108b8e6a75b82e15374e170ff7-48x48.png"
|
87
102
|
_documentation_url = "https://docs.prefect.io/api-ref/prefect/blocks/system/#prefect.blocks.system.DateTime"
|
88
103
|
|
89
|
-
value:
|
104
|
+
value: PydanticDateTime = Field(
|
90
105
|
default=...,
|
91
106
|
description="An ISO 8601-compatible datetime value.",
|
92
107
|
)
|
93
108
|
|
94
109
|
|
95
|
-
class Secret(Block):
|
110
|
+
class Secret(Block, Generic[T]):
|
96
111
|
"""
|
97
112
|
A block that represents a secret value. The value stored in this block will be obfuscated when
|
98
|
-
this block is
|
113
|
+
this block is viewed or edited in the UI.
|
99
114
|
|
100
115
|
Attributes:
|
101
|
-
value: A
|
116
|
+
value: A value that should be kept secret.
|
102
117
|
|
103
118
|
Example:
|
104
119
|
```python
|
105
120
|
from prefect.blocks.system import Secret
|
106
121
|
|
122
|
+
Secret(value="sk-1234567890").save("BLOCK_NAME", overwrite=True)
|
123
|
+
|
107
124
|
secret_block = Secret.load("BLOCK_NAME")
|
108
125
|
|
109
126
|
# Access the stored secret
|
@@ -114,9 +131,28 @@ class Secret(Block):
|
|
114
131
|
_logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/c6f20e556dd16effda9df16551feecfb5822092b-48x48.png"
|
115
132
|
_documentation_url = "https://docs.prefect.io/api-ref/prefect/blocks/system/#prefect.blocks.system.Secret"
|
116
133
|
|
117
|
-
value: SecretStr = Field(
|
118
|
-
default=...,
|
134
|
+
value: Union[SecretStr, PydanticSecret[T]] = Field(
|
135
|
+
default=...,
|
136
|
+
description="A value that should be kept secret.",
|
137
|
+
examples=["sk-1234567890", {"username": "johndoe", "password": "s3cr3t"}],
|
138
|
+
json_schema_extra={
|
139
|
+
"writeOnly": True,
|
140
|
+
"format": "password",
|
141
|
+
},
|
119
142
|
)
|
120
143
|
|
121
|
-
|
122
|
-
|
144
|
+
@field_validator("value", mode="before")
|
145
|
+
def validate_value(
|
146
|
+
cls, value: Union[T, SecretStr, PydanticSecret[T]]
|
147
|
+
) -> Union[SecretStr, PydanticSecret[T]]:
|
148
|
+
if isinstance(value, (PydanticSecret, SecretStr)):
|
149
|
+
return value
|
150
|
+
else:
|
151
|
+
return PydanticSecret[type(value)](value)
|
152
|
+
|
153
|
+
def get(self) -> T:
|
154
|
+
try:
|
155
|
+
value = self.value.get_secret_value()
|
156
|
+
return json.loads(value)
|
157
|
+
except (TypeError, json.JSONDecodeError):
|
158
|
+
return value
|
prefect/client/cloud.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
import re
|
2
|
-
from typing import Any, Dict, List, Optional
|
2
|
+
from typing import Any, Dict, List, Optional, cast
|
3
3
|
|
4
4
|
import anyio
|
5
5
|
import httpx
|
@@ -9,7 +9,11 @@ from starlette import status
|
|
9
9
|
import prefect.context
|
10
10
|
import prefect.settings
|
11
11
|
from prefect.client.base import PrefectHttpxAsyncClient
|
12
|
-
from prefect.client.schemas.objects import
|
12
|
+
from prefect.client.schemas.objects import (
|
13
|
+
IPAllowlist,
|
14
|
+
IPAllowlistMyAccessResponse,
|
15
|
+
Workspace,
|
16
|
+
)
|
13
17
|
from prefect.exceptions import ObjectNotFound, PrefectException
|
14
18
|
from prefect.settings import (
|
15
19
|
PREFECT_API_KEY,
|
@@ -69,6 +73,26 @@ class CloudClient:
|
|
69
73
|
**httpx_settings, enable_csrf_support=False
|
70
74
|
)
|
71
75
|
|
76
|
+
if match := (
|
77
|
+
re.search(PARSE_API_URL_REGEX, host)
|
78
|
+
or re.search(PARSE_API_URL_REGEX, prefect.settings.PREFECT_API_URL.value())
|
79
|
+
):
|
80
|
+
self.account_id, self.workspace_id = match.groups()
|
81
|
+
|
82
|
+
@property
|
83
|
+
def account_base_url(self) -> str:
|
84
|
+
if not self.account_id:
|
85
|
+
raise ValueError("Account ID not set")
|
86
|
+
|
87
|
+
return f"accounts/{self.account_id}"
|
88
|
+
|
89
|
+
@property
|
90
|
+
def workspace_base_url(self) -> str:
|
91
|
+
if not self.workspace_id:
|
92
|
+
raise ValueError("Workspace ID not set")
|
93
|
+
|
94
|
+
return f"{self.account_base_url}/workspaces/{self.workspace_id}"
|
95
|
+
|
72
96
|
async def api_healthcheck(self):
|
73
97
|
"""
|
74
98
|
Attempts to connect to the Cloud API and raises the encountered exception if not
|
@@ -86,11 +110,36 @@ class CloudClient:
|
|
86
110
|
return workspaces
|
87
111
|
|
88
112
|
async def read_worker_metadata(self) -> Dict[str, Any]:
|
89
|
-
|
90
|
-
|
91
|
-
return await self.get(
|
92
|
-
f"accounts/{account_id}/workspaces/{workspace_id}/collections/work_pool_types"
|
113
|
+
response = await self.get(
|
114
|
+
f"{self.workspace_base_url}/collections/work_pool_types"
|
93
115
|
)
|
116
|
+
return cast(Dict[str, Any], response)
|
117
|
+
|
118
|
+
async def read_account_settings(self) -> Dict[str, Any]:
|
119
|
+
response = await self.get(f"{self.account_base_url}/settings")
|
120
|
+
return cast(Dict[str, Any], response)
|
121
|
+
|
122
|
+
async def update_account_settings(self, settings: Dict[str, Any]):
|
123
|
+
await self.request(
|
124
|
+
"PATCH",
|
125
|
+
f"{self.account_base_url}/settings",
|
126
|
+
json=settings,
|
127
|
+
)
|
128
|
+
|
129
|
+
async def read_account_ip_allowlist(self) -> IPAllowlist:
|
130
|
+
response = await self.get(f"{self.account_base_url}/ip_allowlist")
|
131
|
+
return IPAllowlist.model_validate(response)
|
132
|
+
|
133
|
+
async def update_account_ip_allowlist(self, updated_allowlist: IPAllowlist):
|
134
|
+
await self.request(
|
135
|
+
"PUT",
|
136
|
+
f"{self.account_base_url}/ip_allowlist",
|
137
|
+
json=updated_allowlist.model_dump(mode="json"),
|
138
|
+
)
|
139
|
+
|
140
|
+
async def check_ip_allowlist_access(self) -> IPAllowlistMyAccessResponse:
|
141
|
+
response = await self.get(f"{self.account_base_url}/ip_allowlist/my_access")
|
142
|
+
return IPAllowlistMyAccessResponse.model_validate(response)
|
94
143
|
|
95
144
|
async def __aenter__(self):
|
96
145
|
await self._client.__aenter__()
|
@@ -120,7 +169,7 @@ class CloudClient:
|
|
120
169
|
status.HTTP_401_UNAUTHORIZED,
|
121
170
|
status.HTTP_403_FORBIDDEN,
|
122
171
|
):
|
123
|
-
raise CloudUnauthorizedError
|
172
|
+
raise CloudUnauthorizedError(str(exc)) from exc
|
124
173
|
elif exc.response.status_code == status.HTTP_404_NOT_FOUND:
|
125
174
|
raise ObjectNotFound(http_exc=exc) from exc
|
126
175
|
else:
|
prefect/client/collections.py
CHANGED
@@ -29,6 +29,6 @@ def get_collections_metadata_client(
|
|
29
29
|
"""
|
30
30
|
orchestration_client = get_client(httpx_settings=httpx_settings)
|
31
31
|
if orchestration_client.server_type == ServerType.CLOUD:
|
32
|
-
return get_cloud_client(httpx_settings=httpx_settings)
|
32
|
+
return get_cloud_client(httpx_settings=httpx_settings, infer_cloud_url=True)
|
33
33
|
else:
|
34
34
|
return orchestration_client
|
prefect/client/orchestration.py
CHANGED
@@ -939,6 +939,57 @@ class PrefectClient:
|
|
939
939
|
else:
|
940
940
|
raise
|
941
941
|
|
942
|
+
async def increment_v1_concurrency_slots(
|
943
|
+
self,
|
944
|
+
names: List[str],
|
945
|
+
task_run_id: UUID,
|
946
|
+
) -> httpx.Response:
|
947
|
+
"""
|
948
|
+
Increment concurrency limit slots for the specified limits.
|
949
|
+
|
950
|
+
Args:
|
951
|
+
names (List[str]): A list of limit names for which to increment limits.
|
952
|
+
task_run_id (UUID): The task run ID incrementing the limits.
|
953
|
+
"""
|
954
|
+
data = {
|
955
|
+
"names": names,
|
956
|
+
"task_run_id": str(task_run_id),
|
957
|
+
}
|
958
|
+
|
959
|
+
return await self._client.post(
|
960
|
+
"/concurrency_limits/increment",
|
961
|
+
json=data,
|
962
|
+
)
|
963
|
+
|
964
|
+
async def decrement_v1_concurrency_slots(
|
965
|
+
self,
|
966
|
+
names: List[str],
|
967
|
+
task_run_id: UUID,
|
968
|
+
occupancy_seconds: float,
|
969
|
+
) -> httpx.Response:
|
970
|
+
"""
|
971
|
+
Decrement concurrency limit slots for the specified limits.
|
972
|
+
|
973
|
+
Args:
|
974
|
+
names (List[str]): A list of limit names to decrement.
|
975
|
+
task_run_id (UUID): The task run ID that incremented the limits.
|
976
|
+
occupancy_seconds (float): The duration in seconds that the limits
|
977
|
+
were held.
|
978
|
+
|
979
|
+
Returns:
|
980
|
+
httpx.Response: The HTTP response from the server.
|
981
|
+
"""
|
982
|
+
data = {
|
983
|
+
"names": names,
|
984
|
+
"task_run_id": str(task_run_id),
|
985
|
+
"occupancy_seconds": occupancy_seconds,
|
986
|
+
}
|
987
|
+
|
988
|
+
return await self._client.post(
|
989
|
+
"/concurrency_limits/decrement",
|
990
|
+
json=data,
|
991
|
+
)
|
992
|
+
|
942
993
|
async def create_work_queue(
|
943
994
|
self,
|
944
995
|
name: str,
|
@@ -1273,15 +1324,17 @@ class PrefectClient:
|
|
1273
1324
|
`SecretBytes` fields. Note Blocks may not work as expected if
|
1274
1325
|
this is set to `False`.
|
1275
1326
|
"""
|
1327
|
+
block_document_data = block_document.model_dump(
|
1328
|
+
mode="json",
|
1329
|
+
exclude_unset=True,
|
1330
|
+
exclude={"id", "block_schema", "block_type"},
|
1331
|
+
context={"include_secrets": include_secrets},
|
1332
|
+
serialize_as_any=True,
|
1333
|
+
)
|
1276
1334
|
try:
|
1277
1335
|
response = await self._client.post(
|
1278
1336
|
"/block_documents/",
|
1279
|
-
json=
|
1280
|
-
mode="json",
|
1281
|
-
exclude_unset=True,
|
1282
|
-
exclude={"id", "block_schema", "block_type"},
|
1283
|
-
context={"include_secrets": include_secrets},
|
1284
|
-
),
|
1337
|
+
json=block_document_data,
|
1285
1338
|
)
|
1286
1339
|
except httpx.HTTPStatusError as e:
|
1287
1340
|
if e.response.status_code == status.HTTP_409_CONFLICT:
|
@@ -1599,6 +1652,7 @@ class PrefectClient:
|
|
1599
1652
|
name: str,
|
1600
1653
|
version: Optional[str] = None,
|
1601
1654
|
schedules: Optional[List[DeploymentScheduleCreate]] = None,
|
1655
|
+
concurrency_limit: Optional[int] = None,
|
1602
1656
|
parameters: Optional[Dict[str, Any]] = None,
|
1603
1657
|
description: Optional[str] = None,
|
1604
1658
|
work_queue_name: Optional[str] = None,
|
@@ -1656,6 +1710,7 @@ class PrefectClient:
|
|
1656
1710
|
parameter_openapi_schema=parameter_openapi_schema,
|
1657
1711
|
paused=paused,
|
1658
1712
|
schedules=schedules or [],
|
1713
|
+
concurrency_limit=concurrency_limit,
|
1659
1714
|
pull_steps=pull_steps,
|
1660
1715
|
enforce_parameter_schema=enforce_parameter_schema,
|
1661
1716
|
)
|
@@ -1733,6 +1788,12 @@ class PrefectClient:
|
|
1733
1788
|
Returns:
|
1734
1789
|
a [Deployment model][prefect.client.schemas.objects.Deployment] representation of the deployment
|
1735
1790
|
"""
|
1791
|
+
if not isinstance(deployment_id, UUID):
|
1792
|
+
try:
|
1793
|
+
deployment_id = UUID(deployment_id)
|
1794
|
+
except ValueError:
|
1795
|
+
raise ValueError(f"Invalid deployment ID: {deployment_id}")
|
1796
|
+
|
1736
1797
|
try:
|
1737
1798
|
response = await self._client.get(f"/deployments/{deployment_id}")
|
1738
1799
|
except httpx.HTTPStatusError as e:
|
@@ -2612,6 +2673,7 @@ class PrefectClient:
|
|
2612
2673
|
async def create_work_pool(
|
2613
2674
|
self,
|
2614
2675
|
work_pool: WorkPoolCreate,
|
2676
|
+
overwrite: bool = False,
|
2615
2677
|
) -> WorkPool:
|
2616
2678
|
"""
|
2617
2679
|
Creates a work pool with the provided configuration.
|
@@ -2629,7 +2691,24 @@ class PrefectClient:
|
|
2629
2691
|
)
|
2630
2692
|
except httpx.HTTPStatusError as e:
|
2631
2693
|
if e.response.status_code == status.HTTP_409_CONFLICT:
|
2632
|
-
|
2694
|
+
if overwrite:
|
2695
|
+
existing_work_pool = await self.read_work_pool(
|
2696
|
+
work_pool_name=work_pool.name
|
2697
|
+
)
|
2698
|
+
if existing_work_pool.type != work_pool.type:
|
2699
|
+
warnings.warn(
|
2700
|
+
"Overwriting work pool type is not supported. Ignoring provided type.",
|
2701
|
+
category=UserWarning,
|
2702
|
+
)
|
2703
|
+
await self.update_work_pool(
|
2704
|
+
work_pool_name=work_pool.name,
|
2705
|
+
work_pool=WorkPoolUpdate.model_validate(
|
2706
|
+
work_pool.model_dump(exclude={"name", "type"})
|
2707
|
+
),
|
2708
|
+
)
|
2709
|
+
response = await self._client.get(f"/work_pools/{work_pool.name}")
|
2710
|
+
else:
|
2711
|
+
raise prefect.exceptions.ObjectAlreadyExists(http_exc=e) from e
|
2633
2712
|
else:
|
2634
2713
|
raise
|
2635
2714
|
|
@@ -3156,7 +3235,7 @@ class PrefectClient:
|
|
3156
3235
|
return pydantic.TypeAdapter(List[Automation]).validate_python(response.json())
|
3157
3236
|
|
3158
3237
|
async def find_automation(
|
3159
|
-
self, id_or_name: Union[str, UUID]
|
3238
|
+
self, id_or_name: Union[str, UUID]
|
3160
3239
|
) -> Optional[Automation]:
|
3161
3240
|
if isinstance(id_or_name, str):
|
3162
3241
|
try:
|
@@ -4096,3 +4175,27 @@ class SyncPrefectClient:
|
|
4096
4175
|
"occupancy_seconds": occupancy_seconds,
|
4097
4176
|
},
|
4098
4177
|
)
|
4178
|
+
|
4179
|
+
def decrement_v1_concurrency_slots(
|
4180
|
+
self, names: List[str], occupancy_seconds: float, task_run_id: UUID
|
4181
|
+
) -> httpx.Response:
|
4182
|
+
"""
|
4183
|
+
Release the specified concurrency limits.
|
4184
|
+
|
4185
|
+
Args:
|
4186
|
+
names (List[str]): A list of limit names to decrement.
|
4187
|
+
occupancy_seconds (float): The duration in seconds that the slots
|
4188
|
+
were held.
|
4189
|
+
task_run_id (UUID): The task run ID that incremented the limits.
|
4190
|
+
|
4191
|
+
Returns:
|
4192
|
+
httpx.Response: The HTTP response from the server.
|
4193
|
+
"""
|
4194
|
+
return self._client.post(
|
4195
|
+
"/concurrency_limits/decrement",
|
4196
|
+
json={
|
4197
|
+
"names": names,
|
4198
|
+
"occupancy_seconds": occupancy_seconds,
|
4199
|
+
"task_run_id": str(task_run_id),
|
4200
|
+
},
|
4201
|
+
)
|
@@ -19,11 +19,13 @@ from pydantic import (
|
|
19
19
|
ConfigDict,
|
20
20
|
Field,
|
21
21
|
HttpUrl,
|
22
|
+
IPvAnyNetwork,
|
22
23
|
SerializationInfo,
|
23
24
|
field_validator,
|
24
25
|
model_serializer,
|
25
26
|
model_validator,
|
26
27
|
)
|
28
|
+
from pydantic.functional_validators import ModelWrapValidatorHandler
|
27
29
|
from pydantic_extra_types.pendulum_dt import DateTime
|
28
30
|
from typing_extensions import Literal, Self, TypeVar
|
29
31
|
|
@@ -276,11 +278,16 @@ class State(ObjectBaseModel, Generic[R]):
|
|
276
278
|
from prefect.client.schemas.actions import StateCreate
|
277
279
|
from prefect.results import BaseResult
|
278
280
|
|
281
|
+
if isinstance(self.data, BaseResult) and self.data.serialize_to_none is False:
|
282
|
+
data = self.data
|
283
|
+
else:
|
284
|
+
data = None
|
285
|
+
|
279
286
|
return StateCreate(
|
280
287
|
type=self.type,
|
281
288
|
name=self.name,
|
282
289
|
message=self.message,
|
283
|
-
data=
|
290
|
+
data=data,
|
284
291
|
state_details=self.state_details,
|
285
292
|
)
|
286
293
|
|
@@ -848,6 +855,35 @@ class Workspace(PrefectBaseModel):
|
|
848
855
|
return hash(self.handle)
|
849
856
|
|
850
857
|
|
858
|
+
class IPAllowlistEntry(PrefectBaseModel):
|
859
|
+
ip_network: IPvAnyNetwork
|
860
|
+
enabled: bool
|
861
|
+
description: Optional[str] = Field(
|
862
|
+
default=None, description="A description of the IP entry."
|
863
|
+
)
|
864
|
+
last_seen: Optional[str] = Field(
|
865
|
+
default=None,
|
866
|
+
description="The last time this IP was seen accessing Prefect Cloud.",
|
867
|
+
)
|
868
|
+
|
869
|
+
|
870
|
+
class IPAllowlist(PrefectBaseModel):
|
871
|
+
"""
|
872
|
+
A Prefect Cloud IP allowlist.
|
873
|
+
|
874
|
+
Expected payload for an IP allowlist from the Prefect Cloud API.
|
875
|
+
"""
|
876
|
+
|
877
|
+
entries: List[IPAllowlistEntry]
|
878
|
+
|
879
|
+
|
880
|
+
class IPAllowlistMyAccessResponse(PrefectBaseModel):
|
881
|
+
"""Expected payload for an IP allowlist access response from the Prefect Cloud API."""
|
882
|
+
|
883
|
+
allowed: bool
|
884
|
+
detail: str
|
885
|
+
|
886
|
+
|
851
887
|
class BlockType(ObjectBaseModel):
|
852
888
|
"""An ORM representation of a block type"""
|
853
889
|
|
@@ -933,7 +969,9 @@ class BlockDocument(ObjectBaseModel):
|
|
933
969
|
return validate_name_present_on_nonanonymous_blocks(values)
|
934
970
|
|
935
971
|
@model_serializer(mode="wrap")
|
936
|
-
def serialize_data(
|
972
|
+
def serialize_data(
|
973
|
+
self, handler: ModelWrapValidatorHandler, info: SerializationInfo
|
974
|
+
):
|
937
975
|
self.data = visit_collection(
|
938
976
|
self.data,
|
939
977
|
visit_fn=partial(handle_secret_render, context=info.context or {}),
|
prefect/concurrency/asyncio.py
CHANGED
@@ -36,7 +36,8 @@ async def concurrency(
|
|
36
36
|
names: Union[str, List[str]],
|
37
37
|
occupy: int = 1,
|
38
38
|
timeout_seconds: Optional[float] = None,
|
39
|
-
create_if_missing:
|
39
|
+
create_if_missing: bool = True,
|
40
|
+
max_retries: Optional[int] = None,
|
40
41
|
) -> AsyncGenerator[None, None]:
|
41
42
|
"""A context manager that acquires and releases concurrency slots from the
|
42
43
|
given concurrency limits.
|
@@ -47,6 +48,7 @@ async def concurrency(
|
|
47
48
|
timeout_seconds: The number of seconds to wait for the slots to be acquired before
|
48
49
|
raising a `TimeoutError`. A timeout of `None` will wait indefinitely.
|
49
50
|
create_if_missing: Whether to create the concurrency limits if they do not exist.
|
51
|
+
max_retries: The maximum number of retries to acquire the concurrency slots.
|
50
52
|
|
51
53
|
Raises:
|
52
54
|
TimeoutError: If the slots are not acquired within the given timeout.
|
@@ -75,6 +77,7 @@ async def concurrency(
|
|
75
77
|
occupy,
|
76
78
|
timeout_seconds=timeout_seconds,
|
77
79
|
create_if_missing=create_if_missing,
|
80
|
+
max_retries=max_retries,
|
78
81
|
)
|
79
82
|
acquisition_time = pendulum.now("UTC")
|
80
83
|
emitted_events = _emit_concurrency_acquisition_events(limits, occupy)
|
@@ -137,9 +140,12 @@ async def _acquire_concurrency_slots(
|
|
137
140
|
mode: Union[Literal["concurrency"], Literal["rate_limit"]] = "concurrency",
|
138
141
|
timeout_seconds: Optional[float] = None,
|
139
142
|
create_if_missing: Optional[bool] = True,
|
143
|
+
max_retries: Optional[int] = None,
|
140
144
|
) -> List[MinimalConcurrencyLimitResponse]:
|
141
145
|
service = ConcurrencySlotAcquisitionService.instance(frozenset(names))
|
142
|
-
future = service.send(
|
146
|
+
future = service.send(
|
147
|
+
(slots, mode, timeout_seconds, create_if_missing, max_retries)
|
148
|
+
)
|
143
149
|
response_or_exception = await asyncio.wrap_future(future)
|
144
150
|
|
145
151
|
if isinstance(response_or_exception, Exception):
|