prefect-client 3.1.5__py3-none-any.whl → 3.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +3 -0
- prefect/_experimental/__init__.py +0 -0
- prefect/_experimental/lineage.py +181 -0
- prefect/_internal/compatibility/async_dispatch.py +38 -9
- prefect/_internal/compatibility/migration.py +1 -1
- prefect/_internal/concurrency/api.py +52 -52
- prefect/_internal/concurrency/calls.py +59 -35
- prefect/_internal/concurrency/cancellation.py +34 -18
- prefect/_internal/concurrency/event_loop.py +7 -6
- prefect/_internal/concurrency/threads.py +41 -33
- prefect/_internal/concurrency/waiters.py +28 -21
- prefect/_internal/pydantic/v1_schema.py +2 -2
- prefect/_internal/pydantic/v2_schema.py +10 -9
- prefect/_internal/pydantic/v2_validated_func.py +15 -10
- prefect/_internal/retries.py +15 -6
- prefect/_internal/schemas/bases.py +11 -8
- prefect/_internal/schemas/validators.py +7 -5
- prefect/_version.py +3 -3
- prefect/automations.py +53 -47
- prefect/blocks/abstract.py +12 -10
- prefect/blocks/core.py +148 -19
- prefect/blocks/system.py +2 -1
- prefect/cache_policies.py +11 -11
- prefect/client/__init__.py +3 -1
- prefect/client/base.py +36 -37
- prefect/client/cloud.py +26 -19
- prefect/client/collections.py +2 -2
- prefect/client/orchestration.py +430 -273
- prefect/client/schemas/__init__.py +24 -0
- prefect/client/schemas/actions.py +128 -121
- prefect/client/schemas/filters.py +1 -1
- prefect/client/schemas/objects.py +114 -85
- prefect/client/schemas/responses.py +19 -20
- prefect/client/schemas/schedules.py +136 -93
- prefect/client/subscriptions.py +30 -15
- prefect/client/utilities.py +46 -36
- prefect/concurrency/asyncio.py +6 -9
- prefect/concurrency/sync.py +35 -5
- prefect/context.py +40 -32
- prefect/deployments/flow_runs.py +6 -8
- prefect/deployments/runner.py +14 -14
- prefect/deployments/steps/core.py +3 -1
- prefect/deployments/steps/pull.py +60 -12
- prefect/docker/__init__.py +1 -1
- prefect/events/clients.py +55 -4
- prefect/events/filters.py +1 -1
- prefect/events/related.py +2 -1
- prefect/events/schemas/events.py +26 -21
- prefect/events/utilities.py +3 -2
- prefect/events/worker.py +8 -0
- prefect/filesystems.py +3 -3
- prefect/flow_engine.py +87 -87
- prefect/flow_runs.py +7 -5
- prefect/flows.py +218 -176
- prefect/logging/configuration.py +1 -1
- prefect/logging/highlighters.py +1 -2
- prefect/logging/loggers.py +30 -20
- prefect/main.py +17 -24
- prefect/results.py +43 -22
- prefect/runner/runner.py +43 -21
- prefect/runner/server.py +30 -32
- prefect/runner/storage.py +3 -3
- prefect/runner/submit.py +3 -6
- prefect/runner/utils.py +6 -6
- prefect/runtime/flow_run.py +7 -0
- prefect/serializers.py +28 -24
- prefect/settings/constants.py +2 -2
- prefect/settings/legacy.py +1 -1
- prefect/settings/models/experiments.py +5 -0
- prefect/settings/models/server/events.py +10 -0
- prefect/task_engine.py +87 -26
- prefect/task_runners.py +2 -2
- prefect/task_worker.py +43 -25
- prefect/tasks.py +148 -142
- prefect/telemetry/bootstrap.py +15 -2
- prefect/telemetry/instrumentation.py +1 -1
- prefect/telemetry/processors.py +10 -7
- prefect/telemetry/run_telemetry.py +231 -0
- prefect/transactions.py +14 -14
- prefect/types/__init__.py +5 -5
- prefect/utilities/_engine.py +96 -0
- prefect/utilities/annotations.py +25 -18
- prefect/utilities/asyncutils.py +126 -140
- prefect/utilities/callables.py +87 -78
- prefect/utilities/collections.py +278 -117
- prefect/utilities/compat.py +13 -21
- prefect/utilities/context.py +6 -5
- prefect/utilities/dispatch.py +23 -12
- prefect/utilities/dockerutils.py +33 -32
- prefect/utilities/engine.py +126 -239
- prefect/utilities/filesystem.py +18 -15
- prefect/utilities/hashing.py +10 -11
- prefect/utilities/importtools.py +40 -27
- prefect/utilities/math.py +9 -5
- prefect/utilities/names.py +3 -3
- prefect/utilities/processutils.py +121 -57
- prefect/utilities/pydantic.py +41 -36
- prefect/utilities/render_swagger.py +22 -12
- prefect/utilities/schema_tools/__init__.py +2 -1
- prefect/utilities/schema_tools/hydration.py +50 -43
- prefect/utilities/schema_tools/validation.py +52 -42
- prefect/utilities/services.py +13 -12
- prefect/utilities/templating.py +45 -45
- prefect/utilities/text.py +2 -1
- prefect/utilities/timeout.py +4 -4
- prefect/utilities/urls.py +9 -4
- prefect/utilities/visualization.py +46 -24
- prefect/variables.py +136 -27
- prefect/workers/base.py +15 -8
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/METADATA +5 -2
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/RECORD +114 -110
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.5.dist-info → prefect_client-3.1.7.dist-info}/top_level.txt +0 -0
prefect/variables.py
CHANGED
@@ -1,13 +1,14 @@
|
|
1
|
-
from typing import
|
1
|
+
from typing import Optional
|
2
2
|
|
3
3
|
from pydantic import BaseModel, Field
|
4
4
|
|
5
|
+
from prefect._internal.compatibility.async_dispatch import async_dispatch
|
5
6
|
from prefect._internal.compatibility.migration import getattr_migration
|
7
|
+
from prefect.client.orchestration import get_client
|
6
8
|
from prefect.client.schemas.actions import VariableCreate, VariableUpdate
|
7
9
|
from prefect.client.utilities import get_or_create_client
|
8
10
|
from prefect.exceptions import ObjectNotFound
|
9
11
|
from prefect.types import MAX_VARIABLE_NAME_LENGTH, StrictVariableValue
|
10
|
-
from prefect.utilities.asyncutils import sync_compatible
|
11
12
|
|
12
13
|
|
13
14
|
class Variable(BaseModel):
|
@@ -31,19 +32,18 @@ class Variable(BaseModel):
|
|
31
32
|
description="The value of the variable",
|
32
33
|
examples=["my-value"],
|
33
34
|
)
|
34
|
-
tags: Optional[
|
35
|
+
tags: Optional[list[str]] = Field(default=None)
|
35
36
|
|
36
37
|
@classmethod
|
37
|
-
|
38
|
-
async def set(
|
38
|
+
async def aset(
|
39
39
|
cls,
|
40
40
|
name: str,
|
41
41
|
value: StrictVariableValue,
|
42
|
-
tags: Optional[
|
42
|
+
tags: Optional[list[str]] = None,
|
43
43
|
overwrite: bool = False,
|
44
44
|
) -> "Variable":
|
45
45
|
"""
|
46
|
-
|
46
|
+
Asynchronously sets a new variable. If one exists with the same name, must pass `overwrite=True`
|
47
47
|
|
48
48
|
Returns the newly set variable object.
|
49
49
|
|
@@ -60,8 +60,8 @@ class Variable(BaseModel):
|
|
60
60
|
from prefect.variables import Variable
|
61
61
|
|
62
62
|
@flow
|
63
|
-
def my_flow():
|
64
|
-
Variable.
|
63
|
+
async def my_flow():
|
64
|
+
await Variable.aset(name="my_var",value="test_value", tags=["hi", "there"], overwrite=True)
|
65
65
|
```
|
66
66
|
"""
|
67
67
|
client, _ = get_or_create_client()
|
@@ -73,27 +73,76 @@ class Variable(BaseModel):
|
|
73
73
|
raise ValueError(
|
74
74
|
f"Variable {name!r} already exists. Use `overwrite=True` to update it."
|
75
75
|
)
|
76
|
-
await client.update_variable(
|
76
|
+
await client.update_variable(
|
77
|
+
variable=VariableUpdate.model_validate(var_dict)
|
78
|
+
)
|
77
79
|
variable = await client.read_variable_by_name(name)
|
78
|
-
|
79
|
-
|
80
|
-
"value": variable.value,
|
81
|
-
"tags": variable.tags or [],
|
82
|
-
}
|
80
|
+
for key in var_dict.keys():
|
81
|
+
var_dict.update({key: getattr(variable, key)})
|
83
82
|
else:
|
84
|
-
await client.create_variable(
|
83
|
+
await client.create_variable(
|
84
|
+
variable=VariableCreate.model_validate(var_dict)
|
85
|
+
)
|
86
|
+
|
87
|
+
return cls.model_validate(var_dict)
|
88
|
+
|
89
|
+
@classmethod
|
90
|
+
@async_dispatch(aset)
|
91
|
+
def set(
|
92
|
+
cls,
|
93
|
+
name: str,
|
94
|
+
value: StrictVariableValue,
|
95
|
+
tags: Optional[list[str]] = None,
|
96
|
+
overwrite: bool = False,
|
97
|
+
) -> "Variable":
|
98
|
+
"""
|
99
|
+
Sets a new variable. If one exists with the same name, must pass `overwrite=True`
|
100
|
+
|
101
|
+
Returns the newly set variable object.
|
102
|
+
|
103
|
+
Args:
|
104
|
+
- name: The name of the variable to set.
|
105
|
+
- value: The value of the variable to set.
|
106
|
+
- tags: An optional list of strings to associate with the variable.
|
107
|
+
- overwrite: Whether to overwrite the variable if it already exists.
|
108
|
+
|
109
|
+
Example:
|
110
|
+
Set a new variable and overwrite it if it already exists.
|
111
|
+
|
112
|
+
```
|
113
|
+
from prefect.variables import Variable
|
114
|
+
|
115
|
+
@flow
|
116
|
+
def my_flow():
|
117
|
+
Variable.set(name="my_var",value="test_value", tags=["hi", "there"], overwrite=True)
|
118
|
+
```
|
119
|
+
"""
|
120
|
+
with get_client(sync_client=True) as client:
|
121
|
+
variable_exists = client.read_variable_by_name(name)
|
122
|
+
var_dict = {"name": name, "value": value, "tags": tags or []}
|
85
123
|
|
86
|
-
|
124
|
+
if variable_exists:
|
125
|
+
if not overwrite:
|
126
|
+
raise ValueError(
|
127
|
+
f"Variable {name!r} already exists. Use `overwrite=True` to update it."
|
128
|
+
)
|
129
|
+
client.update_variable(variable=VariableUpdate.model_validate(var_dict))
|
130
|
+
variable = client.read_variable_by_name(name)
|
131
|
+
for key in var_dict.keys():
|
132
|
+
var_dict.update({key: getattr(variable, key)})
|
133
|
+
else:
|
134
|
+
client.create_variable(variable=VariableCreate.model_validate(var_dict))
|
135
|
+
|
136
|
+
return cls.model_validate(var_dict)
|
87
137
|
|
88
138
|
@classmethod
|
89
|
-
|
90
|
-
async def get(
|
139
|
+
async def aget(
|
91
140
|
cls,
|
92
141
|
name: str,
|
93
142
|
default: StrictVariableValue = None,
|
94
143
|
) -> StrictVariableValue:
|
95
144
|
"""
|
96
|
-
|
145
|
+
Asynchronously get a variable's value by name.
|
97
146
|
|
98
147
|
If the variable does not exist, return the default value.
|
99
148
|
|
@@ -108,8 +157,8 @@ class Variable(BaseModel):
|
|
108
157
|
from prefect.variables import Variable
|
109
158
|
|
110
159
|
@flow
|
111
|
-
def my_flow():
|
112
|
-
var = Variable.
|
160
|
+
async def my_flow():
|
161
|
+
var = await Variable.aget("my_var")
|
113
162
|
```
|
114
163
|
"""
|
115
164
|
client, _ = get_or_create_client()
|
@@ -118,10 +167,41 @@ class Variable(BaseModel):
|
|
118
167
|
return variable.value if variable else default
|
119
168
|
|
120
169
|
@classmethod
|
121
|
-
@
|
122
|
-
|
170
|
+
@async_dispatch(aget)
|
171
|
+
def get(
|
172
|
+
cls,
|
173
|
+
name: str,
|
174
|
+
default: StrictVariableValue = None,
|
175
|
+
) -> StrictVariableValue:
|
123
176
|
"""
|
124
|
-
|
177
|
+
Get a variable's value by name.
|
178
|
+
|
179
|
+
If the variable does not exist, return the default value.
|
180
|
+
|
181
|
+
Args:
|
182
|
+
- name: The name of the variable value to get.
|
183
|
+
- default: The default value to return if the variable does not exist.
|
184
|
+
|
185
|
+
Example:
|
186
|
+
Get a variable's value by name.
|
187
|
+
```python
|
188
|
+
from prefect import flow
|
189
|
+
from prefect.variables import Variable
|
190
|
+
|
191
|
+
@flow
|
192
|
+
def my_flow():
|
193
|
+
var = Variable.get("my_var")
|
194
|
+
```
|
195
|
+
"""
|
196
|
+
with get_client(sync_client=True) as client:
|
197
|
+
variable = client.read_variable_by_name(name)
|
198
|
+
|
199
|
+
return variable.value if variable else default
|
200
|
+
|
201
|
+
@classmethod
|
202
|
+
async def aunset(cls, name: str) -> bool:
|
203
|
+
"""
|
204
|
+
Asynchronously unset a variable by name.
|
125
205
|
|
126
206
|
Args:
|
127
207
|
- name: The name of the variable to unset.
|
@@ -135,8 +215,8 @@ class Variable(BaseModel):
|
|
135
215
|
from prefect.variables import Variable
|
136
216
|
|
137
217
|
@flow
|
138
|
-
def my_flow():
|
139
|
-
Variable.
|
218
|
+
async def my_flow():
|
219
|
+
await Variable.aunset("my_var")
|
140
220
|
```
|
141
221
|
"""
|
142
222
|
client, _ = get_or_create_client()
|
@@ -146,5 +226,34 @@ class Variable(BaseModel):
|
|
146
226
|
except ObjectNotFound:
|
147
227
|
return False
|
148
228
|
|
229
|
+
@classmethod
|
230
|
+
@async_dispatch(aunset)
|
231
|
+
def unset(cls, name: str) -> bool:
|
232
|
+
"""
|
233
|
+
Unset a variable by name.
|
234
|
+
|
235
|
+
Args:
|
236
|
+
- name: The name of the variable to unset.
|
237
|
+
|
238
|
+
Returns `True` if the variable was deleted, `False` if the variable did not exist.
|
239
|
+
|
240
|
+
Example:
|
241
|
+
Unset a variable by name.
|
242
|
+
```python
|
243
|
+
from prefect import flow
|
244
|
+
from prefect.variables import Variable
|
245
|
+
|
246
|
+
@flow
|
247
|
+
def my_flow():
|
248
|
+
Variable.unset("my_var")
|
249
|
+
```
|
250
|
+
"""
|
251
|
+
with get_client(sync_client=True) as client:
|
252
|
+
try:
|
253
|
+
client.delete_variable_by_name(name=name)
|
254
|
+
return True
|
255
|
+
except ObjectNotFound:
|
256
|
+
return False
|
257
|
+
|
149
258
|
|
150
259
|
__getattr__ = getattr_migration(__name__)
|
prefect/workers/base.py
CHANGED
@@ -53,6 +53,7 @@ from prefect.states import (
|
|
53
53
|
Pending,
|
54
54
|
exception_to_failed_state,
|
55
55
|
)
|
56
|
+
from prefect.types import KeyValueLabels
|
56
57
|
from prefect.utilities.dispatch import get_registry_for_type, register_base_type
|
57
58
|
from prefect.utilities.engine import propose_state
|
58
59
|
from prefect.utilities.services import critical_service_loop
|
@@ -988,7 +989,6 @@ class BaseWorker(abc.ABC):
|
|
988
989
|
try:
|
989
990
|
configuration = await self._get_configuration(flow_run)
|
990
991
|
submitted_event = self._emit_flow_run_submitted_event(configuration)
|
991
|
-
await self._give_worker_labels_to_flow_run(flow_run.id)
|
992
992
|
result = await self.run(
|
993
993
|
flow_run=flow_run,
|
994
994
|
task_status=task_status,
|
@@ -1222,13 +1222,20 @@ class BaseWorker(abc.ABC):
|
|
1222
1222
|
Give this worker's identifying labels to the specified flow run.
|
1223
1223
|
"""
|
1224
1224
|
if self._cloud_client:
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1228
|
-
|
1229
|
-
|
1230
|
-
|
1231
|
-
|
1225
|
+
labels: KeyValueLabels = {
|
1226
|
+
"prefect.worker.name": self.name,
|
1227
|
+
"prefect.worker.type": self.type,
|
1228
|
+
}
|
1229
|
+
|
1230
|
+
if self._work_pool:
|
1231
|
+
labels.update(
|
1232
|
+
{
|
1233
|
+
"prefect.work-pool.name": self._work_pool.name,
|
1234
|
+
"prefect.work-pool.id": str(self._work_pool.id),
|
1235
|
+
}
|
1236
|
+
)
|
1237
|
+
|
1238
|
+
await self._cloud_client.update_flow_run_labels(flow_run_id, labels)
|
1232
1239
|
|
1233
1240
|
async def __aenter__(self):
|
1234
1241
|
self._logger.debug("Entering worker context...")
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: prefect-client
|
3
|
-
Version: 3.1.
|
3
|
+
Version: 3.1.7
|
4
4
|
Summary: Workflow orchestration and management.
|
5
5
|
Home-page: https://www.prefect.io
|
6
6
|
Author: Prefect Technologies, Inc.
|
@@ -51,6 +51,7 @@ Requires-Dist: pydantic-extra-types<3.0.0,>=2.8.2
|
|
51
51
|
Requires-Dist: pydantic-settings>2.2.1
|
52
52
|
Requires-Dist: python-dateutil<3.0.0,>=2.8.2
|
53
53
|
Requires-Dist: python-slugify<9.0,>=5.0
|
54
|
+
Requires-Dist: python-socks[asyncio]<3.0,>=2.5.3
|
54
55
|
Requires-Dist: pyyaml<7.0.0,>=5.4.1
|
55
56
|
Requires-Dist: rfc3339-validator<0.2.0,>=0.1.4
|
56
57
|
Requires-Dist: rich<14.0,>=11.0
|
@@ -165,7 +166,9 @@ Start with our [friendly tutorial](https://docs.prefect.io/tutorials) or explore
|
|
165
166
|
|
166
167
|
## Join the community
|
167
168
|
|
168
|
-
Prefect is made possible by the fastest growing community of thousands of friendly data engineers. Join us in building a new kind of workflow system.
|
169
|
+
Prefect is made possible by the fastest growing community of thousands of friendly data engineers. Join us in building a new kind of workflow system.
|
170
|
+
The [Prefect Slack community](https://prefect.io/slack) is a fantastic place to learn more about Prefect, ask questions, or get help with workflow design.
|
171
|
+
All community forums, including code contributions, issue discussions, and Slack messages are subject to our [Code of Conduct](https://github.com/PrefectHQ/prefect/blob/main/CODE_OF_CONDUCT.md).
|
169
172
|
|
170
173
|
## Contribute
|
171
174
|
|