prefect-client 3.1.8__py3-none-any.whl → 3.1.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +53 -59
- prefect/_internal/concurrency/services.py +6 -4
- prefect/_version.py +3 -3
- prefect/agent.py +3 -1
- prefect/artifacts.py +61 -74
- prefect/automations.py +27 -7
- prefect/client/cloud.py +0 -21
- prefect/client/schemas/objects.py +11 -0
- prefect/client/utilities.py +1 -15
- prefect/context.py +16 -27
- prefect/deployments/deployments.py +4 -2
- prefect/deployments/runner.py +3 -1
- prefect/engine.py +2 -1
- prefect/events/filters.py +2 -8
- prefect/exceptions.py +31 -41
- prefect/filesystems.py +2 -2
- prefect/flow_engine.py +2 -2
- prefect/flows.py +230 -186
- prefect/futures.py +42 -27
- prefect/infrastructure/__init__.py +3 -1
- prefect/infrastructure/base.py +3 -1
- prefect/locking/filesystem.py +8 -7
- prefect/locking/memory.py +5 -3
- prefect/locking/protocol.py +1 -1
- prefect/plugins.py +12 -10
- prefect/results.py +76 -19
- prefect/runner/runner.py +2 -3
- prefect/states.py +22 -10
- prefect/task_engine.py +1 -1
- prefect/telemetry/instrumentation.py +9 -10
- prefect/telemetry/processors.py +6 -6
- prefect/telemetry/services.py +68 -0
- prefect/utilities/engine.py +15 -1
- prefect/utilities/importtools.py +28 -21
- prefect/variables.py +2 -2
- prefect/workers/__init__.py +2 -0
- prefect/workers/base.py +6 -12
- prefect/workers/block.py +3 -1
- prefect/workers/cloud.py +3 -1
- {prefect_client-3.1.8.dist-info → prefect_client-3.1.10.dist-info}/METADATA +1 -1
- {prefect_client-3.1.8.dist-info → prefect_client-3.1.10.dist-info}/RECORD +44 -43
- {prefect_client-3.1.8.dist-info → prefect_client-3.1.10.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.8.dist-info → prefect_client-3.1.10.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.8.dist-info → prefect_client-3.1.10.dist-info}/top_level.txt +0 -0
prefect/context.py
CHANGED
@@ -9,21 +9,10 @@ For more user-accessible information about the current run, see [`prefect.runtim
|
|
9
9
|
import os
|
10
10
|
import sys
|
11
11
|
import warnings
|
12
|
+
from collections.abc import AsyncGenerator, Generator, Mapping
|
12
13
|
from contextlib import ExitStack, asynccontextmanager, contextmanager
|
13
14
|
from contextvars import ContextVar, Token
|
14
|
-
from typing import
|
15
|
-
TYPE_CHECKING,
|
16
|
-
Any,
|
17
|
-
AsyncGenerator,
|
18
|
-
Dict,
|
19
|
-
Generator,
|
20
|
-
Mapping,
|
21
|
-
Optional,
|
22
|
-
Set,
|
23
|
-
Type,
|
24
|
-
TypeVar,
|
25
|
-
Union,
|
26
|
-
)
|
15
|
+
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
|
27
16
|
|
28
17
|
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
29
18
|
from typing_extensions import Self
|
@@ -64,7 +53,7 @@ if TYPE_CHECKING:
|
|
64
53
|
GLOBAL_SETTINGS_CONTEXT = None # type: ignore
|
65
54
|
|
66
55
|
|
67
|
-
def serialize_context() ->
|
56
|
+
def serialize_context() -> dict[str, Any]:
|
68
57
|
"""
|
69
58
|
Serialize the current context for use in a remote execution environment.
|
70
59
|
"""
|
@@ -84,7 +73,7 @@ def serialize_context() -> Dict[str, Any]:
|
|
84
73
|
|
85
74
|
@contextmanager
|
86
75
|
def hydrated_context(
|
87
|
-
serialized_context: Optional[
|
76
|
+
serialized_context: Optional[dict[str, Any]] = None,
|
88
77
|
client: Union[PrefectClient, SyncPrefectClient, None] = None,
|
89
78
|
):
|
90
79
|
with ExitStack() as stack:
|
@@ -148,7 +137,7 @@ class ContextModel(BaseModel):
|
|
148
137
|
self._token = None
|
149
138
|
|
150
139
|
@classmethod
|
151
|
-
def get(cls:
|
140
|
+
def get(cls: type[Self]) -> Optional[Self]:
|
152
141
|
"""Get the current context instance"""
|
153
142
|
return cls.__var__.get(None)
|
154
143
|
|
@@ -173,7 +162,7 @@ class ContextModel(BaseModel):
|
|
173
162
|
new._token = None
|
174
163
|
return new
|
175
164
|
|
176
|
-
def serialize(self, include_secrets: bool = True) ->
|
165
|
+
def serialize(self, include_secrets: bool = True) -> dict[str, Any]:
|
177
166
|
"""
|
178
167
|
Serialize the context model to a dictionary that can be pickled with cloudpickle.
|
179
168
|
"""
|
@@ -314,10 +303,10 @@ class RunContext(ContextModel):
|
|
314
303
|
start_client_metrics_server()
|
315
304
|
|
316
305
|
start_time: DateTime = Field(default_factory=lambda: DateTime.now("UTC"))
|
317
|
-
input_keyset: Optional[
|
306
|
+
input_keyset: Optional[dict[str, dict[str, str]]] = None
|
318
307
|
client: Union[PrefectClient, SyncPrefectClient]
|
319
308
|
|
320
|
-
def serialize(self: Self, include_secrets: bool = True) ->
|
309
|
+
def serialize(self: Self, include_secrets: bool = True) -> dict[str, Any]:
|
321
310
|
return self.model_dump(
|
322
311
|
include={"start_time", "input_keyset"},
|
323
312
|
exclude_unset=True,
|
@@ -344,7 +333,7 @@ class EngineContext(RunContext):
|
|
344
333
|
flow_run: Optional[FlowRun] = None
|
345
334
|
task_runner: TaskRunner[Any]
|
346
335
|
log_prints: bool = False
|
347
|
-
parameters: Optional[
|
336
|
+
parameters: Optional[dict[str, Any]] = None
|
348
337
|
|
349
338
|
# Flag signaling if the flow run context has been serialized and sent
|
350
339
|
# to remote infrastructure.
|
@@ -355,10 +344,10 @@ class EngineContext(RunContext):
|
|
355
344
|
persist_result: bool = Field(default_factory=get_default_persist_setting)
|
356
345
|
|
357
346
|
# Counter for task calls allowing unique
|
358
|
-
task_run_dynamic_keys:
|
347
|
+
task_run_dynamic_keys: dict[str, Union[str, int]] = Field(default_factory=dict)
|
359
348
|
|
360
349
|
# Counter for flow pauses
|
361
|
-
observed_flow_pauses:
|
350
|
+
observed_flow_pauses: dict[str, int] = Field(default_factory=dict)
|
362
351
|
|
363
352
|
# Tracking for result from task runs in this flow run for dependency tracking
|
364
353
|
# Holds the ID of the object returned by the task run and task run state
|
@@ -369,7 +358,7 @@ class EngineContext(RunContext):
|
|
369
358
|
|
370
359
|
__var__: ContextVar[Self] = ContextVar("flow_run")
|
371
360
|
|
372
|
-
def serialize(self: Self, include_secrets: bool = True) ->
|
361
|
+
def serialize(self: Self, include_secrets: bool = True) -> dict[str, Any]:
|
373
362
|
return self.model_dump(
|
374
363
|
include={
|
375
364
|
"flow_run",
|
@@ -403,7 +392,7 @@ class TaskRunContext(RunContext):
|
|
403
392
|
task: "Task[Any, Any]"
|
404
393
|
task_run: TaskRun
|
405
394
|
log_prints: bool = False
|
406
|
-
parameters:
|
395
|
+
parameters: dict[str, Any]
|
407
396
|
|
408
397
|
# Result handling
|
409
398
|
result_store: ResultStore
|
@@ -411,7 +400,7 @@ class TaskRunContext(RunContext):
|
|
411
400
|
|
412
401
|
__var__ = ContextVar("task_run")
|
413
402
|
|
414
|
-
def serialize(self: Self, include_secrets: bool = True) ->
|
403
|
+
def serialize(self: Self, include_secrets: bool = True) -> dict[str, Any]:
|
415
404
|
return self.model_dump(
|
416
405
|
include={
|
417
406
|
"task_run",
|
@@ -437,7 +426,7 @@ class TagsContext(ContextModel):
|
|
437
426
|
current_tags: A set of current tags in the context
|
438
427
|
"""
|
439
428
|
|
440
|
-
current_tags:
|
429
|
+
current_tags: set[str] = Field(default_factory=set)
|
441
430
|
|
442
431
|
@classmethod
|
443
432
|
def get(cls) -> "TagsContext":
|
@@ -512,7 +501,7 @@ def get_settings_context() -> SettingsContext:
|
|
512
501
|
|
513
502
|
|
514
503
|
@contextmanager
|
515
|
-
def tags(*new_tags: str) -> Generator[
|
504
|
+
def tags(*new_tags: str) -> Generator[set[str], None, None]:
|
516
505
|
"""
|
517
506
|
Context manager to add tags to flow and task run calls.
|
518
507
|
|
@@ -1,3 +1,5 @@
|
|
1
|
-
from
|
1
|
+
from typing import Any, Callable
|
2
2
|
|
3
|
-
|
3
|
+
from prefect._internal.compatibility.migration import getattr_migration
|
4
|
+
|
5
|
+
__getattr__: Callable[[str], Any] = getattr_migration(__name__)
|
prefect/deployments/runner.py
CHANGED
@@ -549,7 +549,9 @@ class RunnerDeployment(BaseModel):
|
|
549
549
|
module = importlib.import_module(mod_name)
|
550
550
|
flow_file = getattr(module, "__file__", None)
|
551
551
|
except ModuleNotFoundError as exc:
|
552
|
-
|
552
|
+
# 16458 adds an identifier to the module name, so checking
|
553
|
+
# for "__prefect_loader__" (2 underscores) will not match
|
554
|
+
if "__prefect_loader_" in str(exc):
|
553
555
|
raise ValueError(
|
554
556
|
"Cannot create a RunnerDeployment from a flow that has been"
|
555
557
|
" loaded from an entrypoint. To deploy a flow via"
|
prefect/engine.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
import os
|
2
2
|
import sys
|
3
|
+
from typing import Any, Callable
|
3
4
|
from uuid import UUID
|
4
5
|
|
5
6
|
from prefect._internal.compatibility.migration import getattr_migration
|
@@ -72,4 +73,4 @@ if __name__ == "__main__":
|
|
72
73
|
# Let the exit code be determined by the base exception type
|
73
74
|
raise
|
74
75
|
|
75
|
-
__getattr__ = getattr_migration(__name__)
|
76
|
+
__getattr__: Callable[[str], Any] = getattr_migration(__name__)
|
prefect/events/filters.py
CHANGED
@@ -2,7 +2,7 @@ from typing import List, Optional, Tuple, cast
|
|
2
2
|
from uuid import UUID
|
3
3
|
|
4
4
|
import pendulum
|
5
|
-
from pydantic import Field
|
5
|
+
from pydantic import Field
|
6
6
|
|
7
7
|
from prefect._internal.schemas.bases import PrefectBaseModel
|
8
8
|
from prefect.types import DateTime
|
@@ -41,18 +41,12 @@ class AutomationFilter(PrefectBaseModel):
|
|
41
41
|
class EventDataFilter(PrefectBaseModel, extra="forbid"): # type: ignore[call-arg]
|
42
42
|
"""A base class for filtering event data."""
|
43
43
|
|
44
|
-
_top_level_filter: Optional["EventFilter"] = PrivateAttr(None)
|
45
|
-
|
46
44
|
def get_filters(self) -> List["EventDataFilter"]:
|
47
45
|
filters: List["EventDataFilter"] = [
|
48
46
|
filter
|
49
|
-
for filter in [
|
50
|
-
getattr(self, name) for name, field in self.model_fields.items()
|
51
|
-
]
|
47
|
+
for filter in [getattr(self, name) for name in self.model_fields]
|
52
48
|
if isinstance(filter, EventDataFilter)
|
53
49
|
]
|
54
|
-
for filter in filters:
|
55
|
-
filter._top_level_filter = self._top_level_filter
|
56
50
|
return filters
|
57
51
|
|
58
52
|
def includes(self, event: Event) -> bool:
|
prefect/exceptions.py
CHANGED
@@ -4,19 +4,20 @@ Prefect-specific exceptions.
|
|
4
4
|
|
5
5
|
import inspect
|
6
6
|
import traceback
|
7
|
+
from collections.abc import Iterable
|
7
8
|
from types import ModuleType, TracebackType
|
8
|
-
from typing import Any, Callable,
|
9
|
+
from typing import TYPE_CHECKING, Any, Callable, Optional
|
9
10
|
|
10
11
|
from httpx._exceptions import HTTPStatusError
|
11
12
|
from pydantic import ValidationError
|
12
|
-
from rich.traceback import Traceback
|
13
13
|
from typing_extensions import Self
|
14
14
|
|
15
|
-
|
15
|
+
if TYPE_CHECKING:
|
16
|
+
from prefect.states import State
|
16
17
|
|
17
18
|
|
18
19
|
def _trim_traceback(
|
19
|
-
tb: TracebackType, remove_modules: Iterable[ModuleType]
|
20
|
+
tb: Optional[TracebackType], remove_modules: Iterable[ModuleType]
|
20
21
|
) -> Optional[TracebackType]:
|
21
22
|
"""
|
22
23
|
Utility to remove frames from specific modules from a traceback.
|
@@ -32,8 +33,9 @@ def _trim_traceback(
|
|
32
33
|
A traceback, or `None` if all traceback frames originate from an excluded module
|
33
34
|
|
34
35
|
"""
|
35
|
-
strip_paths = [
|
36
|
-
|
36
|
+
strip_paths = [
|
37
|
+
module.__file__ for module in remove_modules if module.__file__ is not None
|
38
|
+
]
|
37
39
|
while tb and any(
|
38
40
|
module_path in str(tb.tb_frame.f_globals.get("__file__", ""))
|
39
41
|
for module_path in strip_paths
|
@@ -91,7 +93,9 @@ class PausedRun(PrefectException):
|
|
91
93
|
Raised when the result from a paused run is retrieved.
|
92
94
|
"""
|
93
95
|
|
94
|
-
def __init__(
|
96
|
+
def __init__(
|
97
|
+
self, *args: Any, state: Optional["State[Any]"] = None, **kwargs: Any
|
98
|
+
) -> None:
|
95
99
|
super().__init__(*args, **kwargs)
|
96
100
|
self.state = state
|
97
101
|
|
@@ -133,6 +137,8 @@ class ScriptError(PrefectException):
|
|
133
137
|
user_exc: Exception,
|
134
138
|
path: str,
|
135
139
|
) -> None:
|
140
|
+
import prefect.utilities.importtools
|
141
|
+
|
136
142
|
message = f"Script at {str(path)!r} encountered an exception: {user_exc!r}"
|
137
143
|
super().__init__(message)
|
138
144
|
self.user_exc = user_exc
|
@@ -144,30 +150,6 @@ class ScriptError(PrefectException):
|
|
144
150
|
)
|
145
151
|
|
146
152
|
|
147
|
-
class FlowScriptError(PrefectException):
|
148
|
-
"""
|
149
|
-
Raised when a script errors during evaluation while attempting to load a flow.
|
150
|
-
"""
|
151
|
-
|
152
|
-
def __init__(
|
153
|
-
self,
|
154
|
-
user_exc: Exception,
|
155
|
-
script_path: str,
|
156
|
-
) -> None:
|
157
|
-
message = f"Flow script at {script_path!r} encountered an exception"
|
158
|
-
super().__init__(message)
|
159
|
-
|
160
|
-
self.user_exc = user_exc
|
161
|
-
|
162
|
-
def rich_user_traceback(self, **kwargs):
|
163
|
-
trace = Traceback.extract(
|
164
|
-
type(self.user_exc),
|
165
|
-
self.user_exc,
|
166
|
-
self.user_exc.__traceback__.tb_next.tb_next.tb_next.tb_next,
|
167
|
-
)
|
168
|
-
return Traceback(trace, **kwargs)
|
169
|
-
|
170
|
-
|
171
153
|
class ParameterTypeError(PrefectException):
|
172
154
|
"""
|
173
155
|
Raised when a parameter does not pass Pydantic type validation.
|
@@ -196,7 +178,11 @@ class ParameterBindError(TypeError, PrefectException):
|
|
196
178
|
|
197
179
|
@classmethod
|
198
180
|
def from_bind_failure(
|
199
|
-
cls,
|
181
|
+
cls,
|
182
|
+
fn: Callable[..., Any],
|
183
|
+
exc: TypeError,
|
184
|
+
call_args: tuple[Any, ...],
|
185
|
+
call_kwargs: dict[str, Any],
|
200
186
|
) -> Self:
|
201
187
|
fn_signature = str(inspect.signature(fn)).strip("()")
|
202
188
|
|
@@ -214,7 +200,9 @@ class SignatureMismatchError(PrefectException, TypeError):
|
|
214
200
|
super().__init__(msg)
|
215
201
|
|
216
202
|
@classmethod
|
217
|
-
def from_bad_params(
|
203
|
+
def from_bad_params(
|
204
|
+
cls, expected_params: list[str], provided_params: list[str]
|
205
|
+
) -> Self:
|
218
206
|
msg = (
|
219
207
|
f"Function expects parameters {expected_params} but was provided with"
|
220
208
|
f" parameters {provided_params}"
|
@@ -231,14 +219,14 @@ class ObjectNotFound(PrefectException):
|
|
231
219
|
self,
|
232
220
|
http_exc: Exception,
|
233
221
|
help_message: Optional[str] = None,
|
234
|
-
*args,
|
235
|
-
**kwargs,
|
236
|
-
):
|
222
|
+
*args: Any,
|
223
|
+
**kwargs: Any,
|
224
|
+
) -> None:
|
237
225
|
self.http_exc = http_exc
|
238
226
|
self.help_message = help_message
|
239
227
|
super().__init__(help_message, *args, **kwargs)
|
240
228
|
|
241
|
-
def __str__(self):
|
229
|
+
def __str__(self) -> str:
|
242
230
|
return self.help_message or super().__str__()
|
243
231
|
|
244
232
|
|
@@ -247,7 +235,7 @@ class ObjectAlreadyExists(PrefectException):
|
|
247
235
|
Raised when the client receives a 409 (conflict) from the API.
|
248
236
|
"""
|
249
237
|
|
250
|
-
def __init__(self, http_exc: Exception, *args, **kwargs):
|
238
|
+
def __init__(self, http_exc: Exception, *args: Any, **kwargs: Any) -> None:
|
251
239
|
self.http_exc = http_exc
|
252
240
|
super().__init__(*args, **kwargs)
|
253
241
|
|
@@ -304,7 +292,9 @@ class Pause(PrefectSignal):
|
|
304
292
|
Raised when a flow run is PAUSED and needs to exit for resubmission.
|
305
293
|
"""
|
306
294
|
|
307
|
-
def __init__(
|
295
|
+
def __init__(
|
296
|
+
self, *args: Any, state: Optional["State[Any]"] = None, **kwargs: Any
|
297
|
+
) -> None:
|
308
298
|
super().__init__(*args, **kwargs)
|
309
299
|
self.state = state
|
310
300
|
|
@@ -332,7 +322,7 @@ class PrefectHTTPStatusError(HTTPStatusError):
|
|
332
322
|
"""
|
333
323
|
|
334
324
|
@classmethod
|
335
|
-
def from_httpx_error(cls:
|
325
|
+
def from_httpx_error(cls: type[Self], httpx_error: HTTPStatusError) -> Self:
|
336
326
|
"""
|
337
327
|
Generate a `PrefectHTTPStatusError` from an `httpx.HTTPStatusError`.
|
338
328
|
"""
|
@@ -441,7 +431,7 @@ class ProfileSettingsValidationError(PrefectException):
|
|
441
431
|
Raised when a profile settings are invalid.
|
442
432
|
"""
|
443
433
|
|
444
|
-
def __init__(self, errors:
|
434
|
+
def __init__(self, errors: list[tuple[Any, ValidationError]]) -> None:
|
445
435
|
self.errors = errors
|
446
436
|
|
447
437
|
|
prefect/filesystems.py
CHANGED
@@ -2,7 +2,7 @@ import abc
|
|
2
2
|
import urllib.parse
|
3
3
|
from pathlib import Path
|
4
4
|
from shutil import copytree
|
5
|
-
from typing import Any, Dict, Optional
|
5
|
+
from typing import Any, Callable, Dict, Optional
|
6
6
|
|
7
7
|
import anyio
|
8
8
|
import fsspec
|
@@ -544,4 +544,4 @@ class NullFileSystem(BaseModel):
|
|
544
544
|
pass
|
545
545
|
|
546
546
|
|
547
|
-
__getattr__ = getattr_migration(__name__)
|
547
|
+
__getattr__: Callable[[str], Any] = getattr_migration(__name__)
|
prefect/flow_engine.py
CHANGED
@@ -221,7 +221,7 @@ class FlowRunEngine(BaseFlowRunEngine[P, R]):
|
|
221
221
|
return_data=True,
|
222
222
|
max_depth=-1,
|
223
223
|
remove_annotations=True,
|
224
|
-
context={},
|
224
|
+
context={"parameter_name": parameter},
|
225
225
|
)
|
226
226
|
except UpstreamTaskError:
|
227
227
|
raise
|
@@ -784,7 +784,7 @@ class AsyncFlowRunEngine(BaseFlowRunEngine[P, R]):
|
|
784
784
|
return_data=True,
|
785
785
|
max_depth=-1,
|
786
786
|
remove_annotations=True,
|
787
|
-
context={},
|
787
|
+
context={"parameter_name": parameter},
|
788
788
|
)
|
789
789
|
except UpstreamTaskError:
|
790
790
|
raise
|