prefect-client 3.1.6__py3-none-any.whl → 3.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_experimental/__init__.py +0 -0
- prefect/_experimental/lineage.py +181 -0
- prefect/_internal/compatibility/async_dispatch.py +38 -9
- prefect/_internal/pydantic/v2_validated_func.py +15 -10
- prefect/_internal/retries.py +15 -6
- prefect/_internal/schemas/bases.py +2 -1
- prefect/_internal/schemas/validators.py +5 -4
- prefect/_version.py +3 -3
- prefect/blocks/core.py +144 -17
- prefect/blocks/system.py +2 -1
- prefect/client/orchestration.py +88 -0
- prefect/client/schemas/actions.py +5 -5
- prefect/client/schemas/filters.py +1 -1
- prefect/client/schemas/objects.py +5 -5
- prefect/client/schemas/responses.py +1 -2
- prefect/client/schemas/schedules.py +1 -1
- prefect/client/subscriptions.py +2 -1
- prefect/client/utilities.py +15 -1
- prefect/context.py +1 -1
- prefect/deployments/flow_runs.py +3 -3
- prefect/deployments/runner.py +14 -14
- prefect/deployments/steps/core.py +3 -1
- prefect/deployments/steps/pull.py +60 -12
- prefect/events/clients.py +55 -4
- prefect/events/filters.py +1 -1
- prefect/events/related.py +2 -1
- prefect/events/schemas/events.py +1 -1
- prefect/events/utilities.py +2 -0
- prefect/events/worker.py +8 -0
- prefect/flow_engine.py +41 -81
- prefect/flow_runs.py +4 -2
- prefect/flows.py +4 -6
- prefect/results.py +43 -22
- prefect/runner/storage.py +3 -3
- prefect/serializers.py +28 -24
- prefect/settings/models/experiments.py +5 -0
- prefect/task_engine.py +34 -26
- prefect/task_worker.py +43 -25
- prefect/tasks.py +118 -125
- prefect/telemetry/instrumentation.py +1 -1
- prefect/telemetry/processors.py +10 -7
- prefect/telemetry/run_telemetry.py +157 -33
- prefect/types/__init__.py +4 -1
- prefect/variables.py +127 -19
- {prefect_client-3.1.6.dist-info → prefect_client-3.1.7.dist-info}/METADATA +2 -1
- {prefect_client-3.1.6.dist-info → prefect_client-3.1.7.dist-info}/RECORD +49 -47
- {prefect_client-3.1.6.dist-info → prefect_client-3.1.7.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.6.dist-info → prefect_client-3.1.7.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.6.dist-info → prefect_client-3.1.7.dist-info}/top_level.txt +0 -0
prefect/blocks/core.py
CHANGED
@@ -41,6 +41,7 @@ from pydantic.json_schema import GenerateJsonSchema
|
|
41
41
|
from typing_extensions import Literal, ParamSpec, Self, get_args
|
42
42
|
|
43
43
|
import prefect.exceptions
|
44
|
+
from prefect._internal.compatibility.async_dispatch import async_dispatch
|
44
45
|
from prefect.client.schemas import (
|
45
46
|
DEFAULT_BLOCK_SCHEMA_VERSION,
|
46
47
|
BlockDocument,
|
@@ -53,7 +54,7 @@ from prefect.events import emit_event
|
|
53
54
|
from prefect.logging.loggers import disable_logger
|
54
55
|
from prefect.plugins import load_prefect_collections
|
55
56
|
from prefect.types import SecretDict
|
56
|
-
from prefect.utilities.asyncutils import sync_compatible
|
57
|
+
from prefect.utilities.asyncutils import run_coro_as_sync, sync_compatible
|
57
58
|
from prefect.utilities.collections import listrepr, remove_nested_keys, visit_collection
|
58
59
|
from prefect.utilities.dispatch import lookup_type, register_base_type
|
59
60
|
from prefect.utilities.hashing import hash_objects
|
@@ -64,7 +65,7 @@ from prefect.utilities.slugify import slugify
|
|
64
65
|
if TYPE_CHECKING:
|
65
66
|
from pydantic.main import IncEx
|
66
67
|
|
67
|
-
from prefect.client.orchestration import PrefectClient
|
68
|
+
from prefect.client.orchestration import PrefectClient, SyncPrefectClient
|
68
69
|
|
69
70
|
R = TypeVar("R")
|
70
71
|
P = ParamSpec("P")
|
@@ -280,7 +281,7 @@ class Block(BaseModel, ABC):
|
|
280
281
|
json_schema_extra=schema_extra,
|
281
282
|
)
|
282
283
|
|
283
|
-
def __init__(self, *args, **kwargs):
|
284
|
+
def __init__(self, *args: Any, **kwargs: Any):
|
284
285
|
super().__init__(*args, **kwargs)
|
285
286
|
self.block_initialization()
|
286
287
|
|
@@ -628,7 +629,8 @@ class Block(BaseModel, ABC):
|
|
628
629
|
"""Generates a default code example for the current class"""
|
629
630
|
qualified_name = to_qualified_name(cls)
|
630
631
|
module_str = ".".join(qualified_name.split(".")[:-1])
|
631
|
-
|
632
|
+
origin = cls.__pydantic_generic_metadata__.get("origin") or cls
|
633
|
+
class_name = origin.__name__
|
632
634
|
block_variable_name = f'{cls.get_block_type_slug().replace("-", "_")}_block'
|
633
635
|
|
634
636
|
return dedent(
|
@@ -777,12 +779,11 @@ class Block(BaseModel, ABC):
|
|
777
779
|
)
|
778
780
|
|
779
781
|
@classmethod
|
780
|
-
|
781
|
-
async def _get_block_document(
|
782
|
+
async def _aget_block_document(
|
782
783
|
cls,
|
783
784
|
name: str,
|
784
|
-
client:
|
785
|
-
):
|
785
|
+
client: "PrefectClient",
|
786
|
+
) -> tuple[BlockDocument, str]:
|
786
787
|
if cls.__name__ == "Block":
|
787
788
|
block_type_slug, block_document_name = name.split("/", 1)
|
788
789
|
else:
|
@@ -801,6 +802,30 @@ class Block(BaseModel, ABC):
|
|
801
802
|
|
802
803
|
return block_document, block_document_name
|
803
804
|
|
805
|
+
@classmethod
|
806
|
+
def _get_block_document(
|
807
|
+
cls,
|
808
|
+
name: str,
|
809
|
+
client: "SyncPrefectClient",
|
810
|
+
) -> tuple[BlockDocument, str]:
|
811
|
+
if cls.__name__ == "Block":
|
812
|
+
block_type_slug, block_document_name = name.split("/", 1)
|
813
|
+
else:
|
814
|
+
block_type_slug = cls.get_block_type_slug()
|
815
|
+
block_document_name = name
|
816
|
+
|
817
|
+
try:
|
818
|
+
block_document = client.read_block_document_by_name(
|
819
|
+
name=block_document_name, block_type_slug=block_type_slug
|
820
|
+
)
|
821
|
+
except prefect.exceptions.ObjectNotFound as e:
|
822
|
+
raise ValueError(
|
823
|
+
f"Unable to find block document named {block_document_name} for block"
|
824
|
+
f" type {block_type_slug}"
|
825
|
+
) from e
|
826
|
+
|
827
|
+
return block_document, block_document_name
|
828
|
+
|
804
829
|
@classmethod
|
805
830
|
@sync_compatible
|
806
831
|
@inject_client
|
@@ -829,9 +854,97 @@ class Block(BaseModel, ABC):
|
|
829
854
|
return block_document, block_document.name
|
830
855
|
|
831
856
|
@classmethod
|
832
|
-
@sync_compatible
|
833
857
|
@inject_client
|
834
|
-
async def
|
858
|
+
async def aload(
|
859
|
+
cls,
|
860
|
+
name: str,
|
861
|
+
validate: bool = True,
|
862
|
+
client: Optional["PrefectClient"] = None,
|
863
|
+
) -> "Self":
|
864
|
+
"""
|
865
|
+
Retrieves data from the block document with the given name for the block type
|
866
|
+
that corresponds with the current class and returns an instantiated version of
|
867
|
+
the current class with the data stored in the block document.
|
868
|
+
|
869
|
+
If a block document for a given block type is saved with a different schema
|
870
|
+
than the current class calling `aload`, a warning will be raised.
|
871
|
+
|
872
|
+
If the current class schema is a subset of the block document schema, the block
|
873
|
+
can be loaded as normal using the default `validate = True`.
|
874
|
+
|
875
|
+
If the current class schema is a superset of the block document schema, `aload`
|
876
|
+
must be called with `validate` set to False to prevent a validation error. In
|
877
|
+
this case, the block attributes will default to `None` and must be set manually
|
878
|
+
and saved to a new block document before the block can be used as expected.
|
879
|
+
|
880
|
+
Args:
|
881
|
+
name: The name or slug of the block document. A block document slug is a
|
882
|
+
string with the format <block_type_slug>/<block_document_name>
|
883
|
+
validate: If False, the block document will be loaded without Pydantic
|
884
|
+
validating the block schema. This is useful if the block schema has
|
885
|
+
changed client-side since the block document referred to by `name` was saved.
|
886
|
+
client: The client to use to load the block document. If not provided, the
|
887
|
+
default client will be injected.
|
888
|
+
|
889
|
+
Raises:
|
890
|
+
ValueError: If the requested block document is not found.
|
891
|
+
|
892
|
+
Returns:
|
893
|
+
An instance of the current class hydrated with the data stored in the
|
894
|
+
block document with the specified name.
|
895
|
+
|
896
|
+
Examples:
|
897
|
+
Load from a Block subclass with a block document name:
|
898
|
+
```python
|
899
|
+
class Custom(Block):
|
900
|
+
message: str
|
901
|
+
|
902
|
+
Custom(message="Hello!").save("my-custom-message")
|
903
|
+
|
904
|
+
loaded_block = await Custom.aload("my-custom-message")
|
905
|
+
```
|
906
|
+
|
907
|
+
Load from Block with a block document slug:
|
908
|
+
```python
|
909
|
+
class Custom(Block):
|
910
|
+
message: str
|
911
|
+
|
912
|
+
Custom(message="Hello!").save("my-custom-message")
|
913
|
+
|
914
|
+
loaded_block = await Block.aload("custom/my-custom-message")
|
915
|
+
```
|
916
|
+
|
917
|
+
Migrate a block document to a new schema:
|
918
|
+
```python
|
919
|
+
# original class
|
920
|
+
class Custom(Block):
|
921
|
+
message: str
|
922
|
+
|
923
|
+
Custom(message="Hello!").save("my-custom-message")
|
924
|
+
|
925
|
+
# Updated class with new required field
|
926
|
+
class Custom(Block):
|
927
|
+
message: str
|
928
|
+
number_of_ducks: int
|
929
|
+
|
930
|
+
loaded_block = await Custom.aload("my-custom-message", validate=False)
|
931
|
+
|
932
|
+
# Prints UserWarning about schema mismatch
|
933
|
+
|
934
|
+
loaded_block.number_of_ducks = 42
|
935
|
+
|
936
|
+
loaded_block.save("my-custom-message", overwrite=True)
|
937
|
+
```
|
938
|
+
"""
|
939
|
+
if TYPE_CHECKING:
|
940
|
+
assert isinstance(client, PrefectClient)
|
941
|
+
block_document, _ = await cls._aget_block_document(name, client=client)
|
942
|
+
|
943
|
+
return cls._load_from_block_document(block_document, validate=validate)
|
944
|
+
|
945
|
+
@classmethod
|
946
|
+
@async_dispatch(aload)
|
947
|
+
def load(
|
835
948
|
cls,
|
836
949
|
name: str,
|
837
950
|
validate: bool = True,
|
@@ -912,9 +1025,19 @@ class Block(BaseModel, ABC):
|
|
912
1025
|
loaded_block.save("my-custom-message", overwrite=True)
|
913
1026
|
```
|
914
1027
|
"""
|
915
|
-
|
916
|
-
|
917
|
-
|
1028
|
+
# Need to use a `PrefectClient` here to ensure `Block.load` and `Block.aload` signatures match
|
1029
|
+
# TODO: replace with only sync client once all internal calls are updated to use `Block.aload` and `@async_dispatch` is removed
|
1030
|
+
if client is None:
|
1031
|
+
# If a client wasn't provided, we get to use a sync client
|
1032
|
+
from prefect.client.orchestration import get_client
|
1033
|
+
|
1034
|
+
with get_client(sync_client=True) as sync_client:
|
1035
|
+
block_document, _ = cls._get_block_document(name, client=sync_client)
|
1036
|
+
else:
|
1037
|
+
# If a client was provided, reuse it, even though it's async, to avoid excessive client creation
|
1038
|
+
block_document, _ = run_coro_as_sync(
|
1039
|
+
cls._aget_block_document(name, client=client)
|
1040
|
+
)
|
918
1041
|
|
919
1042
|
return cls._load_from_block_document(block_document, validate=validate)
|
920
1043
|
|
@@ -968,14 +1091,16 @@ class Block(BaseModel, ABC):
|
|
968
1091
|
"""
|
969
1092
|
block_document = None
|
970
1093
|
if isinstance(ref, (str, UUID)):
|
971
|
-
block_document, _ = await cls._get_block_document_by_id(ref)
|
1094
|
+
block_document, _ = await cls._get_block_document_by_id(ref, client=client)
|
972
1095
|
elif isinstance(ref, dict):
|
973
1096
|
if block_document_id := ref.get("block_document_id"):
|
974
1097
|
block_document, _ = await cls._get_block_document_by_id(
|
975
|
-
block_document_id
|
1098
|
+
block_document_id, client=client
|
976
1099
|
)
|
977
1100
|
elif block_document_slug := ref.get("block_document_slug"):
|
978
|
-
block_document, _ = await cls._get_block_document(
|
1101
|
+
block_document, _ = await cls._get_block_document(
|
1102
|
+
block_document_slug, client=client
|
1103
|
+
)
|
979
1104
|
|
980
1105
|
if not block_document:
|
981
1106
|
raise ValueError(f"Invalid reference format {ref!r}.")
|
@@ -1220,7 +1345,9 @@ class Block(BaseModel, ABC):
|
|
1220
1345
|
name: str,
|
1221
1346
|
client: Optional["PrefectClient"] = None,
|
1222
1347
|
):
|
1223
|
-
|
1348
|
+
if TYPE_CHECKING:
|
1349
|
+
assert isinstance(client, PrefectClient)
|
1350
|
+
block_document, _ = await cls._aget_block_document(name, client=client)
|
1224
1351
|
|
1225
1352
|
await client.delete_block_document(block_document.id)
|
1226
1353
|
|
prefect/blocks/system.py
CHANGED
@@ -9,10 +9,10 @@ from pydantic import (
|
|
9
9
|
field_validator,
|
10
10
|
)
|
11
11
|
from pydantic import Secret as PydanticSecret
|
12
|
-
from pydantic_extra_types.pendulum_dt import DateTime as PydanticDateTime
|
13
12
|
|
14
13
|
from prefect._internal.compatibility.deprecated import deprecated_class
|
15
14
|
from prefect.blocks.core import Block
|
15
|
+
from prefect.types import DateTime as PydanticDateTime
|
16
16
|
|
17
17
|
_SecretValueType = Union[
|
18
18
|
Annotated[StrictStr, Field(title="string")],
|
@@ -130,6 +130,7 @@ class Secret(Block, Generic[T]):
|
|
130
130
|
|
131
131
|
_logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/c6f20e556dd16effda9df16551feecfb5822092b-48x48.png"
|
132
132
|
_documentation_url = "https://docs.prefect.io/latest/develop/blocks"
|
133
|
+
_description = "A block that represents a secret value. The value stored in this block will be obfuscated when this block is viewed or edited in the UI."
|
133
134
|
|
134
135
|
value: Union[SecretStr, PydanticSecret[T]] = Field(
|
135
136
|
default=...,
|
prefect/client/orchestration.py
CHANGED
@@ -4415,3 +4415,91 @@ class SyncPrefectClient:
|
|
4415
4415
|
json=labels,
|
4416
4416
|
)
|
4417
4417
|
response.raise_for_status()
|
4418
|
+
|
4419
|
+
def read_block_document_by_name(
|
4420
|
+
self,
|
4421
|
+
name: str,
|
4422
|
+
block_type_slug: str,
|
4423
|
+
include_secrets: bool = True,
|
4424
|
+
) -> BlockDocument:
|
4425
|
+
"""
|
4426
|
+
Read the block document with the specified name that corresponds to a
|
4427
|
+
specific block type name.
|
4428
|
+
|
4429
|
+
Args:
|
4430
|
+
name: The block document name.
|
4431
|
+
block_type_slug: The block type slug.
|
4432
|
+
include_secrets (bool): whether to include secret values
|
4433
|
+
on the Block, corresponding to Pydantic's `SecretStr` and
|
4434
|
+
`SecretBytes` fields. These fields are automatically obfuscated
|
4435
|
+
by Pydantic, but users can additionally choose not to receive
|
4436
|
+
their values from the API. Note that any business logic on the
|
4437
|
+
Block may not work if this is `False`.
|
4438
|
+
|
4439
|
+
Raises:
|
4440
|
+
httpx.RequestError: if the block document was not found for any reason
|
4441
|
+
|
4442
|
+
Returns:
|
4443
|
+
A block document or None.
|
4444
|
+
"""
|
4445
|
+
try:
|
4446
|
+
response = self._client.get(
|
4447
|
+
f"/block_types/slug/{block_type_slug}/block_documents/name/{name}",
|
4448
|
+
params=dict(include_secrets=include_secrets),
|
4449
|
+
)
|
4450
|
+
except httpx.HTTPStatusError as e:
|
4451
|
+
if e.response.status_code == status.HTTP_404_NOT_FOUND:
|
4452
|
+
raise prefect.exceptions.ObjectNotFound(http_exc=e) from e
|
4453
|
+
else:
|
4454
|
+
raise
|
4455
|
+
return BlockDocument.model_validate(response.json())
|
4456
|
+
|
4457
|
+
def create_variable(self, variable: VariableCreate) -> Variable:
|
4458
|
+
"""
|
4459
|
+
Creates an variable with the provided configuration.
|
4460
|
+
|
4461
|
+
Args:
|
4462
|
+
variable: Desired configuration for the new variable.
|
4463
|
+
Returns:
|
4464
|
+
Information about the newly created variable.
|
4465
|
+
"""
|
4466
|
+
response = self._client.post(
|
4467
|
+
"/variables/",
|
4468
|
+
json=variable.model_dump(mode="json", exclude_unset=True),
|
4469
|
+
)
|
4470
|
+
return Variable(**response.json())
|
4471
|
+
|
4472
|
+
def update_variable(self, variable: VariableUpdate) -> None:
|
4473
|
+
"""
|
4474
|
+
Updates a variable with the provided configuration.
|
4475
|
+
|
4476
|
+
Args:
|
4477
|
+
variable: Desired configuration for the updated variable.
|
4478
|
+
Returns:
|
4479
|
+
Information about the updated variable.
|
4480
|
+
"""
|
4481
|
+
self._client.patch(
|
4482
|
+
f"/variables/name/{variable.name}",
|
4483
|
+
json=variable.model_dump(mode="json", exclude_unset=True),
|
4484
|
+
)
|
4485
|
+
|
4486
|
+
def read_variable_by_name(self, name: str) -> Optional[Variable]:
|
4487
|
+
"""Reads a variable by name. Returns None if no variable is found."""
|
4488
|
+
try:
|
4489
|
+
response = self._client.get(f"/variables/name/{name}")
|
4490
|
+
return Variable(**response.json())
|
4491
|
+
except httpx.HTTPStatusError as e:
|
4492
|
+
if e.response.status_code == status.HTTP_404_NOT_FOUND:
|
4493
|
+
return None
|
4494
|
+
else:
|
4495
|
+
raise
|
4496
|
+
|
4497
|
+
def delete_variable_by_name(self, name: str) -> None:
|
4498
|
+
"""Deletes a variable by name."""
|
4499
|
+
try:
|
4500
|
+
self._client.delete(f"/variables/name/{name}")
|
4501
|
+
except httpx.HTTPStatusError as e:
|
4502
|
+
if e.response.status_code == 404:
|
4503
|
+
raise prefect.exceptions.ObjectNotFound(http_exc=e) from e
|
4504
|
+
else:
|
4505
|
+
raise
|
@@ -4,7 +4,6 @@ from uuid import UUID, uuid4
|
|
4
4
|
|
5
5
|
import jsonschema
|
6
6
|
from pydantic import Field, field_validator, model_validator
|
7
|
-
from pydantic_extra_types.pendulum_dt import DateTime
|
8
7
|
|
9
8
|
import prefect.client.schemas.objects as objects
|
10
9
|
from prefect._internal.schemas.bases import ActionBaseModel
|
@@ -27,6 +26,7 @@ from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
|
27
26
|
from prefect.settings import PREFECT_DEPLOYMENT_SCHEDULE_MAX_SCHEDULED_RUNS
|
28
27
|
from prefect.types import (
|
29
28
|
MAX_VARIABLE_NAME_LENGTH,
|
29
|
+
DateTime,
|
30
30
|
KeyValueLabelsField,
|
31
31
|
Name,
|
32
32
|
NonEmptyishName,
|
@@ -393,10 +393,10 @@ class DeploymentFlowRunCreate(ActionBaseModel):
|
|
393
393
|
default_factory=objects.FlowRunPolicy
|
394
394
|
)
|
395
395
|
tags: list[str] = Field(default_factory=list)
|
396
|
-
idempotency_key: Optional[str] = Field(None)
|
397
|
-
parent_task_run_id: Optional[UUID] = Field(None)
|
398
|
-
work_queue_name: Optional[str] = Field(None)
|
399
|
-
job_variables: Optional[dict] = Field(None)
|
396
|
+
idempotency_key: Optional[str] = Field(default=None)
|
397
|
+
parent_task_run_id: Optional[UUID] = Field(default=None)
|
398
|
+
work_queue_name: Optional[str] = Field(default=None)
|
399
|
+
job_variables: Optional[dict[str, Any]] = Field(default=None)
|
400
400
|
labels: KeyValueLabelsField = Field(default_factory=dict)
|
401
401
|
|
402
402
|
|
@@ -6,10 +6,10 @@ from typing import List, Optional
|
|
6
6
|
from uuid import UUID
|
7
7
|
|
8
8
|
from pydantic import Field
|
9
|
-
from pydantic_extra_types.pendulum_dt import DateTime
|
10
9
|
|
11
10
|
from prefect._internal.schemas.bases import PrefectBaseModel
|
12
11
|
from prefect.client.schemas.objects import StateType
|
12
|
+
from prefect.types import DateTime
|
13
13
|
from prefect.utilities.collections import AutoEnum
|
14
14
|
|
15
15
|
|
@@ -69,7 +69,7 @@ if TYPE_CHECKING:
|
|
69
69
|
|
70
70
|
DateTime = pendulum.DateTime
|
71
71
|
else:
|
72
|
-
from
|
72
|
+
from prefect.types import DateTime
|
73
73
|
|
74
74
|
|
75
75
|
R = TypeVar("R", default=Any)
|
@@ -207,7 +207,7 @@ class State(ObjectBaseModel, Generic[R]):
|
|
207
207
|
|
208
208
|
type: StateType
|
209
209
|
name: Optional[str] = Field(default=None)
|
210
|
-
timestamp: DateTime = Field(default_factory=lambda:
|
210
|
+
timestamp: DateTime = Field(default_factory=lambda: DateTime.now("UTC"))
|
211
211
|
message: Optional[str] = Field(default=None, examples=["Run started"])
|
212
212
|
state_details: StateDetails = Field(default_factory=StateDetails)
|
213
213
|
data: Annotated[
|
@@ -429,9 +429,9 @@ class State(ObjectBaseModel, Generic[R]):
|
|
429
429
|
return self.model_copy(
|
430
430
|
update={
|
431
431
|
"id": uuid4(),
|
432
|
-
"created":
|
433
|
-
"updated":
|
434
|
-
"timestamp":
|
432
|
+
"created": DateTime.now("utc"),
|
433
|
+
"updated": DateTime.now("utc"),
|
434
|
+
"timestamp": DateTime.now("utc"),
|
435
435
|
},
|
436
436
|
**kwargs,
|
437
437
|
)
|
@@ -3,13 +3,12 @@ from typing import Any, ClassVar, Generic, Optional, TypeVar, Union
|
|
3
3
|
from uuid import UUID
|
4
4
|
|
5
5
|
from pydantic import ConfigDict, Field
|
6
|
-
from pydantic_extra_types.pendulum_dt import DateTime
|
7
6
|
from typing_extensions import Literal
|
8
7
|
|
9
8
|
import prefect.client.schemas.objects as objects
|
10
9
|
from prefect._internal.schemas.bases import ObjectBaseModel, PrefectBaseModel
|
11
10
|
from prefect._internal.schemas.fields import CreatedBy, UpdatedBy
|
12
|
-
from prefect.types import KeyValueLabelsField
|
11
|
+
from prefect.types import DateTime, KeyValueLabelsField
|
13
12
|
from prefect.utilities.collections import AutoEnum
|
14
13
|
from prefect.utilities.names import generate_slug
|
15
14
|
|
prefect/client/subscriptions.py
CHANGED
@@ -10,6 +10,7 @@ from starlette.status import WS_1008_POLICY_VIOLATION
|
|
10
10
|
from typing_extensions import Self
|
11
11
|
|
12
12
|
from prefect._internal.schemas.bases import IDBaseModel
|
13
|
+
from prefect.events.clients import websocket_connect
|
13
14
|
from prefect.logging import get_logger
|
14
15
|
from prefect.settings import PREFECT_API_KEY
|
15
16
|
|
@@ -34,7 +35,7 @@ class Subscription(Generic[S]):
|
|
34
35
|
|
35
36
|
self.keys: list[str] = list(keys)
|
36
37
|
|
37
|
-
self._connect =
|
38
|
+
self._connect = websocket_connect(
|
38
39
|
self.subscription_url,
|
39
40
|
subprotocols=[websockets.Subprotocol("prefect")],
|
40
41
|
)
|
prefect/client/utilities.py
CHANGED
@@ -7,7 +7,7 @@ Utilities for working with clients.
|
|
7
7
|
|
8
8
|
from collections.abc import Awaitable, Coroutine
|
9
9
|
from functools import wraps
|
10
|
-
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
|
10
|
+
from typing import TYPE_CHECKING, Any, Callable, Optional, Union, overload
|
11
11
|
|
12
12
|
from typing_extensions import Concatenate, ParamSpec, TypeGuard, TypeVar
|
13
13
|
|
@@ -71,9 +71,23 @@ def client_injector(
|
|
71
71
|
return wrapper
|
72
72
|
|
73
73
|
|
74
|
+
@overload
|
74
75
|
def inject_client(
|
75
76
|
fn: Callable[P, Coroutine[Any, Any, R]],
|
76
77
|
) -> Callable[P, Coroutine[Any, Any, R]]:
|
78
|
+
...
|
79
|
+
|
80
|
+
|
81
|
+
@overload
|
82
|
+
def inject_client(
|
83
|
+
fn: Callable[P, R],
|
84
|
+
) -> Callable[P, R]:
|
85
|
+
...
|
86
|
+
|
87
|
+
|
88
|
+
def inject_client(
|
89
|
+
fn: Callable[P, Union[Coroutine[Any, Any, R], R]],
|
90
|
+
) -> Callable[P, Union[Coroutine[Any, Any, R], R]]:
|
77
91
|
"""
|
78
92
|
Simple helper to provide a context managed client to an asynchronous function.
|
79
93
|
|
prefect/context.py
CHANGED
@@ -26,7 +26,6 @@ from typing import (
|
|
26
26
|
)
|
27
27
|
|
28
28
|
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
|
29
|
-
from pydantic_extra_types.pendulum_dt import DateTime
|
30
29
|
from typing_extensions import Self
|
31
30
|
|
32
31
|
import prefect.logging
|
@@ -48,6 +47,7 @@ from prefect.settings.legacy import (
|
|
48
47
|
)
|
49
48
|
from prefect.states import State
|
50
49
|
from prefect.task_runners import TaskRunner
|
50
|
+
from prefect.types import DateTime
|
51
51
|
from prefect.utilities.services import start_client_metrics_server
|
52
52
|
|
53
53
|
T = TypeVar("T")
|
prefect/deployments/flow_runs.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
from datetime import datetime
|
2
|
-
from typing import TYPE_CHECKING, Iterable, Optional, Union
|
2
|
+
from typing import TYPE_CHECKING, Any, Iterable, Optional, Union
|
3
3
|
from uuid import UUID
|
4
4
|
|
5
5
|
import anyio
|
@@ -35,7 +35,7 @@ logger = get_logger(__name__)
|
|
35
35
|
async def run_deployment(
|
36
36
|
name: Union[str, UUID],
|
37
37
|
client: Optional["PrefectClient"] = None,
|
38
|
-
parameters: Optional[dict] = None,
|
38
|
+
parameters: Optional[dict[str, Any]] = None,
|
39
39
|
scheduled_time: Optional[datetime] = None,
|
40
40
|
flow_run_name: Optional[str] = None,
|
41
41
|
timeout: Optional[float] = None,
|
@@ -44,7 +44,7 @@ async def run_deployment(
|
|
44
44
|
idempotency_key: Optional[str] = None,
|
45
45
|
work_queue_name: Optional[str] = None,
|
46
46
|
as_subflow: Optional[bool] = True,
|
47
|
-
job_variables: Optional[dict] = None,
|
47
|
+
job_variables: Optional[dict[str, Any]] = None,
|
48
48
|
) -> "FlowRun":
|
49
49
|
"""
|
50
50
|
Create a flow run for a deployment and return it after completion or a timeout.
|
prefect/deployments/runner.py
CHANGED
@@ -33,7 +33,7 @@ import importlib
|
|
33
33
|
import tempfile
|
34
34
|
from datetime import datetime, timedelta
|
35
35
|
from pathlib import Path
|
36
|
-
from typing import TYPE_CHECKING, Any,
|
36
|
+
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union
|
37
37
|
from uuid import UUID
|
38
38
|
|
39
39
|
from pydantic import (
|
@@ -160,7 +160,7 @@ class RunnerDeployment(BaseModel):
|
|
160
160
|
paused: Optional[bool] = Field(
|
161
161
|
default=None, description="Whether or not the deployment is paused."
|
162
162
|
)
|
163
|
-
parameters:
|
163
|
+
parameters: dict[str, Any] = Field(default_factory=dict)
|
164
164
|
entrypoint: Optional[str] = Field(
|
165
165
|
default=None,
|
166
166
|
description=(
|
@@ -198,7 +198,7 @@ class RunnerDeployment(BaseModel):
|
|
198
198
|
" the deployment is registered with a built runner."
|
199
199
|
),
|
200
200
|
)
|
201
|
-
job_variables:
|
201
|
+
job_variables: dict[str, Any] = Field(
|
202
202
|
default_factory=dict,
|
203
203
|
description=(
|
204
204
|
"Job variables used to override the default values of a work pool"
|
@@ -280,7 +280,7 @@ class RunnerDeployment(BaseModel):
|
|
280
280
|
async with get_client() as client:
|
281
281
|
flow_id = await client.create_flow_from_name(self.flow_name)
|
282
282
|
|
283
|
-
create_payload = dict(
|
283
|
+
create_payload: dict[str, Any] = dict(
|
284
284
|
flow_id=flow_id,
|
285
285
|
name=self.name,
|
286
286
|
work_queue_name=self.work_queue_name,
|
@@ -428,7 +428,7 @@ class RunnerDeployment(BaseModel):
|
|
428
428
|
else:
|
429
429
|
return [create_deployment_schedule_create(schedule)]
|
430
430
|
|
431
|
-
def _set_defaults_from_flow(self, flow: "Flow"):
|
431
|
+
def _set_defaults_from_flow(self, flow: "Flow[..., Any]"):
|
432
432
|
self._parameter_openapi_schema = parameter_schema(flow)
|
433
433
|
|
434
434
|
if not self.version:
|
@@ -439,7 +439,7 @@ class RunnerDeployment(BaseModel):
|
|
439
439
|
@classmethod
|
440
440
|
def from_flow(
|
441
441
|
cls,
|
442
|
-
flow: "Flow",
|
442
|
+
flow: "Flow[..., Any]",
|
443
443
|
name: str,
|
444
444
|
interval: Optional[
|
445
445
|
Union[Iterable[Union[int, float, timedelta]], int, float, timedelta]
|
@@ -449,7 +449,7 @@ class RunnerDeployment(BaseModel):
|
|
449
449
|
paused: Optional[bool] = None,
|
450
450
|
schedules: Optional["FlexibleScheduleList"] = None,
|
451
451
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
452
|
-
parameters: Optional[dict] = None,
|
452
|
+
parameters: Optional[dict[str, Any]] = None,
|
453
453
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
454
454
|
description: Optional[str] = None,
|
455
455
|
tags: Optional[List[str]] = None,
|
@@ -457,7 +457,7 @@ class RunnerDeployment(BaseModel):
|
|
457
457
|
enforce_parameter_schema: bool = True,
|
458
458
|
work_pool_name: Optional[str] = None,
|
459
459
|
work_queue_name: Optional[str] = None,
|
460
|
-
job_variables: Optional[
|
460
|
+
job_variables: Optional[dict[str, Any]] = None,
|
461
461
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
462
462
|
) -> "RunnerDeployment":
|
463
463
|
"""
|
@@ -588,7 +588,7 @@ class RunnerDeployment(BaseModel):
|
|
588
588
|
paused: Optional[bool] = None,
|
589
589
|
schedules: Optional["FlexibleScheduleList"] = None,
|
590
590
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
591
|
-
parameters: Optional[dict] = None,
|
591
|
+
parameters: Optional[dict[str, Any]] = None,
|
592
592
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
593
593
|
description: Optional[str] = None,
|
594
594
|
tags: Optional[List[str]] = None,
|
@@ -596,7 +596,7 @@ class RunnerDeployment(BaseModel):
|
|
596
596
|
enforce_parameter_schema: bool = True,
|
597
597
|
work_pool_name: Optional[str] = None,
|
598
598
|
work_queue_name: Optional[str] = None,
|
599
|
-
job_variables: Optional[
|
599
|
+
job_variables: Optional[dict[str, Any]] = None,
|
600
600
|
) -> "RunnerDeployment":
|
601
601
|
"""
|
602
602
|
Configure a deployment for a given flow located at a given entrypoint.
|
@@ -689,7 +689,7 @@ class RunnerDeployment(BaseModel):
|
|
689
689
|
paused: Optional[bool] = None,
|
690
690
|
schedules: Optional["FlexibleScheduleList"] = None,
|
691
691
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
692
|
-
parameters: Optional[dict] = None,
|
692
|
+
parameters: Optional[dict[str, Any]] = None,
|
693
693
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
694
694
|
description: Optional[str] = None,
|
695
695
|
tags: Optional[List[str]] = None,
|
@@ -697,7 +697,7 @@ class RunnerDeployment(BaseModel):
|
|
697
697
|
enforce_parameter_schema: bool = True,
|
698
698
|
work_pool_name: Optional[str] = None,
|
699
699
|
work_queue_name: Optional[str] = None,
|
700
|
-
job_variables: Optional[
|
700
|
+
job_variables: Optional[dict[str, Any]] = None,
|
701
701
|
):
|
702
702
|
"""
|
703
703
|
Create a RunnerDeployment from a flow located at a given entrypoint and stored in a
|
@@ -945,8 +945,8 @@ async def deploy(
|
|
945
945
|
|
946
946
|
console.print(f"Successfully pushed image {image.reference!r}", style="green")
|
947
947
|
|
948
|
-
deployment_exceptions = []
|
949
|
-
deployment_ids = []
|
948
|
+
deployment_exceptions: list[dict[str, Any]] = []
|
949
|
+
deployment_ids: list[UUID] = []
|
950
950
|
image_ref = image.reference if image else None
|
951
951
|
for deployment in track(
|
952
952
|
deployments,
|
@@ -99,7 +99,9 @@ def _get_function_for_step(
|
|
99
99
|
return step_func
|
100
100
|
|
101
101
|
|
102
|
-
async def run_step(
|
102
|
+
async def run_step(
|
103
|
+
step: dict[str, Any], upstream_outputs: Optional[dict[str, Any]] = None
|
104
|
+
) -> dict[str, Any]:
|
103
105
|
"""
|
104
106
|
Runs a step, returns the step's output.
|
105
107
|
|