prefect-client 2.16.7__py3-none-any.whl → 2.16.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/compatibility/experimental.py +9 -8
- prefect/_internal/concurrency/api.py +23 -42
- prefect/_internal/concurrency/waiters.py +25 -22
- prefect/_internal/pydantic/__init__.py +12 -3
- prefect/_internal/pydantic/_base_model.py +7 -4
- prefect/_internal/pydantic/_compat.py +39 -453
- prefect/_internal/pydantic/_flags.py +2 -0
- prefect/_internal/pydantic/_types.py +8 -0
- prefect/_internal/pydantic/utilities/__init__.py +0 -0
- prefect/_internal/pydantic/utilities/model_construct.py +56 -0
- prefect/_internal/pydantic/utilities/model_copy.py +55 -0
- prefect/_internal/pydantic/utilities/model_dump.py +136 -0
- prefect/_internal/pydantic/utilities/model_dump_json.py +112 -0
- prefect/_internal/pydantic/utilities/model_fields.py +50 -0
- prefect/_internal/pydantic/utilities/model_json_schema.py +82 -0
- prefect/_internal/pydantic/utilities/model_rebuild.py +80 -0
- prefect/_internal/pydantic/utilities/model_validate.py +75 -0
- prefect/_internal/pydantic/utilities/model_validate_json.py +68 -0
- prefect/_internal/pydantic/utilities/type_adapter.py +71 -0
- prefect/_internal/schemas/bases.py +1 -17
- prefect/_internal/schemas/validators.py +425 -4
- prefect/blocks/kubernetes.py +7 -3
- prefect/client/cloud.py +1 -1
- prefect/client/orchestration.py +8 -8
- prefect/client/schemas/actions.py +348 -285
- prefect/client/schemas/objects.py +47 -126
- prefect/client/schemas/responses.py +231 -57
- prefect/concurrency/events.py +2 -2
- prefect/context.py +2 -1
- prefect/deployments/base.py +4 -3
- prefect/deployments/runner.py +7 -25
- prefect/deprecated/packaging/base.py +5 -6
- prefect/deprecated/packaging/docker.py +19 -25
- prefect/deprecated/packaging/file.py +10 -5
- prefect/deprecated/packaging/orion.py +9 -4
- prefect/deprecated/packaging/serializers.py +8 -58
- prefect/engine.py +23 -22
- prefect/events/actions.py +16 -1
- prefect/events/related.py +4 -4
- prefect/events/schemas/automations.py +13 -2
- prefect/events/schemas/deployment_triggers.py +73 -5
- prefect/events/schemas/events.py +1 -1
- prefect/flows.py +3 -0
- prefect/infrastructure/provisioners/ecs.py +1 -0
- prefect/logging/configuration.py +2 -2
- prefect/pydantic/__init__.py +48 -2
- prefect/pydantic/main.py +2 -2
- prefect/serializers.py +6 -31
- prefect/settings.py +40 -17
- prefect/software/python.py +3 -5
- prefect/utilities/callables.py +1 -1
- prefect/utilities/collections.py +2 -1
- prefect/utilities/schema_tools/validation.py +2 -2
- prefect/workers/base.py +19 -10
- prefect/workers/block.py +3 -7
- prefect/workers/process.py +2 -5
- {prefect_client-2.16.7.dist-info → prefect_client-2.16.9.dist-info}/METADATA +3 -2
- {prefect_client-2.16.7.dist-info → prefect_client-2.16.9.dist-info}/RECORD +61 -50
- prefect/_internal/schemas/transformations.py +0 -106
- {prefect_client-2.16.7.dist-info → prefect_client-2.16.9.dist-info}/LICENSE +0 -0
- {prefect_client-2.16.7.dist-info → prefect_client-2.16.9.dist-info}/WHEEL +0 -0
- {prefect_client-2.16.7.dist-info → prefect_client-2.16.9.dist-info}/top_level.txt +0 -0
@@ -11,6 +11,7 @@ create them from YAML.
|
|
11
11
|
|
12
12
|
import abc
|
13
13
|
import textwrap
|
14
|
+
import warnings
|
14
15
|
from datetime import timedelta
|
15
16
|
from typing import (
|
16
17
|
Any,
|
@@ -36,8 +37,17 @@ else:
|
|
36
37
|
from pydantic import Field, PrivateAttr, root_validator, validator
|
37
38
|
from pydantic.fields import ModelField
|
38
39
|
|
40
|
+
from prefect._internal.compatibility.experimental import (
|
41
|
+
EXPERIMENTAL_WARNING,
|
42
|
+
PREFECT_EXPERIMENTAL_WARN,
|
43
|
+
ExperimentalFeature,
|
44
|
+
experiment_enabled,
|
45
|
+
)
|
39
46
|
from prefect._internal.schemas.bases import PrefectBaseModel
|
40
47
|
from prefect.events.actions import RunDeployment
|
48
|
+
from prefect.settings import (
|
49
|
+
PREFECT_EXPERIMENTAL_WARN_FLOW_RUN_INFRA_OVERRIDES,
|
50
|
+
)
|
41
51
|
|
42
52
|
from .automations import (
|
43
53
|
Automation,
|
@@ -80,7 +90,13 @@ class BaseDeploymentTrigger(PrefectBaseModel, abc.ABC, extra="ignore"):
|
|
80
90
|
"deployment's default parameters"
|
81
91
|
),
|
82
92
|
)
|
83
|
-
|
93
|
+
job_variables: Optional[Dict[str, Any]] = Field(
|
94
|
+
None,
|
95
|
+
description=(
|
96
|
+
"Job variables to pass to the deployment, or None to use the "
|
97
|
+
"deployment's default job variables"
|
98
|
+
),
|
99
|
+
)
|
84
100
|
_deployment_id: Optional[UUID] = PrivateAttr(default=None)
|
85
101
|
|
86
102
|
def set_deployment_id(self, deployment_id: UUID):
|
@@ -90,11 +106,32 @@ class BaseDeploymentTrigger(PrefectBaseModel, abc.ABC, extra="ignore"):
|
|
90
106
|
return f"prefect.deployment.{self._deployment_id}"
|
91
107
|
|
92
108
|
def actions(self) -> List[RunDeployment]:
|
109
|
+
if self.job_variables is not None and experiment_enabled(
|
110
|
+
"flow_run_infra_overrides"
|
111
|
+
):
|
112
|
+
if (
|
113
|
+
PREFECT_EXPERIMENTAL_WARN
|
114
|
+
and PREFECT_EXPERIMENTAL_WARN_FLOW_RUN_INFRA_OVERRIDES
|
115
|
+
):
|
116
|
+
warnings.warn(
|
117
|
+
EXPERIMENTAL_WARNING.format(
|
118
|
+
feature="Flow run job variables",
|
119
|
+
group="flow_run_infra_overrides",
|
120
|
+
help="To use this feature, update your workers to Prefect 2.16.4 or later. ",
|
121
|
+
),
|
122
|
+
ExperimentalFeature,
|
123
|
+
stacklevel=3,
|
124
|
+
)
|
125
|
+
if not experiment_enabled("flow_run_infra_overrides"):
|
126
|
+
# nullify job_variables if the flag is disabled
|
127
|
+
self.job_variables = None
|
128
|
+
|
93
129
|
assert self._deployment_id
|
94
130
|
return [
|
95
131
|
RunDeployment(
|
96
132
|
parameters=self.parameters,
|
97
133
|
deployment_id=self._deployment_id,
|
134
|
+
job_variables=self.job_variables,
|
98
135
|
)
|
99
136
|
]
|
100
137
|
|
@@ -124,11 +161,11 @@ class DeploymentResourceTrigger(BaseDeploymentTrigger, abc.ABC):
|
|
124
161
|
type: str
|
125
162
|
|
126
163
|
match: ResourceSpecification = Field(
|
127
|
-
default_factory=lambda: ResourceSpecification(
|
164
|
+
default_factory=lambda: ResourceSpecification.parse_obj({}),
|
128
165
|
description="Labels for resources which this trigger will match.",
|
129
166
|
)
|
130
167
|
match_related: ResourceSpecification = Field(
|
131
|
-
default_factory=lambda: ResourceSpecification(
|
168
|
+
default_factory=lambda: ResourceSpecification.parse_obj({}),
|
132
169
|
description="Labels for related resources which this trigger will match.",
|
133
170
|
)
|
134
171
|
|
@@ -254,6 +291,7 @@ class DeploymentMetricTrigger(DeploymentResourceTrigger):
|
|
254
291
|
match_related=self.match_related,
|
255
292
|
posture=self.posture,
|
256
293
|
metric=self.metric,
|
294
|
+
job_variables=self.job_variables,
|
257
295
|
)
|
258
296
|
|
259
297
|
|
@@ -293,6 +331,7 @@ class DeploymentCompoundTrigger(DeploymentCompositeTrigger):
|
|
293
331
|
require=self.require,
|
294
332
|
triggers=self.triggers,
|
295
333
|
within=self.within,
|
334
|
+
job_variables=self.job_variables,
|
296
335
|
)
|
297
336
|
|
298
337
|
|
@@ -306,6 +345,7 @@ class DeploymentSequenceTrigger(DeploymentCompositeTrigger):
|
|
306
345
|
return SequenceTrigger(
|
307
346
|
triggers=self.triggers,
|
308
347
|
within=self.within,
|
348
|
+
job_variables=self.job_variables,
|
309
349
|
)
|
310
350
|
|
311
351
|
|
@@ -343,15 +383,22 @@ class DeploymentTrigger(PrefectBaseModel):
|
|
343
383
|
)
|
344
384
|
description: str = Field("", description="A longer description of this automation")
|
345
385
|
enabled: bool = Field(True, description="Whether this automation will be evaluated")
|
386
|
+
job_variables: Optional[Dict[str, Any]] = Field(
|
387
|
+
None,
|
388
|
+
description=(
|
389
|
+
"Job variables to pass to the run, or None to use the "
|
390
|
+
"deployment's default job variables"
|
391
|
+
),
|
392
|
+
)
|
346
393
|
|
347
394
|
# from ResourceTrigger
|
348
395
|
|
349
396
|
match: ResourceSpecification = Field(
|
350
|
-
default_factory=lambda: ResourceSpecification(
|
397
|
+
default_factory=lambda: ResourceSpecification.parse_obj({}),
|
351
398
|
description="Labels for resources which this trigger will match.",
|
352
399
|
)
|
353
400
|
match_related: ResourceSpecification = Field(
|
354
|
-
default_factory=lambda: ResourceSpecification(
|
401
|
+
default_factory=lambda: ResourceSpecification.parse_obj({}),
|
355
402
|
description="Labels for related resources which this trigger will match.",
|
356
403
|
)
|
357
404
|
|
@@ -472,10 +519,31 @@ class DeploymentTrigger(PrefectBaseModel):
|
|
472
519
|
return f"prefect.deployment.{self._deployment_id}"
|
473
520
|
|
474
521
|
def actions(self) -> List[RunDeployment]:
|
522
|
+
if self.job_variables is not None and experiment_enabled(
|
523
|
+
"flow_run_infra_overrides"
|
524
|
+
):
|
525
|
+
if (
|
526
|
+
PREFECT_EXPERIMENTAL_WARN
|
527
|
+
and PREFECT_EXPERIMENTAL_WARN_FLOW_RUN_INFRA_OVERRIDES
|
528
|
+
):
|
529
|
+
warnings.warn(
|
530
|
+
EXPERIMENTAL_WARNING.format(
|
531
|
+
feature="Flow run job variables",
|
532
|
+
group="flow_run_infra_overrides",
|
533
|
+
help="To use this feature, update your workers to Prefect 2.16.4 or later. ",
|
534
|
+
),
|
535
|
+
ExperimentalFeature,
|
536
|
+
stacklevel=3,
|
537
|
+
)
|
538
|
+
if not experiment_enabled("flow_run_infra_overrides"):
|
539
|
+
# nullify job_variables if the flag is disabled
|
540
|
+
self.job_variables = None
|
541
|
+
|
475
542
|
assert self._deployment_id
|
476
543
|
return [
|
477
544
|
RunDeployment(
|
478
545
|
parameters=self.parameters,
|
479
546
|
deployment_id=self._deployment_id,
|
547
|
+
job_variables=self.job_variables,
|
480
548
|
)
|
481
549
|
]
|
prefect/events/schemas/events.py
CHANGED
@@ -282,4 +282,4 @@ class ResourceSpecification(PrefectBaseModel):
|
|
282
282
|
return len(self.__root__)
|
283
283
|
|
284
284
|
def deepcopy(self) -> "ResourceSpecification":
|
285
|
-
return ResourceSpecification(
|
285
|
+
return ResourceSpecification.parse_obj(copy.deepcopy(self.__root__))
|
prefect/flows.py
CHANGED
@@ -970,6 +970,7 @@ class Flow(Generic[P, R]):
|
|
970
970
|
enforce_parameter_schema: bool = False,
|
971
971
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
972
972
|
print_next_steps: bool = True,
|
973
|
+
ignore_warnings: bool = False,
|
973
974
|
) -> UUID:
|
974
975
|
"""
|
975
976
|
Deploys a flow to run on dynamic infrastructure via a work pool.
|
@@ -1024,6 +1025,7 @@ class Flow(Generic[P, R]):
|
|
1024
1025
|
entrypoint, ensure that the module will be importable in the execution environment.
|
1025
1026
|
print_next_steps_message: Whether or not to print a message with next steps
|
1026
1027
|
after deploying the deployments.
|
1028
|
+
ignore_warnings: Whether or not to ignore warnings about the work pool type.
|
1027
1029
|
|
1028
1030
|
Returns:
|
1029
1031
|
The ID of the created/updated deployment.
|
@@ -1100,6 +1102,7 @@ class Flow(Generic[P, R]):
|
|
1100
1102
|
build=build,
|
1101
1103
|
push=push,
|
1102
1104
|
print_next_steps_message=False,
|
1105
|
+
ignore_warnings=ignore_warnings,
|
1103
1106
|
)
|
1104
1107
|
|
1105
1108
|
if print_next_steps:
|
prefect/logging/configuration.py
CHANGED
@@ -6,7 +6,7 @@ import string
|
|
6
6
|
import warnings
|
7
7
|
from functools import partial
|
8
8
|
from pathlib import Path
|
9
|
-
from typing import Optional
|
9
|
+
from typing import Any, Dict, Optional
|
10
10
|
|
11
11
|
import yaml
|
12
12
|
|
@@ -21,7 +21,7 @@ from prefect.utilities.collections import dict_to_flatdict, flatdict_to_dict
|
|
21
21
|
DEFAULT_LOGGING_SETTINGS_PATH = Path(__file__).parent / "logging.yml"
|
22
22
|
|
23
23
|
# Stores the configuration used to setup logging in this Python process
|
24
|
-
PROCESS_LOGGING_CONFIG:
|
24
|
+
PROCESS_LOGGING_CONFIG: Optional[Dict[str, Any]] = None
|
25
25
|
|
26
26
|
# Regex call to replace non-alphanumeric characters to '_' to create a valid env var
|
27
27
|
to_envvar = partial(re.sub, re.compile(r"[^0-9a-zA-Z]+"), "_")
|
prefect/pydantic/__init__.py
CHANGED
@@ -1,4 +1,50 @@
|
|
1
|
-
|
1
|
+
import typing
|
2
|
+
from prefect._internal.pydantic._flags import HAS_PYDANTIC_V2, USE_PYDANTIC_V2
|
2
3
|
|
4
|
+
if typing.TYPE_CHECKING:
|
5
|
+
# import of virtually everything is supported via `__getattr__` below,
|
6
|
+
# but we need them here for type checking and IDE support
|
7
|
+
from pydantic import validator, root_validator
|
8
|
+
from .main import BaseModel, PrefectBaseModel, FieldInfo, Field
|
3
9
|
|
4
|
-
__all__ = [
|
10
|
+
__all__ = [
|
11
|
+
"BaseModel",
|
12
|
+
"PrefectBaseModel",
|
13
|
+
"Field",
|
14
|
+
"FieldInfo",
|
15
|
+
"validator",
|
16
|
+
"root_validator",
|
17
|
+
]
|
18
|
+
|
19
|
+
_dynamic_imports: "typing.Dict[str, typing.Tuple[str, str]]" = {
|
20
|
+
"BaseModel": ("prefect.pydantic", ".main"),
|
21
|
+
"PrefectBaseModel": ("prefect.pydantic", ".main"),
|
22
|
+
"Field": ("prefect.pydantic", ".main"),
|
23
|
+
"FieldInfo": ("prefect.pydantic", ".main"),
|
24
|
+
}
|
25
|
+
|
26
|
+
|
27
|
+
def __getattr__(attr_name: str) -> object:
|
28
|
+
from importlib import import_module
|
29
|
+
|
30
|
+
if attr_name in _dynamic_imports:
|
31
|
+
# If the attribute is in the dynamic imports, import it from the specified module
|
32
|
+
package, module_name = _dynamic_imports[attr_name]
|
33
|
+
|
34
|
+
# Prevent recursive import
|
35
|
+
if module_name == "__module__":
|
36
|
+
return import_module(f".{attr_name}", package=package)
|
37
|
+
|
38
|
+
# Import the module and return the attribute
|
39
|
+
else:
|
40
|
+
module = import_module(module_name, package=package)
|
41
|
+
return getattr(module, attr_name)
|
42
|
+
|
43
|
+
elif HAS_PYDANTIC_V2 and not USE_PYDANTIC_V2:
|
44
|
+
# In this case, we are using Pydantic v2 but it is not enabled, so we should import from pydantic.v1
|
45
|
+
module = import_module("pydantic.v1")
|
46
|
+
return getattr(module, attr_name)
|
47
|
+
else:
|
48
|
+
# In this case, we are using either Pydantic v1 or Pydantic v2 is enabled, so we should import from pydantic
|
49
|
+
module = import_module("pydantic")
|
50
|
+
return getattr(module, attr_name)
|
prefect/pydantic/main.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
import typing
|
2
2
|
|
3
|
-
from prefect._internal.pydantic._compat import BaseModel
|
3
|
+
from prefect._internal.pydantic._compat import BaseModel, Field, FieldInfo
|
4
4
|
|
5
5
|
|
6
6
|
class PrefectBaseModel(BaseModel):
|
@@ -12,4 +12,4 @@ class PrefectBaseModel(BaseModel):
|
|
12
12
|
return set()
|
13
13
|
|
14
14
|
|
15
|
-
__all__ = ["BaseModel", "PrefectBaseModel"]
|
15
|
+
__all__ = ["BaseModel", "PrefectBaseModel", "Field", "FieldInfo"]
|
prefect/serializers.py
CHANGED
@@ -10,10 +10,10 @@ the instance so the same settings can be used to load saved objects.
|
|
10
10
|
All serializers must implement `dumps` and `loads` which convert objects to bytes and
|
11
11
|
bytes to an object respectively.
|
12
12
|
"""
|
13
|
+
|
13
14
|
import abc
|
14
15
|
import base64
|
15
|
-
import
|
16
|
-
from typing import Any, Generic, Optional, TypeVar
|
16
|
+
from typing import Any, Dict, Generic, Optional, TypeVar
|
17
17
|
|
18
18
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
19
19
|
from prefect._internal.schemas.validators import (
|
@@ -22,6 +22,7 @@ from prefect._internal.schemas.validators import (
|
|
22
22
|
validate_dump_kwargs,
|
23
23
|
validate_load_kwargs,
|
24
24
|
validate_picklelib,
|
25
|
+
validate_picklelib_version,
|
25
26
|
)
|
26
27
|
|
27
28
|
if HAS_PYDANTIC_V2:
|
@@ -112,33 +113,7 @@ class PickleSerializer(Serializer):
|
|
112
113
|
|
113
114
|
@pydantic.root_validator
|
114
115
|
def check_picklelib_version(cls, values):
|
115
|
-
|
116
|
-
Infers a default value for `picklelib_version` if null or ensures it matches
|
117
|
-
the version retrieved from the `pickelib`.
|
118
|
-
"""
|
119
|
-
picklelib = values.get("picklelib")
|
120
|
-
picklelib_version = values.get("picklelib_version")
|
121
|
-
|
122
|
-
if not picklelib:
|
123
|
-
raise ValueError("Unable to check version of unrecognized picklelib module")
|
124
|
-
|
125
|
-
pickler = from_qualified_name(picklelib)
|
126
|
-
pickler_version = getattr(pickler, "__version__", None)
|
127
|
-
|
128
|
-
if not picklelib_version:
|
129
|
-
values["picklelib_version"] = pickler_version
|
130
|
-
elif picklelib_version != pickler_version:
|
131
|
-
warnings.warn(
|
132
|
-
(
|
133
|
-
f"Mismatched {picklelib!r} versions. Found {pickler_version} in the"
|
134
|
-
f" environment but {picklelib_version} was requested. This may"
|
135
|
-
" cause the serializer to fail."
|
136
|
-
),
|
137
|
-
RuntimeWarning,
|
138
|
-
stacklevel=3,
|
139
|
-
)
|
140
|
-
|
141
|
-
return values
|
116
|
+
return validate_picklelib_version(values)
|
142
117
|
|
143
118
|
def dumps(self, obj: Any) -> bytes:
|
144
119
|
pickler = from_qualified_name(self.picklelib)
|
@@ -178,8 +153,8 @@ class JSONSerializer(Serializer):
|
|
178
153
|
"by our default `object_encoder`."
|
179
154
|
),
|
180
155
|
)
|
181
|
-
dumps_kwargs:
|
182
|
-
loads_kwargs:
|
156
|
+
dumps_kwargs: Dict[str, Any] = pydantic.Field(default_factory=dict)
|
157
|
+
loads_kwargs: Dict[str, Any] = pydantic.Field(default_factory=dict)
|
183
158
|
|
184
159
|
@pydantic.validator("dumps_kwargs")
|
185
160
|
def dumps_kwargs_cannot_contain_default(cls, value):
|
prefect/settings.py
CHANGED
@@ -68,6 +68,7 @@ from urllib.parse import urlparse
|
|
68
68
|
import toml
|
69
69
|
|
70
70
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
71
|
+
from prefect._internal.schemas.validators import validate_settings
|
71
72
|
|
72
73
|
if HAS_PYDANTIC_V2:
|
73
74
|
from pydantic.v1 import (
|
@@ -577,6 +578,14 @@ PREFECT_UNIT_TEST_MODE = Setting(
|
|
577
578
|
This variable only exists to facilitate unit testing. If `True`,
|
578
579
|
code is executing in a unit test context. Defaults to `False`.
|
579
580
|
"""
|
581
|
+
PREFECT_UNIT_TEST_LOOP_DEBUG = Setting(
|
582
|
+
bool,
|
583
|
+
default=True,
|
584
|
+
)
|
585
|
+
"""
|
586
|
+
If `True` turns on debug mode for the unit testing event loop.
|
587
|
+
Defaults to `False`.
|
588
|
+
"""
|
580
589
|
|
581
590
|
PREFECT_TEST_SETTING = Setting(
|
582
591
|
Any,
|
@@ -1236,7 +1245,7 @@ Note this setting only applies when calling `prefect server start`; if hosting t
|
|
1236
1245
|
API with another tool you will need to configure this there instead.
|
1237
1246
|
"""
|
1238
1247
|
|
1239
|
-
PREFECT_SERVER_CSRF_PROTECTION_ENABLED = Setting(bool, default=
|
1248
|
+
PREFECT_SERVER_CSRF_PROTECTION_ENABLED = Setting(bool, default=False)
|
1240
1249
|
"""
|
1241
1250
|
Controls the activation of CSRF protection for the Prefect server API.
|
1242
1251
|
|
@@ -1629,8 +1638,26 @@ The directory to serve static files from. This should be used when running into
|
|
1629
1638
|
when attempting to serve the UI from the default directory (for example when running in a Docker container)
|
1630
1639
|
"""
|
1631
1640
|
|
1641
|
+
# Messaging system settings
|
1642
|
+
|
1643
|
+
PREFECT_MESSAGING_BROKER = Setting(
|
1644
|
+
str, default="prefect.server.utilities.messaging.memory"
|
1645
|
+
)
|
1646
|
+
"""
|
1647
|
+
Which message broker implementation to use for the messaging system, should point to a
|
1648
|
+
module that exports a Publisher and Consumer class.
|
1649
|
+
"""
|
1650
|
+
|
1651
|
+
PREFECT_MESSAGING_CACHE = Setting(
|
1652
|
+
str, default="prefect.server.utilities.messaging.memory"
|
1653
|
+
)
|
1654
|
+
"""
|
1655
|
+
Which cache implementation to use for the events system. Should point to a module that
|
1656
|
+
exports a Cache class.
|
1657
|
+
"""
|
1632
1658
|
|
1633
|
-
|
1659
|
+
|
1660
|
+
# Events settings
|
1634
1661
|
|
1635
1662
|
PREFECT_EVENTS_MAXIMUM_LABELS_PER_RESOURCE = Setting(int, default=500)
|
1636
1663
|
"""
|
@@ -1642,13 +1669,22 @@ PREFECT_EVENTS_MAXIMUM_RELATED_RESOURCES = Setting(int, default=500)
|
|
1642
1669
|
The maximum number of related resources an Event may have.
|
1643
1670
|
"""
|
1644
1671
|
|
1672
|
+
PREFECT_EVENTS_MAXIMUM_SIZE_BYTES = Setting(int, default=1_500_000)
|
1673
|
+
"""
|
1674
|
+
The maximum size of an Event when serialized to JSON
|
1675
|
+
"""
|
1676
|
+
|
1677
|
+
PREFECT_API_SERVICES_EVENT_LOGGER_ENABLED = Setting(bool, default=True)
|
1678
|
+
"""
|
1679
|
+
Whether or not to start the event debug logger service in the server application.
|
1680
|
+
"""
|
1645
1681
|
|
1646
1682
|
# Deprecated settings ------------------------------------------------------------------
|
1647
1683
|
|
1648
1684
|
|
1649
1685
|
# Collect all defined settings ---------------------------------------------------------
|
1650
1686
|
|
1651
|
-
SETTING_VARIABLES = {
|
1687
|
+
SETTING_VARIABLES: Dict[str, Any] = {
|
1652
1688
|
name: val for name, val in tuple(globals().items()) if isinstance(val, Setting)
|
1653
1689
|
}
|
1654
1690
|
|
@@ -1947,20 +1983,7 @@ class Profile(BaseModel):
|
|
1947
1983
|
|
1948
1984
|
@validator("settings", pre=True)
|
1949
1985
|
def map_names_to_settings(cls, value):
|
1950
|
-
|
1951
|
-
return value
|
1952
|
-
|
1953
|
-
# Cast string setting names to variables
|
1954
|
-
validated = {}
|
1955
|
-
for setting, val in value.items():
|
1956
|
-
if isinstance(setting, str) and setting in SETTING_VARIABLES:
|
1957
|
-
validated[SETTING_VARIABLES[setting]] = val
|
1958
|
-
elif isinstance(setting, Setting):
|
1959
|
-
validated[setting] = val
|
1960
|
-
else:
|
1961
|
-
raise ValueError(f"Unknown setting {setting!r}.")
|
1962
|
-
|
1963
|
-
return validated
|
1986
|
+
return validate_settings(value)
|
1964
1987
|
|
1965
1988
|
def validate_settings(self) -> None:
|
1966
1989
|
"""
|
prefect/software/python.py
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
import sys
|
2
1
|
from pathlib import Path
|
3
2
|
from typing import List, Type
|
4
3
|
|
5
4
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
5
|
+
from prefect._internal.schemas.validators import infer_python_version
|
6
6
|
|
7
7
|
if HAS_PYDANTIC_V2:
|
8
8
|
from pydantic.v1 import BaseModel, Field, validate_arguments, validator
|
@@ -23,10 +23,8 @@ class PythonEnvironment(BaseModel):
|
|
23
23
|
pip_requirements: List[PipRequirement] = Field(default_factory=list)
|
24
24
|
|
25
25
|
@validator("python_version", pre=True, always=True)
|
26
|
-
def
|
27
|
-
|
28
|
-
return f"{sys.version_info.major}.{sys.version_info.minor}"
|
29
|
-
return value
|
26
|
+
def validate_python_version(cls, value):
|
27
|
+
return infer_python_version(value)
|
30
28
|
|
31
29
|
@classmethod
|
32
30
|
def from_environment(cls: Type[Self], exclude_nested: bool = False) -> Self:
|
prefect/utilities/callables.py
CHANGED
@@ -225,7 +225,7 @@ class ParameterSchema(pydantic.BaseModel):
|
|
225
225
|
type: Literal["object"] = "object"
|
226
226
|
properties: Dict[str, Any] = pydantic.Field(default_factory=dict)
|
227
227
|
required: List[str] = None
|
228
|
-
definitions: Dict[str, Any] = None
|
228
|
+
definitions: Optional[Dict[str, Any]] = None
|
229
229
|
|
230
230
|
def dict(self, *args, **kwargs):
|
231
231
|
"""Exclude `None` fields by default to comply with
|
prefect/utilities/collections.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
Utilities for extensions of and operations on Python collections.
|
3
3
|
"""
|
4
|
+
|
4
5
|
import io
|
5
6
|
import itertools
|
6
7
|
from collections import OrderedDict, defaultdict
|
@@ -158,7 +159,7 @@ def ensure_iterable(obj: Union[T, Iterable[T]]) -> Iterable[T]:
|
|
158
159
|
return [obj]
|
159
160
|
|
160
161
|
|
161
|
-
def listrepr(objs: Iterable, sep=" ") -> str:
|
162
|
+
def listrepr(objs: Iterable[Any], sep: str = " ") -> str:
|
162
163
|
return sep.join(repr(obj) for obj in objs)
|
163
164
|
|
164
165
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
from collections import defaultdict, deque
|
2
2
|
from copy import deepcopy
|
3
|
-
from typing import Dict, List
|
3
|
+
from typing import Any, Dict, List
|
4
4
|
|
5
5
|
import jsonschema
|
6
6
|
from jsonschema.exceptions import ValidationError as JSONSchemaValidationError
|
@@ -129,7 +129,7 @@ def prioritize_placeholder_errors(errors):
|
|
129
129
|
|
130
130
|
|
131
131
|
def build_error_obj(errors: List[JSONSchemaValidationError]) -> Dict:
|
132
|
-
error_response:
|
132
|
+
error_response: Dict[str, Any] = {"errors": []}
|
133
133
|
|
134
134
|
# If multiple errors are present for the same path and one of them
|
135
135
|
# is a placeholder error, we want only want to use the placeholder error.
|
prefect/workers/base.py
CHANGED
@@ -9,6 +9,7 @@ import anyio.abc
|
|
9
9
|
import pendulum
|
10
10
|
|
11
11
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
12
|
+
from prefect._internal.schemas.validators import return_v_or_none
|
12
13
|
|
13
14
|
if HAS_PYDANTIC_V2:
|
14
15
|
from pydantic.v1 import BaseModel, Field, PrivateAttr, validator
|
@@ -108,10 +109,7 @@ class BaseJobConfiguration(BaseModel):
|
|
108
109
|
|
109
110
|
@validator("command")
|
110
111
|
def _coerce_command(cls, v):
|
111
|
-
|
112
|
-
if not v:
|
113
|
-
return None
|
114
|
-
return v
|
112
|
+
return return_v_or_none(v)
|
115
113
|
|
116
114
|
@staticmethod
|
117
115
|
def _get_base_config_defaults(variables: dict) -> dict:
|
@@ -632,6 +630,22 @@ class BaseWorker(abc.ABC):
|
|
632
630
|
|
633
631
|
try:
|
634
632
|
configuration = await self._get_configuration(flow_run)
|
633
|
+
except ObjectNotFound:
|
634
|
+
self._logger.warning(
|
635
|
+
f"Flow run {flow_run.id!r} cannot be cancelled by this worker:"
|
636
|
+
f" associated deployment {flow_run.deployment_id!r} does not exist."
|
637
|
+
)
|
638
|
+
await self._mark_flow_run_as_cancelled(
|
639
|
+
flow_run,
|
640
|
+
state_updates={
|
641
|
+
"message": (
|
642
|
+
"This flow run is missing infrastructure configuration information"
|
643
|
+
" and cancellation cannot be guaranteed."
|
644
|
+
)
|
645
|
+
},
|
646
|
+
)
|
647
|
+
return
|
648
|
+
else:
|
635
649
|
if configuration.is_using_a_runner:
|
636
650
|
self._logger.info(
|
637
651
|
f"Skipping cancellation because flow run {str(flow_run.id)!r} is"
|
@@ -639,11 +653,6 @@ class BaseWorker(abc.ABC):
|
|
639
653
|
" cancellation."
|
640
654
|
)
|
641
655
|
return
|
642
|
-
except ObjectNotFound:
|
643
|
-
self._logger.warning(
|
644
|
-
f"Flow run {flow_run.id!r} cannot be cancelled by this worker:"
|
645
|
-
f" associated deployment {flow_run.deployment_id!r} does not exist."
|
646
|
-
)
|
647
656
|
|
648
657
|
if not flow_run.infrastructure_pid:
|
649
658
|
run_logger.error(
|
@@ -1142,7 +1151,7 @@ class BaseWorker(abc.ABC):
|
|
1142
1151
|
if include_self:
|
1143
1152
|
worker_resource = self._event_resource()
|
1144
1153
|
worker_resource["prefect.resource.role"] = "worker"
|
1145
|
-
related.append(RelatedResource(
|
1154
|
+
related.append(RelatedResource.parse_obj(worker_resource))
|
1146
1155
|
|
1147
1156
|
return related
|
1148
1157
|
|
prefect/workers/block.py
CHANGED
@@ -5,9 +5,9 @@ import anyio
|
|
5
5
|
import anyio.abc
|
6
6
|
|
7
7
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
8
|
+
from prefect._internal.schemas.validators import validate_block_is_infrastructure
|
8
9
|
from prefect.blocks.core import Block
|
9
10
|
from prefect.client.schemas.objects import BlockDocument
|
10
|
-
from prefect.infrastructure.base import Infrastructure
|
11
11
|
from prefect.utilities.collections import get_from_dict
|
12
12
|
from prefect.workers.base import BaseWorker, BaseWorkerResult
|
13
13
|
|
@@ -33,12 +33,8 @@ class BlockWorkerJobConfiguration(BaseModel):
|
|
33
33
|
)
|
34
34
|
|
35
35
|
@validator("block")
|
36
|
-
def
|
37
|
-
|
38
|
-
if not isinstance(v, Infrastructure):
|
39
|
-
raise TypeError("Provided block is not a valid infrastructure block.")
|
40
|
-
|
41
|
-
return v
|
36
|
+
def _validate_infrastructure_block(cls, v):
|
37
|
+
return validate_block_is_infrastructure(v)
|
42
38
|
|
43
39
|
_related_objects: Dict[str, Any] = PrivateAttr(default_factory=dict)
|
44
40
|
|
prefect/workers/process.py
CHANGED
@@ -35,9 +35,9 @@ if HAS_PYDANTIC_V2:
|
|
35
35
|
else:
|
36
36
|
from pydantic import Field, validator
|
37
37
|
|
38
|
+
from prefect._internal.schemas.validators import validate_command
|
38
39
|
from prefect.client.schemas import FlowRun
|
39
40
|
from prefect.exceptions import InfrastructureNotAvailable, InfrastructureNotFound
|
40
|
-
from prefect.utilities.filesystem import relative_path_to_current_platform
|
41
41
|
from prefect.utilities.processutils import get_sys_executable, run_process
|
42
42
|
from prefect.workers.base import (
|
43
43
|
BaseJobConfiguration,
|
@@ -85,10 +85,7 @@ class ProcessJobConfiguration(BaseJobConfiguration):
|
|
85
85
|
|
86
86
|
@validator("working_dir")
|
87
87
|
def validate_command(cls, v):
|
88
|
-
|
89
|
-
if v:
|
90
|
-
return relative_path_to_current_platform(v)
|
91
|
-
return v
|
88
|
+
return validate_command(v)
|
92
89
|
|
93
90
|
def prepare_for_flow_run(
|
94
91
|
self,
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: prefect-client
|
3
|
-
Version: 2.16.
|
3
|
+
Version: 2.16.9
|
4
4
|
Summary: Workflow orchestration and management.
|
5
5
|
Home-page: https://www.prefect.io
|
6
6
|
Author: Prefect Technologies, Inc.
|
@@ -32,7 +32,7 @@ Requires-Dist: croniter <3.0.0,>=1.0.12
|
|
32
32
|
Requires-Dist: fsspec >=2022.5.0
|
33
33
|
Requires-Dist: graphviz >=0.20.1
|
34
34
|
Requires-Dist: griffe >=0.20.0
|
35
|
-
Requires-Dist: httpcore <2.0.0,>=0.
|
35
|
+
Requires-Dist: httpcore <2.0.0,>=1.0.5
|
36
36
|
Requires-Dist: httpx[http2] !=0.23.2,>=0.23
|
37
37
|
Requires-Dist: importlib-resources <6.2.0,>=6.1.3
|
38
38
|
Requires-Dist: jsonpatch <2.0,>=1.32
|
@@ -41,6 +41,7 @@ Requires-Dist: orjson <4.0,>=3.7
|
|
41
41
|
Requires-Dist: packaging <24.3,>=21.3
|
42
42
|
Requires-Dist: pathspec >=0.8.0
|
43
43
|
Requires-Dist: pydantic[email] !=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.10.0
|
44
|
+
Requires-Dist: pydantic-core <3.0.0,>=2.10.0
|
44
45
|
Requires-Dist: python-dateutil <3.0.0,>=2.8.2
|
45
46
|
Requires-Dist: python-slugify <9.0,>=5.0
|
46
47
|
Requires-Dist: pyyaml <7.0.0,>=5.4.1
|