prefect-client 2.20.2__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +74 -110
- prefect/_internal/compatibility/deprecated.py +6 -115
- prefect/_internal/compatibility/experimental.py +4 -79
- prefect/_internal/compatibility/migration.py +166 -0
- prefect/_internal/concurrency/__init__.py +2 -2
- prefect/_internal/concurrency/api.py +1 -35
- prefect/_internal/concurrency/calls.py +0 -6
- prefect/_internal/concurrency/cancellation.py +0 -3
- prefect/_internal/concurrency/event_loop.py +0 -20
- prefect/_internal/concurrency/inspection.py +3 -3
- prefect/_internal/concurrency/primitives.py +1 -0
- prefect/_internal/concurrency/services.py +23 -0
- prefect/_internal/concurrency/threads.py +35 -0
- prefect/_internal/concurrency/waiters.py +0 -28
- prefect/_internal/integrations.py +7 -0
- prefect/_internal/pydantic/__init__.py +0 -45
- prefect/_internal/pydantic/annotations/pendulum.py +2 -2
- prefect/_internal/pydantic/v1_schema.py +21 -22
- prefect/_internal/pydantic/v2_schema.py +0 -2
- prefect/_internal/pydantic/v2_validated_func.py +18 -23
- prefect/_internal/pytz.py +1 -1
- prefect/_internal/retries.py +61 -0
- prefect/_internal/schemas/bases.py +45 -177
- prefect/_internal/schemas/fields.py +1 -43
- prefect/_internal/schemas/validators.py +47 -233
- prefect/agent.py +3 -695
- prefect/artifacts.py +173 -14
- prefect/automations.py +39 -4
- prefect/blocks/abstract.py +1 -1
- prefect/blocks/core.py +423 -164
- prefect/blocks/fields.py +2 -57
- prefect/blocks/notifications.py +43 -28
- prefect/blocks/redis.py +168 -0
- prefect/blocks/system.py +67 -20
- prefect/blocks/webhook.py +2 -9
- prefect/cache_policies.py +239 -0
- prefect/client/__init__.py +4 -0
- prefect/client/base.py +33 -27
- prefect/client/cloud.py +65 -20
- prefect/client/collections.py +1 -1
- prefect/client/orchestration.py +667 -440
- prefect/client/schemas/actions.py +115 -100
- prefect/client/schemas/filters.py +46 -52
- prefect/client/schemas/objects.py +228 -178
- prefect/client/schemas/responses.py +18 -36
- prefect/client/schemas/schedules.py +55 -36
- prefect/client/schemas/sorting.py +2 -0
- prefect/client/subscriptions.py +8 -7
- prefect/client/types/flexible_schedule_list.py +11 -0
- prefect/client/utilities.py +9 -6
- prefect/concurrency/asyncio.py +60 -11
- prefect/concurrency/context.py +24 -0
- prefect/concurrency/events.py +2 -2
- prefect/concurrency/services.py +46 -16
- prefect/concurrency/sync.py +51 -7
- prefect/concurrency/v1/asyncio.py +143 -0
- prefect/concurrency/v1/context.py +27 -0
- prefect/concurrency/v1/events.py +61 -0
- prefect/concurrency/v1/services.py +116 -0
- prefect/concurrency/v1/sync.py +92 -0
- prefect/context.py +246 -149
- prefect/deployments/__init__.py +33 -18
- prefect/deployments/base.py +10 -15
- prefect/deployments/deployments.py +2 -1048
- prefect/deployments/flow_runs.py +178 -0
- prefect/deployments/runner.py +72 -173
- prefect/deployments/schedules.py +31 -25
- prefect/deployments/steps/__init__.py +0 -1
- prefect/deployments/steps/core.py +7 -0
- prefect/deployments/steps/pull.py +15 -21
- prefect/deployments/steps/utility.py +2 -1
- prefect/docker/__init__.py +20 -0
- prefect/docker/docker_image.py +82 -0
- prefect/engine.py +15 -2466
- prefect/events/actions.py +17 -23
- prefect/events/cli/automations.py +20 -7
- prefect/events/clients.py +142 -80
- prefect/events/filters.py +14 -18
- prefect/events/related.py +74 -75
- prefect/events/schemas/__init__.py +0 -5
- prefect/events/schemas/automations.py +55 -46
- prefect/events/schemas/deployment_triggers.py +7 -197
- prefect/events/schemas/events.py +46 -65
- prefect/events/schemas/labelling.py +10 -14
- prefect/events/utilities.py +4 -5
- prefect/events/worker.py +23 -8
- prefect/exceptions.py +15 -0
- prefect/filesystems.py +30 -529
- prefect/flow_engine.py +827 -0
- prefect/flow_runs.py +379 -7
- prefect/flows.py +470 -360
- prefect/futures.py +382 -331
- prefect/infrastructure/__init__.py +5 -26
- prefect/infrastructure/base.py +3 -320
- prefect/infrastructure/provisioners/__init__.py +5 -3
- prefect/infrastructure/provisioners/cloud_run.py +13 -8
- prefect/infrastructure/provisioners/container_instance.py +14 -9
- prefect/infrastructure/provisioners/ecs.py +10 -8
- prefect/infrastructure/provisioners/modal.py +8 -5
- prefect/input/__init__.py +4 -0
- prefect/input/actions.py +2 -4
- prefect/input/run_input.py +9 -9
- prefect/logging/formatters.py +2 -4
- prefect/logging/handlers.py +9 -14
- prefect/logging/loggers.py +5 -5
- prefect/main.py +72 -0
- prefect/plugins.py +2 -64
- prefect/profiles.toml +16 -2
- prefect/records/__init__.py +1 -0
- prefect/records/base.py +223 -0
- prefect/records/filesystem.py +207 -0
- prefect/records/memory.py +178 -0
- prefect/records/result_store.py +64 -0
- prefect/results.py +577 -504
- prefect/runner/runner.py +124 -51
- prefect/runner/server.py +32 -34
- prefect/runner/storage.py +3 -12
- prefect/runner/submit.py +2 -10
- prefect/runner/utils.py +2 -2
- prefect/runtime/__init__.py +1 -0
- prefect/runtime/deployment.py +1 -0
- prefect/runtime/flow_run.py +40 -5
- prefect/runtime/task_run.py +1 -0
- prefect/serializers.py +28 -39
- prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
- prefect/settings.py +209 -332
- prefect/states.py +160 -63
- prefect/task_engine.py +1478 -57
- prefect/task_runners.py +383 -287
- prefect/task_runs.py +240 -0
- prefect/task_worker.py +463 -0
- prefect/tasks.py +684 -374
- prefect/transactions.py +410 -0
- prefect/types/__init__.py +72 -86
- prefect/types/entrypoint.py +13 -0
- prefect/utilities/annotations.py +4 -3
- prefect/utilities/asyncutils.py +227 -148
- prefect/utilities/callables.py +138 -48
- prefect/utilities/collections.py +134 -86
- prefect/utilities/dispatch.py +27 -14
- prefect/utilities/dockerutils.py +11 -4
- prefect/utilities/engine.py +186 -32
- prefect/utilities/filesystem.py +4 -5
- prefect/utilities/importtools.py +26 -27
- prefect/utilities/pydantic.py +128 -38
- prefect/utilities/schema_tools/hydration.py +18 -1
- prefect/utilities/schema_tools/validation.py +30 -0
- prefect/utilities/services.py +35 -9
- prefect/utilities/templating.py +12 -2
- prefect/utilities/timeout.py +20 -5
- prefect/utilities/urls.py +195 -0
- prefect/utilities/visualization.py +1 -0
- prefect/variables.py +78 -59
- prefect/workers/__init__.py +0 -1
- prefect/workers/base.py +237 -244
- prefect/workers/block.py +5 -226
- prefect/workers/cloud.py +6 -0
- prefect/workers/process.py +265 -12
- prefect/workers/server.py +29 -11
- {prefect_client-2.20.2.dist-info → prefect_client-3.0.0.dist-info}/METADATA +30 -26
- prefect_client-3.0.0.dist-info/RECORD +201 -0
- {prefect_client-2.20.2.dist-info → prefect_client-3.0.0.dist-info}/WHEEL +1 -1
- prefect/_internal/pydantic/_base_model.py +0 -51
- prefect/_internal/pydantic/_compat.py +0 -82
- prefect/_internal/pydantic/_flags.py +0 -20
- prefect/_internal/pydantic/_types.py +0 -8
- prefect/_internal/pydantic/utilities/config_dict.py +0 -72
- prefect/_internal/pydantic/utilities/field_validator.py +0 -150
- prefect/_internal/pydantic/utilities/model_construct.py +0 -56
- prefect/_internal/pydantic/utilities/model_copy.py +0 -55
- prefect/_internal/pydantic/utilities/model_dump.py +0 -136
- prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
- prefect/_internal/pydantic/utilities/model_fields.py +0 -50
- prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
- prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
- prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
- prefect/_internal/pydantic/utilities/model_validate.py +0 -75
- prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
- prefect/_internal/pydantic/utilities/model_validator.py +0 -87
- prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
- prefect/_vendor/fastapi/__init__.py +0 -25
- prefect/_vendor/fastapi/applications.py +0 -946
- prefect/_vendor/fastapi/background.py +0 -3
- prefect/_vendor/fastapi/concurrency.py +0 -44
- prefect/_vendor/fastapi/datastructures.py +0 -58
- prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
- prefect/_vendor/fastapi/dependencies/models.py +0 -64
- prefect/_vendor/fastapi/dependencies/utils.py +0 -877
- prefect/_vendor/fastapi/encoders.py +0 -177
- prefect/_vendor/fastapi/exception_handlers.py +0 -40
- prefect/_vendor/fastapi/exceptions.py +0 -46
- prefect/_vendor/fastapi/logger.py +0 -3
- prefect/_vendor/fastapi/middleware/__init__.py +0 -1
- prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
- prefect/_vendor/fastapi/middleware/cors.py +0 -3
- prefect/_vendor/fastapi/middleware/gzip.py +0 -3
- prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
- prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
- prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
- prefect/_vendor/fastapi/openapi/__init__.py +0 -0
- prefect/_vendor/fastapi/openapi/constants.py +0 -2
- prefect/_vendor/fastapi/openapi/docs.py +0 -203
- prefect/_vendor/fastapi/openapi/models.py +0 -480
- prefect/_vendor/fastapi/openapi/utils.py +0 -485
- prefect/_vendor/fastapi/param_functions.py +0 -340
- prefect/_vendor/fastapi/params.py +0 -453
- prefect/_vendor/fastapi/py.typed +0 -0
- prefect/_vendor/fastapi/requests.py +0 -4
- prefect/_vendor/fastapi/responses.py +0 -40
- prefect/_vendor/fastapi/routing.py +0 -1331
- prefect/_vendor/fastapi/security/__init__.py +0 -15
- prefect/_vendor/fastapi/security/api_key.py +0 -98
- prefect/_vendor/fastapi/security/base.py +0 -6
- prefect/_vendor/fastapi/security/http.py +0 -172
- prefect/_vendor/fastapi/security/oauth2.py +0 -227
- prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
- prefect/_vendor/fastapi/security/utils.py +0 -10
- prefect/_vendor/fastapi/staticfiles.py +0 -1
- prefect/_vendor/fastapi/templating.py +0 -3
- prefect/_vendor/fastapi/testclient.py +0 -1
- prefect/_vendor/fastapi/types.py +0 -3
- prefect/_vendor/fastapi/utils.py +0 -235
- prefect/_vendor/fastapi/websockets.py +0 -7
- prefect/_vendor/starlette/__init__.py +0 -1
- prefect/_vendor/starlette/_compat.py +0 -28
- prefect/_vendor/starlette/_exception_handler.py +0 -80
- prefect/_vendor/starlette/_utils.py +0 -88
- prefect/_vendor/starlette/applications.py +0 -261
- prefect/_vendor/starlette/authentication.py +0 -159
- prefect/_vendor/starlette/background.py +0 -43
- prefect/_vendor/starlette/concurrency.py +0 -59
- prefect/_vendor/starlette/config.py +0 -151
- prefect/_vendor/starlette/convertors.py +0 -87
- prefect/_vendor/starlette/datastructures.py +0 -707
- prefect/_vendor/starlette/endpoints.py +0 -130
- prefect/_vendor/starlette/exceptions.py +0 -60
- prefect/_vendor/starlette/formparsers.py +0 -276
- prefect/_vendor/starlette/middleware/__init__.py +0 -17
- prefect/_vendor/starlette/middleware/authentication.py +0 -52
- prefect/_vendor/starlette/middleware/base.py +0 -220
- prefect/_vendor/starlette/middleware/cors.py +0 -176
- prefect/_vendor/starlette/middleware/errors.py +0 -265
- prefect/_vendor/starlette/middleware/exceptions.py +0 -74
- prefect/_vendor/starlette/middleware/gzip.py +0 -113
- prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
- prefect/_vendor/starlette/middleware/sessions.py +0 -82
- prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
- prefect/_vendor/starlette/middleware/wsgi.py +0 -147
- prefect/_vendor/starlette/py.typed +0 -0
- prefect/_vendor/starlette/requests.py +0 -328
- prefect/_vendor/starlette/responses.py +0 -347
- prefect/_vendor/starlette/routing.py +0 -933
- prefect/_vendor/starlette/schemas.py +0 -154
- prefect/_vendor/starlette/staticfiles.py +0 -248
- prefect/_vendor/starlette/status.py +0 -199
- prefect/_vendor/starlette/templating.py +0 -231
- prefect/_vendor/starlette/testclient.py +0 -804
- prefect/_vendor/starlette/types.py +0 -30
- prefect/_vendor/starlette/websockets.py +0 -193
- prefect/blocks/kubernetes.py +0 -119
- prefect/deprecated/__init__.py +0 -0
- prefect/deprecated/data_documents.py +0 -350
- prefect/deprecated/packaging/__init__.py +0 -12
- prefect/deprecated/packaging/base.py +0 -96
- prefect/deprecated/packaging/docker.py +0 -146
- prefect/deprecated/packaging/file.py +0 -92
- prefect/deprecated/packaging/orion.py +0 -80
- prefect/deprecated/packaging/serializers.py +0 -171
- prefect/events/instrument.py +0 -135
- prefect/infrastructure/container.py +0 -824
- prefect/infrastructure/kubernetes.py +0 -920
- prefect/infrastructure/process.py +0 -289
- prefect/manifests.py +0 -20
- prefect/new_flow_engine.py +0 -449
- prefect/new_task_engine.py +0 -423
- prefect/pydantic/__init__.py +0 -76
- prefect/pydantic/main.py +0 -39
- prefect/software/__init__.py +0 -2
- prefect/software/base.py +0 -50
- prefect/software/conda.py +0 -199
- prefect/software/pip.py +0 -122
- prefect/software/python.py +0 -52
- prefect/task_server.py +0 -322
- prefect_client-2.20.2.dist-info/RECORD +0 -294
- /prefect/{_internal/pydantic/utilities → client/types}/__init__.py +0 -0
- /prefect/{_vendor → concurrency/v1}/__init__.py +0 -0
- {prefect_client-2.20.2.dist-info → prefect_client-3.0.0.dist-info}/LICENSE +0 -0
- {prefect_client-2.20.2.dist-info → prefect_client-3.0.0.dist-info}/top_level.txt +0 -0
prefect/deployments/runner.py
CHANGED
@@ -29,7 +29,6 @@ Example:
|
|
29
29
|
|
30
30
|
"""
|
31
31
|
|
32
|
-
import enum
|
33
32
|
import importlib
|
34
33
|
import tempfile
|
35
34
|
from datetime import datetime, timedelta
|
@@ -37,34 +36,32 @@ from pathlib import Path
|
|
37
36
|
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
38
37
|
from uuid import UUID
|
39
38
|
|
40
|
-
import
|
39
|
+
from pydantic import (
|
40
|
+
BaseModel,
|
41
|
+
ConfigDict,
|
42
|
+
Field,
|
43
|
+
PrivateAttr,
|
44
|
+
model_validator,
|
45
|
+
)
|
41
46
|
from rich.console import Console
|
42
47
|
from rich.progress import Progress, SpinnerColumn, TextColumn, track
|
43
48
|
from rich.table import Table
|
44
49
|
|
45
|
-
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
46
|
-
|
47
|
-
if HAS_PYDANTIC_V2:
|
48
|
-
from pydantic.v1 import BaseModel, Field, PrivateAttr, root_validator, validator
|
49
|
-
else:
|
50
|
-
from pydantic import BaseModel, Field, PrivateAttr, root_validator, validator
|
51
|
-
|
52
50
|
from prefect._internal.concurrency.api import create_call, from_async
|
53
51
|
from prefect._internal.schemas.validators import (
|
54
52
|
reconcile_paused_deployment,
|
55
53
|
reconcile_schedules_runner,
|
56
|
-
validate_automation_names,
|
57
54
|
)
|
58
55
|
from prefect.client.orchestration import get_client
|
59
|
-
from prefect.client.schemas.
|
56
|
+
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
60
57
|
from prefect.client.schemas.schedules import (
|
61
58
|
SCHEDULE_TYPES,
|
62
59
|
construct_schedule,
|
63
60
|
)
|
64
61
|
from prefect.deployments.schedules import (
|
65
|
-
|
66
|
-
create_minimal_deployment_schedule,
|
62
|
+
create_deployment_schedule_create,
|
67
63
|
)
|
64
|
+
from prefect.docker.docker_image import DockerImage
|
68
65
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
69
66
|
from prefect.exceptions import (
|
70
67
|
ObjectNotFound,
|
@@ -72,24 +69,19 @@ from prefect.exceptions import (
|
|
72
69
|
)
|
73
70
|
from prefect.runner.storage import RunnerStorage
|
74
71
|
from prefect.settings import (
|
75
|
-
PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE,
|
76
72
|
PREFECT_DEFAULT_WORK_POOL_NAME,
|
77
73
|
PREFECT_UI_URL,
|
78
74
|
)
|
75
|
+
from prefect.types.entrypoint import EntrypointType
|
79
76
|
from prefect.utilities.asyncutils import sync_compatible
|
80
77
|
from prefect.utilities.callables import ParameterSchema, parameter_schema
|
81
78
|
from prefect.utilities.collections import get_from_dict, isiterable
|
82
79
|
from prefect.utilities.dockerutils import (
|
83
|
-
PushError,
|
84
|
-
build_image,
|
85
|
-
docker_client,
|
86
|
-
generate_default_dockerfile,
|
87
80
|
parse_image_tag,
|
88
|
-
split_repository_path,
|
89
81
|
)
|
90
|
-
from prefect.utilities.slugify import slugify
|
91
82
|
|
92
83
|
if TYPE_CHECKING:
|
84
|
+
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
93
85
|
from prefect.flows import Flow
|
94
86
|
|
95
87
|
__all__ = ["RunnerDeployment"]
|
@@ -101,18 +93,6 @@ class DeploymentApplyError(RuntimeError):
|
|
101
93
|
"""
|
102
94
|
|
103
95
|
|
104
|
-
class EntrypointType(enum.Enum):
|
105
|
-
"""
|
106
|
-
Enum representing a entrypoint type.
|
107
|
-
|
108
|
-
File path entrypoints are in the format: `path/to/file.py:function_name`.
|
109
|
-
Module path entrypoints are in the format: `path.to.module.function_name`.
|
110
|
-
"""
|
111
|
-
|
112
|
-
FILE_PATH = "file_path"
|
113
|
-
MODULE_PATH = "module_path"
|
114
|
-
|
115
|
-
|
116
96
|
class RunnerDeployment(BaseModel):
|
117
97
|
"""
|
118
98
|
A Prefect RunnerDeployment definition, used for specifying and building deployments.
|
@@ -126,7 +106,6 @@ class RunnerDeployment(BaseModel):
|
|
126
106
|
are used only for organizational purposes. For delegating work to agents,
|
127
107
|
see `work_queue_name`.
|
128
108
|
schedule: A schedule to run this deployment on, once registered
|
129
|
-
is_schedule_active: Whether or not the schedule is active
|
130
109
|
parameters: A dictionary of parameter values to pass to runs created from this
|
131
110
|
deployment
|
132
111
|
path: The path to the working directory for the workflow, relative to remote
|
@@ -144,8 +123,7 @@ class RunnerDeployment(BaseModel):
|
|
144
123
|
available settings.
|
145
124
|
"""
|
146
125
|
|
147
|
-
|
148
|
-
arbitrary_types_allowed = True
|
126
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
149
127
|
|
150
128
|
name: str = Field(..., description="The name of the deployment.")
|
151
129
|
flow_name: Optional[str] = Field(
|
@@ -161,17 +139,17 @@ class RunnerDeployment(BaseModel):
|
|
161
139
|
default_factory=list,
|
162
140
|
description="One of more tags to apply to this deployment.",
|
163
141
|
)
|
164
|
-
schedules: Optional[List[
|
142
|
+
schedules: Optional[List[DeploymentScheduleCreate]] = Field(
|
165
143
|
default=None,
|
166
144
|
description="The schedules that should cause this deployment to run.",
|
167
145
|
)
|
168
|
-
|
146
|
+
concurrency_limit: Optional[int] = Field(
|
147
|
+
default=None,
|
148
|
+
description="The maximum number of concurrent runs of this deployment.",
|
149
|
+
)
|
169
150
|
paused: Optional[bool] = Field(
|
170
151
|
default=None, description="Whether or not the deployment is paused."
|
171
152
|
)
|
172
|
-
is_schedule_active: Optional[bool] = Field(
|
173
|
-
default=None, description="DEPRECATED: Whether or not the schedule is active."
|
174
|
-
)
|
175
153
|
parameters: Dict[str, Any] = Field(default_factory=dict)
|
176
154
|
entrypoint: Optional[str] = Field(
|
177
155
|
default=None,
|
@@ -184,7 +162,7 @@ class RunnerDeployment(BaseModel):
|
|
184
162
|
description="The triggers that should cause this deployment to run.",
|
185
163
|
)
|
186
164
|
enforce_parameter_schema: bool = Field(
|
187
|
-
default=
|
165
|
+
default=True,
|
188
166
|
description=(
|
189
167
|
"Whether or not the Prefect API should enforce the parameter schema for"
|
190
168
|
" this deployment."
|
@@ -232,16 +210,22 @@ class RunnerDeployment(BaseModel):
|
|
232
210
|
def entrypoint_type(self) -> EntrypointType:
|
233
211
|
return self._entrypoint_type
|
234
212
|
|
235
|
-
@
|
236
|
-
def validate_automation_names(
|
213
|
+
@model_validator(mode="after")
|
214
|
+
def validate_automation_names(self):
|
237
215
|
"""Ensure that each trigger has a name for its automation if none is provided."""
|
238
|
-
|
216
|
+
trigger: Union[DeploymentTriggerTypes, TriggerTypes]
|
217
|
+
for i, trigger in enumerate(self.triggers, start=1):
|
218
|
+
if trigger.name is None:
|
219
|
+
trigger.name = f"{self.name}__automation_{i}"
|
220
|
+
return self
|
239
221
|
|
240
|
-
@
|
222
|
+
@model_validator(mode="before")
|
223
|
+
@classmethod
|
241
224
|
def reconcile_paused(cls, values):
|
242
225
|
return reconcile_paused_deployment(values)
|
243
226
|
|
244
|
-
@
|
227
|
+
@model_validator(mode="before")
|
228
|
+
@classmethod
|
245
229
|
def reconcile_schedules(cls, values):
|
246
230
|
return reconcile_schedules_runner(values)
|
247
231
|
|
@@ -294,6 +278,7 @@ class RunnerDeployment(BaseModel):
|
|
294
278
|
version=self.version,
|
295
279
|
paused=self.paused,
|
296
280
|
schedules=self.schedules,
|
281
|
+
concurrency_limit=self.concurrency_limit,
|
297
282
|
parameters=self.parameters,
|
298
283
|
description=self.description,
|
299
284
|
tags=self.tags,
|
@@ -301,7 +286,9 @@ class RunnerDeployment(BaseModel):
|
|
301
286
|
entrypoint=self.entrypoint,
|
302
287
|
storage_document_id=None,
|
303
288
|
infrastructure_document_id=None,
|
304
|
-
parameter_openapi_schema=self._parameter_openapi_schema.
|
289
|
+
parameter_openapi_schema=self._parameter_openapi_schema.model_dump(
|
290
|
+
exclude_unset=True
|
291
|
+
),
|
305
292
|
enforce_parameter_schema=self.enforce_parameter_schema,
|
306
293
|
)
|
307
294
|
|
@@ -325,26 +312,25 @@ class RunnerDeployment(BaseModel):
|
|
325
312
|
f"Error while applying deployment: {str(exc)}"
|
326
313
|
) from exc
|
327
314
|
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
raise e
|
315
|
+
try:
|
316
|
+
# The triggers defined in the deployment spec are, essentially,
|
317
|
+
# anonymous and attempting truly sync them with cloud is not
|
318
|
+
# feasible. Instead, we remove all automations that are owned
|
319
|
+
# by the deployment, meaning that they were created via this
|
320
|
+
# mechanism below, and then recreate them.
|
321
|
+
await client.delete_resource_owned_automations(
|
322
|
+
f"prefect.deployment.{deployment_id}"
|
323
|
+
)
|
324
|
+
except PrefectHTTPStatusError as e:
|
325
|
+
if e.response.status_code == 404:
|
326
|
+
# This Prefect server does not support automations, so we can safely
|
327
|
+
# ignore this 404 and move on.
|
328
|
+
return deployment_id
|
329
|
+
raise e
|
344
330
|
|
345
|
-
|
346
|
-
|
347
|
-
|
331
|
+
for trigger in self.triggers:
|
332
|
+
trigger.set_deployment_id(deployment_id)
|
333
|
+
await client.create_automation(trigger.as_automation())
|
348
334
|
|
349
335
|
return deployment_id
|
350
336
|
|
@@ -358,8 +344,8 @@ class RunnerDeployment(BaseModel):
|
|
358
344
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
359
345
|
timezone: Optional[str] = None,
|
360
346
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
361
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
362
|
-
) -> Union[List[
|
347
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
348
|
+
) -> Union[List[DeploymentScheduleCreate], "FlexibleScheduleList"]:
|
363
349
|
"""
|
364
350
|
Construct a schedule or schedules from the provided arguments.
|
365
351
|
|
@@ -397,7 +383,7 @@ class RunnerDeployment(BaseModel):
|
|
397
383
|
)
|
398
384
|
if num_schedules > 1:
|
399
385
|
raise ValueError(
|
400
|
-
"Only one of interval, cron, rrule,
|
386
|
+
"Only one of interval, cron, rrule, or schedules can be provided."
|
401
387
|
)
|
402
388
|
elif num_schedules == 0:
|
403
389
|
return []
|
@@ -417,7 +403,7 @@ class RunnerDeployment(BaseModel):
|
|
417
403
|
value = [value]
|
418
404
|
|
419
405
|
return [
|
420
|
-
|
406
|
+
create_deployment_schedule_create(
|
421
407
|
construct_schedule(
|
422
408
|
**{
|
423
409
|
schedule_type: v,
|
@@ -429,7 +415,7 @@ class RunnerDeployment(BaseModel):
|
|
429
415
|
for v in value
|
430
416
|
]
|
431
417
|
else:
|
432
|
-
return [
|
418
|
+
return [create_deployment_schedule_create(schedule)]
|
433
419
|
|
434
420
|
def _set_defaults_from_flow(self, flow: "Flow"):
|
435
421
|
self._parameter_openapi_schema = parameter_schema(flow)
|
@@ -450,15 +436,14 @@ class RunnerDeployment(BaseModel):
|
|
450
436
|
cron: Optional[Union[Iterable[str], str]] = None,
|
451
437
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
452
438
|
paused: Optional[bool] = None,
|
453
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
454
|
-
|
455
|
-
is_schedule_active: Optional[bool] = None,
|
439
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
440
|
+
concurrency_limit: Optional[int] = None,
|
456
441
|
parameters: Optional[dict] = None,
|
457
442
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
458
443
|
description: Optional[str] = None,
|
459
444
|
tags: Optional[List[str]] = None,
|
460
445
|
version: Optional[str] = None,
|
461
|
-
enforce_parameter_schema: bool =
|
446
|
+
enforce_parameter_schema: bool = True,
|
462
447
|
work_pool_name: Optional[str] = None,
|
463
448
|
work_queue_name: Optional[str] = None,
|
464
449
|
job_variables: Optional[Dict[str, Any]] = None,
|
@@ -477,11 +462,6 @@ class RunnerDeployment(BaseModel):
|
|
477
462
|
paused: Whether or not to set this deployment as paused.
|
478
463
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
479
464
|
Used to define multiple schedules or additional scheduling options like `timezone`.
|
480
|
-
schedule: A schedule object of when to execute runs of this flow. Used for
|
481
|
-
advanced scheduling options like timezone.
|
482
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
483
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
484
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
485
465
|
triggers: A list of triggers that should kick of a run of this flow.
|
486
466
|
parameters: A dictionary of default parameter values to pass to runs of this flow.
|
487
467
|
description: A description for the created deployment. Defaults to the flow's
|
@@ -502,7 +482,6 @@ class RunnerDeployment(BaseModel):
|
|
502
482
|
interval=interval,
|
503
483
|
cron=cron,
|
504
484
|
rrule=rrule,
|
505
|
-
schedule=schedule,
|
506
485
|
schedules=schedules,
|
507
486
|
)
|
508
487
|
|
@@ -511,9 +490,8 @@ class RunnerDeployment(BaseModel):
|
|
511
490
|
deployment = cls(
|
512
491
|
name=Path(name).stem,
|
513
492
|
flow_name=flow.name,
|
514
|
-
schedule=schedule,
|
515
493
|
schedules=constructed_schedules,
|
516
|
-
|
494
|
+
concurrency_limit=concurrency_limit,
|
517
495
|
paused=paused,
|
518
496
|
tags=tags or [],
|
519
497
|
triggers=triggers or [],
|
@@ -586,15 +564,14 @@ class RunnerDeployment(BaseModel):
|
|
586
564
|
cron: Optional[Union[Iterable[str], str]] = None,
|
587
565
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
588
566
|
paused: Optional[bool] = None,
|
589
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
590
|
-
|
591
|
-
is_schedule_active: Optional[bool] = None,
|
567
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
568
|
+
concurrency_limit: Optional[int] = None,
|
592
569
|
parameters: Optional[dict] = None,
|
593
570
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
594
571
|
description: Optional[str] = None,
|
595
572
|
tags: Optional[List[str]] = None,
|
596
573
|
version: Optional[str] = None,
|
597
|
-
enforce_parameter_schema: bool =
|
574
|
+
enforce_parameter_schema: bool = True,
|
598
575
|
work_pool_name: Optional[str] = None,
|
599
576
|
work_queue_name: Optional[str] = None,
|
600
577
|
job_variables: Optional[Dict[str, Any]] = None,
|
@@ -613,11 +590,6 @@ class RunnerDeployment(BaseModel):
|
|
613
590
|
paused: Whether or not to set this deployment as paused.
|
614
591
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
615
592
|
Used to define multiple schedules or additional scheduling options like `timezone`.
|
616
|
-
schedule: A schedule object of when to execute runs of this flow. Used for
|
617
|
-
advanced scheduling options like timezone.
|
618
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
619
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
620
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
621
593
|
triggers: A list of triggers that should kick of a run of this flow.
|
622
594
|
parameters: A dictionary of default parameter values to pass to runs of this flow.
|
623
595
|
description: A description for the created deployment. Defaults to the flow's
|
@@ -643,17 +615,15 @@ class RunnerDeployment(BaseModel):
|
|
643
615
|
interval=interval,
|
644
616
|
cron=cron,
|
645
617
|
rrule=rrule,
|
646
|
-
schedule=schedule,
|
647
618
|
schedules=schedules,
|
648
619
|
)
|
649
620
|
|
650
621
|
deployment = cls(
|
651
622
|
name=Path(name).stem,
|
652
623
|
flow_name=flow.name,
|
653
|
-
schedule=schedule,
|
654
624
|
schedules=constructed_schedules,
|
625
|
+
concurrency_limit=concurrency_limit,
|
655
626
|
paused=paused,
|
656
|
-
is_schedule_active=is_schedule_active,
|
657
627
|
tags=tags or [],
|
658
628
|
triggers=triggers or [],
|
659
629
|
parameters=parameters or {},
|
@@ -684,15 +654,14 @@ class RunnerDeployment(BaseModel):
|
|
684
654
|
cron: Optional[Union[Iterable[str], str]] = None,
|
685
655
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
686
656
|
paused: Optional[bool] = None,
|
687
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
688
|
-
|
689
|
-
is_schedule_active: Optional[bool] = None,
|
657
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
658
|
+
concurrency_limit: Optional[int] = None,
|
690
659
|
parameters: Optional[dict] = None,
|
691
660
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
692
661
|
description: Optional[str] = None,
|
693
662
|
tags: Optional[List[str]] = None,
|
694
663
|
version: Optional[str] = None,
|
695
|
-
enforce_parameter_schema: bool =
|
664
|
+
enforce_parameter_schema: bool = True,
|
696
665
|
work_pool_name: Optional[str] = None,
|
697
666
|
work_queue_name: Optional[str] = None,
|
698
667
|
job_variables: Optional[Dict[str, Any]] = None,
|
@@ -711,11 +680,6 @@ class RunnerDeployment(BaseModel):
|
|
711
680
|
or a timedelta object. If a number is given, it will be interpreted as seconds.
|
712
681
|
cron: A cron schedule of when to execute runs of this flow.
|
713
682
|
rrule: An rrule schedule of when to execute runs of this flow.
|
714
|
-
schedule: A schedule object of when to execute runs of this flow. Used for
|
715
|
-
advanced scheduling options like timezone.
|
716
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
717
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
718
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
719
683
|
triggers: A list of triggers that should kick of a run of this flow.
|
720
684
|
parameters: A dictionary of default parameter values to pass to runs of this flow.
|
721
685
|
description: A description for the created deployment. Defaults to the flow's
|
@@ -738,7 +702,6 @@ class RunnerDeployment(BaseModel):
|
|
738
702
|
interval=interval,
|
739
703
|
cron=cron,
|
740
704
|
rrule=rrule,
|
741
|
-
schedule=schedule,
|
742
705
|
schedules=schedules,
|
743
706
|
)
|
744
707
|
|
@@ -756,10 +719,9 @@ class RunnerDeployment(BaseModel):
|
|
756
719
|
deployment = cls(
|
757
720
|
name=Path(name).stem,
|
758
721
|
flow_name=flow.name,
|
759
|
-
schedule=schedule,
|
760
722
|
schedules=constructed_schedules,
|
723
|
+
concurrency_limit=concurrency_limit,
|
761
724
|
paused=paused,
|
762
|
-
is_schedule_active=is_schedule_active,
|
763
725
|
tags=tags or [],
|
764
726
|
triggers=triggers or [],
|
765
727
|
parameters=parameters or {},
|
@@ -781,74 +743,11 @@ class RunnerDeployment(BaseModel):
|
|
781
743
|
return deployment
|
782
744
|
|
783
745
|
|
784
|
-
class DeploymentImage:
|
785
|
-
"""
|
786
|
-
Configuration used to build and push a Docker image for a deployment.
|
787
|
-
|
788
|
-
Attributes:
|
789
|
-
name: The name of the Docker image to build, including the registry and
|
790
|
-
repository.
|
791
|
-
tag: The tag to apply to the built image.
|
792
|
-
dockerfile: The path to the Dockerfile to use for building the image. If
|
793
|
-
not provided, a default Dockerfile will be generated.
|
794
|
-
**build_kwargs: Additional keyword arguments to pass to the Docker build request.
|
795
|
-
See the [`docker-py` documentation](https://docker-py.readthedocs.io/en/stable/images.html#docker.models.images.ImageCollection.build)
|
796
|
-
for more information.
|
797
|
-
|
798
|
-
"""
|
799
|
-
|
800
|
-
def __init__(self, name, tag=None, dockerfile="auto", **build_kwargs):
|
801
|
-
image_name, image_tag = parse_image_tag(name)
|
802
|
-
if tag and image_tag:
|
803
|
-
raise ValueError(
|
804
|
-
f"Only one tag can be provided - both {image_tag!r} and {tag!r} were"
|
805
|
-
" provided as tags."
|
806
|
-
)
|
807
|
-
namespace, repository = split_repository_path(image_name)
|
808
|
-
# if the provided image name does not include a namespace (registry URL or user/org name),
|
809
|
-
# use the default namespace
|
810
|
-
if not namespace:
|
811
|
-
namespace = PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE.value()
|
812
|
-
# join the namespace and repository to create the full image name
|
813
|
-
# ignore namespace if it is None
|
814
|
-
self.name = "/".join(filter(None, [namespace, repository]))
|
815
|
-
self.tag = tag or image_tag or slugify(pendulum.now("utc").isoformat())
|
816
|
-
self.dockerfile = dockerfile
|
817
|
-
self.build_kwargs = build_kwargs
|
818
|
-
|
819
|
-
@property
|
820
|
-
def reference(self):
|
821
|
-
return f"{self.name}:{self.tag}"
|
822
|
-
|
823
|
-
def build(self):
|
824
|
-
full_image_name = self.reference
|
825
|
-
build_kwargs = self.build_kwargs.copy()
|
826
|
-
build_kwargs["context"] = Path.cwd()
|
827
|
-
build_kwargs["tag"] = full_image_name
|
828
|
-
build_kwargs["pull"] = build_kwargs.get("pull", True)
|
829
|
-
|
830
|
-
if self.dockerfile == "auto":
|
831
|
-
with generate_default_dockerfile():
|
832
|
-
build_image(**build_kwargs)
|
833
|
-
else:
|
834
|
-
build_kwargs["dockerfile"] = self.dockerfile
|
835
|
-
build_image(**build_kwargs)
|
836
|
-
|
837
|
-
def push(self):
|
838
|
-
with docker_client() as client:
|
839
|
-
events = client.api.push(
|
840
|
-
repository=self.name, tag=self.tag, stream=True, decode=True
|
841
|
-
)
|
842
|
-
for event in events:
|
843
|
-
if "error" in event:
|
844
|
-
raise PushError(event["error"])
|
845
|
-
|
846
|
-
|
847
746
|
@sync_compatible
|
848
747
|
async def deploy(
|
849
748
|
*deployments: RunnerDeployment,
|
850
749
|
work_pool_name: Optional[str] = None,
|
851
|
-
image: Optional[Union[str,
|
750
|
+
image: Optional[Union[str, DockerImage]] = None,
|
852
751
|
build: bool = True,
|
853
752
|
push: bool = True,
|
854
753
|
print_next_steps_message: bool = True,
|
@@ -870,7 +769,7 @@ async def deploy(
|
|
870
769
|
work_pool_name: The name of the work pool to use for these deployments. Defaults to
|
871
770
|
the value of `PREFECT_DEFAULT_WORK_POOL_NAME`.
|
872
771
|
image: The name of the Docker image to build, including the registry and
|
873
|
-
repository. Pass a
|
772
|
+
repository. Pass a DockerImage instance to customize the Dockerfile used
|
874
773
|
and build arguments.
|
875
774
|
build: Whether or not to build a new image for the flow. If False, the provided
|
876
775
|
image will be used as-is and pulled at runtime.
|
@@ -925,7 +824,7 @@ async def deploy(
|
|
925
824
|
|
926
825
|
if image and isinstance(image, str):
|
927
826
|
image_name, image_tag = parse_image_tag(image)
|
928
|
-
image =
|
827
|
+
image = DockerImage(name=image_name, tag=image_tag)
|
929
828
|
|
930
829
|
try:
|
931
830
|
async with get_client() as client:
|
prefect/deployments/schedules.py
CHANGED
@@ -1,42 +1,44 @@
|
|
1
|
-
from typing import List, Optional, Sequence, Union
|
1
|
+
from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Union
|
2
2
|
|
3
|
-
from prefect.client.schemas.
|
4
|
-
from prefect.client.schemas.schedules import
|
3
|
+
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
4
|
+
from prefect.client.schemas.schedules import is_schedule_type
|
5
5
|
|
6
|
-
|
7
|
-
from prefect.
|
6
|
+
if TYPE_CHECKING:
|
7
|
+
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
SERVER_SCHEDULE_TYPES = ()
|
9
|
+
FlexibleScheduleList = Sequence[
|
10
|
+
Union[DeploymentScheduleCreate, dict[str, Any], "SCHEDULE_TYPES"]
|
11
|
+
]
|
13
12
|
|
14
|
-
FlexibleScheduleList = Sequence[Union[MinimalDeploymentSchedule, dict, SCHEDULE_TYPES]]
|
15
13
|
|
16
|
-
|
17
|
-
|
18
|
-
schedule: SCHEDULE_TYPES,
|
14
|
+
def create_deployment_schedule_create(
|
15
|
+
schedule: "SCHEDULE_TYPES",
|
19
16
|
active: Optional[bool] = True,
|
20
|
-
|
21
|
-
|
17
|
+
max_active_runs: Optional[int] = None,
|
18
|
+
catchup: bool = False,
|
19
|
+
) -> DeploymentScheduleCreate:
|
20
|
+
"""Create a DeploymentScheduleCreate object from common schedule parameters."""
|
21
|
+
return DeploymentScheduleCreate(
|
22
22
|
schedule=schedule,
|
23
23
|
active=active if active is not None else True,
|
24
|
+
max_active_runs=max_active_runs,
|
25
|
+
catchup=catchup,
|
24
26
|
)
|
25
27
|
|
26
28
|
|
27
|
-
def
|
28
|
-
schedules: Optional[FlexibleScheduleList],
|
29
|
-
) -> List[
|
30
|
-
normalized = []
|
29
|
+
def normalize_to_deployment_schedule_create(
|
30
|
+
schedules: Optional["FlexibleScheduleList"],
|
31
|
+
) -> List[DeploymentScheduleCreate]:
|
32
|
+
normalized: list[DeploymentScheduleCreate] = []
|
31
33
|
if schedules is not None:
|
32
34
|
for obj in schedules:
|
33
|
-
if
|
34
|
-
normalized.append(
|
35
|
+
if is_schedule_type(obj):
|
36
|
+
normalized.append(create_deployment_schedule_create(obj))
|
35
37
|
elif isinstance(obj, dict):
|
36
|
-
normalized.append(
|
37
|
-
elif isinstance(obj,
|
38
|
+
normalized.append(create_deployment_schedule_create(**obj))
|
39
|
+
elif isinstance(obj, DeploymentScheduleCreate):
|
38
40
|
normalized.append(obj)
|
39
|
-
elif
|
41
|
+
elif _is_server_schema(obj):
|
40
42
|
raise ValueError(
|
41
43
|
"Server schema schedules are not supported. Please use "
|
42
44
|
"the schedule objects from `prefect.client.schemas.schedules`"
|
@@ -44,7 +46,11 @@ def normalize_to_minimal_deployment_schedules(
|
|
44
46
|
else:
|
45
47
|
raise ValueError(
|
46
48
|
"Invalid schedule provided. Must be a schedule object, a dict,"
|
47
|
-
"
|
49
|
+
"or a `DeploymentScheduleCreate` object"
|
48
50
|
)
|
49
51
|
|
50
52
|
return normalized
|
53
|
+
|
54
|
+
|
55
|
+
def _is_server_schema(obj: Any):
|
56
|
+
return obj.__class__.__module__.startswith("prefect.server.schemas")
|
@@ -10,6 +10,7 @@ Whenever a step is run, the following actions are taken:
|
|
10
10
|
- The step's function is called with the resolved inputs
|
11
11
|
- The step's output is returned and used to resolve inputs for subsequent steps
|
12
12
|
"""
|
13
|
+
|
13
14
|
import os
|
14
15
|
import re
|
15
16
|
import subprocess
|
@@ -21,6 +22,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
|
|
21
22
|
|
22
23
|
from prefect._internal.compatibility.deprecated import PrefectDeprecationWarning
|
23
24
|
from prefect._internal.concurrency.api import Call, from_async
|
25
|
+
from prefect._internal.integrations import KNOWN_EXTRAS_FOR_PACKAGES
|
24
26
|
from prefect.logging.loggers import get_logger
|
25
27
|
from prefect.settings import PREFECT_DEBUG_MODE
|
26
28
|
from prefect.utilities.importtools import import_object
|
@@ -83,6 +85,11 @@ def _get_function_for_step(
|
|
83
85
|
raise
|
84
86
|
|
85
87
|
try:
|
88
|
+
packages = [
|
89
|
+
KNOWN_EXTRAS_FOR_PACKAGES.get(package, package)
|
90
|
+
for package in packages
|
91
|
+
if package
|
92
|
+
]
|
86
93
|
subprocess.check_call([sys.executable, "-m", "pip", "install", *packages])
|
87
94
|
except subprocess.CalledProcessError:
|
88
95
|
get_logger("deployments.steps.core").warning(
|