prefect-client 2.16.8__py3-none-any.whl → 2.17.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +0 -18
- prefect/_internal/compatibility/deprecated.py +108 -5
- prefect/_internal/compatibility/experimental.py +9 -8
- prefect/_internal/concurrency/api.py +23 -42
- prefect/_internal/concurrency/waiters.py +25 -22
- prefect/_internal/pydantic/__init__.py +16 -3
- prefect/_internal/pydantic/_base_model.py +39 -4
- prefect/_internal/pydantic/_compat.py +69 -452
- prefect/_internal/pydantic/_flags.py +5 -0
- prefect/_internal/pydantic/_types.py +8 -0
- prefect/_internal/pydantic/utilities/__init__.py +0 -0
- prefect/_internal/pydantic/utilities/config_dict.py +72 -0
- prefect/_internal/pydantic/utilities/field_validator.py +135 -0
- prefect/_internal/pydantic/utilities/model_construct.py +56 -0
- prefect/_internal/pydantic/utilities/model_copy.py +55 -0
- prefect/_internal/pydantic/utilities/model_dump.py +136 -0
- prefect/_internal/pydantic/utilities/model_dump_json.py +112 -0
- prefect/_internal/pydantic/utilities/model_fields.py +50 -0
- prefect/_internal/pydantic/utilities/model_fields_set.py +29 -0
- prefect/_internal/pydantic/utilities/model_json_schema.py +82 -0
- prefect/_internal/pydantic/utilities/model_rebuild.py +80 -0
- prefect/_internal/pydantic/utilities/model_validate.py +75 -0
- prefect/_internal/pydantic/utilities/model_validate_json.py +68 -0
- prefect/_internal/pydantic/utilities/model_validator.py +79 -0
- prefect/_internal/pydantic/utilities/type_adapter.py +71 -0
- prefect/_internal/schemas/bases.py +1 -17
- prefect/_internal/schemas/validators.py +425 -4
- prefect/agent.py +1 -1
- prefect/blocks/kubernetes.py +7 -3
- prefect/blocks/notifications.py +18 -18
- prefect/blocks/webhook.py +1 -1
- prefect/client/base.py +7 -0
- prefect/client/cloud.py +1 -1
- prefect/client/orchestration.py +51 -11
- prefect/client/schemas/actions.py +367 -297
- prefect/client/schemas/filters.py +28 -28
- prefect/client/schemas/objects.py +78 -147
- prefect/client/schemas/responses.py +240 -60
- prefect/client/schemas/schedules.py +6 -8
- prefect/concurrency/events.py +2 -2
- prefect/context.py +4 -2
- prefect/deployments/base.py +6 -13
- prefect/deployments/deployments.py +34 -9
- prefect/deployments/runner.py +9 -27
- prefect/deprecated/packaging/base.py +5 -6
- prefect/deprecated/packaging/docker.py +19 -25
- prefect/deprecated/packaging/file.py +10 -5
- prefect/deprecated/packaging/orion.py +9 -4
- prefect/deprecated/packaging/serializers.py +8 -58
- prefect/engine.py +55 -618
- prefect/events/actions.py +16 -1
- prefect/events/clients.py +45 -13
- prefect/events/filters.py +19 -2
- prefect/events/related.py +4 -4
- prefect/events/schemas/automations.py +13 -2
- prefect/events/schemas/deployment_triggers.py +73 -5
- prefect/events/schemas/events.py +1 -1
- prefect/events/utilities.py +12 -4
- prefect/events/worker.py +26 -8
- prefect/exceptions.py +3 -8
- prefect/filesystems.py +7 -7
- prefect/flows.py +7 -3
- prefect/infrastructure/provisioners/ecs.py +1 -0
- prefect/logging/configuration.py +2 -2
- prefect/manifests.py +1 -8
- prefect/profiles.toml +1 -1
- prefect/pydantic/__init__.py +74 -2
- prefect/pydantic/main.py +26 -2
- prefect/serializers.py +6 -31
- prefect/settings.py +72 -26
- prefect/software/python.py +3 -5
- prefect/task_server.py +2 -2
- prefect/utilities/callables.py +1 -1
- prefect/utilities/collections.py +2 -1
- prefect/utilities/dispatch.py +1 -0
- prefect/utilities/engine.py +629 -0
- prefect/utilities/pydantic.py +1 -1
- prefect/utilities/schema_tools/validation.py +2 -2
- prefect/utilities/visualization.py +1 -1
- prefect/variables.py +88 -12
- prefect/workers/base.py +20 -11
- prefect/workers/block.py +4 -8
- prefect/workers/process.py +2 -5
- {prefect_client-2.16.8.dist-info → prefect_client-2.17.0.dist-info}/METADATA +4 -3
- {prefect_client-2.16.8.dist-info → prefect_client-2.17.0.dist-info}/RECORD +88 -72
- prefect/_internal/schemas/transformations.py +0 -106
- {prefect_client-2.16.8.dist-info → prefect_client-2.17.0.dist-info}/LICENSE +0 -0
- {prefect_client-2.16.8.dist-info → prefect_client-2.17.0.dist-info}/WHEEL +0 -0
- {prefect_client-2.16.8.dist-info → prefect_client-2.17.0.dist-info}/top_level.txt +0 -0
@@ -2,6 +2,7 @@ import datetime
|
|
2
2
|
from typing import Any, Dict, List, Optional, TypeVar, Union
|
3
3
|
from uuid import UUID
|
4
4
|
|
5
|
+
from prefect._internal.compatibility.deprecated import DeprecatedInfraOverridesField
|
5
6
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
6
7
|
|
7
8
|
if HAS_PYDANTIC_V2:
|
@@ -14,9 +15,9 @@ from typing_extensions import Literal
|
|
14
15
|
import prefect.client.schemas.objects as objects
|
15
16
|
from prefect._internal.schemas.bases import ObjectBaseModel, PrefectBaseModel
|
16
17
|
from prefect._internal.schemas.fields import CreatedBy, DateTimeTZ, UpdatedBy
|
17
|
-
from prefect._internal.schemas.transformations import FieldFrom, copy_model_fields
|
18
18
|
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
19
19
|
from prefect.utilities.collections import AutoEnum
|
20
|
+
from prefect.utilities.names import generate_slug
|
20
21
|
|
21
22
|
R = TypeVar("R")
|
22
23
|
|
@@ -154,43 +155,145 @@ class WorkerFlowRunResponse(PrefectBaseModel):
|
|
154
155
|
flow_run: objects.FlowRun
|
155
156
|
|
156
157
|
|
157
|
-
@copy_model_fields
|
158
158
|
class FlowRunResponse(ObjectBaseModel):
|
159
|
-
name: str =
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
159
|
+
name: str = Field(
|
160
|
+
default_factory=lambda: generate_slug(2),
|
161
|
+
description=(
|
162
|
+
"The name of the flow run. Defaults to a random slug if not specified."
|
163
|
+
),
|
164
|
+
examples=["my-flow-run"],
|
165
|
+
)
|
166
|
+
flow_id: UUID = Field(default=..., description="The id of the flow being run.")
|
167
|
+
state_id: Optional[UUID] = Field(
|
168
|
+
default=None, description="The id of the flow run's current state."
|
169
|
+
)
|
170
|
+
deployment_id: Optional[UUID] = Field(
|
171
|
+
default=None,
|
172
|
+
description=(
|
173
|
+
"The id of the deployment associated with this flow run, if available."
|
174
|
+
),
|
175
|
+
)
|
176
|
+
deployment_version: Optional[str] = Field(
|
177
|
+
default=None,
|
178
|
+
description="The version of the deployment associated with this flow run.",
|
179
|
+
examples=["1.0"],
|
180
|
+
)
|
181
|
+
work_queue_name: Optional[str] = Field(
|
182
|
+
default=None, description="The work queue that handled this flow run."
|
183
|
+
)
|
184
|
+
flow_version: Optional[str] = Field(
|
185
|
+
default=None,
|
186
|
+
description="The version of the flow executed in this flow run.",
|
187
|
+
examples=["1.0"],
|
188
|
+
)
|
189
|
+
parameters: Dict[str, Any] = Field(
|
190
|
+
default_factory=dict, description="Parameters for the flow run."
|
191
|
+
)
|
192
|
+
idempotency_key: Optional[str] = Field(
|
193
|
+
default=None,
|
194
|
+
description=(
|
195
|
+
"An optional idempotency key for the flow run. Used to ensure the same flow"
|
196
|
+
" run is not created multiple times."
|
197
|
+
),
|
198
|
+
)
|
199
|
+
context: Dict[str, Any] = Field(
|
200
|
+
default_factory=dict,
|
201
|
+
description="Additional context for the flow run.",
|
202
|
+
examples=[{"my_var": "my_val"}],
|
203
|
+
)
|
204
|
+
empirical_policy: objects.FlowRunPolicy = Field(
|
205
|
+
default_factory=objects.FlowRunPolicy,
|
206
|
+
)
|
207
|
+
tags: List[str] = Field(
|
208
|
+
default_factory=list,
|
209
|
+
description="A list of tags on the flow run",
|
210
|
+
examples=[["tag-1", "tag-2"]],
|
211
|
+
)
|
212
|
+
parent_task_run_id: Optional[UUID] = Field(
|
213
|
+
default=None,
|
214
|
+
description=(
|
215
|
+
"If the flow run is a subflow, the id of the 'dummy' task in the parent"
|
216
|
+
" flow used to track subflow state."
|
217
|
+
),
|
218
|
+
)
|
219
|
+
run_count: int = Field(
|
220
|
+
default=0, description="The number of times the flow run was executed."
|
221
|
+
)
|
222
|
+
expected_start_time: Optional[DateTimeTZ] = Field(
|
223
|
+
default=None,
|
224
|
+
description="The flow run's expected start time.",
|
225
|
+
)
|
226
|
+
next_scheduled_start_time: Optional[DateTimeTZ] = Field(
|
227
|
+
default=None,
|
228
|
+
description="The next time the flow run is scheduled to start.",
|
229
|
+
)
|
230
|
+
start_time: Optional[DateTimeTZ] = Field(
|
231
|
+
default=None, description="The actual start time."
|
232
|
+
)
|
233
|
+
end_time: Optional[DateTimeTZ] = Field(
|
234
|
+
default=None, description="The actual end time."
|
235
|
+
)
|
236
|
+
total_run_time: datetime.timedelta = Field(
|
237
|
+
default=datetime.timedelta(0),
|
238
|
+
description=(
|
239
|
+
"Total run time. If the flow run was executed multiple times, the time of"
|
240
|
+
" each run will be summed."
|
241
|
+
),
|
242
|
+
)
|
243
|
+
estimated_run_time: datetime.timedelta = Field(
|
244
|
+
default=datetime.timedelta(0),
|
245
|
+
description="A real-time estimate of the total run time.",
|
246
|
+
)
|
247
|
+
estimated_start_time_delta: datetime.timedelta = Field(
|
248
|
+
default=datetime.timedelta(0),
|
249
|
+
description="The difference between actual and expected start time.",
|
250
|
+
)
|
251
|
+
auto_scheduled: bool = Field(
|
252
|
+
default=False,
|
253
|
+
description="Whether or not the flow run was automatically scheduled.",
|
254
|
+
)
|
255
|
+
infrastructure_document_id: Optional[UUID] = Field(
|
256
|
+
default=None,
|
257
|
+
description="The block document defining infrastructure to use this flow run.",
|
258
|
+
)
|
259
|
+
infrastructure_pid: Optional[str] = Field(
|
260
|
+
default=None,
|
261
|
+
description="The id of the flow run as returned by an infrastructure block.",
|
262
|
+
)
|
263
|
+
created_by: Optional[CreatedBy] = Field(
|
264
|
+
default=None,
|
265
|
+
description="Optional information about the creator of this flow run.",
|
266
|
+
)
|
267
|
+
work_queue_id: Optional[UUID] = Field(
|
268
|
+
default=None, description="The id of the run's work pool queue."
|
269
|
+
)
|
270
|
+
|
271
|
+
work_pool_id: Optional[UUID] = Field(
|
272
|
+
description="The work pool with which the queue is associated."
|
273
|
+
)
|
187
274
|
work_pool_name: Optional[str] = Field(
|
188
275
|
default=None,
|
189
276
|
description="The name of the flow run's work pool.",
|
190
|
-
|
277
|
+
examples=["my-work-pool"],
|
278
|
+
)
|
279
|
+
state: Optional[objects.State] = Field(
|
280
|
+
default=None,
|
281
|
+
description="The state of the flow run.",
|
282
|
+
examples=[objects.State(type=objects.StateType.COMPLETED)],
|
283
|
+
)
|
284
|
+
job_variables: Optional[dict] = Field(
|
285
|
+
default=None, description="Job variables for the flow run."
|
286
|
+
)
|
287
|
+
|
288
|
+
# These are server-side optimizations and should not be present on client models
|
289
|
+
# TODO: Deprecate these fields
|
290
|
+
|
291
|
+
state_type: Optional[objects.StateType] = Field(
|
292
|
+
default=None, description="The type of the current flow run state."
|
293
|
+
)
|
294
|
+
state_name: Optional[str] = Field(
|
295
|
+
default=None, description="The name of the current flow run state."
|
191
296
|
)
|
192
|
-
state: Optional[objects.State] = FieldFrom(objects.FlowRun)
|
193
|
-
job_variables: Optional[dict] = FieldFrom(objects.FlowRun)
|
194
297
|
|
195
298
|
def __eq__(self, other: Any) -> bool:
|
196
299
|
"""
|
@@ -199,7 +302,7 @@ class FlowRunResponse(ObjectBaseModel):
|
|
199
302
|
Estimates times are rolling and will always change with repeated queries for
|
200
303
|
a flow run so we ignore them during equality checks.
|
201
304
|
"""
|
202
|
-
if isinstance(other,
|
305
|
+
if isinstance(other, objects.FlowRun):
|
203
306
|
exclude_fields = {"estimated_run_time", "estimated_start_time_delta"}
|
204
307
|
return self.dict(exclude=exclude_fields) == other.dict(
|
205
308
|
exclude=exclude_fields
|
@@ -207,30 +310,108 @@ class FlowRunResponse(ObjectBaseModel):
|
|
207
310
|
return super().__eq__(other)
|
208
311
|
|
209
312
|
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
313
|
+
class DeploymentResponse(DeprecatedInfraOverridesField, ObjectBaseModel):
|
314
|
+
name: str = Field(default=..., description="The name of the deployment.")
|
315
|
+
version: Optional[str] = Field(
|
316
|
+
default=None, description="An optional version for the deployment."
|
317
|
+
)
|
318
|
+
description: Optional[str] = Field(
|
319
|
+
default=None, description="A description for the deployment."
|
320
|
+
)
|
321
|
+
flow_id: UUID = Field(
|
322
|
+
default=..., description="The flow id associated with the deployment."
|
323
|
+
)
|
324
|
+
schedule: Optional[SCHEDULE_TYPES] = Field(
|
325
|
+
default=None, description="A schedule for the deployment."
|
326
|
+
)
|
327
|
+
is_schedule_active: bool = Field(
|
328
|
+
default=True, description="Whether or not the deployment schedule is active."
|
329
|
+
)
|
330
|
+
paused: bool = Field(
|
331
|
+
default=False, description="Whether or not the deployment is paused."
|
332
|
+
)
|
333
|
+
schedules: List[objects.DeploymentSchedule] = Field(
|
334
|
+
default_factory=list, description="A list of schedules for the deployment."
|
335
|
+
)
|
336
|
+
job_variables: Dict[str, Any] = Field(
|
337
|
+
default_factory=dict,
|
338
|
+
description="Overrides to apply to flow run infrastructure at runtime.",
|
339
|
+
)
|
340
|
+
parameters: Dict[str, Any] = Field(
|
341
|
+
default_factory=dict,
|
342
|
+
description="Parameters for flow runs scheduled by the deployment.",
|
343
|
+
)
|
344
|
+
pull_steps: Optional[List[dict]] = Field(
|
345
|
+
default=None,
|
346
|
+
description="Pull steps for cloning and running this deployment.",
|
347
|
+
)
|
348
|
+
tags: List[str] = Field(
|
349
|
+
default_factory=list,
|
350
|
+
description="A list of tags for the deployment",
|
351
|
+
examples=[["tag-1", "tag-2"]],
|
352
|
+
)
|
353
|
+
work_queue_name: Optional[str] = Field(
|
354
|
+
default=None,
|
355
|
+
description=(
|
356
|
+
"The work queue for the deployment. If no work queue is set, work will not"
|
357
|
+
" be scheduled."
|
358
|
+
),
|
359
|
+
)
|
360
|
+
last_polled: Optional[DateTimeTZ] = Field(
|
361
|
+
default=None,
|
362
|
+
description="The last time the deployment was polled for status updates.",
|
363
|
+
)
|
364
|
+
parameter_openapi_schema: Optional[Dict[str, Any]] = Field(
|
365
|
+
default=None,
|
366
|
+
description="The parameter schema of the flow, including defaults.",
|
367
|
+
)
|
368
|
+
path: Optional[str] = Field(
|
369
|
+
default=None,
|
370
|
+
description=(
|
371
|
+
"The path to the working directory for the workflow, relative to remote"
|
372
|
+
" storage or an absolute path."
|
373
|
+
),
|
374
|
+
)
|
375
|
+
entrypoint: Optional[str] = Field(
|
376
|
+
default=None,
|
377
|
+
description=(
|
378
|
+
"The path to the entrypoint for the workflow, relative to the `path`."
|
379
|
+
),
|
380
|
+
)
|
381
|
+
manifest_path: Optional[str] = Field(
|
382
|
+
default=None,
|
383
|
+
description=(
|
384
|
+
"The path to the flow's manifest file, relative to the chosen storage."
|
385
|
+
),
|
386
|
+
)
|
387
|
+
storage_document_id: Optional[UUID] = Field(
|
388
|
+
default=None,
|
389
|
+
description="The block document defining storage used for this flow.",
|
390
|
+
)
|
391
|
+
infrastructure_document_id: Optional[UUID] = Field(
|
392
|
+
default=None,
|
393
|
+
description="The block document defining infrastructure to use for flow runs.",
|
394
|
+
)
|
395
|
+
created_by: Optional[CreatedBy] = Field(
|
396
|
+
default=None,
|
397
|
+
description="Optional information about the creator of this deployment.",
|
398
|
+
)
|
399
|
+
updated_by: Optional[UpdatedBy] = Field(
|
400
|
+
default=None,
|
401
|
+
description="Optional information about the updater of this deployment.",
|
402
|
+
)
|
403
|
+
work_queue_id: UUID = Field(
|
404
|
+
default=None,
|
405
|
+
description=(
|
406
|
+
"The id of the work pool queue to which this deployment is assigned."
|
407
|
+
),
|
408
|
+
)
|
409
|
+
enforce_parameter_schema: bool = Field(
|
410
|
+
default=False,
|
411
|
+
description=(
|
412
|
+
"Whether or not the deployment should enforce the parameter schema."
|
413
|
+
),
|
414
|
+
)
|
234
415
|
work_pool_name: Optional[str] = Field(
|
235
416
|
default=None,
|
236
417
|
description="The name of the deployment's work pool.",
|
@@ -239,12 +420,11 @@ class DeploymentResponse(ObjectBaseModel):
|
|
239
420
|
default=None,
|
240
421
|
description="Current status of the deployment.",
|
241
422
|
)
|
242
|
-
enforce_parameter_schema: bool = FieldFrom(objects.Deployment)
|
243
423
|
|
244
424
|
|
245
425
|
class MinimalConcurrencyLimitResponse(PrefectBaseModel):
|
246
426
|
class Config:
|
247
|
-
extra = "ignore"
|
427
|
+
extra = "ignore" # 2024/4/1
|
248
428
|
|
249
429
|
id: UUID
|
250
430
|
name: str
|
@@ -10,6 +10,8 @@ import dateutil.rrule
|
|
10
10
|
import pendulum
|
11
11
|
|
12
12
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
13
|
+
from prefect._internal.schemas.bases import PrefectBaseModel
|
14
|
+
from prefect._internal.schemas.fields import DateTimeTZ
|
13
15
|
from prefect._internal.schemas.validators import (
|
14
16
|
default_anchor_date,
|
15
17
|
default_timezone,
|
@@ -24,10 +26,6 @@ if HAS_PYDANTIC_V2:
|
|
24
26
|
else:
|
25
27
|
from pydantic import Field, validator
|
26
28
|
|
27
|
-
|
28
|
-
from prefect._internal.schemas.bases import PrefectBaseModel
|
29
|
-
from prefect._internal.schemas.fields import DateTimeTZ
|
30
|
-
|
31
29
|
MAX_ITERATIONS = 1000
|
32
30
|
# approx. 1 years worth of RDATEs + buffer
|
33
31
|
MAX_RRULE_LENGTH = 6500
|
@@ -68,7 +66,7 @@ class IntervalSchedule(PrefectBaseModel):
|
|
68
66
|
|
69
67
|
interval: datetime.timedelta
|
70
68
|
anchor_date: DateTimeTZ = None
|
71
|
-
timezone: Optional[str] = Field(default=None,
|
69
|
+
timezone: Optional[str] = Field(default=None, examples=["America/New_York"])
|
72
70
|
|
73
71
|
@validator("interval")
|
74
72
|
def validate_interval_schedule(cls, v):
|
@@ -111,8 +109,8 @@ class CronSchedule(PrefectBaseModel):
|
|
111
109
|
class Config:
|
112
110
|
extra = "forbid"
|
113
111
|
|
114
|
-
cron: str = Field(default=...,
|
115
|
-
timezone: Optional[str] = Field(default=None,
|
112
|
+
cron: str = Field(default=..., examples=["0 0 * * *"])
|
113
|
+
timezone: Optional[str] = Field(default=None, examples=["America/New_York"])
|
116
114
|
day_or: bool = Field(
|
117
115
|
default=True,
|
118
116
|
description=(
|
@@ -156,7 +154,7 @@ class RRuleSchedule(PrefectBaseModel):
|
|
156
154
|
extra = "forbid"
|
157
155
|
|
158
156
|
rrule: str
|
159
|
-
timezone: Optional[str] = Field(default=None,
|
157
|
+
timezone: Optional[str] = Field(default=None, examples=["America/New_York"])
|
160
158
|
|
161
159
|
@validator("rrule")
|
162
160
|
def validate_rrule_str(cls, v):
|
prefect/concurrency/events.py
CHANGED
prefect/context.py
CHANGED
@@ -5,6 +5,7 @@ These contexts should never be directly mutated by the user.
|
|
5
5
|
|
6
6
|
For more user-accessible information about the current run, see [`prefect.runtime`](../runtime/flow_run).
|
7
7
|
"""
|
8
|
+
|
8
9
|
import os
|
9
10
|
import sys
|
10
11
|
import warnings
|
@@ -18,6 +19,7 @@ from typing import (
|
|
18
19
|
Any,
|
19
20
|
ContextManager,
|
20
21
|
Dict,
|
22
|
+
Generator,
|
21
23
|
List,
|
22
24
|
Optional,
|
23
25
|
Set,
|
@@ -75,7 +77,7 @@ class ContextModel(BaseModel):
|
|
75
77
|
_token: Token = PrivateAttr(None)
|
76
78
|
|
77
79
|
class Config:
|
78
|
-
allow_mutation = False
|
80
|
+
# allow_mutation = False
|
79
81
|
arbitrary_types_allowed = True
|
80
82
|
extra = "forbid"
|
81
83
|
|
@@ -394,7 +396,7 @@ def get_settings_context() -> SettingsContext:
|
|
394
396
|
|
395
397
|
|
396
398
|
@contextmanager
|
397
|
-
def tags(*new_tags: str) -> Set[str]:
|
399
|
+
def tags(*new_tags: str) -> Generator[Set[str], None, None]:
|
398
400
|
"""
|
399
401
|
Context manager to add tags to flow and task run calls.
|
400
402
|
|
prefect/deployments/base.py
CHANGED
@@ -4,6 +4,7 @@ build system for managing flows and deployments.
|
|
4
4
|
|
5
5
|
To get started, follow along with [the deloyments tutorial](/tutorials/deployments/).
|
6
6
|
"""
|
7
|
+
|
7
8
|
import ast
|
8
9
|
import asyncio
|
9
10
|
import json
|
@@ -13,18 +14,10 @@ import subprocess
|
|
13
14
|
import sys
|
14
15
|
from copy import deepcopy
|
15
16
|
from pathlib import Path
|
16
|
-
from typing import Dict, List, Optional, cast
|
17
|
+
from typing import Any, Dict, List, Optional, cast
|
17
18
|
|
18
19
|
import anyio
|
19
20
|
import yaml
|
20
|
-
|
21
|
-
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
22
|
-
|
23
|
-
if HAS_PYDANTIC_V2:
|
24
|
-
from pydantic.v1 import BaseModel
|
25
|
-
else:
|
26
|
-
from pydantic import BaseModel
|
27
|
-
|
28
21
|
from ruamel.yaml import YAML
|
29
22
|
|
30
23
|
from prefect.client.schemas.objects import MinimalDeploymentSchedule
|
@@ -74,7 +67,7 @@ def set_prefect_hidden_dir(path: str = None) -> bool:
|
|
74
67
|
|
75
68
|
|
76
69
|
def create_default_prefect_yaml(
|
77
|
-
path: str, name: str = None, contents:
|
70
|
+
path: str, name: str = None, contents: Optional[Dict[str, Any]] = None
|
78
71
|
) -> bool:
|
79
72
|
"""
|
80
73
|
Creates default `prefect.yaml` file in the provided path if one does not already exist;
|
@@ -221,7 +214,7 @@ def _get_git_branch() -> Optional[str]:
|
|
221
214
|
|
222
215
|
|
223
216
|
def initialize_project(
|
224
|
-
name: str = None, recipe: str = None, inputs:
|
217
|
+
name: str = None, recipe: str = None, inputs: Optional[Dict[str, Any]] = None
|
225
218
|
) -> List[str]:
|
226
219
|
"""
|
227
220
|
Initializes a basic project structure with base files. If no name is provided, the name
|
@@ -408,7 +401,7 @@ def _format_deployment_for_saving_to_prefect_file(
|
|
408
401
|
if deployment.get("schedule"):
|
409
402
|
if isinstance(deployment["schedule"], IntervalSchedule):
|
410
403
|
deployment["schedule"] = _interval_schedule_to_dict(deployment["schedule"])
|
411
|
-
|
404
|
+
else: # all valid SCHEDULE_TYPES are subclasses of BaseModel
|
412
405
|
deployment["schedule"] = deployment["schedule"].dict()
|
413
406
|
|
414
407
|
if "is_schedule_active" in deployment:
|
@@ -423,7 +416,7 @@ def _format_deployment_for_saving_to_prefect_file(
|
|
423
416
|
schedule_config = _interval_schedule_to_dict(
|
424
417
|
deployment_schedule.schedule
|
425
418
|
)
|
426
|
-
|
419
|
+
else: # all valid SCHEDULE_TYPES are subclasses of BaseModel
|
427
420
|
schedule_config = deployment_schedule.schedule.dict()
|
428
421
|
|
429
422
|
schedule_config["active"] = deployment_schedule.active
|
@@ -17,8 +17,11 @@ import pendulum
|
|
17
17
|
import yaml
|
18
18
|
|
19
19
|
from prefect._internal.compatibility.deprecated import (
|
20
|
+
DeprecatedInfraOverridesField,
|
20
21
|
deprecated_callable,
|
21
22
|
deprecated_class,
|
23
|
+
deprecated_parameter,
|
24
|
+
handle_deprecated_infra_overrides_parameter,
|
22
25
|
)
|
23
26
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
24
27
|
from prefect._internal.schemas.validators import (
|
@@ -71,6 +74,11 @@ logger = get_logger("deployments")
|
|
71
74
|
|
72
75
|
|
73
76
|
@sync_compatible
|
77
|
+
@deprecated_parameter(
|
78
|
+
"infra_overrides",
|
79
|
+
start_date="Apr 2024",
|
80
|
+
help="Use `job_variables` instead.",
|
81
|
+
)
|
74
82
|
@inject_client
|
75
83
|
async def run_deployment(
|
76
84
|
name: Union[str, UUID],
|
@@ -84,6 +92,7 @@ async def run_deployment(
|
|
84
92
|
idempotency_key: Optional[str] = None,
|
85
93
|
work_queue_name: Optional[str] = None,
|
86
94
|
as_subflow: Optional[bool] = True,
|
95
|
+
infra_overrides: Optional[dict] = None,
|
87
96
|
job_variables: Optional[dict] = None,
|
88
97
|
) -> FlowRun:
|
89
98
|
"""
|
@@ -103,7 +112,7 @@ async def run_deployment(
|
|
103
112
|
|
104
113
|
Args:
|
105
114
|
name: The deployment id or deployment name in the form:
|
106
|
-
|
115
|
+
`"flow name/deployment name"`
|
107
116
|
parameters: Parameter overrides for this flow run. Merged with the deployment
|
108
117
|
defaults.
|
109
118
|
scheduled_time: The time to schedule the flow run for, defaults to scheduling
|
@@ -122,6 +131,9 @@ async def run_deployment(
|
|
122
131
|
the default work queue for the deployment.
|
123
132
|
as_subflow: Whether to link the flow run as a subflow of the current
|
124
133
|
flow or task run.
|
134
|
+
job_variables: A dictionary of dot delimited infrastructure overrides that
|
135
|
+
will be applied at runtime; for example `env.CONFIG_KEY=config_value` or
|
136
|
+
`namespace='prefect'`
|
125
137
|
"""
|
126
138
|
if timeout is not None and timeout < 0:
|
127
139
|
raise ValueError("`timeout` cannot be negative")
|
@@ -129,6 +141,8 @@ async def run_deployment(
|
|
129
141
|
if scheduled_time is None:
|
130
142
|
scheduled_time = pendulum.now("UTC")
|
131
143
|
|
144
|
+
jv = handle_deprecated_infra_overrides_parameter(job_variables, infra_overrides)
|
145
|
+
|
132
146
|
parameters = parameters or {}
|
133
147
|
|
134
148
|
deployment_id = None
|
@@ -204,7 +218,7 @@ async def run_deployment(
|
|
204
218
|
idempotency_key=idempotency_key,
|
205
219
|
parent_task_run_id=parent_task_run_id,
|
206
220
|
work_queue_name=work_queue_name,
|
207
|
-
job_variables=
|
221
|
+
job_variables=jv,
|
208
222
|
)
|
209
223
|
|
210
224
|
flow_run_id = flow_run.id
|
@@ -334,7 +348,7 @@ def load_deployments_from_yaml(
|
|
334
348
|
" Refer to the upgrade guide for more information:"
|
335
349
|
" https://docs.prefect.io/latest/guides/upgrade-guide-agents-to-workers/.",
|
336
350
|
)
|
337
|
-
class Deployment(BaseModel):
|
351
|
+
class Deployment(DeprecatedInfraOverridesField, BaseModel):
|
338
352
|
"""
|
339
353
|
DEPRECATION WARNING:
|
340
354
|
|
@@ -363,7 +377,7 @@ class Deployment(BaseModel):
|
|
363
377
|
infrastructure: An optional infrastructure block used to configure
|
364
378
|
infrastructure for runs; if not provided, will default to running this
|
365
379
|
deployment in Agent subprocesses
|
366
|
-
|
380
|
+
job_variables: A dictionary of dot delimited infrastructure overrides that
|
367
381
|
will be applied at runtime; for example `env.CONFIG_KEY=config_value` or
|
368
382
|
`namespace='prefect'`
|
369
383
|
storage: An optional remote storage block used to store and retrieve this
|
@@ -405,7 +419,7 @@ class Deployment(BaseModel):
|
|
405
419
|
... version="2",
|
406
420
|
... tags=["aws"],
|
407
421
|
... storage=storage,
|
408
|
-
...
|
422
|
+
... job_variables=dict("env.PREFECT_LOGGING_LEVEL"="DEBUG"),
|
409
423
|
>>> )
|
410
424
|
>>> deployment.apply()
|
411
425
|
|
@@ -429,6 +443,11 @@ class Deployment(BaseModel):
|
|
429
443
|
"schedule",
|
430
444
|
"schedules",
|
431
445
|
"is_schedule_active",
|
446
|
+
# The `infra_overrides` field has been renamed to `job_variables`.
|
447
|
+
# We will continue writing it in the YAML file as `infra_overrides`
|
448
|
+
# instead of `job_variables` for better backwards compat, but we'll
|
449
|
+
# accept either `job_variables` or `infra_overrides` when we read
|
450
|
+
# the file.
|
432
451
|
"infra_overrides",
|
433
452
|
]
|
434
453
|
|
@@ -478,10 +497,16 @@ class Deployment(BaseModel):
|
|
478
497
|
# write the field
|
479
498
|
yaml.dump({field: yaml_dict[field]}, f, sort_keys=False)
|
480
499
|
|
481
|
-
# write non-editable fields
|
500
|
+
# write non-editable fields, excluding `job_variables` because we'll
|
501
|
+
# continue writing it as `infra_overrides` for better backwards compat
|
502
|
+
# with the existing file format.
|
482
503
|
f.write("\n###\n### DO NOT EDIT BELOW THIS LINE\n###\n")
|
483
504
|
yaml.dump(
|
484
|
-
{
|
505
|
+
{
|
506
|
+
k: v
|
507
|
+
for k, v in yaml_dict.items()
|
508
|
+
if k not in self._editable_fields and k != "job_variables"
|
509
|
+
},
|
485
510
|
f,
|
486
511
|
sort_keys=False,
|
487
512
|
)
|
@@ -558,7 +583,7 @@ class Deployment(BaseModel):
|
|
558
583
|
),
|
559
584
|
)
|
560
585
|
infrastructure: Infrastructure = Field(default_factory=Process)
|
561
|
-
|
586
|
+
job_variables: Dict[str, Any] = Field(
|
562
587
|
default_factory=dict,
|
563
588
|
description="Overrides to apply to the base infrastructure block at runtime.",
|
564
589
|
)
|
@@ -869,7 +894,7 @@ class Deployment(BaseModel):
|
|
869
894
|
manifest_path=self.manifest_path, # allows for backwards YAML compat
|
870
895
|
path=self.path,
|
871
896
|
entrypoint=self.entrypoint,
|
872
|
-
|
897
|
+
job_variables=self.job_variables,
|
873
898
|
storage_document_id=storage_document_id,
|
874
899
|
infrastructure_document_id=infrastructure_document_id,
|
875
900
|
parameter_openapi_schema=self.parameter_openapi_schema.dict(),
|