prefect-client 3.1.14__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__main__.py +4 -0
- prefect/_experimental/lineage.py +40 -22
- prefect/_experimental/sla/objects.py +29 -1
- prefect/_internal/compatibility/deprecated.py +4 -4
- prefect/_internal/compatibility/migration.py +1 -1
- prefect/_internal/concurrency/calls.py +1 -2
- prefect/_internal/concurrency/cancellation.py +2 -4
- prefect/_internal/concurrency/services.py +1 -1
- prefect/_internal/concurrency/threads.py +3 -3
- prefect/_internal/schemas/bases.py +3 -11
- prefect/_internal/schemas/validators.py +36 -60
- prefect/_result_records.py +235 -0
- prefect/_version.py +3 -3
- prefect/agent.py +1 -0
- prefect/artifacts.py +408 -105
- prefect/automations.py +4 -8
- prefect/blocks/core.py +1 -1
- prefect/blocks/notifications.py +13 -8
- prefect/cache_policies.py +2 -0
- prefect/client/base.py +7 -8
- prefect/client/collections.py +3 -6
- prefect/client/orchestration/__init__.py +15 -263
- prefect/client/orchestration/_deployments/client.py +14 -6
- prefect/client/orchestration/_flow_runs/client.py +10 -6
- prefect/client/orchestration/_work_pools/__init__.py +0 -0
- prefect/client/orchestration/_work_pools/client.py +598 -0
- prefect/client/orchestration/base.py +9 -2
- prefect/client/schemas/actions.py +77 -3
- prefect/client/schemas/objects.py +22 -50
- prefect/client/schemas/schedules.py +11 -22
- prefect/client/types/flexible_schedule_list.py +2 -1
- prefect/context.py +2 -3
- prefect/deployments/base.py +13 -16
- prefect/deployments/flow_runs.py +1 -1
- prefect/deployments/runner.py +236 -47
- prefect/deployments/schedules.py +7 -1
- prefect/engine.py +4 -9
- prefect/events/clients.py +39 -0
- prefect/events/schemas/automations.py +4 -2
- prefect/events/utilities.py +15 -13
- prefect/exceptions.py +1 -1
- prefect/flow_engine.py +119 -0
- prefect/flow_runs.py +4 -8
- prefect/flows.py +282 -31
- prefect/infrastructure/__init__.py +1 -0
- prefect/infrastructure/base.py +1 -0
- prefect/infrastructure/provisioners/__init__.py +3 -6
- prefect/infrastructure/provisioners/coiled.py +3 -3
- prefect/infrastructure/provisioners/container_instance.py +1 -0
- prefect/infrastructure/provisioners/ecs.py +6 -6
- prefect/infrastructure/provisioners/modal.py +3 -3
- prefect/input/run_input.py +5 -7
- prefect/locking/filesystem.py +4 -3
- prefect/main.py +1 -1
- prefect/results.py +42 -249
- prefect/runner/runner.py +9 -4
- prefect/runner/server.py +5 -5
- prefect/runner/storage.py +12 -10
- prefect/runner/submit.py +2 -4
- prefect/runtime/task_run.py +37 -9
- prefect/schedules.py +231 -0
- prefect/serializers.py +5 -5
- prefect/settings/__init__.py +2 -1
- prefect/settings/base.py +3 -3
- prefect/settings/models/root.py +4 -0
- prefect/settings/models/server/services.py +50 -9
- prefect/settings/sources.py +4 -4
- prefect/states.py +42 -11
- prefect/task_engine.py +10 -10
- prefect/task_runners.py +11 -22
- prefect/task_worker.py +9 -9
- prefect/tasks.py +28 -45
- prefect/telemetry/bootstrap.py +4 -6
- prefect/telemetry/services.py +2 -4
- prefect/types/__init__.py +2 -1
- prefect/types/_datetime.py +28 -1
- prefect/utilities/_engine.py +0 -1
- prefect/utilities/asyncutils.py +4 -8
- prefect/utilities/collections.py +13 -22
- prefect/utilities/dispatch.py +2 -4
- prefect/utilities/dockerutils.py +6 -6
- prefect/utilities/importtools.py +1 -68
- prefect/utilities/names.py +1 -1
- prefect/utilities/processutils.py +3 -6
- prefect/utilities/pydantic.py +4 -6
- prefect/utilities/render_swagger.py +1 -1
- prefect/utilities/schema_tools/hydration.py +6 -5
- prefect/utilities/templating.py +21 -8
- prefect/utilities/visualization.py +2 -4
- prefect/workers/base.py +3 -3
- prefect/workers/block.py +1 -0
- prefect/workers/cloud.py +1 -0
- prefect/workers/process.py +1 -0
- {prefect_client-3.1.14.dist-info → prefect_client-3.2.0.dist-info}/METADATA +1 -1
- {prefect_client-3.1.14.dist-info → prefect_client-3.2.0.dist-info}/RECORD +98 -93
- {prefect_client-3.1.14.dist-info → prefect_client-3.2.0.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.14.dist-info → prefect_client-3.2.0.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.14.dist-info → prefect_client-3.2.0.dist-info}/top_level.txt +0 -0
prefect/deployments/runner.py
CHANGED
@@ -49,6 +49,7 @@ from rich.progress import Progress, SpinnerColumn, TextColumn, track
|
|
49
49
|
from rich.table import Table
|
50
50
|
|
51
51
|
from prefect._experimental.sla.objects import SlaTypes
|
52
|
+
from prefect._internal.compatibility.async_dispatch import async_dispatch
|
52
53
|
from prefect._internal.concurrency.api import create_call, from_async
|
53
54
|
from prefect._internal.schemas.validators import (
|
54
55
|
reconcile_paused_deployment,
|
@@ -56,7 +57,7 @@ from prefect._internal.schemas.validators import (
|
|
56
57
|
)
|
57
58
|
from prefect.client.base import ServerType
|
58
59
|
from prefect.client.orchestration import PrefectClient, get_client
|
59
|
-
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
60
|
+
from prefect.client.schemas.actions import DeploymentScheduleCreate, DeploymentUpdate
|
60
61
|
from prefect.client.schemas.filters import WorkerFilter, WorkerFilterStatus
|
61
62
|
from prefect.client.schemas.objects import (
|
62
63
|
ConcurrencyLimitConfig,
|
@@ -76,13 +77,14 @@ from prefect.exceptions import (
|
|
76
77
|
PrefectHTTPStatusError,
|
77
78
|
)
|
78
79
|
from prefect.runner.storage import RunnerStorage
|
80
|
+
from prefect.schedules import Schedule
|
79
81
|
from prefect.settings import (
|
80
82
|
PREFECT_DEFAULT_WORK_POOL_NAME,
|
81
83
|
PREFECT_UI_URL,
|
82
84
|
)
|
83
85
|
from prefect.types import ListOfNonEmptyStrings
|
84
86
|
from prefect.types.entrypoint import EntrypointType
|
85
|
-
from prefect.utilities.asyncutils import sync_compatible
|
87
|
+
from prefect.utilities.asyncutils import run_coro_as_sync, sync_compatible
|
86
88
|
from prefect.utilities.callables import ParameterSchema, parameter_schema
|
87
89
|
from prefect.utilities.collections import get_from_dict, isiterable
|
88
90
|
from prefect.utilities.dockerutils import (
|
@@ -112,7 +114,7 @@ class RunnerDeployment(BaseModel):
|
|
112
114
|
description: An optional description of the deployment; defaults to the flow's
|
113
115
|
description
|
114
116
|
tags: An optional list of tags to associate with this deployment; note that tags
|
115
|
-
are used only for organizational purposes. For delegating work to
|
117
|
+
are used only for organizational purposes. For delegating work to workers,
|
116
118
|
see `work_queue_name`.
|
117
119
|
schedule: A schedule to run this deployment on, once registered
|
118
120
|
parameters: A dictionary of parameter values to pass to runs created from this
|
@@ -228,6 +230,10 @@ class RunnerDeployment(BaseModel):
|
|
228
230
|
def entrypoint_type(self) -> EntrypointType:
|
229
231
|
return self._entrypoint_type
|
230
232
|
|
233
|
+
@property
|
234
|
+
def full_name(self) -> str:
|
235
|
+
return f"{self.flow_name}/{self.name}"
|
236
|
+
|
231
237
|
@field_validator("name", mode="before")
|
232
238
|
@classmethod
|
233
239
|
def validate_name(cls, value: str) -> str:
|
@@ -254,24 +260,9 @@ class RunnerDeployment(BaseModel):
|
|
254
260
|
def reconcile_schedules(cls, values):
|
255
261
|
return reconcile_schedules_runner(values)
|
256
262
|
|
257
|
-
|
258
|
-
async def apply(
|
263
|
+
async def _create(
|
259
264
|
self, work_pool_name: Optional[str] = None, image: Optional[str] = None
|
260
265
|
) -> UUID:
|
261
|
-
"""
|
262
|
-
Registers this deployment with the API and returns the deployment's ID.
|
263
|
-
|
264
|
-
Args:
|
265
|
-
work_pool_name: The name of the work pool to use for this
|
266
|
-
deployment.
|
267
|
-
image: The registry, name, and tag of the Docker image to
|
268
|
-
use for this deployment. Only used when the deployment is
|
269
|
-
deployed to a work pool.
|
270
|
-
|
271
|
-
Returns:
|
272
|
-
The ID of the created deployment.
|
273
|
-
"""
|
274
|
-
|
275
266
|
work_pool_name = work_pool_name or self.work_pool_name
|
276
267
|
|
277
268
|
if image and not work_pool_name:
|
@@ -323,9 +314,14 @@ class RunnerDeployment(BaseModel):
|
|
323
314
|
if image:
|
324
315
|
create_payload["job_variables"]["image"] = image
|
325
316
|
create_payload["path"] = None if self.storage else self._path
|
326
|
-
|
327
|
-
|
328
|
-
|
317
|
+
if self.storage:
|
318
|
+
pull_steps = self.storage.to_pull_step()
|
319
|
+
if isinstance(pull_steps, list):
|
320
|
+
create_payload["pull_steps"] = pull_steps
|
321
|
+
else:
|
322
|
+
create_payload["pull_steps"] = [pull_steps]
|
323
|
+
else:
|
324
|
+
create_payload["pull_steps"] = []
|
329
325
|
|
330
326
|
try:
|
331
327
|
deployment_id = await client.create_deployment(**create_payload)
|
@@ -338,25 +334,7 @@ class RunnerDeployment(BaseModel):
|
|
338
334
|
f"Error while applying deployment: {str(exc)}"
|
339
335
|
) from exc
|
340
336
|
|
341
|
-
|
342
|
-
# The triggers defined in the deployment spec are, essentially,
|
343
|
-
# anonymous and attempting truly sync them with cloud is not
|
344
|
-
# feasible. Instead, we remove all automations that are owned
|
345
|
-
# by the deployment, meaning that they were created via this
|
346
|
-
# mechanism below, and then recreate them.
|
347
|
-
await client.delete_resource_owned_automations(
|
348
|
-
f"prefect.deployment.{deployment_id}"
|
349
|
-
)
|
350
|
-
except PrefectHTTPStatusError as e:
|
351
|
-
if e.response.status_code == 404:
|
352
|
-
# This Prefect server does not support automations, so we can safely
|
353
|
-
# ignore this 404 and move on.
|
354
|
-
return deployment_id
|
355
|
-
raise e
|
356
|
-
|
357
|
-
for trigger in self.triggers:
|
358
|
-
trigger.set_deployment_id(deployment_id)
|
359
|
-
await client.create_automation(trigger.as_automation())
|
337
|
+
await self._create_triggers(deployment_id, client)
|
360
338
|
|
361
339
|
# We plan to support SLA configuration on the Prefect Server in the future.
|
362
340
|
# For now, we only support it on Prefect Cloud.
|
@@ -369,6 +347,86 @@ class RunnerDeployment(BaseModel):
|
|
369
347
|
|
370
348
|
return deployment_id
|
371
349
|
|
350
|
+
async def _update(self, deployment_id: UUID, client: PrefectClient):
|
351
|
+
parameter_openapi_schema = self._parameter_openapi_schema.model_dump(
|
352
|
+
exclude_unset=True
|
353
|
+
)
|
354
|
+
await client.update_deployment(
|
355
|
+
deployment_id,
|
356
|
+
deployment=DeploymentUpdate(
|
357
|
+
parameter_openapi_schema=parameter_openapi_schema,
|
358
|
+
**self.model_dump(
|
359
|
+
mode="json",
|
360
|
+
exclude_unset=True,
|
361
|
+
exclude={"storage", "name", "flow_name", "triggers"},
|
362
|
+
),
|
363
|
+
),
|
364
|
+
)
|
365
|
+
|
366
|
+
await self._create_triggers(deployment_id, client)
|
367
|
+
|
368
|
+
# We plan to support SLA configuration on the Prefect Server in the future.
|
369
|
+
# For now, we only support it on Prefect Cloud.
|
370
|
+
|
371
|
+
# If we're provided with an empty list, we will call the apply endpoint
|
372
|
+
# to remove existing SLAs for the deployment. If the argument is not provided,
|
373
|
+
# we will not call the endpoint.
|
374
|
+
if self._sla or self._sla == []:
|
375
|
+
await self._create_slas(deployment_id, client)
|
376
|
+
|
377
|
+
return deployment_id
|
378
|
+
|
379
|
+
async def _create_triggers(self, deployment_id: UUID, client: PrefectClient):
|
380
|
+
try:
|
381
|
+
# The triggers defined in the deployment spec are, essentially,
|
382
|
+
# anonymous and attempting truly sync them with cloud is not
|
383
|
+
# feasible. Instead, we remove all automations that are owned
|
384
|
+
# by the deployment, meaning that they were created via this
|
385
|
+
# mechanism below, and then recreate them.
|
386
|
+
await client.delete_resource_owned_automations(
|
387
|
+
f"prefect.deployment.{deployment_id}"
|
388
|
+
)
|
389
|
+
except PrefectHTTPStatusError as e:
|
390
|
+
if e.response.status_code == 404:
|
391
|
+
# This Prefect server does not support automations, so we can safely
|
392
|
+
# ignore this 404 and move on.
|
393
|
+
return deployment_id
|
394
|
+
raise e
|
395
|
+
|
396
|
+
for trigger in self.triggers:
|
397
|
+
trigger.set_deployment_id(deployment_id)
|
398
|
+
await client.create_automation(trigger.as_automation())
|
399
|
+
|
400
|
+
@sync_compatible
|
401
|
+
async def apply(
|
402
|
+
self, work_pool_name: Optional[str] = None, image: Optional[str] = None
|
403
|
+
) -> UUID:
|
404
|
+
"""
|
405
|
+
Registers this deployment with the API and returns the deployment's ID.
|
406
|
+
|
407
|
+
Args:
|
408
|
+
work_pool_name: The name of the work pool to use for this
|
409
|
+
deployment.
|
410
|
+
image: The registry, name, and tag of the Docker image to
|
411
|
+
use for this deployment. Only used when the deployment is
|
412
|
+
deployed to a work pool.
|
413
|
+
|
414
|
+
Returns:
|
415
|
+
The ID of the created deployment.
|
416
|
+
"""
|
417
|
+
|
418
|
+
async with get_client() as client:
|
419
|
+
try:
|
420
|
+
deployment = await client.read_deployment_by_name(self.full_name)
|
421
|
+
except ObjectNotFound:
|
422
|
+
return await self._create(work_pool_name, image)
|
423
|
+
else:
|
424
|
+
if image:
|
425
|
+
self.job_variables["image"] = image
|
426
|
+
if work_pool_name:
|
427
|
+
self.work_pool_name = work_pool_name
|
428
|
+
return await self._update(deployment.id, client)
|
429
|
+
|
372
430
|
async def _create_slas(self, deployment_id: UUID, client: PrefectClient):
|
373
431
|
if not isinstance(self._sla, list):
|
374
432
|
self._sla = [self._sla]
|
@@ -389,7 +447,7 @@ class RunnerDeployment(BaseModel):
|
|
389
447
|
cron: Optional[Union[Iterable[str], str]] = None,
|
390
448
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
391
449
|
timezone: Optional[str] = None,
|
392
|
-
schedule:
|
450
|
+
schedule: Union[SCHEDULE_TYPES, Schedule, None] = None,
|
393
451
|
schedules: Optional["FlexibleScheduleList"] = None,
|
394
452
|
) -> Union[List[DeploymentScheduleCreate], "FlexibleScheduleList"]:
|
395
453
|
"""
|
@@ -421,7 +479,6 @@ class RunnerDeployment(BaseModel):
|
|
421
479
|
this list is returned as-is, bypassing other schedule construction
|
422
480
|
logic.
|
423
481
|
"""
|
424
|
-
|
425
482
|
num_schedules = sum(
|
426
483
|
1
|
427
484
|
for entry in (interval, cron, rrule, schedule, schedules)
|
@@ -429,7 +486,7 @@ class RunnerDeployment(BaseModel):
|
|
429
486
|
)
|
430
487
|
if num_schedules > 1:
|
431
488
|
raise ValueError(
|
432
|
-
"Only one of interval, cron, rrule, or schedules can be provided."
|
489
|
+
"Only one of interval, cron, rrule, schedule, or schedules can be provided."
|
433
490
|
)
|
434
491
|
elif num_schedules == 0:
|
435
492
|
return []
|
@@ -482,6 +539,7 @@ class RunnerDeployment(BaseModel):
|
|
482
539
|
cron: Optional[Union[Iterable[str], str]] = None,
|
483
540
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
484
541
|
paused: Optional[bool] = None,
|
542
|
+
schedule: Optional[Schedule] = None,
|
485
543
|
schedules: Optional["FlexibleScheduleList"] = None,
|
486
544
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
487
545
|
parameters: Optional[dict[str, Any]] = None,
|
@@ -507,6 +565,8 @@ class RunnerDeployment(BaseModel):
|
|
507
565
|
cron: A cron schedule of when to execute runs of this flow.
|
508
566
|
rrule: An rrule schedule of when to execute runs of this flow.
|
509
567
|
paused: Whether or not to set this deployment as paused.
|
568
|
+
schedule: A schedule object defining when to execute runs of this deployment.
|
569
|
+
Used to provide additional scheduling options like `timezone` or `parameters`.
|
510
570
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
511
571
|
Used to define multiple schedules or additional scheduling options like `timezone`.
|
512
572
|
concurrency_limit: The maximum number of concurrent runs this deployment will allow.
|
@@ -532,6 +592,7 @@ class RunnerDeployment(BaseModel):
|
|
532
592
|
cron=cron,
|
533
593
|
rrule=rrule,
|
534
594
|
schedules=schedules,
|
595
|
+
schedule=schedule,
|
535
596
|
)
|
536
597
|
|
537
598
|
job_variables = job_variables or {}
|
@@ -626,6 +687,7 @@ class RunnerDeployment(BaseModel):
|
|
626
687
|
cron: Optional[Union[Iterable[str], str]] = None,
|
627
688
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
628
689
|
paused: Optional[bool] = None,
|
690
|
+
schedule: Optional[Schedule] = None,
|
629
691
|
schedules: Optional["FlexibleScheduleList"] = None,
|
630
692
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
631
693
|
parameters: Optional[dict[str, Any]] = None,
|
@@ -681,6 +743,7 @@ class RunnerDeployment(BaseModel):
|
|
681
743
|
cron=cron,
|
682
744
|
rrule=rrule,
|
683
745
|
schedules=schedules,
|
746
|
+
schedule=schedule,
|
684
747
|
)
|
685
748
|
|
686
749
|
if isinstance(concurrency_limit, ConcurrencyLimitConfig):
|
@@ -717,8 +780,7 @@ class RunnerDeployment(BaseModel):
|
|
717
780
|
return deployment
|
718
781
|
|
719
782
|
@classmethod
|
720
|
-
|
721
|
-
async def from_storage(
|
783
|
+
async def afrom_storage(
|
722
784
|
cls,
|
723
785
|
storage: RunnerStorage,
|
724
786
|
entrypoint: str,
|
@@ -730,6 +792,7 @@ class RunnerDeployment(BaseModel):
|
|
730
792
|
cron: Optional[Union[Iterable[str], str]] = None,
|
731
793
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
732
794
|
paused: Optional[bool] = None,
|
795
|
+
schedule: Optional[Schedule] = None,
|
733
796
|
schedules: Optional["FlexibleScheduleList"] = None,
|
734
797
|
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
735
798
|
parameters: Optional[dict[str, Any]] = None,
|
@@ -742,7 +805,7 @@ class RunnerDeployment(BaseModel):
|
|
742
805
|
work_queue_name: Optional[str] = None,
|
743
806
|
job_variables: Optional[dict[str, Any]] = None,
|
744
807
|
_sla: Optional[Union[SlaTypes, list[SlaTypes]]] = None, # experimental
|
745
|
-
):
|
808
|
+
) -> "RunnerDeployment":
|
746
809
|
"""
|
747
810
|
Create a RunnerDeployment from a flow located at a given entrypoint and stored in a
|
748
811
|
local storage location.
|
@@ -758,6 +821,11 @@ class RunnerDeployment(BaseModel):
|
|
758
821
|
or a timedelta object. If a number is given, it will be interpreted as seconds.
|
759
822
|
cron: A cron schedule of when to execute runs of this flow.
|
760
823
|
rrule: An rrule schedule of when to execute runs of this flow.
|
824
|
+
paused: Whether or not the deployment is paused.
|
825
|
+
schedule: A schedule object defining when to execute runs of this deployment.
|
826
|
+
Used to provide additional scheduling options like `timezone` or `parameters`.
|
827
|
+
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
828
|
+
Used to provide additional scheduling options like `timezone` or `parameters`.
|
761
829
|
triggers: A list of triggers that should kick of a run of this flow.
|
762
830
|
parameters: A dictionary of default parameter values to pass to runs of this flow.
|
763
831
|
description: A description for the created deployment. Defaults to the flow's
|
@@ -782,6 +850,7 @@ class RunnerDeployment(BaseModel):
|
|
782
850
|
cron=cron,
|
783
851
|
rrule=rrule,
|
784
852
|
schedules=schedules,
|
853
|
+
schedule=schedule,
|
785
854
|
)
|
786
855
|
|
787
856
|
if isinstance(concurrency_limit, ConcurrencyLimitConfig):
|
@@ -831,6 +900,126 @@ class RunnerDeployment(BaseModel):
|
|
831
900
|
|
832
901
|
return deployment
|
833
902
|
|
903
|
+
@classmethod
|
904
|
+
@async_dispatch(afrom_storage)
|
905
|
+
def from_storage(
|
906
|
+
cls,
|
907
|
+
storage: RunnerStorage,
|
908
|
+
entrypoint: str,
|
909
|
+
name: str,
|
910
|
+
flow_name: Optional[str] = None,
|
911
|
+
interval: Optional[
|
912
|
+
Union[Iterable[Union[int, float, timedelta]], int, float, timedelta]
|
913
|
+
] = None,
|
914
|
+
cron: Optional[Union[Iterable[str], str]] = None,
|
915
|
+
rrule: Optional[Union[Iterable[str], str]] = None,
|
916
|
+
paused: Optional[bool] = None,
|
917
|
+
schedule: Optional[Schedule] = None,
|
918
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
919
|
+
concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
|
920
|
+
parameters: Optional[dict[str, Any]] = None,
|
921
|
+
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
922
|
+
description: Optional[str] = None,
|
923
|
+
tags: Optional[List[str]] = None,
|
924
|
+
version: Optional[str] = None,
|
925
|
+
enforce_parameter_schema: bool = True,
|
926
|
+
work_pool_name: Optional[str] = None,
|
927
|
+
work_queue_name: Optional[str] = None,
|
928
|
+
job_variables: Optional[dict[str, Any]] = None,
|
929
|
+
_sla: Optional[Union[SlaTypes, list[SlaTypes]]] = None, # experimental
|
930
|
+
) -> "RunnerDeployment":
|
931
|
+
"""
|
932
|
+
Create a RunnerDeployment from a flow located at a given entrypoint and stored in a
|
933
|
+
local storage location.
|
934
|
+
|
935
|
+
Args:
|
936
|
+
entrypoint: The path to a file containing a flow and the name of the flow function in
|
937
|
+
the format `./path/to/file.py:flow_func_name`.
|
938
|
+
name: A name for the deployment
|
939
|
+
flow_name: The name of the flow to deploy
|
940
|
+
storage: A storage object to use for retrieving flow code. If not provided, a
|
941
|
+
URL must be provided.
|
942
|
+
interval: An interval on which to execute the current flow. Accepts either a number
|
943
|
+
or a timedelta object. If a number is given, it will be interpreted as seconds.
|
944
|
+
cron: A cron schedule of when to execute runs of this flow.
|
945
|
+
rrule: An rrule schedule of when to execute runs of this flow.
|
946
|
+
paused: Whether or not the deployment is paused.
|
947
|
+
schedule: A schedule object defining when to execute runs of this deployment.
|
948
|
+
Used to provide additional scheduling options like `timezone` or `parameters`.
|
949
|
+
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
950
|
+
Used to provide additional scheduling options like `timezone` or `parameters`.
|
951
|
+
triggers: A list of triggers that should kick of a run of this flow.
|
952
|
+
parameters: A dictionary of default parameter values to pass to runs of this flow.
|
953
|
+
description: A description for the created deployment. Defaults to the flow's
|
954
|
+
description if not provided.
|
955
|
+
tags: A list of tags to associate with the created deployment for organizational
|
956
|
+
purposes.
|
957
|
+
version: A version for the created deployment. Defaults to the flow's version.
|
958
|
+
enforce_parameter_schema: Whether or not the Prefect API should enforce the
|
959
|
+
parameter schema for this deployment.
|
960
|
+
work_pool_name: The name of the work pool to use for this deployment.
|
961
|
+
work_queue_name: The name of the work queue to use for this deployment's scheduled runs.
|
962
|
+
If not provided the default work queue for the work pool will be used.
|
963
|
+
job_variables: Settings used to override the values specified default base job template
|
964
|
+
of the chosen work pool. Refer to the base job template of the chosen work pool for
|
965
|
+
available settings.
|
966
|
+
_sla: (Experimental) SLA configuration for the deployment. May be removed or modified at any time. Currently only supported on Prefect Cloud.
|
967
|
+
"""
|
968
|
+
from prefect.flows import load_flow_from_entrypoint
|
969
|
+
|
970
|
+
constructed_schedules = cls._construct_deployment_schedules(
|
971
|
+
interval=interval,
|
972
|
+
cron=cron,
|
973
|
+
rrule=rrule,
|
974
|
+
schedules=schedules,
|
975
|
+
schedule=schedule,
|
976
|
+
)
|
977
|
+
|
978
|
+
if isinstance(concurrency_limit, ConcurrencyLimitConfig):
|
979
|
+
concurrency_options = {
|
980
|
+
"collision_strategy": concurrency_limit.collision_strategy
|
981
|
+
}
|
982
|
+
concurrency_limit = concurrency_limit.limit
|
983
|
+
else:
|
984
|
+
concurrency_options = None
|
985
|
+
|
986
|
+
job_variables = job_variables or {}
|
987
|
+
|
988
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
989
|
+
storage.set_base_path(Path(tmpdir))
|
990
|
+
run_coro_as_sync(storage.pull_code())
|
991
|
+
|
992
|
+
full_entrypoint = str(storage.destination / entrypoint)
|
993
|
+
flow = load_flow_from_entrypoint(full_entrypoint)
|
994
|
+
|
995
|
+
deployment = cls(
|
996
|
+
name=Path(name).stem,
|
997
|
+
flow_name=flow_name or flow.name,
|
998
|
+
schedules=constructed_schedules,
|
999
|
+
concurrency_limit=concurrency_limit,
|
1000
|
+
concurrency_options=concurrency_options,
|
1001
|
+
paused=paused,
|
1002
|
+
tags=tags or [],
|
1003
|
+
triggers=triggers or [],
|
1004
|
+
parameters=parameters or {},
|
1005
|
+
description=description,
|
1006
|
+
version=version,
|
1007
|
+
entrypoint=entrypoint,
|
1008
|
+
enforce_parameter_schema=enforce_parameter_schema,
|
1009
|
+
storage=storage,
|
1010
|
+
work_pool_name=work_pool_name,
|
1011
|
+
work_queue_name=work_queue_name,
|
1012
|
+
job_variables=job_variables,
|
1013
|
+
)
|
1014
|
+
deployment._sla = _sla
|
1015
|
+
deployment._path = str(storage.destination).replace(
|
1016
|
+
tmpdir, "$STORAGE_BASE_PATH"
|
1017
|
+
)
|
1018
|
+
|
1019
|
+
cls._set_defaults_from_flow(deployment, flow)
|
1020
|
+
|
1021
|
+
return deployment
|
1022
|
+
|
834
1023
|
|
835
1024
|
@sync_compatible
|
836
1025
|
async def deploy(
|
prefect/deployments/schedules.py
CHANGED
@@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Union
|
|
2
2
|
|
3
3
|
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
4
4
|
from prefect.client.schemas.schedules import is_schedule_type
|
5
|
+
from prefect.schedules import Schedule
|
5
6
|
|
6
7
|
if TYPE_CHECKING:
|
7
8
|
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
@@ -12,10 +13,13 @@ FlexibleScheduleList = Sequence[
|
|
12
13
|
|
13
14
|
|
14
15
|
def create_deployment_schedule_create(
|
15
|
-
schedule: "SCHEDULE_TYPES",
|
16
|
+
schedule: Union["SCHEDULE_TYPES", "Schedule"],
|
16
17
|
active: Optional[bool] = True,
|
17
18
|
) -> DeploymentScheduleCreate:
|
18
19
|
"""Create a DeploymentScheduleCreate object from common schedule parameters."""
|
20
|
+
|
21
|
+
if isinstance(schedule, Schedule):
|
22
|
+
return DeploymentScheduleCreate.from_schedule(schedule)
|
19
23
|
return DeploymentScheduleCreate(
|
20
24
|
schedule=schedule,
|
21
25
|
active=active if active is not None else True,
|
@@ -30,6 +34,8 @@ def normalize_to_deployment_schedule_create(
|
|
30
34
|
for obj in schedules:
|
31
35
|
if is_schedule_type(obj):
|
32
36
|
normalized.append(create_deployment_schedule_create(obj))
|
37
|
+
elif isinstance(obj, Schedule):
|
38
|
+
normalized.append(create_deployment_schedule_create(obj))
|
33
39
|
elif isinstance(obj, dict):
|
34
40
|
normalized.append(create_deployment_schedule_create(**obj))
|
35
41
|
elif isinstance(obj, DeploymentScheduleCreate):
|
prefect/engine.py
CHANGED
@@ -62,18 +62,13 @@ if __name__ == "__main__":
|
|
62
62
|
else:
|
63
63
|
run_flow(flow, flow_run=flow_run, error_logger=run_logger)
|
64
64
|
|
65
|
-
except Abort
|
66
|
-
abort_signal: Abort
|
65
|
+
except Abort:
|
67
66
|
engine_logger.info(
|
68
|
-
|
69
|
-
f" {abort_signal}"
|
67
|
+
"Engine execution of flow run '{flow_run_id}' aborted by orchestrator."
|
70
68
|
)
|
71
69
|
exit(0)
|
72
|
-
except Pause
|
73
|
-
|
74
|
-
engine_logger.info(
|
75
|
-
f"Engine execution of flow run '{flow_run_id}' is paused: {pause_signal}"
|
76
|
-
)
|
70
|
+
except Pause:
|
71
|
+
engine_logger.info(f"Engine execution of flow run '{flow_run_id}' is paused.")
|
77
72
|
exit(0)
|
78
73
|
except Exception:
|
79
74
|
engine_logger.error(
|
prefect/events/clients.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
import abc
|
2
2
|
import asyncio
|
3
3
|
import os
|
4
|
+
import ssl
|
4
5
|
from types import TracebackType
|
5
6
|
from typing import (
|
6
7
|
TYPE_CHECKING,
|
@@ -37,6 +38,7 @@ from prefect.events import Event
|
|
37
38
|
from prefect.logging import get_logger
|
38
39
|
from prefect.settings import (
|
39
40
|
PREFECT_API_KEY,
|
41
|
+
PREFECT_API_TLS_INSECURE_SKIP_VERIFY,
|
40
42
|
PREFECT_API_URL,
|
41
43
|
PREFECT_CLOUD_API_URL,
|
42
44
|
PREFECT_DEBUG_MODE,
|
@@ -120,6 +122,13 @@ class WebsocketProxyConnect(Connect):
|
|
120
122
|
self._host = host
|
121
123
|
self._port = port
|
122
124
|
|
125
|
+
if PREFECT_API_TLS_INSECURE_SKIP_VERIFY:
|
126
|
+
# Create an unverified context for insecure connections
|
127
|
+
ctx = ssl.create_default_context()
|
128
|
+
ctx.check_hostname = False
|
129
|
+
ctx.verify_mode = ssl.CERT_NONE
|
130
|
+
self._kwargs.setdefault("ssl", ctx)
|
131
|
+
|
123
132
|
async def _proxy_connect(self: Self) -> WebSocketClientProtocol:
|
124
133
|
if self._proxy:
|
125
134
|
sock = await self._proxy.connect(
|
@@ -348,16 +357,26 @@ class PrefectEventsClient(EventsClient):
|
|
348
357
|
await self._connect.__aexit__(exc_type, exc_val, exc_tb)
|
349
358
|
return await super().__aexit__(exc_type, exc_val, exc_tb)
|
350
359
|
|
360
|
+
def _log_debug(self, message: str, *args, **kwargs) -> None:
|
361
|
+
message = f"EventsClient(id={id(self)}): " + message
|
362
|
+
logger.debug(message, *args, **kwargs)
|
363
|
+
|
351
364
|
async def _reconnect(self) -> None:
|
365
|
+
logger.debug("Reconnecting websocket connection.")
|
366
|
+
|
352
367
|
if self._websocket:
|
353
368
|
self._websocket = None
|
354
369
|
await self._connect.__aexit__(None, None, None)
|
370
|
+
logger.debug("Cleared existing websocket connection.")
|
355
371
|
|
356
372
|
try:
|
373
|
+
logger.debug("Opening websocket connection.")
|
357
374
|
self._websocket = await self._connect.__aenter__()
|
358
375
|
# make sure we have actually connected
|
376
|
+
logger.debug("Pinging to ensure websocket connected.")
|
359
377
|
pong = await self._websocket.ping()
|
360
378
|
await pong
|
379
|
+
logger.debug("Pong received. Websocket connected.")
|
361
380
|
except Exception as e:
|
362
381
|
# The client is frequently run in a background thread
|
363
382
|
# so we log an additional warning to ensure
|
@@ -375,11 +394,13 @@ class PrefectEventsClient(EventsClient):
|
|
375
394
|
raise
|
376
395
|
|
377
396
|
events_to_resend = self._unconfirmed_events
|
397
|
+
logger.debug("Resending %s unconfirmed events.", len(events_to_resend))
|
378
398
|
# Clear the unconfirmed events here, because they are going back through emit
|
379
399
|
# and will be added again through the normal checkpointing process
|
380
400
|
self._unconfirmed_events = []
|
381
401
|
for event in events_to_resend:
|
382
402
|
await self.emit(event)
|
403
|
+
logger.debug("Finished resending unconfirmed events.")
|
383
404
|
|
384
405
|
async def _checkpoint(self, event: Event) -> None:
|
385
406
|
assert self._websocket
|
@@ -387,11 +408,20 @@ class PrefectEventsClient(EventsClient):
|
|
387
408
|
self._unconfirmed_events.append(event)
|
388
409
|
|
389
410
|
unconfirmed_count = len(self._unconfirmed_events)
|
411
|
+
|
412
|
+
logger.debug(
|
413
|
+
"Added event id=%s to unconfirmed events list. "
|
414
|
+
"There are now %s unconfirmed events.",
|
415
|
+
event.id,
|
416
|
+
unconfirmed_count,
|
417
|
+
)
|
390
418
|
if unconfirmed_count < self._checkpoint_every:
|
391
419
|
return
|
392
420
|
|
421
|
+
logger.debug("Pinging to checkpoint unconfirmed events.")
|
393
422
|
pong = await self._websocket.ping()
|
394
423
|
await pong
|
424
|
+
self._log_debug("Pong received. Events checkpointed.")
|
395
425
|
|
396
426
|
# once the pong returns, we know for sure that we've sent all the messages
|
397
427
|
# we had enqueued prior to that. There could be more that came in after, so
|
@@ -401,7 +431,9 @@ class PrefectEventsClient(EventsClient):
|
|
401
431
|
EVENT_WEBSOCKET_CHECKPOINTS.labels(self.client_name).inc()
|
402
432
|
|
403
433
|
async def _emit(self, event: Event) -> None:
|
434
|
+
self._log_debug("Emitting event id=%s.", event.id)
|
404
435
|
for i in range(self._reconnection_attempts + 1):
|
436
|
+
self._log_debug("Emit reconnection attempt %s.", i)
|
405
437
|
try:
|
406
438
|
# If we're here and the websocket is None, then we've had a failure in a
|
407
439
|
# previous reconnection attempt.
|
@@ -410,14 +442,18 @@ class PrefectEventsClient(EventsClient):
|
|
410
442
|
# from a ConnectionClosed, so reconnect now, resending any unconfirmed
|
411
443
|
# events before we send this one.
|
412
444
|
if not self._websocket or i > 0:
|
445
|
+
self._log_debug("Attempting websocket reconnection.")
|
413
446
|
await self._reconnect()
|
414
447
|
assert self._websocket
|
415
448
|
|
449
|
+
self._log_debug("Sending event id=%s.", event.id)
|
416
450
|
await self._websocket.send(event.model_dump_json())
|
451
|
+
self._log_debug("Checkpointing event id=%s.", event.id)
|
417
452
|
await self._checkpoint(event)
|
418
453
|
|
419
454
|
return
|
420
455
|
except ConnectionClosed:
|
456
|
+
self._log_debug("Got ConnectionClosed error.")
|
421
457
|
if i == self._reconnection_attempts:
|
422
458
|
# this was our final chance, raise the most recent error
|
423
459
|
raise
|
@@ -426,6 +462,9 @@ class PrefectEventsClient(EventsClient):
|
|
426
462
|
# let the first two attempts happen quickly in case this is just
|
427
463
|
# a standard load balancer timeout, but after that, just take a
|
428
464
|
# beat to let things come back around.
|
465
|
+
logger.debug(
|
466
|
+
"Sleeping for 1 second before next reconnection attempt."
|
467
|
+
)
|
429
468
|
await asyncio.sleep(1)
|
430
469
|
|
431
470
|
|
@@ -198,7 +198,9 @@ class EventTrigger(ResourceTrigger):
|
|
198
198
|
"10 seconds"
|
199
199
|
)
|
200
200
|
|
201
|
-
|
201
|
+
if within:
|
202
|
+
data = {**data, "within": within}
|
203
|
+
return data
|
202
204
|
|
203
205
|
def describe_for_cli(self, indent: int = 0) -> str:
|
204
206
|
"""Return a human-readable description of this trigger for the CLI"""
|
@@ -248,7 +250,7 @@ class MetricTriggerQuery(PrefectBaseModel):
|
|
248
250
|
threshold: float = Field(
|
249
251
|
...,
|
250
252
|
description=(
|
251
|
-
"The threshold value against which we'll compare
|
253
|
+
"The threshold value against which we'll compare the query result."
|
252
254
|
),
|
253
255
|
)
|
254
256
|
operator: MetricTriggerOperator = Field(
|