prefect-client 2.17.1__py3-none-any.whl → 2.18.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_internal/compatibility/deprecated.py +2 -0
- prefect/_internal/pydantic/_compat.py +1 -0
- prefect/_internal/pydantic/utilities/field_validator.py +25 -10
- prefect/_internal/pydantic/utilities/model_dump.py +1 -1
- prefect/_internal/pydantic/utilities/model_validate.py +1 -1
- prefect/_internal/pydantic/utilities/model_validator.py +11 -3
- prefect/_internal/schemas/fields.py +31 -12
- prefect/_internal/schemas/validators.py +0 -6
- prefect/_version.py +97 -38
- prefect/blocks/abstract.py +34 -1
- prefect/blocks/core.py +1 -1
- prefect/blocks/notifications.py +16 -7
- prefect/blocks/system.py +2 -3
- prefect/client/base.py +10 -5
- prefect/client/orchestration.py +405 -85
- prefect/client/schemas/actions.py +4 -3
- prefect/client/schemas/objects.py +6 -5
- prefect/client/schemas/schedules.py +2 -6
- prefect/client/schemas/sorting.py +9 -0
- prefect/client/utilities.py +25 -3
- prefect/concurrency/asyncio.py +11 -5
- prefect/concurrency/events.py +3 -3
- prefect/concurrency/services.py +1 -1
- prefect/concurrency/sync.py +9 -5
- prefect/deployments/__init__.py +0 -2
- prefect/deployments/base.py +2 -144
- prefect/deployments/deployments.py +29 -20
- prefect/deployments/runner.py +36 -28
- prefect/deployments/steps/core.py +3 -3
- prefect/deprecated/packaging/serializers.py +5 -4
- prefect/engine.py +3 -1
- prefect/events/__init__.py +45 -0
- prefect/events/actions.py +250 -18
- prefect/events/cli/automations.py +201 -0
- prefect/events/clients.py +179 -21
- prefect/events/filters.py +30 -3
- prefect/events/instrument.py +40 -40
- prefect/events/related.py +2 -1
- prefect/events/schemas/automations.py +126 -8
- prefect/events/schemas/deployment_triggers.py +23 -277
- prefect/events/schemas/events.py +7 -7
- prefect/events/utilities.py +3 -1
- prefect/events/worker.py +21 -8
- prefect/exceptions.py +1 -1
- prefect/flows.py +33 -18
- prefect/input/actions.py +9 -9
- prefect/input/run_input.py +49 -37
- prefect/logging/__init__.py +2 -2
- prefect/logging/loggers.py +64 -1
- prefect/new_flow_engine.py +293 -0
- prefect/new_task_engine.py +374 -0
- prefect/results.py +32 -12
- prefect/runner/runner.py +3 -2
- prefect/serializers.py +62 -31
- prefect/server/api/collections_data/views/aggregate-worker-metadata.json +44 -3
- prefect/settings.py +32 -10
- prefect/states.py +25 -19
- prefect/tasks.py +17 -0
- prefect/types/__init__.py +90 -0
- prefect/utilities/asyncutils.py +37 -0
- prefect/utilities/engine.py +6 -4
- prefect/utilities/pydantic.py +34 -15
- prefect/utilities/schema_tools/hydration.py +88 -19
- prefect/utilities/schema_tools/validation.py +1 -1
- prefect/variables.py +4 -4
- {prefect_client-2.17.1.dist-info → prefect_client-2.18.1.dist-info}/METADATA +1 -1
- {prefect_client-2.17.1.dist-info → prefect_client-2.18.1.dist-info}/RECORD +71 -67
- /prefect/{concurrency/common.py → events/cli/__init__.py} +0 -0
- {prefect_client-2.17.1.dist-info → prefect_client-2.18.1.dist-info}/LICENSE +0 -0
- {prefect_client-2.17.1.dist-info → prefect_client-2.18.1.dist-info}/WHEEL +0 -0
- {prefect_client-2.17.1.dist-info → prefect_client-2.18.1.dist-info}/top_level.txt +0 -0
@@ -19,11 +19,12 @@ from prefect._internal.compatibility.deprecated import (
|
|
19
19
|
DeprecatedInfraOverridesField,
|
20
20
|
)
|
21
21
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
22
|
+
from prefect.types import NonNegativeInteger, PositiveInteger
|
22
23
|
|
23
24
|
if HAS_PYDANTIC_V2:
|
24
|
-
from pydantic.v1 import Field, HttpUrl,
|
25
|
+
from pydantic.v1 import Field, HttpUrl, root_validator, validator
|
25
26
|
else:
|
26
|
-
from pydantic import Field, HttpUrl,
|
27
|
+
from pydantic import Field, HttpUrl, root_validator, validator
|
27
28
|
|
28
29
|
from typing_extensions import Literal
|
29
30
|
|
@@ -1188,10 +1189,10 @@ class WorkQueue(ObjectBaseModel):
|
|
1188
1189
|
is_paused: bool = Field(
|
1189
1190
|
default=False, description="Whether or not the work queue is paused."
|
1190
1191
|
)
|
1191
|
-
concurrency_limit: Optional[
|
1192
|
+
concurrency_limit: Optional[NonNegativeInteger] = Field(
|
1192
1193
|
default=None, description="An optional concurrency limit for the work queue."
|
1193
1194
|
)
|
1194
|
-
priority:
|
1195
|
+
priority: PositiveInteger = Field(
|
1195
1196
|
default=1,
|
1196
1197
|
description=(
|
1197
1198
|
"The queue's priority. Lower values are higher priority (1 is the highest)."
|
@@ -1351,7 +1352,7 @@ class WorkPool(ObjectBaseModel):
|
|
1351
1352
|
default=False,
|
1352
1353
|
description="Pausing the work pool stops the delivery of all work.",
|
1353
1354
|
)
|
1354
|
-
concurrency_limit: Optional[
|
1355
|
+
concurrency_limit: Optional[NonNegativeInteger] = Field(
|
1355
1356
|
default=None, description="A concurrency limit for the work pool."
|
1356
1357
|
)
|
1357
1358
|
status: Optional[WorkPoolStatus] = Field(
|
@@ -15,11 +15,11 @@ from prefect._internal.schemas.fields import DateTimeTZ
|
|
15
15
|
from prefect._internal.schemas.validators import (
|
16
16
|
default_anchor_date,
|
17
17
|
default_timezone,
|
18
|
-
interval_schedule_must_be_positive,
|
19
18
|
validate_cron_string,
|
20
19
|
validate_rrule_string,
|
21
20
|
validate_rrule_timezone,
|
22
21
|
)
|
22
|
+
from prefect.types import PositiveDuration
|
23
23
|
|
24
24
|
if HAS_PYDANTIC_V2:
|
25
25
|
from pydantic.v1 import Field, validator
|
@@ -64,14 +64,10 @@ class IntervalSchedule(PrefectBaseModel):
|
|
64
64
|
extra = "forbid"
|
65
65
|
exclude_none = True
|
66
66
|
|
67
|
-
interval:
|
67
|
+
interval: PositiveDuration
|
68
68
|
anchor_date: DateTimeTZ = None
|
69
69
|
timezone: Optional[str] = Field(default=None, examples=["America/New_York"])
|
70
70
|
|
71
|
-
@validator("interval")
|
72
|
-
def validate_interval_schedule(cls, v):
|
73
|
-
return interval_schedule_must_be_positive(v)
|
74
|
-
|
75
71
|
@validator("anchor_date", always=True)
|
76
72
|
def validate_anchor_date(cls, v):
|
77
73
|
return default_anchor_date(v)
|
@@ -27,6 +27,15 @@ class TaskRunSort(AutoEnum):
|
|
27
27
|
END_TIME_DESC = AutoEnum.auto()
|
28
28
|
|
29
29
|
|
30
|
+
class AutomationSort(AutoEnum):
|
31
|
+
"""Defines automation sorting options."""
|
32
|
+
|
33
|
+
CREATED_DESC = AutoEnum.auto()
|
34
|
+
UPDATED_DESC = AutoEnum.auto()
|
35
|
+
NAME_ASC = AutoEnum.auto()
|
36
|
+
NAME_DESC = AutoEnum.auto()
|
37
|
+
|
38
|
+
|
30
39
|
class LogSort(AutoEnum):
|
31
40
|
"""Defines log sorting options."""
|
32
41
|
|
prefect/client/utilities.py
CHANGED
@@ -6,14 +6,25 @@ Utilities for working with clients.
|
|
6
6
|
# circular imports for decorators such as `inject_client` which are widely used.
|
7
7
|
|
8
8
|
from functools import wraps
|
9
|
-
from typing import
|
10
|
-
|
11
|
-
|
9
|
+
from typing import (
|
10
|
+
TYPE_CHECKING,
|
11
|
+
Any,
|
12
|
+
Awaitable,
|
13
|
+
Callable,
|
14
|
+
Coroutine,
|
15
|
+
Optional,
|
16
|
+
Tuple,
|
17
|
+
TypeVar,
|
18
|
+
cast,
|
19
|
+
)
|
20
|
+
|
21
|
+
from typing_extensions import Concatenate, ParamSpec
|
12
22
|
|
13
23
|
if TYPE_CHECKING:
|
14
24
|
from prefect.client.orchestration import PrefectClient
|
15
25
|
|
16
26
|
P = ParamSpec("P")
|
27
|
+
R = TypeVar("R")
|
17
28
|
|
18
29
|
|
19
30
|
def get_or_create_client(
|
@@ -52,6 +63,17 @@ def get_or_create_client(
|
|
52
63
|
return get_httpx_client(), False
|
53
64
|
|
54
65
|
|
66
|
+
def client_injector(
|
67
|
+
func: Callable[Concatenate["PrefectClient", P], Awaitable[R]],
|
68
|
+
) -> Callable[P, Awaitable[R]]:
|
69
|
+
@wraps(func)
|
70
|
+
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
71
|
+
client, _ = get_or_create_client()
|
72
|
+
return await func(client, *args, **kwargs)
|
73
|
+
|
74
|
+
return wrapper
|
75
|
+
|
76
|
+
|
55
77
|
def inject_client(
|
56
78
|
fn: Callable[P, Coroutine[Any, Any, Any]],
|
57
79
|
) -> Callable[P, Coroutine[Any, Any, Any]]:
|
prefect/concurrency/asyncio.py
CHANGED
@@ -1,10 +1,16 @@
|
|
1
1
|
import asyncio
|
2
2
|
from contextlib import asynccontextmanager
|
3
|
-
from typing import List, Literal, Union
|
3
|
+
from typing import List, Literal, Union, cast
|
4
4
|
|
5
5
|
import httpx
|
6
6
|
import pendulum
|
7
7
|
|
8
|
+
try:
|
9
|
+
from pendulum import Interval
|
10
|
+
except ImportError:
|
11
|
+
# pendulum < 3
|
12
|
+
from pendulum.period import Period as Interval # type: ignore
|
13
|
+
|
8
14
|
from prefect import get_client
|
9
15
|
from prefect.client.schemas.responses import MinimalConcurrencyLimitResponse
|
10
16
|
|
@@ -30,10 +36,10 @@ async def concurrency(names: Union[str, List[str]], occupy: int = 1):
|
|
30
36
|
try:
|
31
37
|
yield
|
32
38
|
finally:
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
39
|
+
occupancy_period = cast(Interval, (pendulum.now("UTC") - acquisition_time))
|
40
|
+
await _release_concurrency_slots(
|
41
|
+
names, occupy, occupancy_period.total_seconds()
|
42
|
+
)
|
37
43
|
_emit_concurrency_release_events(limits, occupy, emitted_events)
|
38
44
|
|
39
45
|
|
prefect/concurrency/events.py
CHANGED
@@ -12,11 +12,11 @@ def _emit_concurrency_event(
|
|
12
12
|
slots: int,
|
13
13
|
follows: Union[Event, None] = None,
|
14
14
|
) -> Union[Event, None]:
|
15
|
-
resource = {
|
15
|
+
resource: Dict[str, str] = {
|
16
16
|
"prefect.resource.id": f"prefect.concurrency-limit.{primary_limit.id}",
|
17
17
|
"prefect.resource.name": primary_limit.name,
|
18
|
-
"slots-acquired": slots,
|
19
|
-
"limit": primary_limit.limit,
|
18
|
+
"slots-acquired": str(slots),
|
19
|
+
"limit": str(primary_limit.limit),
|
20
20
|
}
|
21
21
|
|
22
22
|
related = [
|
prefect/concurrency/services.py
CHANGED
@@ -64,7 +64,7 @@ class ConcurrencySlotAcquisitionService(QueueService):
|
|
64
64
|
raise RuntimeError("Cannot put items in a stopped service instance.")
|
65
65
|
|
66
66
|
logger.debug("Service %r enqueuing item %r", self, item)
|
67
|
-
future = concurrent.futures.Future()
|
67
|
+
future: concurrent.futures.Future = concurrent.futures.Future()
|
68
68
|
|
69
69
|
occupy, mode = item
|
70
70
|
self._queue.put_nowait((occupy, mode, future))
|
prefect/concurrency/sync.py
CHANGED
@@ -1,8 +1,14 @@
|
|
1
1
|
from contextlib import contextmanager
|
2
|
-
from typing import List, Union
|
2
|
+
from typing import List, Union, cast
|
3
3
|
|
4
4
|
import pendulum
|
5
5
|
|
6
|
+
try:
|
7
|
+
from pendulum import Interval
|
8
|
+
except ImportError:
|
9
|
+
# pendulum < 3
|
10
|
+
from pendulum.period import Period as Interval # type: ignore
|
11
|
+
|
6
12
|
from prefect._internal.concurrency.api import create_call, from_sync
|
7
13
|
from prefect._internal.concurrency.event_loop import get_running_loop
|
8
14
|
from prefect.client.schemas.responses import MinimalConcurrencyLimitResponse
|
@@ -30,11 +36,9 @@ def concurrency(names: Union[str, List[str]], occupy: int = 1):
|
|
30
36
|
try:
|
31
37
|
yield
|
32
38
|
finally:
|
33
|
-
|
34
|
-
pendulum.now("UTC") - acquisition_time
|
35
|
-
).total_seconds()
|
39
|
+
occupancy_period = cast(Interval, pendulum.now("UTC") - acquisition_time)
|
36
40
|
_call_async_function_from_sync(
|
37
|
-
_release_concurrency_slots, names, occupy,
|
41
|
+
_release_concurrency_slots, names, occupy, occupancy_period.total_seconds()
|
38
42
|
)
|
39
43
|
_emit_concurrency_release_events(limits, occupy, emitted_events)
|
40
44
|
|
prefect/deployments/__init__.py
CHANGED
prefect/deployments/base.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
"""
|
2
|
-
Core primitives for managing Prefect
|
2
|
+
Core primitives for managing Prefect deployments via `prefect deploy`, providing a minimally opinionated
|
3
3
|
build system for managing flows and deployments.
|
4
4
|
|
5
5
|
To get started, follow along with [the deloyments tutorial](/tutorials/deployments/).
|
@@ -7,7 +7,6 @@ To get started, follow along with [the deloyments tutorial](/tutorials/deploymen
|
|
7
7
|
|
8
8
|
import ast
|
9
9
|
import asyncio
|
10
|
-
import json
|
11
10
|
import math
|
12
11
|
import os
|
13
12
|
import subprocess
|
@@ -22,50 +21,13 @@ from ruamel.yaml import YAML
|
|
22
21
|
|
23
22
|
from prefect.client.schemas.objects import MinimalDeploymentSchedule
|
24
23
|
from prefect.client.schemas.schedules import IntervalSchedule
|
25
|
-
from prefect.flows import load_flow_from_entrypoint
|
26
24
|
from prefect.logging import get_logger
|
27
25
|
from prefect.settings import PREFECT_DEBUG_MODE
|
28
|
-
from prefect.utilities.asyncutils import LazySemaphore
|
26
|
+
from prefect.utilities.asyncutils import LazySemaphore
|
29
27
|
from prefect.utilities.filesystem import create_default_ignore_file, get_open_file_limit
|
30
28
|
from prefect.utilities.templating import apply_values
|
31
29
|
|
32
30
|
|
33
|
-
def find_prefect_directory(path: Path = None) -> Optional[Path]:
|
34
|
-
"""
|
35
|
-
Given a path, recurses upward looking for .prefect/ directories.
|
36
|
-
|
37
|
-
Once found, returns absolute path to the ./prefect directory, which is assumed to reside within the
|
38
|
-
root for the current project.
|
39
|
-
|
40
|
-
If one is never found, `None` is returned.
|
41
|
-
"""
|
42
|
-
path = Path(path or ".").resolve()
|
43
|
-
parent = path.parent.resolve()
|
44
|
-
while path != parent:
|
45
|
-
prefect_dir = path.joinpath(".prefect")
|
46
|
-
if prefect_dir.is_dir():
|
47
|
-
return prefect_dir
|
48
|
-
|
49
|
-
path = parent.resolve()
|
50
|
-
parent = path.parent.resolve()
|
51
|
-
|
52
|
-
|
53
|
-
def set_prefect_hidden_dir(path: str = None) -> bool:
|
54
|
-
"""
|
55
|
-
Creates default `.prefect/` directory if one does not already exist.
|
56
|
-
Returns boolean specifying whether or not a directory was created.
|
57
|
-
|
58
|
-
If a path is provided, the directory will be created in that location.
|
59
|
-
"""
|
60
|
-
path = Path(path or ".") / ".prefect"
|
61
|
-
|
62
|
-
# use exists so that we dont accidentally overwrite a file
|
63
|
-
if path.exists():
|
64
|
-
return False
|
65
|
-
path.mkdir(mode=0o0700)
|
66
|
-
return True
|
67
|
-
|
68
|
-
|
69
31
|
def create_default_prefect_yaml(
|
70
32
|
path: str, name: str = None, contents: Optional[Dict[str, Any]] = None
|
71
33
|
) -> bool:
|
@@ -270,114 +232,10 @@ def initialize_project(
|
|
270
232
|
files.append(".prefectignore")
|
271
233
|
if create_default_prefect_yaml(".", name=project_name, contents=configuration):
|
272
234
|
files.append("prefect.yaml")
|
273
|
-
if set_prefect_hidden_dir():
|
274
|
-
files.append(".prefect/")
|
275
235
|
|
276
236
|
return files
|
277
237
|
|
278
238
|
|
279
|
-
async def register_flow(entrypoint: str, force: bool = False):
|
280
|
-
"""
|
281
|
-
Register a flow with this project from an entrypoint.
|
282
|
-
|
283
|
-
Args:
|
284
|
-
entrypoint (str): the entrypoint to the flow to register
|
285
|
-
force (bool, optional): whether or not to overwrite an existing flow with the same name
|
286
|
-
|
287
|
-
Raises:
|
288
|
-
ValueError: if `force` is `False` and registration would overwrite an existing flow
|
289
|
-
"""
|
290
|
-
try:
|
291
|
-
fpath, obj_name = entrypoint.rsplit(":", 1)
|
292
|
-
except ValueError as exc:
|
293
|
-
if str(exc) == "not enough values to unpack (expected 2, got 1)":
|
294
|
-
missing_flow_name_msg = (
|
295
|
-
"Your flow entrypoint must include the name of the function that is"
|
296
|
-
f" the entrypoint to your flow.\nTry {entrypoint}:<flow_name> as your"
|
297
|
-
f" entrypoint. If you meant to specify '{entrypoint}' as the deployment"
|
298
|
-
f" name, try `prefect deploy -n {entrypoint}`."
|
299
|
-
)
|
300
|
-
raise ValueError(missing_flow_name_msg)
|
301
|
-
else:
|
302
|
-
raise exc
|
303
|
-
|
304
|
-
flow = await run_sync_in_worker_thread(load_flow_from_entrypoint, entrypoint)
|
305
|
-
|
306
|
-
fpath = Path(fpath).absolute()
|
307
|
-
prefect_dir = find_prefect_directory()
|
308
|
-
if not prefect_dir:
|
309
|
-
raise FileNotFoundError(
|
310
|
-
"No .prefect directory could be found - run `prefect project"
|
311
|
-
" init` to create one."
|
312
|
-
)
|
313
|
-
|
314
|
-
entrypoint = f"{fpath.relative_to(prefect_dir.parent)!s}:{obj_name}"
|
315
|
-
|
316
|
-
flows_file = prefect_dir / "flows.json"
|
317
|
-
if flows_file.exists():
|
318
|
-
with flows_file.open(mode="r") as f:
|
319
|
-
flows = json.load(f)
|
320
|
-
else:
|
321
|
-
flows = {}
|
322
|
-
|
323
|
-
## quality control
|
324
|
-
if flow.name in flows and flows[flow.name] != entrypoint:
|
325
|
-
if not force:
|
326
|
-
raise ValueError(
|
327
|
-
f"Conflicting entry found for flow with name {flow.name!r}.\nExisting"
|
328
|
-
f" entrypoint: {flows[flow.name]}\nAttempted entrypoint:"
|
329
|
-
f" {entrypoint}\n\nYou can try removing the existing entry for"
|
330
|
-
f" {flow.name!r} from your [yellow]~/.prefect/flows.json[/yellow]."
|
331
|
-
)
|
332
|
-
|
333
|
-
flows[flow.name] = entrypoint
|
334
|
-
|
335
|
-
with flows_file.open(mode="w") as f:
|
336
|
-
json.dump(flows, f, sort_keys=True, indent=2)
|
337
|
-
|
338
|
-
return flow
|
339
|
-
|
340
|
-
|
341
|
-
def _copy_deployments_into_prefect_file():
|
342
|
-
"""
|
343
|
-
Copy deployments from the `deloyment.yaml` file into the `prefect.yaml` file.
|
344
|
-
|
345
|
-
Used to migrate users from the old `prefect.yaml` + `deployment.yaml` structure
|
346
|
-
to a single `prefect.yaml` file.
|
347
|
-
"""
|
348
|
-
prefect_file = Path("prefect.yaml")
|
349
|
-
deployment_file = Path("deployment.yaml")
|
350
|
-
if not deployment_file.exists() or not prefect_file.exists():
|
351
|
-
raise FileNotFoundError(
|
352
|
-
"Could not find `prefect.yaml` or `deployment.yaml` files."
|
353
|
-
)
|
354
|
-
|
355
|
-
with deployment_file.open(mode="r") as f:
|
356
|
-
raw_deployment_file_contents = f.read()
|
357
|
-
parsed_deployment_file_contents = yaml.safe_load(raw_deployment_file_contents)
|
358
|
-
|
359
|
-
deployments = parsed_deployment_file_contents.get("deployments")
|
360
|
-
|
361
|
-
with prefect_file.open(mode="a") as f:
|
362
|
-
# If deployment.yaml is empty, write an empty deployments list to prefect.yaml.
|
363
|
-
if not parsed_deployment_file_contents:
|
364
|
-
f.write("\n")
|
365
|
-
f.write(yaml.dump({"deployments": []}, sort_keys=False))
|
366
|
-
# If there is no 'deployments' key in deployment.yaml, assume that the
|
367
|
-
# entire file is a single deployment.
|
368
|
-
elif not deployments:
|
369
|
-
f.write("\n")
|
370
|
-
f.write(
|
371
|
-
yaml.dump(
|
372
|
-
{"deployments": [parsed_deployment_file_contents]}, sort_keys=False
|
373
|
-
)
|
374
|
-
)
|
375
|
-
# Write all of deployment.yaml to prefect.yaml.
|
376
|
-
else:
|
377
|
-
f.write("\n")
|
378
|
-
f.write(raw_deployment_file_contents)
|
379
|
-
|
380
|
-
|
381
239
|
def _format_deployment_for_saving_to_prefect_file(
|
382
240
|
deployment: Dict,
|
383
241
|
) -> Dict:
|
@@ -16,6 +16,13 @@ import anyio
|
|
16
16
|
import pendulum
|
17
17
|
import yaml
|
18
18
|
|
19
|
+
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
20
|
+
|
21
|
+
if HAS_PYDANTIC_V2:
|
22
|
+
from pydantic.v1 import BaseModel, Field, parse_obj_as, root_validator, validator
|
23
|
+
else:
|
24
|
+
from pydantic import BaseModel, Field, parse_obj_as, root_validator, validator
|
25
|
+
|
19
26
|
from prefect._internal.compatibility.deprecated import (
|
20
27
|
DeprecatedInfraOverridesField,
|
21
28
|
deprecated_callable,
|
@@ -23,7 +30,6 @@ from prefect._internal.compatibility.deprecated import (
|
|
23
30
|
deprecated_parameter,
|
24
31
|
handle_deprecated_infra_overrides_parameter,
|
25
32
|
)
|
26
|
-
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
27
33
|
from prefect._internal.schemas.validators import (
|
28
34
|
handle_openapi_schema,
|
29
35
|
infrastructure_must_have_capabilities,
|
@@ -32,16 +38,10 @@ from prefect._internal.schemas.validators import (
|
|
32
38
|
validate_automation_names,
|
33
39
|
validate_deprecated_schedule_fields,
|
34
40
|
)
|
35
|
-
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
36
|
-
|
37
|
-
if HAS_PYDANTIC_V2:
|
38
|
-
from pydantic.v1 import BaseModel, Field, parse_obj_as, root_validator, validator
|
39
|
-
else:
|
40
|
-
from pydantic import BaseModel, Field, parse_obj_as, root_validator, validator
|
41
|
-
|
42
41
|
from prefect.blocks.core import Block
|
43
42
|
from prefect.blocks.fields import SecretDict
|
44
|
-
from prefect.client.orchestration import PrefectClient,
|
43
|
+
from prefect.client.orchestration import PrefectClient, get_client
|
44
|
+
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
45
45
|
from prefect.client.schemas.objects import (
|
46
46
|
FlowRun,
|
47
47
|
MinimalDeploymentSchedule,
|
@@ -53,11 +53,12 @@ from prefect.deployments.schedules import (
|
|
53
53
|
FlexibleScheduleList,
|
54
54
|
)
|
55
55
|
from prefect.deployments.steps.core import run_steps
|
56
|
-
from prefect.events import DeploymentTriggerTypes
|
56
|
+
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
57
57
|
from prefect.exceptions import (
|
58
58
|
BlockMissingCapabilities,
|
59
59
|
ObjectAlreadyExists,
|
60
60
|
ObjectNotFound,
|
61
|
+
PrefectHTTPStatusError,
|
61
62
|
)
|
62
63
|
from prefect.filesystems import LocalFileSystem
|
63
64
|
from prefect.flows import Flow, load_flow_from_entrypoint
|
@@ -609,7 +610,7 @@ class Deployment(DeprecatedInfraOverridesField, BaseModel):
|
|
609
610
|
description="The parameter schema of the flow, including defaults.",
|
610
611
|
)
|
611
612
|
timestamp: datetime = Field(default_factory=partial(pendulum.now, "UTC"))
|
612
|
-
triggers: List[DeploymentTriggerTypes] = Field(
|
613
|
+
triggers: List[Union[DeploymentTriggerTypes, TriggerTypes]] = Field(
|
613
614
|
default_factory=list,
|
614
615
|
description="The triggers that should cause this deployment to run.",
|
615
616
|
)
|
@@ -901,15 +902,23 @@ class Deployment(DeprecatedInfraOverridesField, BaseModel):
|
|
901
902
|
enforce_parameter_schema=self.enforce_parameter_schema,
|
902
903
|
)
|
903
904
|
|
904
|
-
if client.server_type
|
905
|
-
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
|
905
|
+
if client.server_type.supports_automations():
|
906
|
+
try:
|
907
|
+
# The triggers defined in the deployment spec are, essentially,
|
908
|
+
# anonymous and attempting truly sync them with cloud is not
|
909
|
+
# feasible. Instead, we remove all automations that are owned
|
910
|
+
# by the deployment, meaning that they were created via this
|
911
|
+
# mechanism below, and then recreate them.
|
912
|
+
await client.delete_resource_owned_automations(
|
913
|
+
f"prefect.deployment.{deployment_id}"
|
914
|
+
)
|
915
|
+
except PrefectHTTPStatusError as e:
|
916
|
+
if e.response.status_code == 404:
|
917
|
+
# This Prefect server does not support automations, so we can safely
|
918
|
+
# ignore this 404 and move on.
|
919
|
+
return deployment_id
|
920
|
+
raise e
|
921
|
+
|
913
922
|
for trigger in self.triggers:
|
914
923
|
trigger.set_deployment_id(deployment_id)
|
915
924
|
await client.create_automation(trigger.as_automation())
|
prefect/deployments/runner.py
CHANGED
@@ -42,27 +42,20 @@ from rich.console import Console
|
|
42
42
|
from rich.progress import Progress, SpinnerColumn, TextColumn, track
|
43
43
|
from rich.table import Table
|
44
44
|
|
45
|
-
from prefect._internal.concurrency.api import create_call, from_async
|
46
45
|
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
47
|
-
from prefect._internal.schemas.validators import (
|
48
|
-
reconcile_paused_deployment,
|
49
|
-
reconcile_schedules_runner,
|
50
|
-
validate_automation_names,
|
51
|
-
)
|
52
|
-
from prefect.runner.storage import RunnerStorage
|
53
|
-
from prefect.settings import (
|
54
|
-
PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE,
|
55
|
-
PREFECT_DEFAULT_WORK_POOL_NAME,
|
56
|
-
PREFECT_UI_URL,
|
57
|
-
)
|
58
|
-
from prefect.utilities.collections import get_from_dict, isiterable
|
59
46
|
|
60
47
|
if HAS_PYDANTIC_V2:
|
61
48
|
from pydantic.v1 import BaseModel, Field, PrivateAttr, root_validator, validator
|
62
49
|
else:
|
63
50
|
from pydantic import BaseModel, Field, PrivateAttr, root_validator, validator
|
64
51
|
|
65
|
-
from prefect.
|
52
|
+
from prefect._internal.concurrency.api import create_call, from_async
|
53
|
+
from prefect._internal.schemas.validators import (
|
54
|
+
reconcile_paused_deployment,
|
55
|
+
reconcile_schedules_runner,
|
56
|
+
validate_automation_names,
|
57
|
+
)
|
58
|
+
from prefect.client.orchestration import get_client
|
66
59
|
from prefect.client.schemas.objects import MinimalDeploymentSchedule
|
67
60
|
from prefect.client.schemas.schedules import (
|
68
61
|
SCHEDULE_TYPES,
|
@@ -72,13 +65,20 @@ from prefect.deployments.schedules import (
|
|
72
65
|
FlexibleScheduleList,
|
73
66
|
create_minimal_deployment_schedule,
|
74
67
|
)
|
75
|
-
from prefect.events import DeploymentTriggerTypes
|
68
|
+
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
76
69
|
from prefect.exceptions import (
|
77
70
|
ObjectNotFound,
|
78
71
|
PrefectHTTPStatusError,
|
79
72
|
)
|
73
|
+
from prefect.runner.storage import RunnerStorage
|
74
|
+
from prefect.settings import (
|
75
|
+
PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE,
|
76
|
+
PREFECT_DEFAULT_WORK_POOL_NAME,
|
77
|
+
PREFECT_UI_URL,
|
78
|
+
)
|
80
79
|
from prefect.utilities.asyncutils import sync_compatible
|
81
80
|
from prefect.utilities.callables import ParameterSchema, parameter_schema
|
81
|
+
from prefect.utilities.collections import get_from_dict, isiterable
|
82
82
|
from prefect.utilities.dockerutils import (
|
83
83
|
PushError,
|
84
84
|
build_image,
|
@@ -179,7 +179,7 @@ class RunnerDeployment(BaseModel):
|
|
179
179
|
"The path to the entrypoint for the workflow, relative to the `path`."
|
180
180
|
),
|
181
181
|
)
|
182
|
-
triggers: List[DeploymentTriggerTypes] = Field(
|
182
|
+
triggers: List[Union[DeploymentTriggerTypes, TriggerTypes]] = Field(
|
183
183
|
default_factory=list,
|
184
184
|
description="The triggers that should cause this deployment to run.",
|
185
185
|
)
|
@@ -325,15 +325,23 @@ class RunnerDeployment(BaseModel):
|
|
325
325
|
f"Error while applying deployment: {str(exc)}"
|
326
326
|
) from exc
|
327
327
|
|
328
|
-
if client.server_type
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
328
|
+
if client.server_type.supports_automations():
|
329
|
+
try:
|
330
|
+
# The triggers defined in the deployment spec are, essentially,
|
331
|
+
# anonymous and attempting truly sync them with cloud is not
|
332
|
+
# feasible. Instead, we remove all automations that are owned
|
333
|
+
# by the deployment, meaning that they were created via this
|
334
|
+
# mechanism below, and then recreate them.
|
335
|
+
await client.delete_resource_owned_automations(
|
336
|
+
f"prefect.deployment.{deployment_id}"
|
337
|
+
)
|
338
|
+
except PrefectHTTPStatusError as e:
|
339
|
+
if e.response.status_code == 404:
|
340
|
+
# This Prefect server does not support automations, so we can safely
|
341
|
+
# ignore this 404 and move on.
|
342
|
+
return deployment_id
|
343
|
+
raise e
|
344
|
+
|
337
345
|
for trigger in self.triggers:
|
338
346
|
trigger.set_deployment_id(deployment_id)
|
339
347
|
await client.create_automation(trigger.as_automation())
|
@@ -446,7 +454,7 @@ class RunnerDeployment(BaseModel):
|
|
446
454
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
447
455
|
is_schedule_active: Optional[bool] = None,
|
448
456
|
parameters: Optional[dict] = None,
|
449
|
-
triggers: Optional[List[DeploymentTriggerTypes]] = None,
|
457
|
+
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
450
458
|
description: Optional[str] = None,
|
451
459
|
tags: Optional[List[str]] = None,
|
452
460
|
version: Optional[str] = None,
|
@@ -582,7 +590,7 @@ class RunnerDeployment(BaseModel):
|
|
582
590
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
583
591
|
is_schedule_active: Optional[bool] = None,
|
584
592
|
parameters: Optional[dict] = None,
|
585
|
-
triggers: Optional[List[DeploymentTriggerTypes]] = None,
|
593
|
+
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
586
594
|
description: Optional[str] = None,
|
587
595
|
tags: Optional[List[str]] = None,
|
588
596
|
version: Optional[str] = None,
|
@@ -680,7 +688,7 @@ class RunnerDeployment(BaseModel):
|
|
680
688
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
681
689
|
is_schedule_active: Optional[bool] = None,
|
682
690
|
parameters: Optional[dict] = None,
|
683
|
-
triggers: Optional[List[DeploymentTriggerTypes]] = None,
|
691
|
+
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
684
692
|
description: Optional[str] = None,
|
685
693
|
tags: Optional[List[str]] = None,
|
686
694
|
version: Optional[str] = None,
|
@@ -1,11 +1,11 @@
|
|
1
1
|
"""
|
2
|
-
Core primitives for running Prefect
|
2
|
+
Core primitives for running Prefect deployment steps.
|
3
3
|
|
4
|
-
|
4
|
+
Deployment steps are YAML representations of Python functions along with their inputs.
|
5
5
|
|
6
6
|
Whenever a step is run, the following actions are taken:
|
7
7
|
|
8
|
-
- The step's inputs and block / variable references are resolved (see [the
|
8
|
+
- The step's inputs and block / variable references are resolved (see [the `prefect deploy` documentation](/guides/prefect-deploy/#templating-options) for more details)
|
9
9
|
- The step's function is imported; if it cannot be found, the `requires` keyword is used to install the necessary packages
|
10
10
|
- The step's function is called with the resolved inputs
|
11
11
|
- The step's output is returned and used to resolve inputs for subsequent steps
|
@@ -55,20 +55,21 @@ class PickleSerializer(Serializer):
|
|
55
55
|
|
56
56
|
picklelib: str = "cloudpickle"
|
57
57
|
picklelib_version: str = None
|
58
|
+
|
58
59
|
pickle_modules: List[str] = pydantic.Field(default_factory=list)
|
59
60
|
|
60
61
|
@pydantic.validator("picklelib")
|
61
62
|
def check_picklelib(cls, value):
|
62
63
|
return validate_picklelib(value)
|
63
64
|
|
64
|
-
@pydantic.root_validator
|
65
|
-
def check_picklelib_version(cls, values):
|
66
|
-
return validate_picklelib_version(values)
|
67
|
-
|
68
65
|
@pydantic.root_validator
|
69
66
|
def check_picklelib_and_modules(cls, values):
|
70
67
|
return validate_picklelib_and_modules(values)
|
71
68
|
|
69
|
+
@pydantic.root_validator
|
70
|
+
def check_picklelib_version(cls, values):
|
71
|
+
return validate_picklelib_version(values)
|
72
|
+
|
72
73
|
def dumps(self, obj: Any) -> bytes:
|
73
74
|
pickler = from_qualified_name(self.picklelib)
|
74
75
|
|