prefect-client 3.1.12__py3-none-any.whl → 3.1.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_experimental/sla/client.py +53 -27
- prefect/_experimental/sla/objects.py +10 -2
- prefect/_internal/concurrency/services.py +2 -2
- prefect/_internal/concurrency/threads.py +6 -0
- prefect/_internal/retries.py +6 -3
- prefect/_internal/schemas/validators.py +6 -4
- prefect/_version.py +3 -3
- prefect/artifacts.py +4 -1
- prefect/automations.py +1 -1
- prefect/blocks/abstract.py +5 -2
- prefect/blocks/notifications.py +1 -0
- prefect/cache_policies.py +20 -20
- prefect/client/utilities.py +3 -3
- prefect/deployments/base.py +7 -4
- prefect/deployments/flow_runs.py +5 -1
- prefect/deployments/runner.py +6 -11
- prefect/deployments/steps/core.py +1 -1
- prefect/deployments/steps/pull.py +8 -3
- prefect/deployments/steps/utility.py +2 -2
- prefect/docker/docker_image.py +13 -9
- prefect/engine.py +19 -10
- prefect/events/cli/automations.py +4 -4
- prefect/events/clients.py +17 -14
- prefect/events/schemas/automations.py +12 -8
- prefect/events/schemas/events.py +5 -1
- prefect/events/worker.py +1 -1
- prefect/filesystems.py +1 -1
- prefect/flow_engine.py +17 -9
- prefect/flows.py +118 -73
- prefect/futures.py +14 -7
- prefect/infrastructure/provisioners/__init__.py +2 -0
- prefect/infrastructure/provisioners/cloud_run.py +4 -4
- prefect/infrastructure/provisioners/coiled.py +249 -0
- prefect/infrastructure/provisioners/container_instance.py +4 -3
- prefect/infrastructure/provisioners/ecs.py +55 -43
- prefect/infrastructure/provisioners/modal.py +5 -4
- prefect/input/actions.py +5 -1
- prefect/input/run_input.py +157 -43
- prefect/logging/configuration.py +3 -3
- prefect/logging/filters.py +2 -2
- prefect/logging/formatters.py +15 -11
- prefect/logging/handlers.py +24 -14
- prefect/logging/highlighters.py +5 -5
- prefect/logging/loggers.py +28 -18
- prefect/main.py +3 -1
- prefect/results.py +166 -86
- prefect/runner/runner.py +34 -27
- prefect/runner/server.py +3 -1
- prefect/runner/storage.py +18 -18
- prefect/runner/submit.py +19 -12
- prefect/runtime/deployment.py +15 -8
- prefect/runtime/flow_run.py +19 -6
- prefect/runtime/task_run.py +7 -3
- prefect/settings/base.py +17 -7
- prefect/settings/legacy.py +4 -4
- prefect/settings/models/api.py +4 -3
- prefect/settings/models/cli.py +4 -3
- prefect/settings/models/client.py +7 -4
- prefect/settings/models/cloud.py +4 -3
- prefect/settings/models/deployments.py +4 -3
- prefect/settings/models/experiments.py +4 -3
- prefect/settings/models/flows.py +4 -3
- prefect/settings/models/internal.py +4 -3
- prefect/settings/models/logging.py +8 -6
- prefect/settings/models/results.py +4 -3
- prefect/settings/models/root.py +11 -16
- prefect/settings/models/runner.py +8 -5
- prefect/settings/models/server/api.py +6 -3
- prefect/settings/models/server/database.py +120 -25
- prefect/settings/models/server/deployments.py +4 -3
- prefect/settings/models/server/ephemeral.py +7 -4
- prefect/settings/models/server/events.py +6 -3
- prefect/settings/models/server/flow_run_graph.py +4 -3
- prefect/settings/models/server/root.py +4 -3
- prefect/settings/models/server/services.py +15 -12
- prefect/settings/models/server/tasks.py +7 -4
- prefect/settings/models/server/ui.py +4 -3
- prefect/settings/models/tasks.py +10 -5
- prefect/settings/models/testing.py +4 -3
- prefect/settings/models/worker.py +7 -4
- prefect/settings/profiles.py +13 -12
- prefect/settings/sources.py +20 -19
- prefect/states.py +17 -13
- prefect/task_engine.py +43 -33
- prefect/task_runners.py +35 -23
- prefect/task_runs.py +20 -11
- prefect/task_worker.py +12 -7
- prefect/tasks.py +30 -24
- prefect/telemetry/bootstrap.py +4 -1
- prefect/telemetry/run_telemetry.py +15 -13
- prefect/transactions.py +3 -3
- prefect/types/__init__.py +3 -1
- prefect/utilities/_deprecated.py +38 -0
- prefect/utilities/engine.py +11 -4
- prefect/utilities/filesystem.py +2 -2
- prefect/utilities/generics.py +1 -1
- prefect/utilities/pydantic.py +21 -36
- prefect/workers/base.py +52 -30
- prefect/workers/process.py +20 -15
- prefect/workers/server.py +4 -5
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.13.dist-info}/METADATA +2 -2
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.13.dist-info}/RECORD +105 -103
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.13.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.13.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.13.dist-info}/top_level.txt +0 -0
@@ -7,60 +7,86 @@ from prefect.client.orchestration.base import BaseAsyncClient, BaseClient
|
|
7
7
|
if TYPE_CHECKING:
|
8
8
|
from uuid import UUID
|
9
9
|
|
10
|
-
from prefect._experimental.sla.objects import SlaTypes
|
10
|
+
from prefect._experimental.sla.objects import SlaMergeResponse, SlaTypes
|
11
11
|
|
12
12
|
|
13
13
|
class SlaClient(BaseClient):
|
14
|
-
def
|
14
|
+
def apply_slas_for_deployment(
|
15
|
+
self, deployment_id: "UUID", slas: "list[SlaTypes]"
|
16
|
+
) -> "SlaMergeResponse":
|
15
17
|
"""
|
16
|
-
|
18
|
+
Applies service level agreements for a deployment. Performs matching by SLA name. If a SLA with the same name already exists, it will be updated. If a SLA with the same name does not exist, it will be created. Existing SLAs that are not in the list will be deleted.
|
17
19
|
Args:
|
18
|
-
|
20
|
+
deployment_id: The ID of the deployment to update SLAs for
|
21
|
+
slas: List of SLAs to associate with the deployment
|
19
22
|
Raises:
|
20
|
-
httpx.RequestError: if the
|
23
|
+
httpx.RequestError: if the SLAs were not updated for any reason
|
21
24
|
Returns:
|
22
|
-
the
|
25
|
+
SlaMergeResponse: The response from the backend, containing the names of the created, updated, and deleted SLAs
|
23
26
|
"""
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
)
|
27
|
+
resource_id = f"prefect.deployment.{deployment_id}"
|
28
|
+
|
29
|
+
for sla in slas:
|
30
|
+
sla.set_deployment_id(deployment_id)
|
31
|
+
|
32
|
+
slas_spec_list = [
|
33
|
+
sla.model_dump(mode="json", exclude_unset=True) for sla in slas
|
34
|
+
]
|
28
35
|
|
29
36
|
response = self.request(
|
30
37
|
"POST",
|
31
|
-
"/slas/",
|
32
|
-
json=
|
38
|
+
f"/slas/apply-resource-slas/{resource_id}",
|
39
|
+
json=slas_spec_list,
|
33
40
|
)
|
34
41
|
response.raise_for_status()
|
35
42
|
|
36
|
-
|
43
|
+
response_json = response.json()
|
37
44
|
|
38
|
-
|
45
|
+
from prefect._experimental.sla.objects import SlaMergeResponse
|
46
|
+
|
47
|
+
return SlaMergeResponse(
|
48
|
+
created=[sla.get("name") for sla in response_json.get("created")],
|
49
|
+
updated=[sla.get("name") for sla in response_json.get("updated")],
|
50
|
+
deleted=[sla.get("name") for sla in response_json.get("deleted")],
|
51
|
+
)
|
39
52
|
|
40
53
|
|
41
54
|
class SlaAsyncClient(BaseAsyncClient):
|
42
|
-
async def
|
55
|
+
async def apply_slas_for_deployment(
|
56
|
+
self, deployment_id: "UUID", slas: "list[SlaTypes]"
|
57
|
+
) -> "UUID":
|
43
58
|
"""
|
44
|
-
|
59
|
+
Applies service level agreements for a deployment. Performs matching by SLA name. If a SLA with the same name already exists, it will be updated. If a SLA with the same name does not exist, it will be created. Existing SLAs that are not in the list will be deleted.
|
45
60
|
Args:
|
46
|
-
|
61
|
+
deployment_id: The ID of the deployment to update SLAs for
|
62
|
+
slas: List of SLAs to associate with the deployment
|
47
63
|
Raises:
|
48
|
-
httpx.RequestError: if the
|
64
|
+
httpx.RequestError: if the SLAs were not updated for any reason
|
49
65
|
Returns:
|
50
|
-
the
|
66
|
+
SlaMergeResponse: The response from the backend, containing the names of the created, updated, and deleted SLAs
|
51
67
|
"""
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
)
|
68
|
+
resource_id = f"prefect.deployment.{deployment_id}"
|
69
|
+
|
70
|
+
for sla in slas:
|
71
|
+
sla.set_deployment_id(deployment_id)
|
72
|
+
|
73
|
+
slas_spec_list = [
|
74
|
+
sla.model_dump(mode="json", exclude_unset=True) for sla in slas
|
75
|
+
]
|
56
76
|
|
57
77
|
response = await self.request(
|
58
78
|
"POST",
|
59
|
-
"/slas/",
|
60
|
-
json=
|
79
|
+
f"/slas/apply-resource-slas/{resource_id}",
|
80
|
+
json=slas_spec_list,
|
61
81
|
)
|
62
82
|
response.raise_for_status()
|
63
83
|
|
64
|
-
|
84
|
+
response_json = response.json()
|
65
85
|
|
66
|
-
|
86
|
+
from prefect._experimental.sla.objects import SlaMergeResponse
|
87
|
+
|
88
|
+
return SlaMergeResponse(
|
89
|
+
created=[sla.get("name") for sla in response_json.get("created")],
|
90
|
+
updated=[sla.get("name") for sla in response_json.get("updated")],
|
91
|
+
deleted=[sla.get("name") for sla in response_json.get("deleted")],
|
92
|
+
)
|
@@ -5,7 +5,7 @@ from typing import Literal, Optional, Union
|
|
5
5
|
from uuid import UUID
|
6
6
|
|
7
7
|
from pydantic import Field, PrivateAttr, computed_field
|
8
|
-
from typing_extensions import TypeAlias
|
8
|
+
from typing_extensions import Self, TypeAlias
|
9
9
|
|
10
10
|
from prefect._internal.schemas.bases import PrefectBaseModel
|
11
11
|
|
@@ -28,7 +28,7 @@ class ServiceLevelAgreement(PrefectBaseModel, abc.ABC):
|
|
28
28
|
description="Whether the SLA is enabled.",
|
29
29
|
)
|
30
30
|
|
31
|
-
def set_deployment_id(self, deployment_id: UUID):
|
31
|
+
def set_deployment_id(self, deployment_id: UUID) -> Self:
|
32
32
|
self._deployment_id = deployment_id
|
33
33
|
return self
|
34
34
|
|
@@ -49,5 +49,13 @@ class TimeToCompletionSla(ServiceLevelAgreement):
|
|
49
49
|
)
|
50
50
|
|
51
51
|
|
52
|
+
class SlaMergeResponse(PrefectBaseModel):
|
53
|
+
"""A response object for the apply_slas_for_deployment method. Contains the names of the created, updated, and deleted SLAs."""
|
54
|
+
|
55
|
+
created: list[str]
|
56
|
+
updated: list[str]
|
57
|
+
deleted: list[str]
|
58
|
+
|
59
|
+
|
52
60
|
# Concrete SLA types
|
53
61
|
SlaTypes: TypeAlias = Union[TimeToCompletionSla]
|
@@ -32,7 +32,7 @@ class _QueueServiceBase(abc.ABC, Generic[T]):
|
|
32
32
|
self._task: Optional[asyncio.Task[None]] = None
|
33
33
|
self._stopped: bool = False
|
34
34
|
self._started: bool = False
|
35
|
-
self._key = hash(args)
|
35
|
+
self._key = hash((self.__class__, *args))
|
36
36
|
self._lock = threading.Lock()
|
37
37
|
self._queue_get_thread = WorkerThread(
|
38
38
|
# TODO: This thread should not need to be a daemon but when it is not, it
|
@@ -256,7 +256,7 @@ class _QueueServiceBase(abc.ABC, Generic[T]):
|
|
256
256
|
If an instance already exists with the given arguments, it will be returned.
|
257
257
|
"""
|
258
258
|
with cls._instance_lock:
|
259
|
-
key = hash(args)
|
259
|
+
key = hash((cls, *args))
|
260
260
|
if key not in cls._instances:
|
261
261
|
cls._instances[key] = cls._new_instance(*args)
|
262
262
|
|
@@ -2,6 +2,8 @@
|
|
2
2
|
Utilities for managing worker threads.
|
3
3
|
"""
|
4
4
|
|
5
|
+
from __future__ import annotations
|
6
|
+
|
5
7
|
import asyncio
|
6
8
|
import atexit
|
7
9
|
import concurrent.futures
|
@@ -197,6 +199,10 @@ class EventLoopThread(Portal):
|
|
197
199
|
def running(self) -> bool:
|
198
200
|
return not self._shutdown_event.is_set()
|
199
201
|
|
202
|
+
@property
|
203
|
+
def loop(self) -> asyncio.AbstractEventLoop | None:
|
204
|
+
return self._loop
|
205
|
+
|
200
206
|
def _entrypoint(self):
|
201
207
|
"""
|
202
208
|
Entrypoint for the thread.
|
prefect/_internal/retries.py
CHANGED
@@ -29,7 +29,7 @@ def retry_async_fn(
|
|
29
29
|
retry_on_exceptions: tuple[type[Exception], ...] = (Exception,),
|
30
30
|
operation_name: Optional[str] = None,
|
31
31
|
) -> Callable[
|
32
|
-
[Callable[P, Coroutine[Any, Any, R]]], Callable[P, Coroutine[Any, Any,
|
32
|
+
[Callable[P, Coroutine[Any, Any, R]]], Callable[P, Coroutine[Any, Any, R]]
|
33
33
|
]:
|
34
34
|
"""A decorator for retrying an async function.
|
35
35
|
|
@@ -48,9 +48,9 @@ def retry_async_fn(
|
|
48
48
|
|
49
49
|
def decorator(
|
50
50
|
func: Callable[P, Coroutine[Any, Any, R]],
|
51
|
-
) -> Callable[P, Coroutine[Any, Any,
|
51
|
+
) -> Callable[P, Coroutine[Any, Any, R]]:
|
52
52
|
@wraps(func)
|
53
|
-
async def wrapper(*args: P.args, **kwargs: P.kwargs) ->
|
53
|
+
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
54
54
|
name = operation_name or func.__name__
|
55
55
|
for attempt in range(max_attempts):
|
56
56
|
try:
|
@@ -67,6 +67,9 @@ def retry_async_fn(
|
|
67
67
|
f"Retrying in {delay:.2f} seconds..."
|
68
68
|
)
|
69
69
|
await asyncio.sleep(delay)
|
70
|
+
# Technically unreachable, but this raise helps pyright know that this function
|
71
|
+
# won't return None.
|
72
|
+
raise Exception(f"Function {name!r} failed after {max_attempts} attempts")
|
70
73
|
|
71
74
|
return wrapper
|
72
75
|
|
@@ -6,6 +6,8 @@ format.
|
|
6
6
|
This will be subject to consolidation and refactoring over the next few months.
|
7
7
|
"""
|
8
8
|
|
9
|
+
from __future__ import annotations
|
10
|
+
|
9
11
|
import os
|
10
12
|
import re
|
11
13
|
import urllib.parse
|
@@ -627,18 +629,18 @@ def validate_name_present_on_nonanonymous_blocks(values: M) -> M:
|
|
627
629
|
|
628
630
|
|
629
631
|
@overload
|
630
|
-
def
|
632
|
+
def validate_working_dir(v: str) -> Path:
|
631
633
|
...
|
632
634
|
|
633
635
|
|
634
636
|
@overload
|
635
|
-
def
|
637
|
+
def validate_working_dir(v: None) -> None:
|
636
638
|
...
|
637
639
|
|
638
640
|
|
639
|
-
def
|
641
|
+
def validate_working_dir(v: Optional[Path | str]) -> Optional[Path]:
|
640
642
|
"""Make sure that the working directory is formatted for the current platform."""
|
641
|
-
if v
|
643
|
+
if isinstance(v, str):
|
642
644
|
return relative_path_to_current_platform(v)
|
643
645
|
return v
|
644
646
|
|
prefect/_version.py
CHANGED
@@ -8,11 +8,11 @@ import json
|
|
8
8
|
|
9
9
|
version_json = '''
|
10
10
|
{
|
11
|
-
"date": "2025-01-
|
11
|
+
"date": "2025-01-17T08:46:53-0800",
|
12
12
|
"dirty": true,
|
13
13
|
"error": null,
|
14
|
-
"full-revisionid": "
|
15
|
-
"version": "3.1.
|
14
|
+
"full-revisionid": "16e85ce3c281778f5ab6487a73377eed63bcac8b",
|
15
|
+
"version": "3.1.13"
|
16
16
|
}
|
17
17
|
''' # END VERSION_JSON
|
18
18
|
|
prefect/artifacts.py
CHANGED
@@ -20,7 +20,10 @@ from prefect.logging.loggers import get_logger
|
|
20
20
|
from prefect.utilities.asyncutils import sync_compatible
|
21
21
|
from prefect.utilities.context import get_task_and_flow_run_ids
|
22
22
|
|
23
|
-
|
23
|
+
if TYPE_CHECKING:
|
24
|
+
import logging
|
25
|
+
|
26
|
+
logger: "logging.Logger" = get_logger("artifacts")
|
24
27
|
|
25
28
|
if TYPE_CHECKING:
|
26
29
|
from prefect.client.orchestration import PrefectClient
|
prefect/automations.py
CHANGED
prefect/blocks/abstract.py
CHANGED
@@ -15,7 +15,7 @@ from typing import (
|
|
15
15
|
Union,
|
16
16
|
)
|
17
17
|
|
18
|
-
from typing_extensions import Self, TypeAlias
|
18
|
+
from typing_extensions import TYPE_CHECKING, Self, TypeAlias
|
19
19
|
|
20
20
|
from prefect.blocks.core import Block
|
21
21
|
from prefect.exceptions import MissingContextError
|
@@ -26,7 +26,10 @@ T = TypeVar("T")
|
|
26
26
|
if sys.version_info >= (3, 12):
|
27
27
|
LoggingAdapter = logging.LoggerAdapter[logging.Logger]
|
28
28
|
else:
|
29
|
-
|
29
|
+
if TYPE_CHECKING:
|
30
|
+
LoggingAdapter = logging.LoggerAdapter[logging.Logger]
|
31
|
+
else:
|
32
|
+
LoggingAdapter = logging.LoggerAdapter
|
30
33
|
|
31
34
|
LoggerOrAdapter: TypeAlias = Union[Logger, LoggingAdapter]
|
32
35
|
|
prefect/blocks/notifications.py
CHANGED
@@ -880,6 +880,7 @@ class SendgridEmail(AbstractAppriseNotificationBlock):
|
|
880
880
|
sendgrid_block = SendgridEmail.load("BLOCK_NAME")
|
881
881
|
|
882
882
|
sendgrid_block.notify("Hello from Prefect!")
|
883
|
+
```
|
883
884
|
"""
|
884
885
|
|
885
886
|
_description = "Enables sending notifications via Sendgrid email service."
|
prefect/cache_policies.py
CHANGED
@@ -2,7 +2,7 @@ import inspect
|
|
2
2
|
from copy import deepcopy
|
3
3
|
from dataclasses import dataclass, field
|
4
4
|
from pathlib import Path
|
5
|
-
from typing import TYPE_CHECKING, Any, Callable, Dict,
|
5
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, Literal, Optional, Union
|
6
6
|
|
7
7
|
from typing_extensions import Self
|
8
8
|
|
@@ -73,8 +73,8 @@ class CachePolicy:
|
|
73
73
|
def compute_key(
|
74
74
|
self,
|
75
75
|
task_ctx: TaskRunContext,
|
76
|
-
inputs:
|
77
|
-
flow_parameters:
|
76
|
+
inputs: dict[str, Any],
|
77
|
+
flow_parameters: dict[str, Any],
|
78
78
|
**kwargs: Any,
|
79
79
|
) -> Optional[str]:
|
80
80
|
raise NotImplementedError
|
@@ -132,14 +132,14 @@ class CacheKeyFnPolicy(CachePolicy):
|
|
132
132
|
|
133
133
|
# making it optional for tests
|
134
134
|
cache_key_fn: Optional[
|
135
|
-
Callable[["TaskRunContext",
|
135
|
+
Callable[["TaskRunContext", dict[str, Any]], Optional[str]]
|
136
136
|
] = None
|
137
137
|
|
138
138
|
def compute_key(
|
139
139
|
self,
|
140
140
|
task_ctx: TaskRunContext,
|
141
|
-
inputs:
|
142
|
-
flow_parameters:
|
141
|
+
inputs: dict[str, Any],
|
142
|
+
flow_parameters: dict[str, Any],
|
143
143
|
**kwargs: Any,
|
144
144
|
) -> Optional[str]:
|
145
145
|
if self.cache_key_fn:
|
@@ -155,13 +155,13 @@ class CompoundCachePolicy(CachePolicy):
|
|
155
155
|
Any keys that return `None` will be ignored.
|
156
156
|
"""
|
157
157
|
|
158
|
-
policies:
|
158
|
+
policies: list[CachePolicy] = field(default_factory=list)
|
159
159
|
|
160
160
|
def compute_key(
|
161
161
|
self,
|
162
162
|
task_ctx: TaskRunContext,
|
163
|
-
inputs:
|
164
|
-
flow_parameters:
|
163
|
+
inputs: dict[str, Any],
|
164
|
+
flow_parameters: dict[str, Any],
|
165
165
|
**kwargs: Any,
|
166
166
|
) -> Optional[str]:
|
167
167
|
keys: list[str] = []
|
@@ -189,8 +189,8 @@ class _None(CachePolicy):
|
|
189
189
|
def compute_key(
|
190
190
|
self,
|
191
191
|
task_ctx: TaskRunContext,
|
192
|
-
inputs:
|
193
|
-
flow_parameters:
|
192
|
+
inputs: dict[str, Any],
|
193
|
+
flow_parameters: dict[str, Any],
|
194
194
|
**kwargs: Any,
|
195
195
|
) -> Optional[str]:
|
196
196
|
return None
|
@@ -209,8 +209,8 @@ class TaskSource(CachePolicy):
|
|
209
209
|
def compute_key(
|
210
210
|
self,
|
211
211
|
task_ctx: TaskRunContext,
|
212
|
-
inputs: Optional[
|
213
|
-
flow_parameters: Optional[
|
212
|
+
inputs: Optional[dict[str, Any]],
|
213
|
+
flow_parameters: Optional[dict[str, Any]],
|
214
214
|
**kwargs: Any,
|
215
215
|
) -> Optional[str]:
|
216
216
|
if not task_ctx:
|
@@ -236,8 +236,8 @@ class FlowParameters(CachePolicy):
|
|
236
236
|
def compute_key(
|
237
237
|
self,
|
238
238
|
task_ctx: TaskRunContext,
|
239
|
-
inputs:
|
240
|
-
flow_parameters:
|
239
|
+
inputs: dict[str, Any],
|
240
|
+
flow_parameters: dict[str, Any],
|
241
241
|
**kwargs: Any,
|
242
242
|
) -> Optional[str]:
|
243
243
|
if not flow_parameters:
|
@@ -255,8 +255,8 @@ class RunId(CachePolicy):
|
|
255
255
|
def compute_key(
|
256
256
|
self,
|
257
257
|
task_ctx: TaskRunContext,
|
258
|
-
inputs:
|
259
|
-
flow_parameters:
|
258
|
+
inputs: dict[str, Any],
|
259
|
+
flow_parameters: dict[str, Any],
|
260
260
|
**kwargs: Any,
|
261
261
|
) -> Optional[str]:
|
262
262
|
if not task_ctx:
|
@@ -273,13 +273,13 @@ class Inputs(CachePolicy):
|
|
273
273
|
Policy that computes a cache key based on a hash of the runtime inputs provided to the task..
|
274
274
|
"""
|
275
275
|
|
276
|
-
exclude:
|
276
|
+
exclude: list[str] = field(default_factory=list)
|
277
277
|
|
278
278
|
def compute_key(
|
279
279
|
self,
|
280
280
|
task_ctx: TaskRunContext,
|
281
|
-
inputs:
|
282
|
-
flow_parameters:
|
281
|
+
inputs: dict[str, Any],
|
282
|
+
flow_parameters: dict[str, Any],
|
283
283
|
**kwargs: Any,
|
284
284
|
) -> Optional[str]:
|
285
285
|
hashed_inputs = {}
|
prefect/client/utilities.py
CHANGED
@@ -5,7 +5,7 @@ Utilities for working with clients.
|
|
5
5
|
# This module must not import from `prefect.client` when it is imported to avoid
|
6
6
|
# circular imports for decorators such as `inject_client` which are widely used.
|
7
7
|
|
8
|
-
from collections.abc import
|
8
|
+
from collections.abc import Coroutine
|
9
9
|
from functools import wraps
|
10
10
|
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
|
11
11
|
|
@@ -61,8 +61,8 @@ def get_or_create_client(
|
|
61
61
|
|
62
62
|
|
63
63
|
def client_injector(
|
64
|
-
func: Callable[Concatenate["PrefectClient", P],
|
65
|
-
) -> Callable[P,
|
64
|
+
func: Callable[Concatenate["PrefectClient", P], Coroutine[Any, Any, R]],
|
65
|
+
) -> Callable[P, Coroutine[Any, Any, R]]:
|
66
66
|
@wraps(func)
|
67
67
|
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
68
68
|
client, _ = get_or_create_client()
|
prefect/deployments/base.py
CHANGED
@@ -19,6 +19,7 @@ from prefect.client.schemas.actions import DeploymentScheduleCreate
|
|
19
19
|
from prefect.client.schemas.objects import ConcurrencyLimitStrategy
|
20
20
|
from prefect.client.schemas.schedules import IntervalSchedule
|
21
21
|
from prefect.utilities._git import get_git_branch, get_git_remote_origin_url
|
22
|
+
from prefect.utilities.annotations import NotSet
|
22
23
|
from prefect.utilities.filesystem import create_default_ignore_file
|
23
24
|
from prefect.utilities.templating import apply_values
|
24
25
|
|
@@ -113,7 +114,9 @@ def create_default_prefect_yaml(
|
|
113
114
|
return True
|
114
115
|
|
115
116
|
|
116
|
-
def configure_project_by_recipe(
|
117
|
+
def configure_project_by_recipe(
|
118
|
+
recipe: str, **formatting_kwargs: Any
|
119
|
+
) -> dict[str, Any] | type[NotSet]:
|
117
120
|
"""
|
118
121
|
Given a recipe name, returns a dictionary representing base configuration options.
|
119
122
|
|
@@ -131,13 +134,13 @@ def configure_project_by_recipe(recipe: str, **formatting_kwargs) -> dict:
|
|
131
134
|
raise ValueError(f"Unknown recipe {recipe!r} provided.")
|
132
135
|
|
133
136
|
with recipe_path.open(mode="r") as f:
|
134
|
-
config = yaml.safe_load(f)
|
137
|
+
config: dict[str, Any] = yaml.safe_load(f)
|
135
138
|
|
136
|
-
|
139
|
+
templated_config = apply_values(
|
137
140
|
template=config, values=formatting_kwargs, remove_notset=False
|
138
141
|
)
|
139
142
|
|
140
|
-
return
|
143
|
+
return templated_config
|
141
144
|
|
142
145
|
|
143
146
|
def initialize_project(
|
prefect/deployments/flow_runs.py
CHANGED
prefect/deployments/runner.py
CHANGED
@@ -36,7 +36,6 @@ from pathlib import Path
|
|
36
36
|
from typing import TYPE_CHECKING, Any, ClassVar, Iterable, List, Optional, Union
|
37
37
|
from uuid import UUID
|
38
38
|
|
39
|
-
from exceptiongroup import ExceptionGroup # novermin
|
40
39
|
from pydantic import (
|
41
40
|
BaseModel,
|
42
41
|
ConfigDict,
|
@@ -361,7 +360,11 @@ class RunnerDeployment(BaseModel):
|
|
361
360
|
|
362
361
|
# We plan to support SLA configuration on the Prefect Server in the future.
|
363
362
|
# For now, we only support it on Prefect Cloud.
|
364
|
-
|
363
|
+
|
364
|
+
# If we're provided with an empty list, we will call the apply endpoint
|
365
|
+
# to remove existing SLAs for the deployment. If the argument is not provided,
|
366
|
+
# we will not call the endpoint.
|
367
|
+
if self._sla or self._sla == []:
|
365
368
|
await self._create_slas(deployment_id, client)
|
366
369
|
|
367
370
|
return deployment_id
|
@@ -371,15 +374,7 @@ class RunnerDeployment(BaseModel):
|
|
371
374
|
self._sla = [self._sla]
|
372
375
|
|
373
376
|
if client.server_type == ServerType.CLOUD:
|
374
|
-
|
375
|
-
for sla in self._sla:
|
376
|
-
try:
|
377
|
-
sla.set_deployment_id(deployment_id)
|
378
|
-
await client.create_sla(sla)
|
379
|
-
except Exception as e:
|
380
|
-
exceptions.append(e)
|
381
|
-
if exceptions:
|
382
|
-
raise ExceptionGroup("Failed to create SLAs", exceptions) # novermin
|
377
|
+
await client.apply_slas_for_deployment(deployment_id, self._sla)
|
383
378
|
else:
|
384
379
|
raise ValueError(
|
385
380
|
"SLA configuration is currently only supported on Prefect Cloud."
|
@@ -141,7 +141,7 @@ async def run_steps(
|
|
141
141
|
steps: List[Dict[str, Any]],
|
142
142
|
upstream_outputs: Optional[Dict[str, Any]] = None,
|
143
143
|
print_function: Any = print,
|
144
|
-
):
|
144
|
+
) -> dict[str, Any]:
|
145
145
|
upstream_outputs = deepcopy(upstream_outputs) if upstream_outputs else {}
|
146
146
|
for step in steps:
|
147
147
|
if not step:
|
@@ -12,7 +12,10 @@ from prefect.logging.loggers import get_logger
|
|
12
12
|
from prefect.runner.storage import BlockStorageAdapter, GitRepository, RemoteStorage
|
13
13
|
from prefect.utilities.asyncutils import run_coro_as_sync
|
14
14
|
|
15
|
-
|
15
|
+
if TYPE_CHECKING:
|
16
|
+
import logging
|
17
|
+
|
18
|
+
deployment_logger: "logging.Logger" = get_logger("deployment")
|
16
19
|
|
17
20
|
if TYPE_CHECKING:
|
18
21
|
from prefect.blocks.core import Block
|
@@ -197,7 +200,7 @@ def git_clone(
|
|
197
200
|
return dict(directory=str(storage.destination.relative_to(Path.cwd())))
|
198
201
|
|
199
202
|
|
200
|
-
async def pull_from_remote_storage(url: str, **settings: Any):
|
203
|
+
async def pull_from_remote_storage(url: str, **settings: Any) -> dict[str, Any]:
|
201
204
|
"""
|
202
205
|
Pulls code from a remote storage location into the current working directory.
|
203
206
|
|
@@ -239,7 +242,9 @@ async def pull_from_remote_storage(url: str, **settings: Any):
|
|
239
242
|
return {"directory": directory}
|
240
243
|
|
241
244
|
|
242
|
-
async def pull_with_block(
|
245
|
+
async def pull_with_block(
|
246
|
+
block_document_name: str, block_type_slug: str
|
247
|
+
) -> dict[str, Any]:
|
243
248
|
"""
|
244
249
|
Pulls code using a block.
|
245
250
|
|
@@ -26,7 +26,7 @@ import shlex
|
|
26
26
|
import string
|
27
27
|
import subprocess
|
28
28
|
import sys
|
29
|
-
from typing import Dict, Optional
|
29
|
+
from typing import Any, Dict, Optional
|
30
30
|
|
31
31
|
from anyio import create_task_group
|
32
32
|
from anyio.streams.text import TextReceiveStream
|
@@ -205,7 +205,7 @@ async def pip_install_requirements(
|
|
205
205
|
directory: Optional[str] = None,
|
206
206
|
requirements_file: str = "requirements.txt",
|
207
207
|
stream_output: bool = True,
|
208
|
-
):
|
208
|
+
) -> dict[str, Any]:
|
209
209
|
"""
|
210
210
|
Installs dependencies from a requirements.txt file.
|
211
211
|
|
prefect/docker/docker_image.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
from pathlib import Path
|
2
|
-
from typing import Optional
|
2
|
+
from typing import Any, Optional
|
3
3
|
|
4
4
|
from pendulum import now as pendulum_now
|
5
5
|
|
@@ -34,7 +34,11 @@ class DockerImage:
|
|
34
34
|
"""
|
35
35
|
|
36
36
|
def __init__(
|
37
|
-
self,
|
37
|
+
self,
|
38
|
+
name: str,
|
39
|
+
tag: Optional[str] = None,
|
40
|
+
dockerfile: str = "auto",
|
41
|
+
**build_kwargs: Any,
|
38
42
|
):
|
39
43
|
image_name, image_tag = parse_image_tag(name)
|
40
44
|
if tag and image_tag:
|
@@ -49,16 +53,16 @@ class DockerImage:
|
|
49
53
|
namespace = PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE.value()
|
50
54
|
# join the namespace and repository to create the full image name
|
51
55
|
# ignore namespace if it is None
|
52
|
-
self.name = "/".join(filter(None, [namespace, repository]))
|
53
|
-
self.tag = tag or image_tag or slugify(pendulum_now("utc").isoformat())
|
54
|
-
self.dockerfile = dockerfile
|
55
|
-
self.build_kwargs = build_kwargs
|
56
|
+
self.name: str = "/".join(filter(None, [namespace, repository]))
|
57
|
+
self.tag: str = tag or image_tag or slugify(pendulum_now("utc").isoformat())
|
58
|
+
self.dockerfile: str = dockerfile
|
59
|
+
self.build_kwargs: dict[str, Any] = build_kwargs
|
56
60
|
|
57
61
|
@property
|
58
|
-
def reference(self):
|
62
|
+
def reference(self) -> str:
|
59
63
|
return f"{self.name}:{self.tag}"
|
60
64
|
|
61
|
-
def build(self):
|
65
|
+
def build(self) -> None:
|
62
66
|
full_image_name = self.reference
|
63
67
|
build_kwargs = self.build_kwargs.copy()
|
64
68
|
build_kwargs["context"] = Path.cwd()
|
@@ -72,7 +76,7 @@ class DockerImage:
|
|
72
76
|
build_kwargs["dockerfile"] = self.dockerfile
|
73
77
|
build_image(**build_kwargs)
|
74
78
|
|
75
|
-
def push(self):
|
79
|
+
def push(self) -> None:
|
76
80
|
with docker_client() as client:
|
77
81
|
events = client.api.push(
|
78
82
|
repository=self.name, tag=self.tag, stream=True, decode=True
|