prefect-client 3.0.0rc3__py3-none-any.whl → 3.0.0rc5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +0 -3
- prefect/client/schemas/schedules.py +9 -2
- prefect/client/subscriptions.py +3 -3
- prefect/client/types/__init__.py +0 -0
- prefect/client/types/flexible_schedule_list.py +11 -0
- prefect/concurrency/asyncio.py +14 -4
- prefect/concurrency/services.py +29 -22
- prefect/concurrency/sync.py +3 -5
- prefect/context.py +0 -114
- prefect/deployments/__init__.py +1 -1
- prefect/deployments/runner.py +11 -93
- prefect/deployments/schedules.py +5 -7
- prefect/docker/__init__.py +20 -0
- prefect/docker/docker_image.py +82 -0
- prefect/flow_engine.py +96 -20
- prefect/flows.py +36 -95
- prefect/futures.py +22 -2
- prefect/infrastructure/provisioners/cloud_run.py +2 -2
- prefect/infrastructure/provisioners/container_instance.py +2 -2
- prefect/infrastructure/provisioners/ecs.py +2 -2
- prefect/records/result_store.py +5 -1
- prefect/results.py +111 -42
- prefect/runner/runner.py +5 -3
- prefect/runner/server.py +6 -2
- prefect/settings.py +1 -1
- prefect/states.py +13 -3
- prefect/task_engine.py +7 -6
- prefect/task_runs.py +23 -9
- prefect/task_worker.py +128 -19
- prefect/tasks.py +20 -16
- prefect/transactions.py +8 -10
- prefect/types/__init__.py +10 -3
- prefect/types/entrypoint.py +13 -0
- prefect/utilities/collections.py +120 -57
- prefect/utilities/dockerutils.py +2 -1
- prefect/utilities/urls.py +5 -5
- {prefect_client-3.0.0rc3.dist-info → prefect_client-3.0.0rc5.dist-info}/METADATA +2 -2
- {prefect_client-3.0.0rc3.dist-info → prefect_client-3.0.0rc5.dist-info}/RECORD +41 -37
- prefect/blocks/kubernetes.py +0 -115
- {prefect_client-3.0.0rc3.dist-info → prefect_client-3.0.0rc5.dist-info}/LICENSE +0 -0
- {prefect_client-3.0.0rc3.dist-info → prefect_client-3.0.0rc5.dist-info}/WHEEL +0 -0
- {prefect_client-3.0.0rc3.dist-info → prefect_client-3.0.0rc5.dist-info}/top_level.txt +0 -0
prefect/__init__.py
CHANGED
@@ -43,15 +43,12 @@ import prefect.runtime
|
|
43
43
|
|
44
44
|
# Import modules that register types
|
45
45
|
import prefect.serializers
|
46
|
-
import prefect.blocks.kubernetes
|
47
46
|
import prefect.blocks.notifications
|
48
47
|
import prefect.blocks.system
|
49
48
|
|
50
49
|
# Initialize the process-wide profile and registry at import time
|
51
50
|
import prefect.context
|
52
51
|
|
53
|
-
prefect.context.initialize_object_registry()
|
54
|
-
|
55
52
|
# Perform any forward-ref updates needed for Pydantic models
|
56
53
|
import prefect.client.schemas
|
57
54
|
|
@@ -3,13 +3,14 @@ Schedule schemas
|
|
3
3
|
"""
|
4
4
|
|
5
5
|
import datetime
|
6
|
-
from typing import Annotated, Optional, Union
|
6
|
+
from typing import Annotated, Any, Optional, Union
|
7
7
|
|
8
8
|
import dateutil
|
9
9
|
import dateutil.rrule
|
10
10
|
import pendulum
|
11
11
|
from pydantic import AfterValidator, ConfigDict, Field, field_validator, model_validator
|
12
12
|
from pydantic_extra_types.pendulum_dt import DateTime
|
13
|
+
from typing_extensions import TypeAlias, TypeGuard
|
13
14
|
|
14
15
|
from prefect._internal.schemas.bases import PrefectBaseModel
|
15
16
|
from prefect._internal.schemas.validators import (
|
@@ -279,7 +280,13 @@ class NoSchedule(PrefectBaseModel):
|
|
279
280
|
model_config = ConfigDict(extra="forbid")
|
280
281
|
|
281
282
|
|
282
|
-
SCHEDULE_TYPES = Union[
|
283
|
+
SCHEDULE_TYPES: TypeAlias = Union[
|
284
|
+
IntervalSchedule, CronSchedule, RRuleSchedule, NoSchedule
|
285
|
+
]
|
286
|
+
|
287
|
+
|
288
|
+
def is_schedule_type(obj: Any) -> TypeGuard[SCHEDULE_TYPES]:
|
289
|
+
return isinstance(obj, (IntervalSchedule, CronSchedule, RRuleSchedule, NoSchedule))
|
283
290
|
|
284
291
|
|
285
292
|
def construct_schedule(
|
prefect/client/subscriptions.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
import asyncio
|
2
|
-
from typing import Any, Dict, Generic,
|
2
|
+
from typing import Any, Dict, Generic, Iterable, Optional, Type, TypeVar
|
3
3
|
|
4
4
|
import orjson
|
5
5
|
import websockets
|
@@ -21,7 +21,7 @@ class Subscription(Generic[S]):
|
|
21
21
|
self,
|
22
22
|
model: Type[S],
|
23
23
|
path: str,
|
24
|
-
keys:
|
24
|
+
keys: Iterable[str],
|
25
25
|
client_id: Optional[str] = None,
|
26
26
|
base_url: Optional[str] = None,
|
27
27
|
):
|
@@ -30,7 +30,7 @@ class Subscription(Generic[S]):
|
|
30
30
|
base_url = base_url.replace("http", "ws", 1)
|
31
31
|
self.subscription_url = f"{base_url}{path}"
|
32
32
|
|
33
|
-
self.keys = keys
|
33
|
+
self.keys = list(keys)
|
34
34
|
|
35
35
|
self._connect = websockets.connect(
|
36
36
|
self.subscription_url,
|
File without changes
|
@@ -0,0 +1,11 @@
|
|
1
|
+
from typing import TYPE_CHECKING, Any, Sequence, Union
|
2
|
+
|
3
|
+
from typing_extensions import TypeAlias
|
4
|
+
|
5
|
+
if TYPE_CHECKING:
|
6
|
+
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
7
|
+
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
8
|
+
|
9
|
+
FlexibleScheduleList: TypeAlias = Sequence[
|
10
|
+
Union[DeploymentScheduleCreate, dict[str, Any], "SCHEDULE_TYPES"]
|
11
|
+
]
|
prefect/concurrency/asyncio.py
CHANGED
@@ -13,7 +13,6 @@ except ImportError:
|
|
13
13
|
|
14
14
|
from prefect.client.orchestration import get_client
|
15
15
|
from prefect.client.schemas.responses import MinimalConcurrencyLimitResponse
|
16
|
-
from prefect.utilities.timeout import timeout_async
|
17
16
|
|
18
17
|
from .events import (
|
19
18
|
_emit_concurrency_acquisition_events,
|
@@ -26,6 +25,10 @@ class ConcurrencySlotAcquisitionError(Exception):
|
|
26
25
|
"""Raised when an unhandlable occurs while acquiring concurrency slots."""
|
27
26
|
|
28
27
|
|
28
|
+
class AcquireConcurrencySlotTimeoutError(TimeoutError):
|
29
|
+
"""Raised when acquiring a concurrency slot times out."""
|
30
|
+
|
31
|
+
|
29
32
|
@asynccontextmanager
|
30
33
|
async def concurrency(
|
31
34
|
names: Union[str, List[str]],
|
@@ -58,8 +61,9 @@ async def concurrency(
|
|
58
61
|
```
|
59
62
|
"""
|
60
63
|
names = names if isinstance(names, list) else [names]
|
61
|
-
|
62
|
-
|
64
|
+
limits = await _acquire_concurrency_slots(
|
65
|
+
names, occupy, timeout_seconds=timeout_seconds
|
66
|
+
)
|
63
67
|
acquisition_time = pendulum.now("UTC")
|
64
68
|
emitted_events = _emit_concurrency_acquisition_events(limits, occupy)
|
65
69
|
|
@@ -91,12 +95,18 @@ async def _acquire_concurrency_slots(
|
|
91
95
|
names: List[str],
|
92
96
|
slots: int,
|
93
97
|
mode: Union[Literal["concurrency"], Literal["rate_limit"]] = "concurrency",
|
98
|
+
timeout_seconds: Optional[float] = None,
|
94
99
|
) -> List[MinimalConcurrencyLimitResponse]:
|
95
100
|
service = ConcurrencySlotAcquisitionService.instance(frozenset(names))
|
96
|
-
future = service.send((slots, mode))
|
101
|
+
future = service.send((slots, mode, timeout_seconds))
|
97
102
|
response_or_exception = await asyncio.wrap_future(future)
|
98
103
|
|
99
104
|
if isinstance(response_or_exception, Exception):
|
105
|
+
if isinstance(response_or_exception, TimeoutError):
|
106
|
+
raise AcquireConcurrencySlotTimeoutError(
|
107
|
+
f"Attempt to acquire concurrency slots timed out after {timeout_seconds} second(s)"
|
108
|
+
) from response_or_exception
|
109
|
+
|
100
110
|
raise ConcurrencySlotAcquisitionError(
|
101
111
|
f"Unable to acquire concurrency slots on {names!r}"
|
102
112
|
) from response_or_exception
|
prefect/concurrency/services.py
CHANGED
@@ -4,6 +4,7 @@ from contextlib import asynccontextmanager
|
|
4
4
|
from typing import (
|
5
5
|
TYPE_CHECKING,
|
6
6
|
FrozenSet,
|
7
|
+
Optional,
|
7
8
|
Tuple,
|
8
9
|
)
|
9
10
|
|
@@ -13,6 +14,7 @@ from starlette import status
|
|
13
14
|
from prefect._internal.concurrency import logger
|
14
15
|
from prefect._internal.concurrency.services import QueueService
|
15
16
|
from prefect.client.orchestration import get_client
|
17
|
+
from prefect.utilities.timeout import timeout_async
|
16
18
|
|
17
19
|
if TYPE_CHECKING:
|
18
20
|
from prefect.client.orchestration import PrefectClient
|
@@ -30,10 +32,12 @@ class ConcurrencySlotAcquisitionService(QueueService):
|
|
30
32
|
self._client = client
|
31
33
|
yield
|
32
34
|
|
33
|
-
async def _handle(
|
34
|
-
|
35
|
+
async def _handle(
|
36
|
+
self, item: Tuple[int, str, Optional[float], concurrent.futures.Future]
|
37
|
+
):
|
38
|
+
occupy, mode, timeout_seconds, future = item
|
35
39
|
try:
|
36
|
-
response = await self.acquire_slots(occupy, mode)
|
40
|
+
response = await self.acquire_slots(occupy, mode, timeout_seconds)
|
37
41
|
except Exception as exc:
|
38
42
|
# If the request to the increment endpoint fails in a non-standard
|
39
43
|
# way, we need to set the future's result so that the caller can
|
@@ -43,25 +47,28 @@ class ConcurrencySlotAcquisitionService(QueueService):
|
|
43
47
|
else:
|
44
48
|
future.set_result(response)
|
45
49
|
|
46
|
-
async def acquire_slots(
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
50
|
+
async def acquire_slots(
|
51
|
+
self, slots: int, mode: str, timeout_seconds: Optional[float] = None
|
52
|
+
) -> httpx.Response:
|
53
|
+
with timeout_async(seconds=timeout_seconds):
|
54
|
+
while True:
|
55
|
+
try:
|
56
|
+
response = await self._client.increment_concurrency_slots(
|
57
|
+
names=self.concurrency_limit_names, slots=slots, mode=mode
|
58
|
+
)
|
59
|
+
except Exception as exc:
|
60
|
+
if (
|
61
|
+
isinstance(exc, httpx.HTTPStatusError)
|
62
|
+
and exc.response.status_code == status.HTTP_423_LOCKED
|
63
|
+
):
|
64
|
+
retry_after = float(exc.response.headers["Retry-After"])
|
65
|
+
await asyncio.sleep(retry_after)
|
66
|
+
else:
|
67
|
+
raise exc
|
59
68
|
else:
|
60
|
-
|
61
|
-
else:
|
62
|
-
return response
|
69
|
+
return response
|
63
70
|
|
64
|
-
def send(self, item: Tuple[int, str]):
|
71
|
+
def send(self, item: Tuple[int, str, Optional[float]]) -> concurrent.futures.Future:
|
65
72
|
with self._lock:
|
66
73
|
if self._stopped:
|
67
74
|
raise RuntimeError("Cannot put items in a stopped service instance.")
|
@@ -69,7 +76,7 @@ class ConcurrencySlotAcquisitionService(QueueService):
|
|
69
76
|
logger.debug("Service %r enqueuing item %r", self, item)
|
70
77
|
future: concurrent.futures.Future = concurrent.futures.Future()
|
71
78
|
|
72
|
-
occupy, mode = item
|
73
|
-
self._queue.put_nowait((occupy, mode, future))
|
79
|
+
occupy, mode, timeout_seconds = item
|
80
|
+
self._queue.put_nowait((occupy, mode, timeout_seconds, future))
|
74
81
|
|
75
82
|
return future
|
prefect/concurrency/sync.py
CHANGED
@@ -12,7 +12,6 @@ except ImportError:
|
|
12
12
|
from prefect._internal.concurrency.api import create_call, from_sync
|
13
13
|
from prefect._internal.concurrency.event_loop import get_running_loop
|
14
14
|
from prefect.client.schemas.responses import MinimalConcurrencyLimitResponse
|
15
|
-
from prefect.utilities.timeout import timeout
|
16
15
|
|
17
16
|
from .asyncio import (
|
18
17
|
_acquire_concurrency_slots,
|
@@ -57,10 +56,9 @@ def concurrency(
|
|
57
56
|
"""
|
58
57
|
names = names if isinstance(names, list) else [names]
|
59
58
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
)
|
59
|
+
limits: List[MinimalConcurrencyLimitResponse] = _call_async_function_from_sync(
|
60
|
+
_acquire_concurrency_slots, names, occupy, timeout_seconds=timeout_seconds
|
61
|
+
)
|
64
62
|
acquisition_time = pendulum.now("UTC")
|
65
63
|
emitted_events = _emit_concurrency_acquisition_events(limits, occupy)
|
66
64
|
|
prefect/context.py
CHANGED
@@ -9,21 +9,16 @@ For more user-accessible information about the current run, see [`prefect.runtim
|
|
9
9
|
import os
|
10
10
|
import sys
|
11
11
|
import warnings
|
12
|
-
from collections import defaultdict
|
13
12
|
from contextlib import ExitStack, contextmanager
|
14
13
|
from contextvars import ContextVar, Token
|
15
|
-
from functools import update_wrapper
|
16
14
|
from pathlib import Path
|
17
15
|
from typing import (
|
18
16
|
TYPE_CHECKING,
|
19
17
|
Any,
|
20
|
-
ContextManager,
|
21
18
|
Dict,
|
22
19
|
Generator,
|
23
|
-
List,
|
24
20
|
Optional,
|
25
21
|
Set,
|
26
|
-
Tuple,
|
27
22
|
Type,
|
28
23
|
TypeVar,
|
29
24
|
Union,
|
@@ -46,7 +41,6 @@ from prefect.settings import PREFECT_HOME, Profile, Settings
|
|
46
41
|
from prefect.states import State
|
47
42
|
from prefect.task_runners import TaskRunner
|
48
43
|
from prefect.utilities.asyncutils import run_coro_as_sync
|
49
|
-
from prefect.utilities.importtools import load_script_as_module
|
50
44
|
|
51
45
|
T = TypeVar("T")
|
52
46
|
|
@@ -180,86 +174,6 @@ class ContextModel(BaseModel):
|
|
180
174
|
return self.model_dump(exclude_unset=True)
|
181
175
|
|
182
176
|
|
183
|
-
class PrefectObjectRegistry(ContextModel):
|
184
|
-
"""
|
185
|
-
A context that acts as a registry for all Prefect objects that are
|
186
|
-
registered during load and execution.
|
187
|
-
|
188
|
-
Attributes:
|
189
|
-
start_time: The time the object registry was created.
|
190
|
-
block_code_execution: If set, flow calls will be ignored.
|
191
|
-
capture_failures: If set, failures during __init__ will be silenced and tracked.
|
192
|
-
"""
|
193
|
-
|
194
|
-
start_time: DateTime = Field(default_factory=lambda: pendulum.now("UTC"))
|
195
|
-
|
196
|
-
_instance_registry: Dict[Type[T], List[T]] = PrivateAttr(
|
197
|
-
default_factory=lambda: defaultdict(list)
|
198
|
-
)
|
199
|
-
|
200
|
-
# Failures will be a tuple of (exception, instance, args, kwargs)
|
201
|
-
_instance_init_failures: Dict[
|
202
|
-
Type[T], List[Tuple[Exception, T, Tuple, Dict]]
|
203
|
-
] = PrivateAttr(default_factory=lambda: defaultdict(list))
|
204
|
-
|
205
|
-
block_code_execution: bool = False
|
206
|
-
capture_failures: bool = False
|
207
|
-
|
208
|
-
__var__ = ContextVar("object_registry")
|
209
|
-
|
210
|
-
def get_instances(self, type_: Type[T]) -> List[T]:
|
211
|
-
instances = []
|
212
|
-
for registered_type, type_instances in self._instance_registry.items():
|
213
|
-
if type_ in registered_type.mro():
|
214
|
-
instances.extend(type_instances)
|
215
|
-
return instances
|
216
|
-
|
217
|
-
def get_instance_failures(
|
218
|
-
self, type_: Type[T]
|
219
|
-
) -> List[Tuple[Exception, T, Tuple, Dict]]:
|
220
|
-
failures = []
|
221
|
-
for type__ in type_.mro():
|
222
|
-
failures.extend(self._instance_init_failures[type__])
|
223
|
-
return failures
|
224
|
-
|
225
|
-
def register_instance(self, object):
|
226
|
-
# TODO: Consider using a 'Set' to avoid duplicate entries
|
227
|
-
self._instance_registry[type(object)].append(object)
|
228
|
-
|
229
|
-
def register_init_failure(
|
230
|
-
self, exc: Exception, object: Any, init_args: Tuple, init_kwargs: Dict
|
231
|
-
):
|
232
|
-
self._instance_init_failures[type(object)].append(
|
233
|
-
(exc, object, init_args, init_kwargs)
|
234
|
-
)
|
235
|
-
|
236
|
-
@classmethod
|
237
|
-
def register_instances(cls, type_: Type[T]) -> Type[T]:
|
238
|
-
"""
|
239
|
-
Decorator for a class that adds registration to the `PrefectObjectRegistry`
|
240
|
-
on initialization of instances.
|
241
|
-
"""
|
242
|
-
original_init = type_.__init__
|
243
|
-
|
244
|
-
def __register_init__(__self__: T, *args: Any, **kwargs: Any) -> None:
|
245
|
-
registry = cls.get()
|
246
|
-
try:
|
247
|
-
original_init(__self__, *args, **kwargs)
|
248
|
-
except Exception as exc:
|
249
|
-
if not registry or not registry.capture_failures:
|
250
|
-
raise
|
251
|
-
else:
|
252
|
-
registry.register_init_failure(exc, __self__, args, kwargs)
|
253
|
-
else:
|
254
|
-
if registry:
|
255
|
-
registry.register_instance(__self__)
|
256
|
-
|
257
|
-
update_wrapper(__register_init__, original_init)
|
258
|
-
|
259
|
-
type_.__init__ = __register_init__
|
260
|
-
return type_
|
261
|
-
|
262
|
-
|
263
177
|
class ClientContext(ContextModel):
|
264
178
|
"""
|
265
179
|
A context for managing the Prefect client instances.
|
@@ -594,23 +508,6 @@ def tags(*new_tags: str) -> Generator[Set[str], None, None]:
|
|
594
508
|
yield new_tags
|
595
509
|
|
596
510
|
|
597
|
-
def registry_from_script(
|
598
|
-
path: str,
|
599
|
-
block_code_execution: bool = True,
|
600
|
-
capture_failures: bool = True,
|
601
|
-
) -> PrefectObjectRegistry:
|
602
|
-
"""
|
603
|
-
Return a fresh registry with instances populated from execution of a script.
|
604
|
-
"""
|
605
|
-
with PrefectObjectRegistry(
|
606
|
-
block_code_execution=block_code_execution,
|
607
|
-
capture_failures=capture_failures,
|
608
|
-
) as registry:
|
609
|
-
load_script_as_module(path)
|
610
|
-
|
611
|
-
return registry
|
612
|
-
|
613
|
-
|
614
511
|
@contextmanager
|
615
512
|
def use_profile(
|
616
513
|
profile: Union[Profile, str],
|
@@ -711,14 +608,3 @@ def root_settings_context():
|
|
711
608
|
|
712
609
|
|
713
610
|
GLOBAL_SETTINGS_CONTEXT: SettingsContext = root_settings_context()
|
714
|
-
GLOBAL_OBJECT_REGISTRY: Optional[ContextManager[PrefectObjectRegistry]] = None
|
715
|
-
|
716
|
-
|
717
|
-
def initialize_object_registry():
|
718
|
-
global GLOBAL_OBJECT_REGISTRY
|
719
|
-
|
720
|
-
if GLOBAL_OBJECT_REGISTRY:
|
721
|
-
return
|
722
|
-
|
723
|
-
GLOBAL_OBJECT_REGISTRY = PrefectObjectRegistry()
|
724
|
-
GLOBAL_OBJECT_REGISTRY.__enter__()
|
prefect/deployments/__init__.py
CHANGED
prefect/deployments/runner.py
CHANGED
@@ -29,7 +29,6 @@ Example:
|
|
29
29
|
|
30
30
|
"""
|
31
31
|
|
32
|
-
import enum
|
33
32
|
import importlib
|
34
33
|
import tempfile
|
35
34
|
from datetime import datetime, timedelta
|
@@ -37,7 +36,6 @@ from pathlib import Path
|
|
37
36
|
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
38
37
|
from uuid import UUID
|
39
38
|
|
40
|
-
import pendulum
|
41
39
|
from pydantic import (
|
42
40
|
BaseModel,
|
43
41
|
ConfigDict,
|
@@ -61,9 +59,9 @@ from prefect.client.schemas.schedules import (
|
|
61
59
|
construct_schedule,
|
62
60
|
)
|
63
61
|
from prefect.deployments.schedules import (
|
64
|
-
FlexibleScheduleList,
|
65
62
|
create_deployment_schedule_create,
|
66
63
|
)
|
64
|
+
from prefect.docker.docker_image import DockerImage
|
67
65
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
68
66
|
from prefect.exceptions import (
|
69
67
|
ObjectNotFound,
|
@@ -71,24 +69,19 @@ from prefect.exceptions import (
|
|
71
69
|
)
|
72
70
|
from prefect.runner.storage import RunnerStorage
|
73
71
|
from prefect.settings import (
|
74
|
-
PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE,
|
75
72
|
PREFECT_DEFAULT_WORK_POOL_NAME,
|
76
73
|
PREFECT_UI_URL,
|
77
74
|
)
|
75
|
+
from prefect.types.entrypoint import EntrypointType
|
78
76
|
from prefect.utilities.asyncutils import sync_compatible
|
79
77
|
from prefect.utilities.callables import ParameterSchema, parameter_schema
|
80
78
|
from prefect.utilities.collections import get_from_dict, isiterable
|
81
79
|
from prefect.utilities.dockerutils import (
|
82
|
-
PushError,
|
83
|
-
build_image,
|
84
|
-
docker_client,
|
85
|
-
generate_default_dockerfile,
|
86
80
|
parse_image_tag,
|
87
|
-
split_repository_path,
|
88
81
|
)
|
89
|
-
from prefect.utilities.slugify import slugify
|
90
82
|
|
91
83
|
if TYPE_CHECKING:
|
84
|
+
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
92
85
|
from prefect.flows import Flow
|
93
86
|
|
94
87
|
__all__ = ["RunnerDeployment"]
|
@@ -100,18 +93,6 @@ class DeploymentApplyError(RuntimeError):
|
|
100
93
|
"""
|
101
94
|
|
102
95
|
|
103
|
-
class EntrypointType(enum.Enum):
|
104
|
-
"""
|
105
|
-
Enum representing a entrypoint type.
|
106
|
-
|
107
|
-
File path entrypoints are in the format: `path/to/file.py:function_name`.
|
108
|
-
Module path entrypoints are in the format: `path.to.module.function_name`.
|
109
|
-
"""
|
110
|
-
|
111
|
-
FILE_PATH = "file_path"
|
112
|
-
MODULE_PATH = "module_path"
|
113
|
-
|
114
|
-
|
115
96
|
class RunnerDeployment(BaseModel):
|
116
97
|
"""
|
117
98
|
A Prefect RunnerDeployment definition, used for specifying and building deployments.
|
@@ -363,8 +344,8 @@ class RunnerDeployment(BaseModel):
|
|
363
344
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
364
345
|
timezone: Optional[str] = None,
|
365
346
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
366
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
367
|
-
) -> Union[List[DeploymentScheduleCreate], FlexibleScheduleList]:
|
347
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
348
|
+
) -> Union[List[DeploymentScheduleCreate], "FlexibleScheduleList"]:
|
368
349
|
"""
|
369
350
|
Construct a schedule or schedules from the provided arguments.
|
370
351
|
|
@@ -455,7 +436,7 @@ class RunnerDeployment(BaseModel):
|
|
455
436
|
cron: Optional[Union[Iterable[str], str]] = None,
|
456
437
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
457
438
|
paused: Optional[bool] = None,
|
458
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
439
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
459
440
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
460
441
|
is_schedule_active: Optional[bool] = None,
|
461
442
|
parameters: Optional[dict] = None,
|
@@ -591,7 +572,7 @@ class RunnerDeployment(BaseModel):
|
|
591
572
|
cron: Optional[Union[Iterable[str], str]] = None,
|
592
573
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
593
574
|
paused: Optional[bool] = None,
|
594
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
575
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
595
576
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
596
577
|
is_schedule_active: Optional[bool] = None,
|
597
578
|
parameters: Optional[dict] = None,
|
@@ -689,7 +670,7 @@ class RunnerDeployment(BaseModel):
|
|
689
670
|
cron: Optional[Union[Iterable[str], str]] = None,
|
690
671
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
691
672
|
paused: Optional[bool] = None,
|
692
|
-
schedules: Optional[FlexibleScheduleList] = None,
|
673
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
693
674
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
694
675
|
is_schedule_active: Optional[bool] = None,
|
695
676
|
parameters: Optional[dict] = None,
|
@@ -786,74 +767,11 @@ class RunnerDeployment(BaseModel):
|
|
786
767
|
return deployment
|
787
768
|
|
788
769
|
|
789
|
-
class DeploymentImage:
|
790
|
-
"""
|
791
|
-
Configuration used to build and push a Docker image for a deployment.
|
792
|
-
|
793
|
-
Attributes:
|
794
|
-
name: The name of the Docker image to build, including the registry and
|
795
|
-
repository.
|
796
|
-
tag: The tag to apply to the built image.
|
797
|
-
dockerfile: The path to the Dockerfile to use for building the image. If
|
798
|
-
not provided, a default Dockerfile will be generated.
|
799
|
-
**build_kwargs: Additional keyword arguments to pass to the Docker build request.
|
800
|
-
See the [`docker-py` documentation](https://docker-py.readthedocs.io/en/stable/images.html#docker.models.images.ImageCollection.build)
|
801
|
-
for more information.
|
802
|
-
|
803
|
-
"""
|
804
|
-
|
805
|
-
def __init__(self, name, tag=None, dockerfile="auto", **build_kwargs):
|
806
|
-
image_name, image_tag = parse_image_tag(name)
|
807
|
-
if tag and image_tag:
|
808
|
-
raise ValueError(
|
809
|
-
f"Only one tag can be provided - both {image_tag!r} and {tag!r} were"
|
810
|
-
" provided as tags."
|
811
|
-
)
|
812
|
-
namespace, repository = split_repository_path(image_name)
|
813
|
-
# if the provided image name does not include a namespace (registry URL or user/org name),
|
814
|
-
# use the default namespace
|
815
|
-
if not namespace:
|
816
|
-
namespace = PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE.value()
|
817
|
-
# join the namespace and repository to create the full image name
|
818
|
-
# ignore namespace if it is None
|
819
|
-
self.name = "/".join(filter(None, [namespace, repository]))
|
820
|
-
self.tag = tag or image_tag or slugify(pendulum.now("utc").isoformat())
|
821
|
-
self.dockerfile = dockerfile
|
822
|
-
self.build_kwargs = build_kwargs
|
823
|
-
|
824
|
-
@property
|
825
|
-
def reference(self):
|
826
|
-
return f"{self.name}:{self.tag}"
|
827
|
-
|
828
|
-
def build(self):
|
829
|
-
full_image_name = self.reference
|
830
|
-
build_kwargs = self.build_kwargs.copy()
|
831
|
-
build_kwargs["context"] = Path.cwd()
|
832
|
-
build_kwargs["tag"] = full_image_name
|
833
|
-
build_kwargs["pull"] = build_kwargs.get("pull", True)
|
834
|
-
|
835
|
-
if self.dockerfile == "auto":
|
836
|
-
with generate_default_dockerfile():
|
837
|
-
build_image(**build_kwargs)
|
838
|
-
else:
|
839
|
-
build_kwargs["dockerfile"] = self.dockerfile
|
840
|
-
build_image(**build_kwargs)
|
841
|
-
|
842
|
-
def push(self):
|
843
|
-
with docker_client() as client:
|
844
|
-
events = client.api.push(
|
845
|
-
repository=self.name, tag=self.tag, stream=True, decode=True
|
846
|
-
)
|
847
|
-
for event in events:
|
848
|
-
if "error" in event:
|
849
|
-
raise PushError(event["error"])
|
850
|
-
|
851
|
-
|
852
770
|
@sync_compatible
|
853
771
|
async def deploy(
|
854
772
|
*deployments: RunnerDeployment,
|
855
773
|
work_pool_name: Optional[str] = None,
|
856
|
-
image: Optional[Union[str,
|
774
|
+
image: Optional[Union[str, DockerImage]] = None,
|
857
775
|
build: bool = True,
|
858
776
|
push: bool = True,
|
859
777
|
print_next_steps_message: bool = True,
|
@@ -875,7 +793,7 @@ async def deploy(
|
|
875
793
|
work_pool_name: The name of the work pool to use for these deployments. Defaults to
|
876
794
|
the value of `PREFECT_DEFAULT_WORK_POOL_NAME`.
|
877
795
|
image: The name of the Docker image to build, including the registry and
|
878
|
-
repository. Pass a
|
796
|
+
repository. Pass a DockerImage instance to customize the Dockerfile used
|
879
797
|
and build arguments.
|
880
798
|
build: Whether or not to build a new image for the flow. If False, the provided
|
881
799
|
image will be used as-is and pulled at runtime.
|
@@ -930,7 +848,7 @@ async def deploy(
|
|
930
848
|
|
931
849
|
if image and isinstance(image, str):
|
932
850
|
image_name, image_tag = parse_image_tag(image)
|
933
|
-
image =
|
851
|
+
image = DockerImage(name=image_name, tag=image_tag)
|
934
852
|
|
935
853
|
try:
|
936
854
|
async with get_client() as client:
|
prefect/deployments/schedules.py
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
from typing import TYPE_CHECKING, Any, List, Optional
|
1
|
+
from typing import TYPE_CHECKING, Any, List, Optional
|
2
2
|
|
3
3
|
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
4
|
+
from prefect.client.schemas.schedules import is_schedule_type
|
4
5
|
|
5
6
|
if TYPE_CHECKING:
|
6
7
|
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
7
|
-
|
8
|
-
FlexibleScheduleList = Sequence[Union[DeploymentScheduleCreate, dict, "SCHEDULE_TYPES"]]
|
8
|
+
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
9
9
|
|
10
10
|
|
11
11
|
def create_deployment_schedule_create(
|
@@ -26,12 +26,10 @@ def create_deployment_schedule_create(
|
|
26
26
|
def normalize_to_deployment_schedule_create(
|
27
27
|
schedules: Optional["FlexibleScheduleList"],
|
28
28
|
) -> List[DeploymentScheduleCreate]:
|
29
|
-
|
30
|
-
|
31
|
-
normalized = []
|
29
|
+
normalized: list[DeploymentScheduleCreate] = []
|
32
30
|
if schedules is not None:
|
33
31
|
for obj in schedules:
|
34
|
-
if
|
32
|
+
if is_schedule_type(obj):
|
35
33
|
normalized.append(create_deployment_schedule_create(obj))
|
36
34
|
elif isinstance(obj, dict):
|
37
35
|
normalized.append(create_deployment_schedule_create(**obj))
|
@@ -0,0 +1,20 @@
|
|
1
|
+
from typing import TYPE_CHECKING
|
2
|
+
|
3
|
+
if TYPE_CHECKING:
|
4
|
+
from prefect.docker.docker_image import DockerImage
|
5
|
+
|
6
|
+
__all__ = ["DockerImage"]
|
7
|
+
|
8
|
+
_public_api: dict[str, tuple[str, str]] = {
|
9
|
+
"DockerImage": ("prefect.docker.docker_image", "DockerImage"),
|
10
|
+
}
|
11
|
+
|
12
|
+
|
13
|
+
def __getattr__(name: str) -> object:
|
14
|
+
from importlib import import_module
|
15
|
+
|
16
|
+
if name in _public_api:
|
17
|
+
module, attr = _public_api[name]
|
18
|
+
return getattr(import_module(module), attr)
|
19
|
+
|
20
|
+
raise ImportError(f"module {__name__!r} has no attribute {name!r}")
|