prefect-client 3.1.11__py3-none-any.whl → 3.1.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_experimental/sla/__init__.py +0 -0
- prefect/_experimental/sla/client.py +92 -0
- prefect/_experimental/sla/objects.py +61 -0
- prefect/_internal/concurrency/services.py +2 -2
- prefect/_internal/concurrency/threads.py +6 -0
- prefect/_internal/retries.py +6 -3
- prefect/_internal/schemas/validators.py +6 -4
- prefect/_version.py +3 -3
- prefect/artifacts.py +4 -1
- prefect/automations.py +236 -30
- prefect/blocks/__init__.py +3 -3
- prefect/blocks/abstract.py +57 -31
- prefect/blocks/core.py +181 -82
- prefect/blocks/notifications.py +134 -73
- prefect/blocks/redis.py +13 -9
- prefect/blocks/system.py +24 -11
- prefect/blocks/webhook.py +7 -5
- prefect/cache_policies.py +23 -22
- prefect/client/orchestration/__init__.py +103 -2006
- prefect/client/orchestration/_automations/__init__.py +0 -0
- prefect/client/orchestration/_automations/client.py +329 -0
- prefect/client/orchestration/_blocks_documents/__init__.py +0 -0
- prefect/client/orchestration/_blocks_documents/client.py +334 -0
- prefect/client/orchestration/_blocks_schemas/__init__.py +0 -0
- prefect/client/orchestration/_blocks_schemas/client.py +200 -0
- prefect/client/orchestration/_blocks_types/__init__.py +0 -0
- prefect/client/orchestration/_blocks_types/client.py +380 -0
- prefect/client/orchestration/_deployments/__init__.py +0 -0
- prefect/client/orchestration/_deployments/client.py +1128 -0
- prefect/client/orchestration/_flow_runs/__init__.py +0 -0
- prefect/client/orchestration/_flow_runs/client.py +903 -0
- prefect/client/orchestration/_flows/__init__.py +0 -0
- prefect/client/orchestration/_flows/client.py +343 -0
- prefect/client/orchestration/_logs/client.py +16 -14
- prefect/client/schemas/__init__.py +68 -28
- prefect/client/schemas/objects.py +5 -5
- prefect/client/utilities.py +3 -3
- prefect/context.py +15 -1
- prefect/deployments/base.py +13 -4
- prefect/deployments/flow_runs.py +5 -1
- prefect/deployments/runner.py +37 -1
- prefect/deployments/steps/core.py +1 -1
- prefect/deployments/steps/pull.py +8 -3
- prefect/deployments/steps/utility.py +2 -2
- prefect/docker/docker_image.py +13 -9
- prefect/engine.py +33 -11
- prefect/events/cli/automations.py +4 -4
- prefect/events/clients.py +17 -14
- prefect/events/schemas/automations.py +12 -8
- prefect/events/schemas/events.py +5 -1
- prefect/events/worker.py +1 -1
- prefect/filesystems.py +7 -3
- prefect/flow_engine.py +64 -47
- prefect/flows.py +128 -74
- prefect/futures.py +14 -7
- prefect/infrastructure/provisioners/__init__.py +2 -0
- prefect/infrastructure/provisioners/cloud_run.py +4 -4
- prefect/infrastructure/provisioners/coiled.py +249 -0
- prefect/infrastructure/provisioners/container_instance.py +4 -3
- prefect/infrastructure/provisioners/ecs.py +55 -43
- prefect/infrastructure/provisioners/modal.py +5 -4
- prefect/input/actions.py +5 -1
- prefect/input/run_input.py +157 -43
- prefect/logging/configuration.py +3 -3
- prefect/logging/filters.py +2 -2
- prefect/logging/formatters.py +15 -11
- prefect/logging/handlers.py +24 -14
- prefect/logging/highlighters.py +5 -5
- prefect/logging/loggers.py +28 -18
- prefect/logging/logging.yml +1 -1
- prefect/main.py +3 -1
- prefect/results.py +166 -86
- prefect/runner/runner.py +38 -29
- prefect/runner/server.py +3 -1
- prefect/runner/storage.py +18 -18
- prefect/runner/submit.py +19 -12
- prefect/runtime/deployment.py +15 -8
- prefect/runtime/flow_run.py +19 -6
- prefect/runtime/task_run.py +7 -3
- prefect/settings/base.py +17 -7
- prefect/settings/legacy.py +4 -4
- prefect/settings/models/api.py +4 -3
- prefect/settings/models/cli.py +4 -3
- prefect/settings/models/client.py +7 -4
- prefect/settings/models/cloud.py +9 -3
- prefect/settings/models/deployments.py +4 -3
- prefect/settings/models/experiments.py +4 -8
- prefect/settings/models/flows.py +4 -3
- prefect/settings/models/internal.py +4 -3
- prefect/settings/models/logging.py +8 -6
- prefect/settings/models/results.py +4 -3
- prefect/settings/models/root.py +11 -16
- prefect/settings/models/runner.py +8 -5
- prefect/settings/models/server/api.py +6 -3
- prefect/settings/models/server/database.py +120 -25
- prefect/settings/models/server/deployments.py +4 -3
- prefect/settings/models/server/ephemeral.py +7 -4
- prefect/settings/models/server/events.py +6 -3
- prefect/settings/models/server/flow_run_graph.py +4 -3
- prefect/settings/models/server/root.py +4 -3
- prefect/settings/models/server/services.py +15 -12
- prefect/settings/models/server/tasks.py +7 -4
- prefect/settings/models/server/ui.py +4 -3
- prefect/settings/models/tasks.py +10 -5
- prefect/settings/models/testing.py +4 -3
- prefect/settings/models/worker.py +7 -4
- prefect/settings/profiles.py +13 -12
- prefect/settings/sources.py +20 -19
- prefect/states.py +74 -51
- prefect/task_engine.py +43 -33
- prefect/task_runners.py +85 -72
- prefect/task_runs.py +20 -11
- prefect/task_worker.py +14 -9
- prefect/tasks.py +36 -28
- prefect/telemetry/bootstrap.py +13 -9
- prefect/telemetry/run_telemetry.py +15 -13
- prefect/telemetry/services.py +4 -0
- prefect/transactions.py +3 -3
- prefect/types/__init__.py +3 -1
- prefect/utilities/_deprecated.py +38 -0
- prefect/utilities/engine.py +11 -4
- prefect/utilities/filesystem.py +2 -2
- prefect/utilities/generics.py +1 -1
- prefect/utilities/pydantic.py +21 -36
- prefect/utilities/templating.py +25 -1
- prefect/workers/base.py +58 -33
- prefect/workers/process.py +20 -15
- prefect/workers/server.py +4 -5
- {prefect_client-3.1.11.dist-info → prefect_client-3.1.13.dist-info}/METADATA +3 -3
- {prefect_client-3.1.11.dist-info → prefect_client-3.1.13.dist-info}/RECORD +133 -114
- {prefect_client-3.1.11.dist-info → prefect_client-3.1.13.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.11.dist-info → prefect_client-3.1.13.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.11.dist-info → prefect_client-3.1.13.dist-info}/top_level.txt +0 -0
prefect/flows.py
CHANGED
@@ -2,6 +2,8 @@
|
|
2
2
|
Module containing the base workflow class and decorator - for most use cases, using the [`@flow` decorator][prefect.flows.flow] is preferred.
|
3
3
|
"""
|
4
4
|
|
5
|
+
from __future__ import annotations
|
6
|
+
|
5
7
|
# This file requires type-checking with pyright because mypy does not yet support PEP612
|
6
8
|
# See https://github.com/python/mypy/issues/8645
|
7
9
|
import ast
|
@@ -27,6 +29,7 @@ from typing import (
|
|
27
29
|
Iterable,
|
28
30
|
NoReturn,
|
29
31
|
Optional,
|
32
|
+
Protocol,
|
30
33
|
Tuple,
|
31
34
|
Type,
|
32
35
|
TypeVar,
|
@@ -41,14 +44,14 @@ from pydantic.v1 import BaseModel as V1BaseModel
|
|
41
44
|
from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
|
42
45
|
from pydantic.v1.errors import ConfigError # TODO
|
43
46
|
from rich.console import Console
|
44
|
-
from typing_extensions import Literal, ParamSpec
|
47
|
+
from typing_extensions import Literal, ParamSpec
|
45
48
|
|
49
|
+
from prefect._experimental.sla.objects import SlaTypes
|
46
50
|
from prefect._internal.concurrency.api import create_call, from_async
|
47
51
|
from prefect.blocks.core import Block
|
48
52
|
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
49
|
-
from prefect.client.schemas.filters import WorkerFilter
|
53
|
+
from prefect.client.schemas.filters import WorkerFilter, WorkerFilterStatus
|
50
54
|
from prefect.client.schemas.objects import ConcurrencyLimitConfig, FlowRun
|
51
|
-
from prefect.client.schemas.objects import Flow as FlowSchema
|
52
55
|
from prefect.client.utilities import client_injector
|
53
56
|
from prefect.docker.docker_image import DockerImage
|
54
57
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
@@ -83,6 +86,7 @@ from prefect.utilities.asyncutils import (
|
|
83
86
|
sync_compatible,
|
84
87
|
)
|
85
88
|
from prefect.utilities.callables import (
|
89
|
+
ParameterSchema,
|
86
90
|
get_call_parameters,
|
87
91
|
parameter_schema,
|
88
92
|
parameters_to_args_kwargs,
|
@@ -105,19 +109,31 @@ R = TypeVar("R") # The return type of the user's function
|
|
105
109
|
P = ParamSpec("P") # The parameters of the flow
|
106
110
|
F = TypeVar("F", bound="Flow[Any, Any]") # The type of the flow
|
107
111
|
|
108
|
-
StateHookCallable: TypeAlias = Callable[
|
109
|
-
[FlowSchema, FlowRun, State], Union[Awaitable[None], None]
|
110
|
-
]
|
111
112
|
|
112
|
-
|
113
|
+
class FlowStateHook(Protocol, Generic[P, R]):
|
114
|
+
"""
|
115
|
+
A callable that is invoked when a flow enters a given state.
|
116
|
+
"""
|
117
|
+
|
118
|
+
__name__: str
|
119
|
+
|
120
|
+
def __call__(
|
121
|
+
self, flow: Flow[P, R], flow_run: FlowRun, state: State
|
122
|
+
) -> Awaitable[None] | None:
|
123
|
+
...
|
124
|
+
|
113
125
|
|
114
126
|
if TYPE_CHECKING:
|
127
|
+
import logging
|
128
|
+
|
115
129
|
from prefect.client.orchestration import PrefectClient
|
116
130
|
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
117
131
|
from prefect.deployments.runner import RunnerDeployment
|
118
132
|
from prefect.flows import FlowRun
|
119
133
|
from prefect.runner.storage import RunnerStorage
|
120
134
|
|
135
|
+
logger: "logging.Logger" = get_logger("flows")
|
136
|
+
|
121
137
|
|
122
138
|
class Flow(Generic[P, R]):
|
123
139
|
"""
|
@@ -187,7 +203,7 @@ class Flow(Generic[P, R]):
|
|
187
203
|
retries: Optional[int] = None,
|
188
204
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
189
205
|
task_runner: Union[
|
190
|
-
Type[TaskRunner[PrefectFuture[
|
206
|
+
Type[TaskRunner[PrefectFuture[Any]]], TaskRunner[PrefectFuture[Any]], None
|
191
207
|
] = None,
|
192
208
|
description: Optional[str] = None,
|
193
209
|
timeout_seconds: Union[int, float, None] = None,
|
@@ -197,13 +213,13 @@ class Flow(Generic[P, R]):
|
|
197
213
|
result_serializer: Optional[ResultSerializer] = None,
|
198
214
|
cache_result_in_memory: bool = True,
|
199
215
|
log_prints: Optional[bool] = None,
|
200
|
-
on_completion: Optional[list[
|
201
|
-
on_failure: Optional[list[
|
202
|
-
on_cancellation: Optional[list[
|
203
|
-
on_crashed: Optional[list[
|
204
|
-
on_running: Optional[list[
|
216
|
+
on_completion: Optional[list[FlowStateHook[P, R]]] = None,
|
217
|
+
on_failure: Optional[list[FlowStateHook[P, R]]] = None,
|
218
|
+
on_cancellation: Optional[list[FlowStateHook[P, R]]] = None,
|
219
|
+
on_crashed: Optional[list[FlowStateHook[P, R]]] = None,
|
220
|
+
on_running: Optional[list[FlowStateHook[P, R]]] = None,
|
205
221
|
):
|
206
|
-
if name is not None and not isinstance(name, str):
|
222
|
+
if name is not None and not isinstance(name, str): # pyright: ignore[reportUnnecessaryIsInstance]
|
207
223
|
raise TypeError(
|
208
224
|
"Expected string for flow parameter 'name'; got {} instead. {}".format(
|
209
225
|
type(name).__name__,
|
@@ -257,7 +273,7 @@ class Flow(Generic[P, R]):
|
|
257
273
|
if not callable(fn):
|
258
274
|
raise TypeError("'fn' must be callable")
|
259
275
|
|
260
|
-
self.name = name or fn.__name__.replace("_", "-").replace(
|
276
|
+
self.name: str = name or fn.__name__.replace("_", "-").replace(
|
261
277
|
"<lambda>",
|
262
278
|
"unknown-lambda", # prefect API will not accept "<" or ">" in flow names
|
263
279
|
)
|
@@ -271,57 +287,64 @@ class Flow(Generic[P, R]):
|
|
271
287
|
)
|
272
288
|
self.flow_run_name = flow_run_name
|
273
289
|
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
290
|
+
if task_runner is None:
|
291
|
+
self.task_runner: TaskRunner[PrefectFuture[Any]] = cast(
|
292
|
+
TaskRunner[PrefectFuture[Any]], ThreadPoolTaskRunner()
|
293
|
+
)
|
294
|
+
else:
|
295
|
+
self.task_runner: TaskRunner[PrefectFuture[Any]] = (
|
296
|
+
task_runner() if isinstance(task_runner, type) else task_runner
|
297
|
+
)
|
279
298
|
|
280
299
|
self.log_prints = log_prints
|
281
300
|
|
282
|
-
self.description = description or inspect.getdoc(fn)
|
301
|
+
self.description: str | None = description or inspect.getdoc(fn)
|
283
302
|
update_wrapper(self, fn)
|
284
303
|
self.fn = fn
|
285
304
|
|
286
305
|
# the flow is considered async if its function is async or an async
|
287
306
|
# generator
|
288
|
-
self.isasync = asyncio.iscoroutinefunction(
|
307
|
+
self.isasync: bool = asyncio.iscoroutinefunction(
|
289
308
|
self.fn
|
290
309
|
) or inspect.isasyncgenfunction(self.fn)
|
291
310
|
|
292
311
|
# the flow is considered a generator if its function is a generator or
|
293
312
|
# an async generator
|
294
|
-
self.isgenerator = inspect.isgeneratorfunction(
|
313
|
+
self.isgenerator: bool = inspect.isgeneratorfunction(
|
295
314
|
self.fn
|
296
315
|
) or inspect.isasyncgenfunction(self.fn)
|
297
316
|
|
298
317
|
raise_for_reserved_arguments(self.fn, ["return_state", "wait_for"])
|
299
318
|
|
300
319
|
# Version defaults to a hash of the function's file
|
301
|
-
flow_file = inspect.getsourcefile(self.fn)
|
302
320
|
if not version:
|
303
321
|
try:
|
322
|
+
flow_file = inspect.getsourcefile(self.fn)
|
323
|
+
if flow_file is None:
|
324
|
+
raise FileNotFoundError
|
304
325
|
version = file_hash(flow_file)
|
305
326
|
except (FileNotFoundError, TypeError, OSError):
|
306
327
|
pass # `getsourcefile` can return null values and "<stdin>" for objects in repls
|
307
328
|
self.version = version
|
308
329
|
|
309
|
-
self.timeout_seconds
|
330
|
+
self.timeout_seconds: float | None = (
|
331
|
+
float(timeout_seconds) if timeout_seconds else None
|
332
|
+
)
|
310
333
|
|
311
334
|
# FlowRunPolicy settings
|
312
335
|
# TODO: We can instantiate a `FlowRunPolicy` and add Pydantic bound checks to
|
313
336
|
# validate that the user passes positive numbers here
|
314
|
-
self.retries = (
|
337
|
+
self.retries: int = (
|
315
338
|
retries if retries is not None else PREFECT_FLOW_DEFAULT_RETRIES.value()
|
316
339
|
)
|
317
340
|
|
318
|
-
self.retry_delay_seconds = (
|
341
|
+
self.retry_delay_seconds: float | int = (
|
319
342
|
retry_delay_seconds
|
320
343
|
if retry_delay_seconds is not None
|
321
344
|
else PREFECT_FLOW_DEFAULT_RETRY_DELAY_SECONDS.value()
|
322
345
|
)
|
323
346
|
|
324
|
-
self.parameters = parameter_schema(self.fn)
|
347
|
+
self.parameters: ParameterSchema = parameter_schema(self.fn)
|
325
348
|
self.should_validate_parameters = validate_parameters
|
326
349
|
|
327
350
|
if self.should_validate_parameters:
|
@@ -352,11 +375,11 @@ class Flow(Generic[P, R]):
|
|
352
375
|
self.result_storage = result_storage
|
353
376
|
self.result_serializer = result_serializer
|
354
377
|
self.cache_result_in_memory = cache_result_in_memory
|
355
|
-
self.on_completion_hooks = on_completion or []
|
356
|
-
self.on_failure_hooks = on_failure or []
|
357
|
-
self.on_cancellation_hooks = on_cancellation or []
|
358
|
-
self.on_crashed_hooks = on_crashed or []
|
359
|
-
self.on_running_hooks = on_running or []
|
378
|
+
self.on_completion_hooks: list[FlowStateHook[P, R]] = on_completion or []
|
379
|
+
self.on_failure_hooks: list[FlowStateHook[P, R]] = on_failure or []
|
380
|
+
self.on_cancellation_hooks: list[FlowStateHook[P, R]] = on_cancellation or []
|
381
|
+
self.on_crashed_hooks: list[FlowStateHook[P, R]] = on_crashed or []
|
382
|
+
self.on_running_hooks: list[FlowStateHook[P, R]] = on_running or []
|
360
383
|
|
361
384
|
# Used for flows loaded from remote storage
|
362
385
|
self._storage: Optional["RunnerStorage"] = None
|
@@ -373,7 +396,7 @@ class Flow(Generic[P, R]):
|
|
373
396
|
def ismethod(self) -> bool:
|
374
397
|
return hasattr(self.fn, "__prefect_self__")
|
375
398
|
|
376
|
-
def __get__(self, instance: Any, owner: Any):
|
399
|
+
def __get__(self, instance: Any, owner: Any) -> "Flow[P, R]":
|
377
400
|
"""
|
378
401
|
Implement the descriptor protocol so that the flow can be used as an instance method.
|
379
402
|
When an instance method is loaded, this method is called with the "self" instance as
|
@@ -388,7 +411,7 @@ class Flow(Generic[P, R]):
|
|
388
411
|
# of the flow's function. This will allow it to be automatically added to the flow's parameters
|
389
412
|
else:
|
390
413
|
bound_flow = copy(self)
|
391
|
-
bound_flow.fn
|
414
|
+
setattr(bound_flow.fn, "__prefect_self__", instance)
|
392
415
|
return bound_flow
|
393
416
|
|
394
417
|
def with_options(
|
@@ -401,7 +424,7 @@ class Flow(Generic[P, R]):
|
|
401
424
|
description: Optional[str] = None,
|
402
425
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
403
426
|
task_runner: Union[
|
404
|
-
Type[TaskRunner[PrefectFuture[
|
427
|
+
Type[TaskRunner[PrefectFuture[Any]]], TaskRunner[PrefectFuture[Any]], None
|
405
428
|
] = None,
|
406
429
|
timeout_seconds: Union[int, float, None] = None,
|
407
430
|
validate_parameters: Optional[bool] = None,
|
@@ -410,11 +433,11 @@ class Flow(Generic[P, R]):
|
|
410
433
|
result_serializer: Optional[ResultSerializer] = NotSet, # type: ignore
|
411
434
|
cache_result_in_memory: Optional[bool] = None,
|
412
435
|
log_prints: Optional[bool] = NotSet, # type: ignore
|
413
|
-
on_completion: Optional[list[
|
414
|
-
on_failure: Optional[list[
|
415
|
-
on_cancellation: Optional[list[
|
416
|
-
on_crashed: Optional[list[
|
417
|
-
on_running: Optional[list[
|
436
|
+
on_completion: Optional[list[FlowStateHook[P, R]]] = None,
|
437
|
+
on_failure: Optional[list[FlowStateHook[P, R]]] = None,
|
438
|
+
on_cancellation: Optional[list[FlowStateHook[P, R]]] = None,
|
439
|
+
on_crashed: Optional[list[FlowStateHook[P, R]]] = None,
|
440
|
+
on_running: Optional[list[FlowStateHook[P, R]]] = None,
|
418
441
|
) -> "Flow[P, R]":
|
419
442
|
"""
|
420
443
|
Create a new flow from the current object, updating provided options.
|
@@ -470,13 +493,18 @@ class Flow(Generic[P, R]):
|
|
470
493
|
>>> state = my_flow.with_options(task_runner=ThreadPoolTaskRunner)(1, 3)
|
471
494
|
>>> assert state.result() == 4
|
472
495
|
"""
|
496
|
+
new_task_runner = (
|
497
|
+
task_runner() if isinstance(task_runner, type) else task_runner
|
498
|
+
)
|
499
|
+
if new_task_runner is None:
|
500
|
+
new_task_runner = self.task_runner
|
473
501
|
new_flow = Flow(
|
474
502
|
fn=self.fn,
|
475
503
|
name=name or self.name,
|
476
504
|
description=description or self.description,
|
477
505
|
flow_run_name=flow_run_name or self.flow_run_name,
|
478
506
|
version=version or self.version,
|
479
|
-
task_runner=
|
507
|
+
task_runner=new_task_runner,
|
480
508
|
retries=retries if retries is not None else self.retries,
|
481
509
|
retry_delay_seconds=(
|
482
510
|
retry_delay_seconds
|
@@ -530,7 +558,7 @@ class Flow(Generic[P, R]):
|
|
530
558
|
ParameterTypeError: if the provided parameters are not valid
|
531
559
|
"""
|
532
560
|
|
533
|
-
def resolve_block_reference(data: Any) -> Any:
|
561
|
+
def resolve_block_reference(data: Any | dict[str, Any]) -> Any:
|
534
562
|
if isinstance(data, dict) and "$ref" in data:
|
535
563
|
return Block.load_from_ref(data["$ref"], _sync=True)
|
536
564
|
return data
|
@@ -593,7 +621,9 @@ class Flow(Generic[P, R]):
|
|
593
621
|
}
|
594
622
|
return cast_parameters
|
595
623
|
|
596
|
-
def serialize_parameters(
|
624
|
+
def serialize_parameters(
|
625
|
+
self, parameters: dict[str, Any | PrefectFuture[Any] | State]
|
626
|
+
) -> dict[str, Any]:
|
597
627
|
"""
|
598
628
|
Convert parameters to a serializable form.
|
599
629
|
|
@@ -601,10 +631,10 @@ class Flow(Generic[P, R]):
|
|
601
631
|
converting everything directly to a string. This maintains basic types like
|
602
632
|
integers during API roundtrips.
|
603
633
|
"""
|
604
|
-
serialized_parameters = {}
|
634
|
+
serialized_parameters: dict[str, Any] = {}
|
605
635
|
for key, value in parameters.items():
|
606
636
|
# do not serialize the bound self object
|
607
|
-
if self.ismethod and value is self.fn
|
637
|
+
if self.ismethod and value is getattr(self.fn, "__prefect_self__", None):
|
608
638
|
continue
|
609
639
|
if isinstance(value, (PrefectFuture, State)):
|
610
640
|
# Don't call jsonable_encoder() on a PrefectFuture or State to
|
@@ -651,6 +681,7 @@ class Flow(Generic[P, R]):
|
|
651
681
|
work_queue_name: Optional[str] = None,
|
652
682
|
job_variables: Optional[dict[str, Any]] = None,
|
653
683
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
684
|
+
_sla: Optional[Union[SlaTypes, list[SlaTypes]]] = None, # experimental
|
654
685
|
) -> "RunnerDeployment":
|
655
686
|
"""
|
656
687
|
Creates a runner deployment object for this flow.
|
@@ -681,6 +712,7 @@ class Flow(Generic[P, R]):
|
|
681
712
|
of the chosen work pool. Refer to the base job template of the chosen work pool for
|
682
713
|
entrypoint_type: Type of entrypoint to use for the deployment. When using a module path
|
683
714
|
entrypoint, ensure that the module will be importable in the execution environment.
|
715
|
+
_sla: (Experimental) SLA configuration for the deployment. May be removed or modified at any time. Currently only supported on Prefect Cloud.
|
684
716
|
|
685
717
|
Examples:
|
686
718
|
Prepare two deployments and serve them:
|
@@ -728,6 +760,7 @@ class Flow(Generic[P, R]):
|
|
728
760
|
work_pool_name=work_pool_name,
|
729
761
|
work_queue_name=work_queue_name,
|
730
762
|
job_variables=job_variables,
|
763
|
+
_sla=_sla,
|
731
764
|
) # type: ignore # TODO: remove sync_compatible
|
732
765
|
else:
|
733
766
|
return RunnerDeployment.from_flow(
|
@@ -749,25 +782,26 @@ class Flow(Generic[P, R]):
|
|
749
782
|
work_queue_name=work_queue_name,
|
750
783
|
job_variables=job_variables,
|
751
784
|
entrypoint_type=entrypoint_type,
|
785
|
+
_sla=_sla,
|
752
786
|
)
|
753
787
|
|
754
|
-
def on_completion(self, fn:
|
788
|
+
def on_completion(self, fn: FlowStateHook[P, R]) -> FlowStateHook[P, R]:
|
755
789
|
self.on_completion_hooks.append(fn)
|
756
790
|
return fn
|
757
791
|
|
758
|
-
def on_cancellation(self, fn:
|
792
|
+
def on_cancellation(self, fn: FlowStateHook[P, R]) -> FlowStateHook[P, R]:
|
759
793
|
self.on_cancellation_hooks.append(fn)
|
760
794
|
return fn
|
761
795
|
|
762
|
-
def on_crashed(self, fn:
|
796
|
+
def on_crashed(self, fn: FlowStateHook[P, R]) -> FlowStateHook[P, R]:
|
763
797
|
self.on_crashed_hooks.append(fn)
|
764
798
|
return fn
|
765
799
|
|
766
|
-
def on_running(self, fn:
|
800
|
+
def on_running(self, fn: FlowStateHook[P, R]) -> FlowStateHook[P, R]:
|
767
801
|
self.on_running_hooks.append(fn)
|
768
802
|
return fn
|
769
803
|
|
770
|
-
def on_failure(self, fn:
|
804
|
+
def on_failure(self, fn: FlowStateHook[P, R]) -> FlowStateHook[P, R]:
|
771
805
|
self.on_failure_hooks.append(fn)
|
772
806
|
return fn
|
773
807
|
|
@@ -798,7 +832,7 @@ class Flow(Generic[P, R]):
|
|
798
832
|
limit: Optional[int] = None,
|
799
833
|
webserver: bool = False,
|
800
834
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
801
|
-
):
|
835
|
+
) -> None:
|
802
836
|
"""
|
803
837
|
Creates a deployment for this flow and starts a runner to monitor for scheduled work.
|
804
838
|
|
@@ -1061,6 +1095,7 @@ class Flow(Generic[P, R]):
|
|
1061
1095
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
1062
1096
|
print_next_steps: bool = True,
|
1063
1097
|
ignore_warnings: bool = False,
|
1098
|
+
_sla: Optional[Union[SlaTypes, list[SlaTypes]]] = None,
|
1064
1099
|
) -> UUID:
|
1065
1100
|
"""
|
1066
1101
|
Deploys a flow to run on dynamic infrastructure via a work pool.
|
@@ -1112,7 +1147,7 @@ class Flow(Generic[P, R]):
|
|
1112
1147
|
print_next_steps_message: Whether or not to print a message with next steps
|
1113
1148
|
after deploying the deployments.
|
1114
1149
|
ignore_warnings: Whether or not to ignore warnings about the work pool type.
|
1115
|
-
|
1150
|
+
_sla: (Experimental) SLA configuration for the deployment. May be removed or modified at any time. Currently only supported on Prefect Cloud.
|
1116
1151
|
Returns:
|
1117
1152
|
The ID of the created/updated deployment.
|
1118
1153
|
|
@@ -1165,7 +1200,9 @@ class Flow(Generic[P, R]):
|
|
1165
1200
|
work_pool = await client.read_work_pool(work_pool_name)
|
1166
1201
|
active_workers = await client.read_workers_for_work_pool(
|
1167
1202
|
work_pool_name,
|
1168
|
-
worker_filter=WorkerFilter(
|
1203
|
+
worker_filter=WorkerFilter(
|
1204
|
+
status=WorkerFilterStatus(any_=["ONLINE"])
|
1205
|
+
),
|
1169
1206
|
)
|
1170
1207
|
except ObjectNotFound as exc:
|
1171
1208
|
raise ValueError(
|
@@ -1173,7 +1210,7 @@ class Flow(Generic[P, R]):
|
|
1173
1210
|
" deploying this flow."
|
1174
1211
|
) from exc
|
1175
1212
|
|
1176
|
-
|
1213
|
+
to_deployment_coro = self.to_deployment(
|
1177
1214
|
name=name,
|
1178
1215
|
interval=interval,
|
1179
1216
|
cron=cron,
|
@@ -1190,11 +1227,17 @@ class Flow(Generic[P, R]):
|
|
1190
1227
|
work_queue_name=work_queue_name,
|
1191
1228
|
job_variables=job_variables,
|
1192
1229
|
entrypoint_type=entrypoint_type,
|
1230
|
+
_sla=_sla,
|
1193
1231
|
)
|
1194
1232
|
|
1233
|
+
if TYPE_CHECKING:
|
1234
|
+
assert inspect.isawaitable(to_deployment_coro)
|
1235
|
+
|
1236
|
+
deployment = await to_deployment_coro
|
1237
|
+
|
1195
1238
|
from prefect.deployments.runner import deploy
|
1196
1239
|
|
1197
|
-
|
1240
|
+
deploy_coro = deploy(
|
1198
1241
|
deployment,
|
1199
1242
|
work_pool_name=work_pool_name,
|
1200
1243
|
image=image,
|
@@ -1203,6 +1246,10 @@ class Flow(Generic[P, R]):
|
|
1203
1246
|
print_next_steps_message=False,
|
1204
1247
|
ignore_warnings=ignore_warnings,
|
1205
1248
|
)
|
1249
|
+
if TYPE_CHECKING:
|
1250
|
+
assert inspect.isawaitable(deploy_coro)
|
1251
|
+
|
1252
|
+
deployment_ids = await deploy_coro
|
1206
1253
|
|
1207
1254
|
if print_next_steps:
|
1208
1255
|
console = Console()
|
@@ -1433,11 +1480,11 @@ class FlowDecorator:
|
|
1433
1480
|
result_serializer: Optional[ResultSerializer] = None,
|
1434
1481
|
cache_result_in_memory: bool = True,
|
1435
1482
|
log_prints: Optional[bool] = None,
|
1436
|
-
on_completion: Optional[list[
|
1437
|
-
on_failure: Optional[list[
|
1438
|
-
on_cancellation: Optional[list[
|
1439
|
-
on_crashed: Optional[list[
|
1440
|
-
on_running: Optional[list[
|
1483
|
+
on_completion: Optional[list[FlowStateHook[..., Any]]] = None,
|
1484
|
+
on_failure: Optional[list[FlowStateHook[..., Any]]] = None,
|
1485
|
+
on_cancellation: Optional[list[FlowStateHook[..., Any]]] = None,
|
1486
|
+
on_crashed: Optional[list[FlowStateHook[..., Any]]] = None,
|
1487
|
+
on_running: Optional[list[FlowStateHook[..., Any]]] = None,
|
1441
1488
|
) -> Callable[[Callable[P, R]], Flow[P, R]]:
|
1442
1489
|
...
|
1443
1490
|
|
@@ -1460,11 +1507,11 @@ class FlowDecorator:
|
|
1460
1507
|
result_serializer: Optional[ResultSerializer] = None,
|
1461
1508
|
cache_result_in_memory: bool = True,
|
1462
1509
|
log_prints: Optional[bool] = None,
|
1463
|
-
on_completion: Optional[list[
|
1464
|
-
on_failure: Optional[list[
|
1465
|
-
on_cancellation: Optional[list[
|
1466
|
-
on_crashed: Optional[list[
|
1467
|
-
on_running: Optional[list[
|
1510
|
+
on_completion: Optional[list[FlowStateHook[..., Any]]] = None,
|
1511
|
+
on_failure: Optional[list[FlowStateHook[..., Any]]] = None,
|
1512
|
+
on_cancellation: Optional[list[FlowStateHook[..., Any]]] = None,
|
1513
|
+
on_crashed: Optional[list[FlowStateHook[..., Any]]] = None,
|
1514
|
+
on_running: Optional[list[FlowStateHook[..., Any]]] = None,
|
1468
1515
|
) -> Callable[[Callable[P, R]], Flow[P, R]]:
|
1469
1516
|
...
|
1470
1517
|
|
@@ -1486,11 +1533,11 @@ class FlowDecorator:
|
|
1486
1533
|
result_serializer: Optional[ResultSerializer] = None,
|
1487
1534
|
cache_result_in_memory: bool = True,
|
1488
1535
|
log_prints: Optional[bool] = None,
|
1489
|
-
on_completion: Optional[list[
|
1490
|
-
on_failure: Optional[list[
|
1491
|
-
on_cancellation: Optional[list[
|
1492
|
-
on_crashed: Optional[list[
|
1493
|
-
on_running: Optional[list[
|
1536
|
+
on_completion: Optional[list[FlowStateHook[..., Any]]] = None,
|
1537
|
+
on_failure: Optional[list[FlowStateHook[..., Any]]] = None,
|
1538
|
+
on_cancellation: Optional[list[FlowStateHook[..., Any]]] = None,
|
1539
|
+
on_crashed: Optional[list[FlowStateHook[..., Any]]] = None,
|
1540
|
+
on_running: Optional[list[FlowStateHook[..., Any]]] = None,
|
1494
1541
|
) -> Union[Flow[P, R], Callable[[Callable[P, R]], Flow[P, R]]]:
|
1495
1542
|
"""
|
1496
1543
|
Decorator to designate a function as a Prefect workflow.
|
@@ -1664,7 +1711,7 @@ class FlowDecorator:
|
|
1664
1711
|
...
|
1665
1712
|
|
1666
1713
|
|
1667
|
-
flow = FlowDecorator()
|
1714
|
+
flow: FlowDecorator = FlowDecorator()
|
1668
1715
|
|
1669
1716
|
|
1670
1717
|
def _raise_on_name_with_banned_characters(name: Optional[str]) -> Optional[str]:
|
@@ -1916,7 +1963,11 @@ async def aserve(
|
|
1916
1963
|
|
1917
1964
|
runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
|
1918
1965
|
for deployment in args:
|
1919
|
-
|
1966
|
+
add_deployment_coro = runner.add_deployment(deployment)
|
1967
|
+
if TYPE_CHECKING:
|
1968
|
+
assert inspect.isawaitable(add_deployment_coro)
|
1969
|
+
|
1970
|
+
await add_deployment_coro
|
1920
1971
|
|
1921
1972
|
if print_starting_message:
|
1922
1973
|
_display_serve_start_message(*args)
|
@@ -1962,13 +2013,16 @@ async def load_flow_from_flow_run(
|
|
1962
2013
|
ignore_storage: bool = False,
|
1963
2014
|
storage_base_path: Optional[str] = None,
|
1964
2015
|
use_placeholder_flow: bool = True,
|
1965
|
-
) -> Flow[
|
2016
|
+
) -> Flow[..., Any]:
|
1966
2017
|
"""
|
1967
2018
|
Load a flow from the location/script provided in a deployment's storage document.
|
1968
2019
|
|
1969
2020
|
If `ignore_storage=True` is provided, no pull from remote storage occurs. This flag
|
1970
2021
|
is largely for testing, and assumes the flow is already available locally.
|
1971
2022
|
"""
|
2023
|
+
if flow_run.deployment_id is None:
|
2024
|
+
raise ValueError("Flow run does not have an associated deployment")
|
2025
|
+
|
1972
2026
|
deployment = await client.read_deployment(flow_run.deployment_id)
|
1973
2027
|
|
1974
2028
|
if deployment.entrypoint is None:
|
prefect/futures.py
CHANGED
@@ -5,7 +5,7 @@ import threading
|
|
5
5
|
import uuid
|
6
6
|
from collections.abc import Generator, Iterator
|
7
7
|
from functools import partial
|
8
|
-
from typing import Any, Callable, Generic, Optional, Union
|
8
|
+
from typing import TYPE_CHECKING, Any, Callable, Generic, Optional, Union
|
9
9
|
|
10
10
|
from typing_extensions import NamedTuple, Self, TypeVar
|
11
11
|
|
@@ -22,7 +22,10 @@ from prefect.utilities.timeout import timeout as timeout_context
|
|
22
22
|
F = TypeVar("F")
|
23
23
|
R = TypeVar("R")
|
24
24
|
|
25
|
-
|
25
|
+
if TYPE_CHECKING:
|
26
|
+
import logging
|
27
|
+
|
28
|
+
logger: "logging.Logger" = get_logger(__name__)
|
26
29
|
|
27
30
|
|
28
31
|
class PrefectFuture(abc.ABC, Generic[R]):
|
@@ -90,7 +93,7 @@ class PrefectFuture(abc.ABC, Generic[R]):
|
|
90
93
|
"""
|
91
94
|
|
92
95
|
@abc.abstractmethod
|
93
|
-
def add_done_callback(self, fn: Callable[["PrefectFuture[R]"], None]):
|
96
|
+
def add_done_callback(self, fn: Callable[["PrefectFuture[R]"], None]) -> None:
|
94
97
|
"""
|
95
98
|
Add a callback to be run when the future completes or is cancelled.
|
96
99
|
|
@@ -173,7 +176,7 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future[
|
|
173
176
|
_result = run_coro_as_sync(_result)
|
174
177
|
return _result
|
175
178
|
|
176
|
-
def __del__(self):
|
179
|
+
def __del__(self) -> None:
|
177
180
|
if self._final_state or self._wrapped_future.done():
|
178
181
|
return
|
179
182
|
try:
|
@@ -202,7 +205,7 @@ class PrefectDistributedFuture(PrefectFuture[R]):
|
|
202
205
|
def wait(self, timeout: Optional[float] = None) -> None:
|
203
206
|
return run_coro_as_sync(self.wait_async(timeout=timeout))
|
204
207
|
|
205
|
-
async def wait_async(self, timeout: Optional[float] = None):
|
208
|
+
async def wait_async(self, timeout: Optional[float] = None) -> None:
|
206
209
|
if self._final_state:
|
207
210
|
logger.debug(
|
208
211
|
"Final state already set for %s. Returning...", self.task_run_id
|
@@ -216,6 +219,10 @@ class PrefectDistributedFuture(PrefectFuture[R]):
|
|
216
219
|
# Read task run to see if it is still running
|
217
220
|
async with get_client() as client:
|
218
221
|
task_run = await client.read_task_run(task_run_id=self._task_run_id)
|
222
|
+
if task_run.state is None:
|
223
|
+
raise RuntimeError(
|
224
|
+
f"Task run {self.task_run_id} has no state which means it hasn't started yet."
|
225
|
+
)
|
219
226
|
if task_run.state.is_final():
|
220
227
|
logger.debug(
|
221
228
|
"Task run %s already finished. Returning...",
|
@@ -260,7 +267,7 @@ class PrefectDistributedFuture(PrefectFuture[R]):
|
|
260
267
|
raise_on_failure=raise_on_failure, fetch=True
|
261
268
|
)
|
262
269
|
|
263
|
-
def add_done_callback(self, fn: Callable[[PrefectFuture[R]], None]):
|
270
|
+
def add_done_callback(self, fn: Callable[[PrefectFuture[R]], None]) -> None:
|
264
271
|
if self._final_state:
|
265
272
|
fn(self)
|
266
273
|
return
|
@@ -278,7 +285,7 @@ class PrefectDistributedFuture(PrefectFuture[R]):
|
|
278
285
|
return False
|
279
286
|
return self.task_run_id == other.task_run_id
|
280
287
|
|
281
|
-
def __hash__(self):
|
288
|
+
def __hash__(self) -> int:
|
282
289
|
return hash(self.task_run_id)
|
283
290
|
|
284
291
|
|
@@ -1,5 +1,6 @@
|
|
1
1
|
from typing import TYPE_CHECKING, Any, Dict, Optional, Protocol, Type
|
2
2
|
|
3
|
+
from prefect.infrastructure.provisioners.coiled import CoiledPushProvisioner
|
3
4
|
from prefect.infrastructure.provisioners.modal import ModalPushProvisioner
|
4
5
|
from .cloud_run import CloudRunPushProvisioner
|
5
6
|
from .container_instance import ContainerInstancePushProvisioner
|
@@ -15,6 +16,7 @@ _provisioners = {
|
|
15
16
|
"azure-container-instance:push": ContainerInstancePushProvisioner,
|
16
17
|
"ecs:push": ElasticContainerServicePushProvisioner,
|
17
18
|
"modal:push": ModalPushProvisioner,
|
19
|
+
"coiled:push": CoiledPushProvisioner,
|
18
20
|
}
|
19
21
|
|
20
22
|
|
@@ -33,7 +33,7 @@ if TYPE_CHECKING:
|
|
33
33
|
|
34
34
|
class CloudRunPushProvisioner:
|
35
35
|
def __init__(self):
|
36
|
-
self._console = Console()
|
36
|
+
self._console: Console = Console()
|
37
37
|
self._project = None
|
38
38
|
self._region = None
|
39
39
|
self._service_account_name = "prefect-cloud-run"
|
@@ -41,14 +41,14 @@ class CloudRunPushProvisioner:
|
|
41
41
|
self._image_repository_name = "prefect-images"
|
42
42
|
|
43
43
|
@property
|
44
|
-
def console(self):
|
44
|
+
def console(self) -> Console:
|
45
45
|
return self._console
|
46
46
|
|
47
47
|
@console.setter
|
48
|
-
def console(self, value):
|
48
|
+
def console(self, value: Console) -> None:
|
49
49
|
self._console = value
|
50
50
|
|
51
|
-
async def _run_command(self, command: str, *args, **kwargs):
|
51
|
+
async def _run_command(self, command: str, *args: Any, **kwargs: Any) -> str:
|
52
52
|
result = await run_process(shlex.split(command), check=False, *args, **kwargs)
|
53
53
|
|
54
54
|
if result.returncode != 0:
|