prefect-client 2.14.18__py3-none-any.whl → 2.14.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/client/orchestration.py +1 -1
- prefect/client/schemas/filters.py +16 -0
- prefect/client/schemas/objects.py +1 -0
- prefect/client/subscriptions.py +82 -0
- prefect/context.py +5 -1
- prefect/engine.py +83 -53
- prefect/filesystems.py +18 -3
- prefect/flows.py +1 -2
- prefect/input/run_input.py +115 -44
- prefect/results.py +24 -1
- prefect/runner/runner.py +3 -2
- prefect/settings.py +12 -0
- prefect/task_engine.py +70 -0
- prefect/task_server.py +208 -0
- prefect/tasks.py +21 -4
- {prefect_client-2.14.18.dist-info → prefect_client-2.14.20.dist-info}/METADATA +1 -1
- {prefect_client-2.14.18.dist-info → prefect_client-2.14.20.dist-info}/RECORD +20 -17
- {prefect_client-2.14.18.dist-info → prefect_client-2.14.20.dist-info}/LICENSE +0 -0
- {prefect_client-2.14.18.dist-info → prefect_client-2.14.20.dist-info}/WHEEL +0 -0
- {prefect_client-2.14.18.dist-info → prefect_client-2.14.20.dist-info}/top_level.txt +0 -0
prefect/client/orchestration.py
CHANGED
@@ -316,6 +316,19 @@ class FlowRunFilter(PrefectBaseModel, OperatorMixin):
|
|
316
316
|
)
|
317
317
|
|
318
318
|
|
319
|
+
class TaskRunFilterFlowRunId(PrefectBaseModel):
|
320
|
+
"""Filter by `TaskRun.flow_run_id`."""
|
321
|
+
|
322
|
+
any_: Optional[List[UUID]] = Field(
|
323
|
+
default=None, description="A list of flow run ids to include"
|
324
|
+
)
|
325
|
+
|
326
|
+
is_null_: bool = Field(
|
327
|
+
default=False,
|
328
|
+
description="If true, only include task runs without a flow run id",
|
329
|
+
)
|
330
|
+
|
331
|
+
|
319
332
|
class TaskRunFilterId(PrefectBaseModel):
|
320
333
|
"""Filter by `TaskRun.id`."""
|
321
334
|
|
@@ -428,6 +441,9 @@ class TaskRunFilter(PrefectBaseModel, OperatorMixin):
|
|
428
441
|
subflow_runs: Optional[TaskRunFilterSubFlowRuns] = Field(
|
429
442
|
default=None, description="Filter criteria for `TaskRun.subflow_run`"
|
430
443
|
)
|
444
|
+
flow_run_id: Optional[TaskRunFilterFlowRunId] = Field(
|
445
|
+
default=None, description="Filter criteria for `TaskRun.flow_run_id`"
|
446
|
+
)
|
431
447
|
|
432
448
|
|
433
449
|
class DeploymentFilterId(PrefectBaseModel):
|
@@ -0,0 +1,82 @@
|
|
1
|
+
import asyncio
|
2
|
+
from typing import Generic, Type, TypeVar
|
3
|
+
|
4
|
+
import orjson
|
5
|
+
import websockets
|
6
|
+
import websockets.exceptions
|
7
|
+
from starlette.status import WS_1008_POLICY_VIOLATION
|
8
|
+
from typing_extensions import Self
|
9
|
+
|
10
|
+
from prefect._internal.schemas.bases import IDBaseModel
|
11
|
+
from prefect.settings import PREFECT_API_KEY, PREFECT_API_URL
|
12
|
+
|
13
|
+
S = TypeVar("S", bound=IDBaseModel)
|
14
|
+
|
15
|
+
|
16
|
+
class Subscription(Generic[S]):
|
17
|
+
def __init__(self, model: Type[S], path: str):
|
18
|
+
self.model = model
|
19
|
+
|
20
|
+
base_url = PREFECT_API_URL.value().replace("http", "ws", 1)
|
21
|
+
self.subscription_url = f"{base_url}{path}"
|
22
|
+
|
23
|
+
self._connect = websockets.connect(
|
24
|
+
self.subscription_url,
|
25
|
+
subprotocols=["prefect"],
|
26
|
+
)
|
27
|
+
self._websocket = None
|
28
|
+
|
29
|
+
def __aiter__(self) -> Self:
|
30
|
+
return self
|
31
|
+
|
32
|
+
async def __anext__(self) -> S:
|
33
|
+
while True:
|
34
|
+
try:
|
35
|
+
await self._ensure_connected()
|
36
|
+
message = await self._websocket.recv()
|
37
|
+
|
38
|
+
message_data = orjson.loads(message)
|
39
|
+
|
40
|
+
if message_data.get("type") == "ping":
|
41
|
+
await self._websocket.send(orjson.dumps({"type": "pong"}).decode())
|
42
|
+
continue
|
43
|
+
|
44
|
+
return self.model.parse_raw(message)
|
45
|
+
except (
|
46
|
+
ConnectionRefusedError,
|
47
|
+
websockets.exceptions.ConnectionClosedError,
|
48
|
+
):
|
49
|
+
self._websocket = None
|
50
|
+
if hasattr(self._connect, "protocol"):
|
51
|
+
await self._connect.__aexit__(None, None, None)
|
52
|
+
await asyncio.sleep(0.5)
|
53
|
+
|
54
|
+
async def _ensure_connected(self):
|
55
|
+
if self._websocket:
|
56
|
+
return
|
57
|
+
|
58
|
+
websocket = await self._connect.__aenter__()
|
59
|
+
|
60
|
+
await websocket.send(
|
61
|
+
orjson.dumps({"type": "auth", "token": PREFECT_API_KEY.value()}).decode()
|
62
|
+
)
|
63
|
+
|
64
|
+
try:
|
65
|
+
auth = orjson.loads(await websocket.recv())
|
66
|
+
assert auth["type"] == "auth_success"
|
67
|
+
except (
|
68
|
+
AssertionError,
|
69
|
+
websockets.exceptions.ConnectionClosedError,
|
70
|
+
) as e:
|
71
|
+
if isinstance(e, AssertionError) or e.code == WS_1008_POLICY_VIOLATION:
|
72
|
+
raise Exception(
|
73
|
+
"Unable to authenticate to the subscription. Please "
|
74
|
+
"ensure the provided `PREFECT_API_KEY` you are using is "
|
75
|
+
"valid for this environment."
|
76
|
+
) from e
|
77
|
+
raise
|
78
|
+
else:
|
79
|
+
self._websocket = websocket
|
80
|
+
|
81
|
+
def __repr__(self) -> str:
|
82
|
+
return f"{type(self).__name__}[{self.model.__name__}]"
|
prefect/context.py
CHANGED
@@ -214,7 +214,7 @@ class RunContext(ContextModel):
|
|
214
214
|
client: PrefectClient
|
215
215
|
|
216
216
|
|
217
|
-
class
|
217
|
+
class EngineContext(RunContext):
|
218
218
|
"""
|
219
219
|
The context for a flow run. Data in this context is only available from within a
|
220
220
|
flow run function.
|
@@ -233,6 +233,7 @@ class FlowRunContext(RunContext):
|
|
233
233
|
|
234
234
|
flow: Optional["Flow"] = None
|
235
235
|
flow_run: Optional[FlowRun] = None
|
236
|
+
autonomous_task_run: Optional[TaskRun] = None
|
236
237
|
task_runner: BaseTaskRunner
|
237
238
|
log_prints: bool = False
|
238
239
|
parameters: Dict[str, Any]
|
@@ -266,6 +267,9 @@ class FlowRunContext(RunContext):
|
|
266
267
|
__var__ = ContextVar("flow_run")
|
267
268
|
|
268
269
|
|
270
|
+
FlowRunContext = EngineContext # for backwards compatibility
|
271
|
+
|
272
|
+
|
269
273
|
class TaskRunContext(RunContext):
|
270
274
|
"""
|
271
275
|
The context for a task run. Data in this context is only available from within a
|
prefect/engine.py
CHANGED
@@ -155,7 +155,7 @@ from prefect.exceptions import (
|
|
155
155
|
)
|
156
156
|
from prefect.flows import Flow, load_flow_from_entrypoint
|
157
157
|
from prefect.futures import PrefectFuture, call_repr, resolve_futures_to_states
|
158
|
-
from prefect.input import
|
158
|
+
from prefect.input import keyset_from_paused_state
|
159
159
|
from prefect.input.run_input import run_input_subclass_from_type
|
160
160
|
from prefect.logging.configuration import setup_logging
|
161
161
|
from prefect.logging.handlers import APILogHandler
|
@@ -169,6 +169,7 @@ from prefect.logging.loggers import (
|
|
169
169
|
from prefect.results import BaseResult, ResultFactory, UnknownResult
|
170
170
|
from prefect.settings import (
|
171
171
|
PREFECT_DEBUG_MODE,
|
172
|
+
PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING,
|
172
173
|
PREFECT_LOGGING_LOG_PRINTS,
|
173
174
|
PREFECT_TASK_INTROSPECTION_WARN_THRESHOLD,
|
174
175
|
PREFECT_TASKS_REFRESH_CACHE,
|
@@ -179,6 +180,7 @@ from prefect.states import (
|
|
179
180
|
Paused,
|
180
181
|
Pending,
|
181
182
|
Running,
|
183
|
+
Scheduled,
|
182
184
|
State,
|
183
185
|
Suspended,
|
184
186
|
exception_to_crashed_state,
|
@@ -210,9 +212,10 @@ from prefect.utilities.pydantic import PartialModel
|
|
210
212
|
from prefect.utilities.text import truncated_to
|
211
213
|
|
212
214
|
R = TypeVar("R")
|
213
|
-
T = TypeVar("T"
|
215
|
+
T = TypeVar("T")
|
214
216
|
EngineReturnType = Literal["future", "state", "result"]
|
215
217
|
|
218
|
+
NUM_CHARS_DYNAMIC_KEY = 8
|
216
219
|
|
217
220
|
API_HEALTHCHECKS = {}
|
218
221
|
UNTRACKABLE_TYPES = {bool, type(None), type(...), type(NotImplemented)}
|
@@ -984,18 +987,6 @@ async def pause_flow_run(
|
|
984
987
|
...
|
985
988
|
|
986
989
|
|
987
|
-
@overload
|
988
|
-
async def pause_flow_run(
|
989
|
-
wait_for_input: Type[Any],
|
990
|
-
flow_run_id: UUID = None,
|
991
|
-
timeout: int = 3600,
|
992
|
-
poll_interval: int = 10,
|
993
|
-
reschedule: bool = False,
|
994
|
-
key: str = None,
|
995
|
-
) -> Any:
|
996
|
-
...
|
997
|
-
|
998
|
-
|
999
990
|
@sync_compatible
|
1000
991
|
@deprecated_parameter(
|
1001
992
|
"flow_run_id", start_date="Dec 2023", help="Use `suspend_flow_run` instead."
|
@@ -1010,13 +1001,13 @@ async def pause_flow_run(
|
|
1010
1001
|
"wait_for_input", group="flow_run_input", when=lambda y: y is not None
|
1011
1002
|
)
|
1012
1003
|
async def pause_flow_run(
|
1013
|
-
wait_for_input: Optional[
|
1004
|
+
wait_for_input: Optional[Type[T]] = None,
|
1014
1005
|
flow_run_id: UUID = None,
|
1015
1006
|
timeout: int = 3600,
|
1016
1007
|
poll_interval: int = 10,
|
1017
1008
|
reschedule: bool = False,
|
1018
1009
|
key: str = None,
|
1019
|
-
):
|
1010
|
+
) -> Optional[T]:
|
1020
1011
|
"""
|
1021
1012
|
Pauses the current flow run by blocking execution until resumed.
|
1022
1013
|
|
@@ -1099,8 +1090,8 @@ async def _in_process_pause(
|
|
1099
1090
|
reschedule=False,
|
1100
1091
|
key: str = None,
|
1101
1092
|
client=None,
|
1102
|
-
wait_for_input: Optional[
|
1103
|
-
) -> Optional[
|
1093
|
+
wait_for_input: Optional[T] = None,
|
1094
|
+
) -> Optional[T]:
|
1104
1095
|
if TaskRunContext.get():
|
1105
1096
|
raise RuntimeError("Cannot pause task runs.")
|
1106
1097
|
|
@@ -1231,29 +1222,18 @@ async def suspend_flow_run(
|
|
1231
1222
|
...
|
1232
1223
|
|
1233
1224
|
|
1234
|
-
@overload
|
1235
|
-
async def suspend_flow_run(
|
1236
|
-
wait_for_input: Type[Any],
|
1237
|
-
flow_run_id: Optional[UUID] = None,
|
1238
|
-
timeout: Optional[int] = 3600,
|
1239
|
-
key: Optional[str] = None,
|
1240
|
-
client: PrefectClient = None,
|
1241
|
-
) -> Any:
|
1242
|
-
...
|
1243
|
-
|
1244
|
-
|
1245
1225
|
@sync_compatible
|
1246
1226
|
@inject_client
|
1247
1227
|
@experimental_parameter(
|
1248
1228
|
"wait_for_input", group="flow_run_input", when=lambda y: y is not None
|
1249
1229
|
)
|
1250
1230
|
async def suspend_flow_run(
|
1251
|
-
wait_for_input: Optional[
|
1231
|
+
wait_for_input: Optional[Type[T]] = None,
|
1252
1232
|
flow_run_id: Optional[UUID] = None,
|
1253
1233
|
timeout: Optional[int] = 3600,
|
1254
1234
|
key: Optional[str] = None,
|
1255
1235
|
client: PrefectClient = None,
|
1256
|
-
):
|
1236
|
+
) -> Optional[T]:
|
1257
1237
|
"""
|
1258
1238
|
Suspends a flow run by stopping code execution until resumed.
|
1259
1239
|
|
@@ -1382,16 +1362,19 @@ def enter_task_run_engine(
|
|
1382
1362
|
return_type: EngineReturnType,
|
1383
1363
|
task_runner: Optional[BaseTaskRunner],
|
1384
1364
|
mapped: bool,
|
1385
|
-
) -> Union[PrefectFuture, Awaitable[PrefectFuture]]:
|
1386
|
-
"""
|
1387
|
-
Sync entrypoint for task calls
|
1388
|
-
"""
|
1365
|
+
) -> Union[PrefectFuture, Awaitable[PrefectFuture], TaskRun]:
|
1366
|
+
"""Sync entrypoint for task calls"""
|
1389
1367
|
|
1390
1368
|
flow_run_context = FlowRunContext.get()
|
1369
|
+
|
1391
1370
|
if not flow_run_context:
|
1371
|
+
if PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING.value():
|
1372
|
+
return _create_autonomous_task_run(task=task, parameters=parameters)
|
1373
|
+
|
1392
1374
|
raise RuntimeError(
|
1393
|
-
"Tasks cannot be run outside of a flow
|
1394
|
-
"
|
1375
|
+
"Tasks cannot be run outside of a flow"
|
1376
|
+
" - if you meant to submit an autonomous task, you need to set"
|
1377
|
+
" `prefect config set PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING=true`"
|
1395
1378
|
)
|
1396
1379
|
|
1397
1380
|
if TaskRunContext.get():
|
@@ -1603,14 +1586,22 @@ async def create_task_run_future(
|
|
1603
1586
|
|
1604
1587
|
# Generate a name for the future
|
1605
1588
|
dynamic_key = _dynamic_key_for_task_run(flow_run_context, task)
|
1606
|
-
task_run_name =
|
1589
|
+
task_run_name = (
|
1590
|
+
f"{task.name}-{dynamic_key}"
|
1591
|
+
if flow_run_context and flow_run_context.flow_run
|
1592
|
+
else f"{task.name}-{dynamic_key[:NUM_CHARS_DYNAMIC_KEY]}" # autonomous task run
|
1593
|
+
)
|
1607
1594
|
|
1608
1595
|
# Generate a future
|
1609
1596
|
future = PrefectFuture(
|
1610
1597
|
name=task_run_name,
|
1611
1598
|
key=uuid4(),
|
1612
1599
|
task_runner=task_runner,
|
1613
|
-
asynchronous=
|
1600
|
+
asynchronous=(
|
1601
|
+
task.isasync and flow_run_context.flow.isasync
|
1602
|
+
if flow_run_context and flow_run_context.flow
|
1603
|
+
else task.isasync
|
1604
|
+
),
|
1614
1605
|
)
|
1615
1606
|
|
1616
1607
|
# Create and submit the task run in the background
|
@@ -1650,14 +1641,18 @@ async def create_task_run_then_submit(
|
|
1650
1641
|
task_runner: BaseTaskRunner,
|
1651
1642
|
extra_task_inputs: Dict[str, Set[TaskRunInput]],
|
1652
1643
|
) -> None:
|
1653
|
-
task_run =
|
1654
|
-
|
1655
|
-
|
1656
|
-
|
1657
|
-
|
1658
|
-
|
1659
|
-
|
1660
|
-
|
1644
|
+
task_run = (
|
1645
|
+
await create_task_run(
|
1646
|
+
task=task,
|
1647
|
+
name=task_run_name,
|
1648
|
+
flow_run_context=flow_run_context,
|
1649
|
+
parameters=parameters,
|
1650
|
+
dynamic_key=task_run_dynamic_key,
|
1651
|
+
wait_for=wait_for,
|
1652
|
+
extra_task_inputs=extra_task_inputs,
|
1653
|
+
)
|
1654
|
+
if not flow_run_context.autonomous_task_run
|
1655
|
+
else flow_run_context.autonomous_task_run
|
1661
1656
|
)
|
1662
1657
|
|
1663
1658
|
# Attach the task run to the future to support `get_state` operations
|
@@ -1698,7 +1693,7 @@ async def create_task_run(
|
|
1698
1693
|
task_run = await flow_run_context.client.create_task_run(
|
1699
1694
|
task=task,
|
1700
1695
|
name=name,
|
1701
|
-
flow_run_id=flow_run_context.flow_run.id,
|
1696
|
+
flow_run_id=flow_run_context.flow_run.id if flow_run_context.flow_run else None,
|
1702
1697
|
dynamic_key=dynamic_key,
|
1703
1698
|
state=Pending(),
|
1704
1699
|
extra_tags=TagsContext.get().current_tags,
|
@@ -1721,7 +1716,10 @@ async def submit_task_run(
|
|
1721
1716
|
) -> PrefectFuture:
|
1722
1717
|
logger = get_run_logger(flow_run_context)
|
1723
1718
|
|
1724
|
-
if
|
1719
|
+
if (
|
1720
|
+
task_runner.concurrency_type == TaskConcurrencyType.SEQUENTIAL
|
1721
|
+
and not flow_run_context.autonomous_task_run
|
1722
|
+
):
|
1725
1723
|
logger.info(f"Executing {task_run.name!r} immediately...")
|
1726
1724
|
|
1727
1725
|
future = await task_runner.submit(
|
@@ -1799,7 +1797,7 @@ async def begin_task_run(
|
|
1799
1797
|
# worker, the flow run timeout will not be raised in the worker process.
|
1800
1798
|
interruptible = maybe_flow_run_context.timeout_scope is not None
|
1801
1799
|
else:
|
1802
|
-
# Otherwise, retrieve a new
|
1800
|
+
# Otherwise, retrieve a new clien`t
|
1803
1801
|
client = await stack.enter_async_context(get_client())
|
1804
1802
|
interruptible = False
|
1805
1803
|
await stack.enter_async_context(anyio.create_task_group())
|
@@ -2153,7 +2151,6 @@ async def orchestrate_task_run(
|
|
2153
2151
|
await _check_task_failure_retriable(task, task_run, terminal_state)
|
2154
2152
|
)
|
2155
2153
|
state = await propose_state(client, terminal_state, task_run_id=task_run.id)
|
2156
|
-
|
2157
2154
|
last_event = _emit_task_run_state_change_event(
|
2158
2155
|
task_run=task_run,
|
2159
2156
|
initial_state=last_state,
|
@@ -2203,7 +2200,7 @@ async def orchestrate_task_run(
|
|
2203
2200
|
level=logging.INFO if state.is_completed() else logging.ERROR,
|
2204
2201
|
msg=f"Finished in state {display_state}",
|
2205
2202
|
)
|
2206
|
-
|
2203
|
+
logger.warning(f"Task run {task_run.name!r} finished in state {display_state}")
|
2207
2204
|
return state
|
2208
2205
|
|
2209
2206
|
|
@@ -2572,7 +2569,12 @@ async def propose_state(
|
|
2572
2569
|
|
2573
2570
|
|
2574
2571
|
def _dynamic_key_for_task_run(context: FlowRunContext, task: Task) -> int:
|
2575
|
-
if
|
2572
|
+
if context.flow_run is None: # this is an autonomous task run
|
2573
|
+
context.task_run_dynamic_keys[task.task_key] = getattr(
|
2574
|
+
task, "dynamic_key", str(uuid4())
|
2575
|
+
)
|
2576
|
+
|
2577
|
+
elif task.task_key not in context.task_run_dynamic_keys:
|
2576
2578
|
context.task_run_dynamic_keys[task.task_key] = 0
|
2577
2579
|
else:
|
2578
2580
|
context.task_run_dynamic_keys[task.task_key] += 1
|
@@ -2912,6 +2914,34 @@ def _emit_task_run_state_change_event(
|
|
2912
2914
|
)
|
2913
2915
|
|
2914
2916
|
|
2917
|
+
@sync_compatible
|
2918
|
+
async def _create_autonomous_task_run(
|
2919
|
+
task: Task, parameters: Dict[str, Any]
|
2920
|
+
) -> TaskRun:
|
2921
|
+
async with get_client() as client:
|
2922
|
+
scheduled = Scheduled()
|
2923
|
+
if parameters:
|
2924
|
+
parameters_id = uuid4()
|
2925
|
+
scheduled.state_details.task_parameters_id = parameters_id
|
2926
|
+
|
2927
|
+
# TODO: We want to use result storage for parameters, but we'll need
|
2928
|
+
# a better way to use it than this.
|
2929
|
+
task.persist_result = True
|
2930
|
+
factory = await ResultFactory.from_task(task, client=client)
|
2931
|
+
await factory.store_parameters(parameters_id, parameters)
|
2932
|
+
|
2933
|
+
task_run = await client.create_task_run(
|
2934
|
+
task=task,
|
2935
|
+
flow_run_id=None,
|
2936
|
+
dynamic_key=f"{task.task_key}-{str(uuid4())[:NUM_CHARS_DYNAMIC_KEY]}",
|
2937
|
+
state=scheduled,
|
2938
|
+
)
|
2939
|
+
|
2940
|
+
engine_logger.debug(f"Submitted run of task {task.name!r} for execution")
|
2941
|
+
|
2942
|
+
return task_run
|
2943
|
+
|
2944
|
+
|
2915
2945
|
if __name__ == "__main__":
|
2916
2946
|
try:
|
2917
2947
|
flow_run_id = UUID(
|
prefect/filesystems.py
CHANGED
@@ -683,12 +683,27 @@ class Azure(WritableFileSystem, WritableDeploymentStorage):
|
|
683
683
|
" require ADLFS to use DefaultAzureCredentials."
|
684
684
|
),
|
685
685
|
)
|
686
|
-
|
686
|
+
azure_storage_container: Optional[SecretStr] = Field(
|
687
|
+
default=None,
|
688
|
+
title="Azure storage container",
|
689
|
+
description=(
|
690
|
+
"Blob Container in Azure Storage Account. If set the 'bucket_path' will"
|
691
|
+
" be interpreted using the following URL format:"
|
692
|
+
"'az://<container>@<storage_account>.dfs.core.windows.net/<bucket_path>'."
|
693
|
+
),
|
694
|
+
)
|
687
695
|
_remote_file_system: RemoteFileSystem = None
|
688
696
|
|
689
697
|
@property
|
690
698
|
def basepath(self) -> str:
|
691
|
-
|
699
|
+
if self.azure_storage_container:
|
700
|
+
return (
|
701
|
+
f"az://{self.azure_storage_container.get_secret_value()}"
|
702
|
+
f"@{self.azure_storage_account_name.get_secret_value()}"
|
703
|
+
f".dfs.core.windows.net/{self.bucket_path}"
|
704
|
+
)
|
705
|
+
else:
|
706
|
+
return f"az://{self.bucket_path}"
|
692
707
|
|
693
708
|
@property
|
694
709
|
def filesystem(self) -> RemoteFileSystem:
|
@@ -713,7 +728,7 @@ class Azure(WritableFileSystem, WritableDeploymentStorage):
|
|
713
728
|
)
|
714
729
|
settings["anon"] = self.azure_storage_anon
|
715
730
|
self._remote_file_system = RemoteFileSystem(
|
716
|
-
basepath=
|
731
|
+
basepath=self.basepath, settings=settings
|
717
732
|
)
|
718
733
|
return self._remote_file_system
|
719
734
|
|
prefect/flows.py
CHANGED
@@ -65,7 +65,6 @@ else:
|
|
65
65
|
V2ValidationError = None
|
66
66
|
|
67
67
|
from rich.console import Console
|
68
|
-
from rich.panel import Panel
|
69
68
|
from typing_extensions import Literal, ParamSpec
|
70
69
|
|
71
70
|
from prefect._internal.schemas.validators import raise_on_name_with_banned_characters
|
@@ -785,7 +784,7 @@ class Flow(Generic[P, R]):
|
|
785
784
|
)
|
786
785
|
|
787
786
|
console = Console()
|
788
|
-
console.print(
|
787
|
+
console.print(help_message, soft_wrap=True)
|
789
788
|
await runner.start(webserver=webserver)
|
790
789
|
|
791
790
|
@classmethod
|
prefect/input/run_input.py
CHANGED
@@ -70,6 +70,8 @@ from typing import (
|
|
70
70
|
Type,
|
71
71
|
TypeVar,
|
72
72
|
Union,
|
73
|
+
cast,
|
74
|
+
overload,
|
73
75
|
)
|
74
76
|
from uuid import UUID, uuid4
|
75
77
|
|
@@ -93,8 +95,12 @@ if TYPE_CHECKING:
|
|
93
95
|
if HAS_PYDANTIC_V2:
|
94
96
|
from prefect._internal.pydantic.v2_schema import create_v2_schema
|
95
97
|
|
96
|
-
|
97
|
-
|
98
|
+
R = TypeVar("R", bound="RunInput")
|
99
|
+
T = TypeVar("T")
|
100
|
+
|
101
|
+
Keyset = Dict[
|
102
|
+
Union[Literal["description"], Literal["response"], Literal["schema"]], str
|
103
|
+
]
|
98
104
|
|
99
105
|
|
100
106
|
def keyset_from_paused_state(state: "State") -> Keyset:
|
@@ -123,6 +129,7 @@ def keyset_from_base_key(base_key: str) -> Keyset:
|
|
123
129
|
- Dict[str, str]: the keyset
|
124
130
|
"""
|
125
131
|
return {
|
132
|
+
"description": f"{base_key}-description",
|
126
133
|
"response": f"{base_key}-response",
|
127
134
|
"schema": f"{base_key}-schema",
|
128
135
|
}
|
@@ -138,6 +145,7 @@ class RunInput(pydantic.BaseModel):
|
|
138
145
|
class Config:
|
139
146
|
extra = "forbid"
|
140
147
|
|
148
|
+
_description: Optional[str] = pydantic.PrivateAttr(default=None)
|
141
149
|
_metadata: RunInputMetadata = pydantic.PrivateAttr()
|
142
150
|
|
143
151
|
@property
|
@@ -168,6 +176,14 @@ class RunInput(pydantic.BaseModel):
|
|
168
176
|
key=keyset["schema"], value=schema, flow_run_id=flow_run_id
|
169
177
|
)
|
170
178
|
|
179
|
+
description = cls._description if isinstance(cls._description, str) else None
|
180
|
+
if description:
|
181
|
+
await create_flow_run_input(
|
182
|
+
key=keyset["description"],
|
183
|
+
value=description,
|
184
|
+
flow_run_id=flow_run_id,
|
185
|
+
)
|
186
|
+
|
171
187
|
@classmethod
|
172
188
|
@sync_compatible
|
173
189
|
async def load(cls, keyset: Keyset, flow_run_id: Optional[UUID] = None):
|
@@ -206,18 +222,27 @@ class RunInput(pydantic.BaseModel):
|
|
206
222
|
return instance
|
207
223
|
|
208
224
|
@classmethod
|
209
|
-
def with_initial_data(
|
225
|
+
def with_initial_data(
|
226
|
+
cls: Type[R], description: Optional[str] = None, **kwargs: Any
|
227
|
+
) -> Type[R]:
|
210
228
|
"""
|
211
229
|
Create a new `RunInput` subclass with the given initial data as field
|
212
230
|
defaults.
|
213
231
|
|
214
232
|
Args:
|
215
|
-
-
|
233
|
+
- description (str, optional): a description to show when resuming
|
234
|
+
a flow run that requires input
|
235
|
+
- kwargs (Any): the initial data to populate the subclass
|
216
236
|
"""
|
217
237
|
fields = {}
|
218
238
|
for key, value in kwargs.items():
|
219
239
|
fields[key] = (type(value), value)
|
220
|
-
|
240
|
+
model = pydantic.create_model(cls.__name__, **fields, __base__=cls)
|
241
|
+
|
242
|
+
if description is not None:
|
243
|
+
model._description = description
|
244
|
+
|
245
|
+
return model
|
221
246
|
|
222
247
|
@sync_compatible
|
223
248
|
async def respond(
|
@@ -295,12 +320,12 @@ class RunInput(pydantic.BaseModel):
|
|
295
320
|
return type(f"{model_cls.__name__}RunInput", (RunInput, model_cls), {}) # type: ignore
|
296
321
|
|
297
322
|
|
298
|
-
class AutomaticRunInput(RunInput):
|
299
|
-
value:
|
323
|
+
class AutomaticRunInput(RunInput, Generic[T]):
|
324
|
+
value: T
|
300
325
|
|
301
326
|
@classmethod
|
302
327
|
@sync_compatible
|
303
|
-
async def load(cls, keyset: Keyset, flow_run_id: Optional[UUID] = None):
|
328
|
+
async def load(cls, keyset: Keyset, flow_run_id: Optional[UUID] = None) -> T:
|
304
329
|
"""
|
305
330
|
Load the run input response from the given key.
|
306
331
|
|
@@ -312,7 +337,7 @@ class AutomaticRunInput(RunInput):
|
|
312
337
|
return instance.value
|
313
338
|
|
314
339
|
@classmethod
|
315
|
-
def subclass_from_type(cls, _type: Type[T]) -> Type["AutomaticRunInput"]:
|
340
|
+
def subclass_from_type(cls, _type: Type[T]) -> Type["AutomaticRunInput[T]"]:
|
316
341
|
"""
|
317
342
|
Create a new `AutomaticRunInput` subclass from the given type.
|
318
343
|
"""
|
@@ -353,33 +378,30 @@ class AutomaticRunInput(RunInput):
|
|
353
378
|
return GetAutomaticInputHandler(run_input_cls=cls, *args, **kwargs)
|
354
379
|
|
355
380
|
|
356
|
-
def run_input_subclass_from_type(
|
381
|
+
def run_input_subclass_from_type(
|
382
|
+
_type: Union[Type[R], Type[T], pydantic.BaseModel]
|
383
|
+
) -> Union[Type[AutomaticRunInput[T]], Type[R]]:
|
357
384
|
"""
|
358
385
|
Create a new `RunInput` subclass from the given type.
|
359
386
|
"""
|
360
387
|
try:
|
361
|
-
|
388
|
+
if issubclass(_type, RunInput):
|
389
|
+
return cast(Type[R], _type)
|
390
|
+
elif issubclass(_type, pydantic.BaseModel):
|
391
|
+
return cast(Type[R], RunInput.subclass_from_base_model_type(_type))
|
362
392
|
except TypeError:
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
if issubclass(_type, RunInput):
|
370
|
-
return _type
|
371
|
-
elif issubclass(_type, pydantic.BaseModel):
|
372
|
-
return RunInput.subclass_from_base_model_type(_type)
|
373
|
-
else:
|
374
|
-
# As a fall-through for a type that isn't a `RunInput` subclass or
|
375
|
-
# `pydantic.BaseModel` subclass, pass it through to Pydantic.
|
376
|
-
return AutomaticRunInput.subclass_from_type(_type)
|
393
|
+
pass
|
394
|
+
|
395
|
+
# Could be something like a typing._GenericAlias or any other type that
|
396
|
+
# isn't a `RunInput` subclass or `pydantic.BaseModel` subclass. Try passing
|
397
|
+
# it to AutomaticRunInput to see if we can create a model from it.
|
398
|
+
return cast(Type[AutomaticRunInput[T]], AutomaticRunInput.subclass_from_type(_type))
|
377
399
|
|
378
400
|
|
379
|
-
class GetInputHandler(Generic[
|
401
|
+
class GetInputHandler(Generic[R]):
|
380
402
|
def __init__(
|
381
403
|
self,
|
382
|
-
run_input_cls: Type[
|
404
|
+
run_input_cls: Type[R],
|
383
405
|
key_prefix: str,
|
384
406
|
timeout: Optional[float] = 3600,
|
385
407
|
poll_interval: float = 10,
|
@@ -401,7 +423,7 @@ class GetInputHandler(Generic[T]):
|
|
401
423
|
def __iter__(self):
|
402
424
|
return self
|
403
425
|
|
404
|
-
def __next__(self) ->
|
426
|
+
def __next__(self) -> R:
|
405
427
|
try:
|
406
428
|
return self.next()
|
407
429
|
except TimeoutError:
|
@@ -412,7 +434,7 @@ class GetInputHandler(Generic[T]):
|
|
412
434
|
def __aiter__(self):
|
413
435
|
return self
|
414
436
|
|
415
|
-
async def __anext__(self) ->
|
437
|
+
async def __anext__(self) -> R:
|
416
438
|
try:
|
417
439
|
return await self.next()
|
418
440
|
except TimeoutError:
|
@@ -433,11 +455,11 @@ class GetInputHandler(Generic[T]):
|
|
433
455
|
|
434
456
|
return flow_run_inputs
|
435
457
|
|
436
|
-
def to_instance(self, flow_run_input: "FlowRunInput") ->
|
458
|
+
def to_instance(self, flow_run_input: "FlowRunInput") -> R:
|
437
459
|
return self.run_input_cls.load_from_flow_run_input(flow_run_input)
|
438
460
|
|
439
461
|
@sync_compatible
|
440
|
-
async def next(self) ->
|
462
|
+
async def next(self) -> R:
|
441
463
|
flow_run_inputs = await self.filter_for_inputs()
|
442
464
|
if flow_run_inputs:
|
443
465
|
return self.to_instance(flow_run_inputs[0])
|
@@ -450,12 +472,22 @@ class GetInputHandler(Generic[T]):
|
|
450
472
|
return self.to_instance(flow_run_inputs[0])
|
451
473
|
|
452
474
|
|
453
|
-
class GetAutomaticInputHandler(GetInputHandler):
|
475
|
+
class GetAutomaticInputHandler(GetInputHandler, Generic[T]):
|
454
476
|
def __init__(self, *args, **kwargs):
|
455
477
|
self.with_metadata = kwargs.pop("with_metadata", False)
|
456
478
|
super().__init__(*args, **kwargs)
|
457
479
|
|
458
|
-
def
|
480
|
+
def __next__(self) -> T:
|
481
|
+
return cast(T, super().__next__())
|
482
|
+
|
483
|
+
async def __anext__(self) -> T:
|
484
|
+
return cast(T, await super().__anext__())
|
485
|
+
|
486
|
+
@sync_compatible
|
487
|
+
async def next(self) -> T:
|
488
|
+
return cast(T, await super().next())
|
489
|
+
|
490
|
+
def to_instance(self, flow_run_input: "FlowRunInput") -> T:
|
459
491
|
run_input = self.run_input_cls.load_from_flow_run_input(flow_run_input)
|
460
492
|
|
461
493
|
if self.with_metadata:
|
@@ -500,8 +532,9 @@ async def send_input(
|
|
500
532
|
)
|
501
533
|
|
502
534
|
|
535
|
+
@overload
|
503
536
|
def receive_input(
|
504
|
-
input_type:
|
537
|
+
input_type: Type[R],
|
505
538
|
timeout: Optional[float] = 3600,
|
506
539
|
poll_interval: float = 10,
|
507
540
|
raise_timeout_error: bool = False,
|
@@ -509,14 +542,52 @@ def receive_input(
|
|
509
542
|
key_prefix: Optional[str] = None,
|
510
543
|
flow_run_id: Optional[UUID] = None,
|
511
544
|
with_metadata: bool = False,
|
512
|
-
):
|
545
|
+
) -> GetInputHandler[R]:
|
546
|
+
...
|
547
|
+
|
548
|
+
|
549
|
+
@overload
|
550
|
+
def receive_input(
|
551
|
+
input_type: Type[T],
|
552
|
+
timeout: Optional[float] = 3600,
|
553
|
+
poll_interval: float = 10,
|
554
|
+
raise_timeout_error: bool = False,
|
555
|
+
exclude_keys: Optional[Set[str]] = None,
|
556
|
+
key_prefix: Optional[str] = None,
|
557
|
+
flow_run_id: Optional[UUID] = None,
|
558
|
+
with_metadata: bool = False,
|
559
|
+
) -> GetAutomaticInputHandler[T]:
|
560
|
+
...
|
561
|
+
|
562
|
+
|
563
|
+
def receive_input(
|
564
|
+
input_type: Union[Type[R], Type[T]],
|
565
|
+
timeout: Optional[float] = 3600,
|
566
|
+
poll_interval: float = 10,
|
567
|
+
raise_timeout_error: bool = False,
|
568
|
+
exclude_keys: Optional[Set[str]] = None,
|
569
|
+
key_prefix: Optional[str] = None,
|
570
|
+
flow_run_id: Optional[UUID] = None,
|
571
|
+
with_metadata: bool = False,
|
572
|
+
) -> Union[GetAutomaticInputHandler[T], GetInputHandler[R]]:
|
513
573
|
input_cls = run_input_subclass_from_type(input_type)
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
574
|
+
|
575
|
+
if issubclass(input_cls, AutomaticRunInput):
|
576
|
+
return input_cls.receive(
|
577
|
+
timeout=timeout,
|
578
|
+
poll_interval=poll_interval,
|
579
|
+
raise_timeout_error=raise_timeout_error,
|
580
|
+
exclude_keys=exclude_keys,
|
581
|
+
key_prefix=key_prefix,
|
582
|
+
flow_run_id=flow_run_id,
|
583
|
+
with_metadata=with_metadata,
|
584
|
+
)
|
585
|
+
else:
|
586
|
+
return input_cls.receive(
|
587
|
+
timeout=timeout,
|
588
|
+
poll_interval=poll_interval,
|
589
|
+
raise_timeout_error=raise_timeout_error,
|
590
|
+
exclude_keys=exclude_keys,
|
591
|
+
key_prefix=key_prefix,
|
592
|
+
flow_run_id=flow_run_id,
|
593
|
+
)
|
prefect/results.py
CHANGED
@@ -5,6 +5,7 @@ from typing import (
|
|
5
5
|
TYPE_CHECKING,
|
6
6
|
Any,
|
7
7
|
Callable,
|
8
|
+
Dict,
|
8
9
|
Generic,
|
9
10
|
Optional,
|
10
11
|
Tuple,
|
@@ -12,6 +13,7 @@ from typing import (
|
|
12
13
|
TypeVar,
|
13
14
|
Union,
|
14
15
|
)
|
16
|
+
from uuid import UUID
|
15
17
|
|
16
18
|
from typing_extensions import Self
|
17
19
|
|
@@ -51,7 +53,7 @@ if TYPE_CHECKING:
|
|
51
53
|
|
52
54
|
ResultStorage = Union[WritableFileSystem, str]
|
53
55
|
ResultSerializer = Union[Serializer, str]
|
54
|
-
LITERAL_TYPES = {type(None), bool}
|
56
|
+
LITERAL_TYPES = {type(None), bool, UUID}
|
55
57
|
|
56
58
|
|
57
59
|
def DEFAULT_STORAGE_KEY_FN():
|
@@ -383,6 +385,27 @@ class ResultFactory(pydantic.BaseModel):
|
|
383
385
|
cache_object=should_cache_object,
|
384
386
|
)
|
385
387
|
|
388
|
+
@sync_compatible
|
389
|
+
async def store_parameters(self, identifier: UUID, parameters: Dict[str, Any]):
|
390
|
+
assert (
|
391
|
+
self.storage_block_id is not None
|
392
|
+
), "Unexpected storage block ID. Was it persisted?"
|
393
|
+
data = self.serializer.dumps(parameters)
|
394
|
+
blob = PersistedResultBlob(serializer=self.serializer, data=data)
|
395
|
+
await self.storage_block.write_path(
|
396
|
+
f"parameters/{identifier}", content=blob.to_bytes()
|
397
|
+
)
|
398
|
+
|
399
|
+
@sync_compatible
|
400
|
+
async def read_parameters(self, identifier: UUID) -> Dict[str, Any]:
|
401
|
+
assert (
|
402
|
+
self.storage_block_id is not None
|
403
|
+
), "Unexpected storage block ID. Was it persisted?"
|
404
|
+
blob = PersistedResultBlob.parse_raw(
|
405
|
+
await self.storage_block.read_path(f"parameters/{identifier}")
|
406
|
+
)
|
407
|
+
return self.serializer.loads(blob.data)
|
408
|
+
|
386
409
|
|
387
410
|
@add_type_dispatch
|
388
411
|
class BaseResult(pydantic.BaseModel, abc.ABC, Generic[R]):
|
prefect/runner/runner.py
CHANGED
@@ -52,7 +52,6 @@ import anyio.abc
|
|
52
52
|
import pendulum
|
53
53
|
import sniffio
|
54
54
|
from rich.console import Console, Group
|
55
|
-
from rich.panel import Panel
|
56
55
|
from rich.table import Table
|
57
56
|
|
58
57
|
from prefect._internal.concurrency.api import (
|
@@ -1260,7 +1259,9 @@ async def serve(
|
|
1260
1259
|
)
|
1261
1260
|
|
1262
1261
|
console = Console()
|
1263
|
-
console.print(
|
1262
|
+
console.print(
|
1263
|
+
Group(help_message_top, table, help_message_bottom), soft_wrap=True
|
1264
|
+
)
|
1264
1265
|
|
1265
1266
|
await runner.start()
|
1266
1267
|
|
prefect/settings.py
CHANGED
@@ -1408,6 +1408,13 @@ PREFECT_WORKER_WEBSERVER_PORT = Setting(
|
|
1408
1408
|
"""
|
1409
1409
|
The port the worker's webserver should bind to.
|
1410
1410
|
"""
|
1411
|
+
PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS = Setting(
|
1412
|
+
bool,
|
1413
|
+
default=True,
|
1414
|
+
)
|
1415
|
+
"""
|
1416
|
+
Whether or not to delete failed task submissions from the database.
|
1417
|
+
"""
|
1411
1418
|
|
1412
1419
|
PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS = Setting(bool, default=False)
|
1413
1420
|
"""
|
@@ -1435,6 +1442,11 @@ PREFECT_EXPERIMENTAL_WARN_WORKSPACE_DASHBOARD = Setting(bool, default=False)
|
|
1435
1442
|
Whether or not to warn when the experimental workspace dashboard is enabled.
|
1436
1443
|
"""
|
1437
1444
|
|
1445
|
+
PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING = Setting(bool, default=False)
|
1446
|
+
"""
|
1447
|
+
Whether or not to enable experimental task scheduling.
|
1448
|
+
"""
|
1449
|
+
|
1438
1450
|
# Defaults -----------------------------------------------------------------------------
|
1439
1451
|
|
1440
1452
|
PREFECT_DEFAULT_RESULT_STORAGE_BLOCK = Setting(
|
prefect/task_engine.py
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
from contextlib import AsyncExitStack
|
2
|
+
from typing import (
|
3
|
+
Any,
|
4
|
+
Dict,
|
5
|
+
Iterable,
|
6
|
+
Optional,
|
7
|
+
Type,
|
8
|
+
)
|
9
|
+
|
10
|
+
import anyio
|
11
|
+
from anyio import start_blocking_portal
|
12
|
+
from typing_extensions import Literal
|
13
|
+
|
14
|
+
from prefect._internal.concurrency.api import create_call, from_async, from_sync
|
15
|
+
from prefect.client.orchestration import get_client
|
16
|
+
from prefect.client.schemas.objects import TaskRun
|
17
|
+
from prefect.context import EngineContext
|
18
|
+
from prefect.engine import (
|
19
|
+
begin_task_map,
|
20
|
+
get_task_call_return_value,
|
21
|
+
)
|
22
|
+
from prefect.futures import PrefectFuture
|
23
|
+
from prefect.results import ResultFactory
|
24
|
+
from prefect.task_runners import BaseTaskRunner, SequentialTaskRunner
|
25
|
+
from prefect.tasks import Task
|
26
|
+
from prefect.utilities.asyncutils import sync_compatible
|
27
|
+
|
28
|
+
EngineReturnType = Literal["future", "state", "result"]
|
29
|
+
|
30
|
+
|
31
|
+
@sync_compatible
|
32
|
+
async def submit_autonomous_task_to_engine(
|
33
|
+
task: Task,
|
34
|
+
task_run: TaskRun,
|
35
|
+
parameters: Optional[Dict] = None,
|
36
|
+
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
37
|
+
mapped: bool = False,
|
38
|
+
return_type: EngineReturnType = "future",
|
39
|
+
task_runner: Optional[Type[BaseTaskRunner]] = None,
|
40
|
+
) -> Any:
|
41
|
+
parameters = parameters or {}
|
42
|
+
async with AsyncExitStack() as stack:
|
43
|
+
with EngineContext(
|
44
|
+
flow=None,
|
45
|
+
flow_run=None,
|
46
|
+
autonomous_task_run=task_run,
|
47
|
+
task_runner=await stack.enter_async_context(
|
48
|
+
(task_runner if task_runner else SequentialTaskRunner()).start()
|
49
|
+
),
|
50
|
+
client=await stack.enter_async_context(get_client()),
|
51
|
+
parameters=parameters,
|
52
|
+
result_factory=await ResultFactory.from_task(task),
|
53
|
+
background_tasks=await stack.enter_async_context(anyio.create_task_group()),
|
54
|
+
sync_portal=(
|
55
|
+
stack.enter_context(start_blocking_portal()) if task.isasync else None
|
56
|
+
),
|
57
|
+
) as flow_run_context:
|
58
|
+
begin_run = create_call(
|
59
|
+
begin_task_map if mapped else get_task_call_return_value,
|
60
|
+
task=task,
|
61
|
+
flow_run_context=flow_run_context,
|
62
|
+
parameters=parameters,
|
63
|
+
wait_for=wait_for,
|
64
|
+
return_type=return_type,
|
65
|
+
task_runner=task_runner,
|
66
|
+
)
|
67
|
+
if task.isasync:
|
68
|
+
return await from_async.wait_for_call_in_loop_thread(begin_run)
|
69
|
+
else:
|
70
|
+
return from_sync.wait_for_call_in_loop_thread(begin_run)
|
prefect/task_server.py
ADDED
@@ -0,0 +1,208 @@
|
|
1
|
+
import asyncio
|
2
|
+
import signal
|
3
|
+
import sys
|
4
|
+
from functools import partial
|
5
|
+
from typing import Iterable, Optional
|
6
|
+
|
7
|
+
import anyio
|
8
|
+
import anyio.abc
|
9
|
+
import pendulum
|
10
|
+
|
11
|
+
from prefect import Task, get_client
|
12
|
+
from prefect._internal.concurrency.api import create_call, from_sync
|
13
|
+
from prefect.client.schemas.objects import TaskRun
|
14
|
+
from prefect.client.subscriptions import Subscription
|
15
|
+
from prefect.logging.loggers import get_logger
|
16
|
+
from prefect.results import ResultFactory
|
17
|
+
from prefect.settings import (
|
18
|
+
PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING,
|
19
|
+
PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS,
|
20
|
+
)
|
21
|
+
from prefect.task_engine import submit_autonomous_task_to_engine
|
22
|
+
from prefect.utilities.asyncutils import asyncnullcontext, sync_compatible
|
23
|
+
from prefect.utilities.collections import distinct
|
24
|
+
from prefect.utilities.processutils import _register_signal
|
25
|
+
|
26
|
+
logger = get_logger("task_server")
|
27
|
+
|
28
|
+
|
29
|
+
class TaskServer:
|
30
|
+
"""This class is responsible for serving tasks that may be executed autonomously
|
31
|
+
(i.e., without a parent flow run).
|
32
|
+
|
33
|
+
When `start()` is called, the task server will subscribe to the task run scheduling
|
34
|
+
topic and poll for scheduled task runs. When a scheduled task run is found, it
|
35
|
+
will submit the task run to the engine for execution, using `submit_autonomous_task_to_engine`
|
36
|
+
to construct a minimal `EngineContext` for the task run.
|
37
|
+
|
38
|
+
Args:
|
39
|
+
- tasks: A list of tasks to serve. These tasks will be submitted to the engine
|
40
|
+
when a scheduled task run is found.
|
41
|
+
- tags: A list of tags to apply to the task server. Defaults to `["autonomous"]`.
|
42
|
+
"""
|
43
|
+
|
44
|
+
def __init__(
|
45
|
+
self,
|
46
|
+
*tasks: Task,
|
47
|
+
tags: Optional[Iterable[str]] = None,
|
48
|
+
):
|
49
|
+
self.tasks: list[Task] = tasks
|
50
|
+
self.tags: Iterable[str] = tags or ["autonomous"]
|
51
|
+
self.last_polled: Optional[pendulum.DateTime] = None
|
52
|
+
self.started = False
|
53
|
+
self.stopping = False
|
54
|
+
|
55
|
+
self._client = get_client()
|
56
|
+
|
57
|
+
if not asyncio.get_event_loop().is_running():
|
58
|
+
raise RuntimeError(
|
59
|
+
"TaskServer must be initialized within an async context."
|
60
|
+
)
|
61
|
+
|
62
|
+
self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
|
63
|
+
|
64
|
+
def handle_sigterm(self, signum, frame):
|
65
|
+
"""
|
66
|
+
Shuts down the task server when a SIGTERM is received.
|
67
|
+
"""
|
68
|
+
logger.info("SIGTERM received, initiating graceful shutdown...")
|
69
|
+
from_sync.call_in_loop_thread(create_call(self.stop))
|
70
|
+
|
71
|
+
sys.exit(0)
|
72
|
+
|
73
|
+
@sync_compatible
|
74
|
+
async def start(self) -> None:
|
75
|
+
"""
|
76
|
+
Starts a task server, which runs the tasks provided in the constructor.
|
77
|
+
"""
|
78
|
+
_register_signal(signal.SIGTERM, self.handle_sigterm)
|
79
|
+
|
80
|
+
async with asyncnullcontext() if self.started else self:
|
81
|
+
await self._subscribe_to_task_scheduling()
|
82
|
+
|
83
|
+
@sync_compatible
|
84
|
+
async def stop(self):
|
85
|
+
"""Stops the task server's polling cycle."""
|
86
|
+
if not self.started:
|
87
|
+
raise RuntimeError(
|
88
|
+
"Task server has not yet started. Please start the task server by"
|
89
|
+
" calling .start()"
|
90
|
+
)
|
91
|
+
|
92
|
+
logger.info("Stopping task server...")
|
93
|
+
self.started = False
|
94
|
+
self.stopping = True
|
95
|
+
|
96
|
+
async def _subscribe_to_task_scheduling(self):
|
97
|
+
subscription = Subscription(TaskRun, "/task_runs/subscriptions/scheduled")
|
98
|
+
logger.debug(f"Created: {subscription}")
|
99
|
+
async for task_run in subscription:
|
100
|
+
logger.info(f"Received task run: {task_run.id} - {task_run.name}")
|
101
|
+
await self._submit_pending_task_run(task_run)
|
102
|
+
|
103
|
+
async def _submit_pending_task_run(self, task_run: TaskRun):
|
104
|
+
logger.debug(
|
105
|
+
f"Found task run: {task_run.name!r} in state: {task_run.state.name!r}"
|
106
|
+
)
|
107
|
+
|
108
|
+
task = next((t for t in self.tasks if t.name in task_run.task_key), None)
|
109
|
+
|
110
|
+
if not task:
|
111
|
+
if PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS.value():
|
112
|
+
logger.warning(
|
113
|
+
f"Task {task_run.name!r} not found in task server registry."
|
114
|
+
)
|
115
|
+
await self._client._client.delete(f"/task_runs/{task_run.id}")
|
116
|
+
|
117
|
+
return
|
118
|
+
|
119
|
+
# The ID of the parameters for this run are stored in the Scheduled state's
|
120
|
+
# state_details. If there is no parameters_id, then the task was created
|
121
|
+
# without parameters.
|
122
|
+
parameters = {}
|
123
|
+
if hasattr(task_run.state.state_details, "task_parameters_id"):
|
124
|
+
parameters_id = task_run.state.state_details.task_parameters_id
|
125
|
+
task.persist_result = True
|
126
|
+
factory = await ResultFactory.from_task(task)
|
127
|
+
try:
|
128
|
+
parameters = await factory.read_parameters(parameters_id)
|
129
|
+
except Exception as exc:
|
130
|
+
logger.exception(
|
131
|
+
f"Failed to read parameters for task run {task_run.id!r}",
|
132
|
+
exc_info=exc,
|
133
|
+
)
|
134
|
+
if PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS.value():
|
135
|
+
logger.info(
|
136
|
+
f"Deleting task run {task_run.id!r} because it failed to submit"
|
137
|
+
)
|
138
|
+
await self._client._client.delete(f"/task_runs/{task_run.id}")
|
139
|
+
return
|
140
|
+
|
141
|
+
logger.debug(
|
142
|
+
f"Submitting run {task_run.name!r} of task {task.name!r} to engine"
|
143
|
+
)
|
144
|
+
|
145
|
+
task_run.tags = distinct(task_run.tags + list(self.tags))
|
146
|
+
|
147
|
+
self._runs_task_group.start_soon(
|
148
|
+
partial(
|
149
|
+
submit_autonomous_task_to_engine,
|
150
|
+
task=task,
|
151
|
+
task_run=task_run,
|
152
|
+
parameters=parameters,
|
153
|
+
)
|
154
|
+
)
|
155
|
+
|
156
|
+
async def __aenter__(self):
|
157
|
+
logger.debug("Starting task server...")
|
158
|
+
self._client = get_client()
|
159
|
+
await self._client.__aenter__()
|
160
|
+
await self._runs_task_group.__aenter__()
|
161
|
+
|
162
|
+
self.started = True
|
163
|
+
return self
|
164
|
+
|
165
|
+
async def __aexit__(self, *exc_info):
|
166
|
+
logger.debug("Stopping task server...")
|
167
|
+
self.started = False
|
168
|
+
if self._runs_task_group:
|
169
|
+
await self._runs_task_group.__aexit__(*exc_info)
|
170
|
+
if self._client:
|
171
|
+
await self._client.__aexit__(*exc_info)
|
172
|
+
|
173
|
+
|
174
|
+
@sync_compatible
|
175
|
+
async def serve(*tasks: Task, tags: Optional[Iterable[str]] = None):
|
176
|
+
"""Serve the provided tasks so that they may be executed autonomously.
|
177
|
+
|
178
|
+
Args:
|
179
|
+
- tasks: A list of tasks to serve. When a scheduled task run is found for a
|
180
|
+
given task, the task run will be submitted to the engine for execution.
|
181
|
+
- tags: A list of tags to apply to the task server. Defaults to `["autonomous"]`.
|
182
|
+
|
183
|
+
Example:
|
184
|
+
```python
|
185
|
+
from prefect import task
|
186
|
+
from prefect.task_server import serve
|
187
|
+
|
188
|
+
@task(log_prints=True)
|
189
|
+
def say(message: str):
|
190
|
+
print(message)
|
191
|
+
|
192
|
+
@task(log_prints=True)
|
193
|
+
def yell(message: str):
|
194
|
+
print(message.upper())
|
195
|
+
|
196
|
+
# starts a long-lived process that listens scheduled runs of these tasks
|
197
|
+
if __name__ == "__main__":
|
198
|
+
serve(say, yell)
|
199
|
+
```
|
200
|
+
"""
|
201
|
+
if not PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING.value():
|
202
|
+
raise RuntimeError(
|
203
|
+
"To enable task scheduling, set PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING"
|
204
|
+
" to True."
|
205
|
+
)
|
206
|
+
|
207
|
+
task_server = TaskServer(*tasks, tags=tags)
|
208
|
+
await task_server.start()
|
prefect/tasks.py
CHANGED
@@ -660,18 +660,35 @@ class Task(Generic[P, R]):
|
|
660
660
|
) -> State[T]:
|
661
661
|
...
|
662
662
|
|
663
|
+
@overload
|
664
|
+
def submit(
|
665
|
+
self: "Task[P, T]",
|
666
|
+
*args: P.args,
|
667
|
+
**kwargs: P.kwargs,
|
668
|
+
) -> TaskRun:
|
669
|
+
...
|
670
|
+
|
671
|
+
@overload
|
672
|
+
def submit(
|
673
|
+
self: "Task[P, Coroutine[Any, Any, T]]",
|
674
|
+
*args: P.args,
|
675
|
+
**kwargs: P.kwargs,
|
676
|
+
) -> Awaitable[TaskRun]:
|
677
|
+
...
|
678
|
+
|
663
679
|
def submit(
|
664
680
|
self,
|
665
681
|
*args: Any,
|
666
682
|
return_state: bool = False,
|
667
683
|
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
668
684
|
**kwargs: Any,
|
669
|
-
) -> Union[PrefectFuture, Awaitable[PrefectFuture]]:
|
685
|
+
) -> Union[PrefectFuture, Awaitable[PrefectFuture], TaskRun, Awaitable[TaskRun]]:
|
670
686
|
"""
|
671
|
-
Submit a run of the task to
|
687
|
+
Submit a run of the task to the engine.
|
688
|
+
|
689
|
+
If writing an async task, this call must be awaited.
|
672
690
|
|
673
|
-
|
674
|
-
be awaited.
|
691
|
+
If called from within a flow function,
|
675
692
|
|
676
693
|
Will create a new task run in the backing API and submit the task to the flow's
|
677
694
|
task runner. This call only blocks execution while the task is being submitted,
|
@@ -2,23 +2,25 @@ prefect/.prefectignore,sha256=awSprvKT0vI8a64mEOLrMxhxqcO-b0ERQeYpA2rNKVQ,390
|
|
2
2
|
prefect/__init__.py,sha256=CbIj8-fhzFJKbvXPadpc73SwIhNXiR_SVzQW4_k52jY,5339
|
3
3
|
prefect/_version.py,sha256=fQguBh1dzT7Baahj504O5RrsLlSyg3Zrx42OpgdPnFc,22378
|
4
4
|
prefect/agent.py,sha256=b557LEcKxcBrgAGOlEDlOPclAkucDj1RhzywBSYxYpI,27487
|
5
|
-
prefect/context.py,sha256=
|
6
|
-
prefect/engine.py,sha256=
|
5
|
+
prefect/context.py,sha256=vg88xl2Awzf755FJM9jHlNEJB_bvbdxIUHkKmTELvBQ,18088
|
6
|
+
prefect/engine.py,sha256=HCOhrvZuap10vGEXHhT5I36X9FIXvHQ64HyGsAwSAUE,108831
|
7
7
|
prefect/exceptions.py,sha256=84rpsDLp0cn_v2gE1TnK_NZXh27NJtzgZQtARVKyVEE,10953
|
8
|
-
prefect/filesystems.py,sha256=
|
8
|
+
prefect/filesystems.py,sha256=AXFFsga4JIp06Hsw7970B6Z0s5HlR7UpUfqAFZl11k4,34782
|
9
9
|
prefect/flow_runs.py,sha256=-XcKLrAZG35PnQxp5ReWDQ97kdgaNEtuB3fdwWZb9T0,2801
|
10
|
-
prefect/flows.py,sha256
|
10
|
+
prefect/flows.py,sha256=-S8boTTBuqQohcZN5Q3HyQCf9qRoKCRGnaSu9SFJ7Ug,64808
|
11
11
|
prefect/futures.py,sha256=uqNlykBSRrXQO1pQ6mZWLMqwkFCLhvMLrEFR4eHs--I,12589
|
12
12
|
prefect/manifests.py,sha256=xfwEEozSEqPK2Lro4dfgdTnjVbQx-aCECNBnf7vO7ZQ,808
|
13
13
|
prefect/plugins.py,sha256=0C-D3-dKi06JZ44XEGmLjCiAkefbE_lKX-g3urzdbQ4,4163
|
14
14
|
prefect/profiles.toml,sha256=1Tz7nKBDTDXL_6KPJSeB7ok0Vx_aQJ_p0AUmbnzDLzw,39
|
15
15
|
prefect/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
16
|
-
prefect/results.py,sha256=
|
16
|
+
prefect/results.py,sha256=w_tVJ9i_XB1jgJuu56L504ecPBw5eeGtkWwLADIeDKk,22288
|
17
17
|
prefect/serializers.py,sha256=sSbe40Ipj-d6VuzBae5k2ao9lkMUZpIXcLtD7f2a7cE,10852
|
18
|
-
prefect/settings.py,sha256
|
18
|
+
prefect/settings.py,sha256=urRa3rG-jD1BuRjQ2Hm6ROk4-ofKtmPqgjtFD67zoxk,65846
|
19
19
|
prefect/states.py,sha256=-Ud4AUom3Qu-HQ4hOLvfVZuuF-b_ibaqtzmL7V949Ac,20839
|
20
|
+
prefect/task_engine.py,sha256=IqImIWtqT_DXBPKdpbWnq8dyxYXerF68wGIPGFcH98s,2475
|
20
21
|
prefect/task_runners.py,sha256=HXUg5UqhZRN2QNBqMdGE1lKhwFhT8TaRN75ScgLbnw8,11012
|
21
|
-
prefect/
|
22
|
+
prefect/task_server.py,sha256=xZDW2-xZ9S23yv_ijgrcQCdJUmPlgPrtpTqbZFwW1Lw,7450
|
23
|
+
prefect/tasks.py,sha256=lDTr2puBBKj53cAcc96rFCyZkeX0VQh83eYGszyyb0I,47049
|
22
24
|
prefect/variables.py,sha256=57h-cJ15ZXWrdQiOnoEQmUVlAe59hmIaa57ZcGNBzao,914
|
23
25
|
prefect/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
24
26
|
prefect/_internal/_logging.py,sha256=HvNHY-8P469o5u4LYEDBTem69XZEt1QUeUaLToijpak,810
|
@@ -104,12 +106,13 @@ prefect/client/base.py,sha256=19VMAsq6Wvp1ZUwAb2OAT4pMQ0CFWsHBwqY3kZfPR2w,12209
|
|
104
106
|
prefect/client/cloud.py,sha256=vlGivNaOIS0YNc0OnVKEx2L88SRU8pal8GYMoPHXyrU,3955
|
105
107
|
prefect/client/collections.py,sha256=I9EgbTg4Fn57gn8vwP_WdDmgnATbx9gfkm2jjhCORjw,1037
|
106
108
|
prefect/client/constants.py,sha256=Z_GG8KF70vbbXxpJuqW5pLnwzujTVeHbcYYRikNmGH0,29
|
107
|
-
prefect/client/orchestration.py,sha256=
|
109
|
+
prefect/client/orchestration.py,sha256=i8vGi2x-soE_POndRFRlsq5EoIn2bRHBS_D8QsNkzNg,103706
|
110
|
+
prefect/client/subscriptions.py,sha256=eaoM0aqa7TUIXIhoJ29qlPSg_9R94O6KI48-BvNkcgU,2624
|
108
111
|
prefect/client/utilities.py,sha256=ejALWrVYuqW-A2zKJkAuRXDkhZ5e8fsiEkn-wI1tzF0,1998
|
109
112
|
prefect/client/schemas/__init__.py,sha256=KlyqFV-hMulMkNstBn_0ijoHoIwJZaBj6B1r07UmgvE,607
|
110
113
|
prefect/client/schemas/actions.py,sha256=R4MUsb_1GuEsYLoLnU8jIfCBobJJFbDydaZE24mkqTc,25206
|
111
|
-
prefect/client/schemas/filters.py,sha256=
|
112
|
-
prefect/client/schemas/objects.py,sha256=
|
114
|
+
prefect/client/schemas/filters.py,sha256=r6gnxZREnmE8Glt2SF6vPxHr0SIeiFBjTrrN32cw-Mo,35514
|
115
|
+
prefect/client/schemas/objects.py,sha256=meib2ZRmECf_Gn3TJL8l8_Yf8VFWbJK-IvBvrjRQWoE,53888
|
113
116
|
prefect/client/schemas/responses.py,sha256=nSYhg2Kl477RdczNsA731vpcJqF93WDnM6eMya3F7qI,9152
|
114
117
|
prefect/client/schemas/schedules.py,sha256=ncGWmmBzZvf5G4AL27E0kWGiJxGX-haR2_-GUNvFlv4,14829
|
115
118
|
prefect/client/schemas/sorting.py,sha256=Y-ea8k_vTUKAPKIxqGebwLSXM7x1s5mJ_4-sDd1Ivi8,2276
|
@@ -150,7 +153,7 @@ prefect/infrastructure/provisioners/ecs.py,sha256=qFHRLMuU0HduCEcuU0ZiEhnKeGFnk1
|
|
150
153
|
prefect/infrastructure/provisioners/modal.py,sha256=mLblDjWWszXXMXWXYzkR_5s3nFFL6c3GvVX-VmIeU5A,9035
|
151
154
|
prefect/input/__init__.py,sha256=TPJ9UfG9_SiBze23sQwU1MnWI8AgyEMNihotgTebFQ0,627
|
152
155
|
prefect/input/actions.py,sha256=yITDUOsnNyslJKDinae6zKcX_1_3QMw8SFu7aTynjPM,3894
|
153
|
-
prefect/input/run_input.py,sha256=
|
156
|
+
prefect/input/run_input.py,sha256=qVb8hPcM9fazrMLT7XDHNFAB-CD6fQvnwvrDod3aOdo,18022
|
154
157
|
prefect/logging/__init__.py,sha256=EnbHzgJE_-e4VM3jG5s7MCABYvZ7UGjntC6NfSdTqLg,112
|
155
158
|
prefect/logging/configuration.py,sha256=Qy0r7_j7b8_klsBEn2_f-eSrTQ_EzaBrFwGnwdtgcK8,3436
|
156
159
|
prefect/logging/formatters.py,sha256=pJKHSo_D_DXXor8R7dnPBCOSwQMhRKfP-35UHdIcOyE,4081
|
@@ -165,7 +168,7 @@ prefect/packaging/file.py,sha256=LdYUpAJfBzaYABCwVs4jMKVyo2DC6psEFGpwJ-iKUd4,227
|
|
165
168
|
prefect/packaging/orion.py,sha256=ctWh8s3UztYfOTsZ0sfumebI0dbNDOTriDNXohtEC-k,1935
|
166
169
|
prefect/packaging/serializers.py,sha256=1x5GjcBSYrE-YMmrpYYZi2ObTs7MM6YEM3LS0e6mHAk,6321
|
167
170
|
prefect/runner/__init__.py,sha256=d3DFUXy5BYd8Z4cppNN_6RTSddmr-KfnQ5Yw5vh8WL8,96
|
168
|
-
prefect/runner/runner.py,sha256=
|
171
|
+
prefect/runner/runner.py,sha256=naKqAUl5cboL0xedksuNWgabAzAAy-AxNcL25N_C8KQ,47326
|
169
172
|
prefect/runner/server.py,sha256=AqbvszD2OQkQe_5ydlyXZGYriSZiYDE7vpbRATstJ-Q,10648
|
170
173
|
prefect/runner/storage.py,sha256=iZey8Am51c1fZFpS9iVXWYpKiM_lSocvaJEOZVExhvA,22428
|
171
174
|
prefect/runner/submit.py,sha256=w53VdsqfwjW-M3e8hUAAoVlNrXsvGuuyGpEN0wi3vX0,8537
|
@@ -210,8 +213,8 @@ prefect/workers/block.py,sha256=lvKlaWdA-DCCXDX23HHK9M5urEq4x2wmpKtU9ft3a7k,7767
|
|
210
213
|
prefect/workers/process.py,sha256=Kxj_eZYh6R8t8253LYIIafiG7dodCF8RZABwd3Ng_R0,10253
|
211
214
|
prefect/workers/server.py,sha256=WVZJxR8nTMzK0ov0BD0xw5OyQpT26AxlXbsGQ1OrxeQ,1551
|
212
215
|
prefect/workers/utilities.py,sha256=VfPfAlGtTuDj0-Kb8WlMgAuOfgXCdrGAnKMapPSBrwc,2483
|
213
|
-
prefect_client-2.14.
|
214
|
-
prefect_client-2.14.
|
215
|
-
prefect_client-2.14.
|
216
|
-
prefect_client-2.14.
|
217
|
-
prefect_client-2.14.
|
216
|
+
prefect_client-2.14.20.dist-info/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
|
217
|
+
prefect_client-2.14.20.dist-info/METADATA,sha256=-tM7M4vqRCG31YMFta1-ZuXEOKVNlXRwmvQSBOuPrX0,8143
|
218
|
+
prefect_client-2.14.20.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
219
|
+
prefect_client-2.14.20.dist-info/top_level.txt,sha256=MJZYJgFdbRc2woQCeB4vM6T33tr01TmkEhRcns6H_H4,8
|
220
|
+
prefect_client-2.14.20.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|