prefect-client 3.1.12__py3-none-any.whl → 3.1.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/_experimental/lineage.py +63 -0
- prefect/_experimental/sla/client.py +53 -27
- prefect/_experimental/sla/objects.py +10 -2
- prefect/_internal/concurrency/services.py +2 -2
- prefect/_internal/concurrency/threads.py +6 -0
- prefect/_internal/retries.py +6 -3
- prefect/_internal/schemas/validators.py +6 -4
- prefect/_version.py +3 -3
- prefect/artifacts.py +4 -1
- prefect/automations.py +1 -1
- prefect/blocks/abstract.py +5 -2
- prefect/blocks/notifications.py +1 -0
- prefect/cache_policies.py +70 -22
- prefect/client/orchestration/_automations/client.py +4 -0
- prefect/client/orchestration/_deployments/client.py +3 -3
- prefect/client/utilities.py +3 -3
- prefect/context.py +16 -6
- prefect/deployments/base.py +7 -4
- prefect/deployments/flow_runs.py +5 -1
- prefect/deployments/runner.py +6 -11
- prefect/deployments/steps/core.py +1 -1
- prefect/deployments/steps/pull.py +8 -3
- prefect/deployments/steps/utility.py +2 -2
- prefect/docker/docker_image.py +13 -9
- prefect/engine.py +19 -10
- prefect/events/cli/automations.py +4 -4
- prefect/events/clients.py +17 -14
- prefect/events/filters.py +34 -34
- prefect/events/schemas/automations.py +12 -8
- prefect/events/schemas/events.py +5 -1
- prefect/events/worker.py +1 -1
- prefect/filesystems.py +1 -1
- prefect/flow_engine.py +172 -123
- prefect/flows.py +119 -74
- prefect/futures.py +14 -7
- prefect/infrastructure/provisioners/__init__.py +2 -0
- prefect/infrastructure/provisioners/cloud_run.py +4 -4
- prefect/infrastructure/provisioners/coiled.py +249 -0
- prefect/infrastructure/provisioners/container_instance.py +4 -3
- prefect/infrastructure/provisioners/ecs.py +55 -43
- prefect/infrastructure/provisioners/modal.py +5 -4
- prefect/input/actions.py +5 -1
- prefect/input/run_input.py +157 -43
- prefect/logging/configuration.py +5 -8
- prefect/logging/filters.py +2 -2
- prefect/logging/formatters.py +15 -11
- prefect/logging/handlers.py +24 -14
- prefect/logging/highlighters.py +5 -5
- prefect/logging/loggers.py +29 -20
- prefect/main.py +3 -1
- prefect/results.py +166 -86
- prefect/runner/runner.py +112 -84
- prefect/runner/server.py +3 -1
- prefect/runner/storage.py +18 -18
- prefect/runner/submit.py +19 -12
- prefect/runtime/deployment.py +15 -8
- prefect/runtime/flow_run.py +19 -6
- prefect/runtime/task_run.py +7 -3
- prefect/settings/base.py +17 -7
- prefect/settings/legacy.py +4 -4
- prefect/settings/models/api.py +4 -3
- prefect/settings/models/cli.py +4 -3
- prefect/settings/models/client.py +7 -4
- prefect/settings/models/cloud.py +4 -3
- prefect/settings/models/deployments.py +4 -3
- prefect/settings/models/experiments.py +4 -3
- prefect/settings/models/flows.py +4 -3
- prefect/settings/models/internal.py +4 -3
- prefect/settings/models/logging.py +8 -6
- prefect/settings/models/results.py +4 -3
- prefect/settings/models/root.py +11 -16
- prefect/settings/models/runner.py +8 -5
- prefect/settings/models/server/api.py +6 -3
- prefect/settings/models/server/database.py +120 -25
- prefect/settings/models/server/deployments.py +4 -3
- prefect/settings/models/server/ephemeral.py +7 -4
- prefect/settings/models/server/events.py +6 -3
- prefect/settings/models/server/flow_run_graph.py +4 -3
- prefect/settings/models/server/root.py +4 -3
- prefect/settings/models/server/services.py +15 -12
- prefect/settings/models/server/tasks.py +7 -4
- prefect/settings/models/server/ui.py +4 -3
- prefect/settings/models/tasks.py +10 -5
- prefect/settings/models/testing.py +4 -3
- prefect/settings/models/worker.py +7 -4
- prefect/settings/profiles.py +13 -12
- prefect/settings/sources.py +20 -19
- prefect/states.py +17 -13
- prefect/task_engine.py +43 -33
- prefect/task_runners.py +35 -23
- prefect/task_runs.py +20 -11
- prefect/task_worker.py +12 -7
- prefect/tasks.py +67 -25
- prefect/telemetry/bootstrap.py +4 -1
- prefect/telemetry/run_telemetry.py +15 -13
- prefect/transactions.py +3 -3
- prefect/types/__init__.py +9 -6
- prefect/types/_datetime.py +19 -0
- prefect/utilities/_deprecated.py +38 -0
- prefect/utilities/engine.py +11 -4
- prefect/utilities/filesystem.py +2 -2
- prefect/utilities/generics.py +1 -1
- prefect/utilities/pydantic.py +21 -36
- prefect/workers/base.py +52 -30
- prefect/workers/process.py +20 -15
- prefect/workers/server.py +4 -5
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.14.dist-info}/METADATA +2 -2
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.14.dist-info}/RECORD +111 -108
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.14.dist-info}/LICENSE +0 -0
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.14.dist-info}/WHEEL +0 -0
- {prefect_client-3.1.12.dist-info → prefect_client-3.1.14.dist-info}/top_level.txt +0 -0
prefect/runner/runner.py
CHANGED
@@ -32,8 +32,11 @@ Example:
|
|
32
32
|
|
33
33
|
"""
|
34
34
|
|
35
|
+
from __future__ import annotations
|
36
|
+
|
35
37
|
import asyncio
|
36
38
|
import datetime
|
39
|
+
import inspect
|
37
40
|
import logging
|
38
41
|
import os
|
39
42
|
import shutil
|
@@ -63,13 +66,14 @@ import anyio
|
|
63
66
|
import anyio.abc
|
64
67
|
import pendulum
|
65
68
|
from cachetools import LRUCache
|
69
|
+
from typing_extensions import Self
|
66
70
|
|
67
71
|
from prefect._internal.concurrency.api import (
|
68
72
|
create_call,
|
69
73
|
from_async,
|
70
74
|
from_sync,
|
71
75
|
)
|
72
|
-
from prefect.client.orchestration import get_client
|
76
|
+
from prefect.client.orchestration import PrefectClient, get_client
|
73
77
|
from prefect.client.schemas.filters import (
|
74
78
|
FlowRunFilter,
|
75
79
|
FlowRunFilterId,
|
@@ -79,21 +83,16 @@ from prefect.client.schemas.filters import (
|
|
79
83
|
)
|
80
84
|
from prefect.client.schemas.objects import (
|
81
85
|
ConcurrencyLimitConfig,
|
82
|
-
FlowRun,
|
83
86
|
State,
|
84
87
|
StateType,
|
85
88
|
)
|
86
89
|
from prefect.client.schemas.objects import Flow as APIFlow
|
87
|
-
from prefect.concurrency.asyncio import (
|
88
|
-
AcquireConcurrencySlotTimeoutError,
|
89
|
-
ConcurrencySlotAcquisitionError,
|
90
|
-
)
|
91
90
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
92
91
|
from prefect.events.related import tags_as_related_resources
|
93
92
|
from prefect.events.schemas.events import RelatedResource
|
94
93
|
from prefect.events.utilities import emit_event
|
95
94
|
from prefect.exceptions import Abort, ObjectNotFound
|
96
|
-
from prefect.flows import Flow, load_flow_from_flow_run
|
95
|
+
from prefect.flows import Flow, FlowStateHook, load_flow_from_flow_run
|
97
96
|
from prefect.logging.loggers import PrefectLogAdapter, flow_run_logger, get_logger
|
98
97
|
from prefect.runner.storage import RunnerStorage
|
99
98
|
from prefect.settings import (
|
@@ -114,7 +113,6 @@ from prefect.utilities.asyncutils import (
|
|
114
113
|
)
|
115
114
|
from prefect.utilities.engine import propose_state
|
116
115
|
from prefect.utilities.processutils import (
|
117
|
-
_register_signal,
|
118
116
|
get_sys_executable,
|
119
117
|
run_process,
|
120
118
|
)
|
@@ -125,6 +123,9 @@ from prefect.utilities.services import (
|
|
125
123
|
from prefect.utilities.slugify import slugify
|
126
124
|
|
127
125
|
if TYPE_CHECKING:
|
126
|
+
import concurrent.futures
|
127
|
+
|
128
|
+
from prefect.client.schemas.objects import FlowRun
|
128
129
|
from prefect.client.schemas.responses import DeploymentResponse
|
129
130
|
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
130
131
|
from prefect.deployments.runner import RunnerDeployment
|
@@ -195,36 +196,36 @@ class Runner:
|
|
195
196
|
|
196
197
|
if name and ("/" in name or "%" in name):
|
197
198
|
raise ValueError("Runner name cannot contain '/' or '%'")
|
198
|
-
self.name = Path(name).stem if name is not None else f"runner-{uuid4()}"
|
199
|
-
self._logger = get_logger("runner")
|
200
|
-
|
201
|
-
self.started = False
|
202
|
-
self.stopping = False
|
203
|
-
self.pause_on_shutdown = pause_on_shutdown
|
204
|
-
self.limit = limit or settings.runner.process_limit
|
205
|
-
self.webserver = webserver
|
206
|
-
|
207
|
-
self.query_seconds = query_seconds or settings.runner.poll_frequency
|
208
|
-
self._prefetch_seconds = prefetch_seconds
|
209
|
-
self.heartbeat_seconds = (
|
199
|
+
self.name: str = Path(name).stem if name is not None else f"runner-{uuid4()}"
|
200
|
+
self._logger: "logging.Logger" = get_logger("runner")
|
201
|
+
|
202
|
+
self.started: bool = False
|
203
|
+
self.stopping: bool = False
|
204
|
+
self.pause_on_shutdown: bool = pause_on_shutdown
|
205
|
+
self.limit: int | None = limit or settings.runner.process_limit
|
206
|
+
self.webserver: bool = webserver
|
207
|
+
|
208
|
+
self.query_seconds: float = query_seconds or settings.runner.poll_frequency
|
209
|
+
self._prefetch_seconds: float = prefetch_seconds
|
210
|
+
self.heartbeat_seconds: float | None = (
|
210
211
|
heartbeat_seconds or settings.runner.heartbeat_frequency
|
211
212
|
)
|
212
213
|
if self.heartbeat_seconds is not None and self.heartbeat_seconds < 30:
|
213
214
|
raise ValueError("Heartbeat must be 30 seconds or greater.")
|
214
215
|
|
215
|
-
self._limiter:
|
216
|
-
self._client = get_client()
|
216
|
+
self._limiter: anyio.CapacityLimiter | None = None
|
217
|
+
self._client: PrefectClient = get_client()
|
217
218
|
self._submitting_flow_run_ids: set[UUID] = set()
|
218
219
|
self._cancelling_flow_run_ids: set[UUID] = set()
|
219
|
-
self._scheduled_task_scopes: set[
|
220
|
+
self._scheduled_task_scopes: set[anyio.abc.CancelScope] = set()
|
220
221
|
self._deployment_ids: set[UUID] = set()
|
221
222
|
self._flow_run_process_map: dict[UUID, ProcessMapEntry] = dict()
|
222
223
|
|
223
224
|
self._tmp_dir: Path = (
|
224
225
|
Path(tempfile.gettempdir()) / "runner_storage" / str(uuid4())
|
225
226
|
)
|
226
|
-
self._storage_objs:
|
227
|
-
self._deployment_storage_map:
|
227
|
+
self._storage_objs: list[RunnerStorage] = []
|
228
|
+
self._deployment_storage_map: dict[UUID, RunnerStorage] = {}
|
228
229
|
|
229
230
|
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
230
231
|
|
@@ -246,10 +247,16 @@ class Runner:
|
|
246
247
|
Args:
|
247
248
|
deployment: A deployment for the runner to register.
|
248
249
|
"""
|
249
|
-
|
250
|
+
apply_coro = deployment.apply()
|
251
|
+
if TYPE_CHECKING:
|
252
|
+
assert inspect.isawaitable(apply_coro)
|
253
|
+
deployment_id = await apply_coro
|
250
254
|
storage = deployment.storage
|
251
255
|
if storage is not None:
|
252
|
-
|
256
|
+
add_storage_coro = self._add_storage(storage)
|
257
|
+
if TYPE_CHECKING:
|
258
|
+
assert inspect.isawaitable(add_storage_coro)
|
259
|
+
storage = await add_storage_coro
|
253
260
|
self._deployment_storage_map[deployment_id] = storage
|
254
261
|
self._deployment_ids.add(deployment_id)
|
255
262
|
|
@@ -317,7 +324,7 @@ class Runner:
|
|
317
324
|
)
|
318
325
|
name = self.name if name is None else name
|
319
326
|
|
320
|
-
|
327
|
+
to_deployment_coro = flow.to_deployment(
|
321
328
|
name=name,
|
322
329
|
interval=interval,
|
323
330
|
cron=cron,
|
@@ -333,7 +340,14 @@ class Runner:
|
|
333
340
|
entrypoint_type=entrypoint_type,
|
334
341
|
concurrency_limit=concurrency_limit,
|
335
342
|
)
|
336
|
-
|
343
|
+
if TYPE_CHECKING:
|
344
|
+
assert inspect.isawaitable(to_deployment_coro)
|
345
|
+
deployment = await to_deployment_coro
|
346
|
+
|
347
|
+
add_deployment_coro = self.add_deployment(deployment)
|
348
|
+
if TYPE_CHECKING:
|
349
|
+
assert inspect.isawaitable(add_deployment_coro)
|
350
|
+
return await add_deployment_coro
|
337
351
|
|
338
352
|
@sync_compatible
|
339
353
|
async def _add_storage(self, storage: RunnerStorage) -> RunnerStorage:
|
@@ -360,7 +374,7 @@ class Runner:
|
|
360
374
|
else:
|
361
375
|
return next(s for s in self._storage_objs if s == storage)
|
362
376
|
|
363
|
-
def handle_sigterm(self, **kwargs: Any) -> None:
|
377
|
+
def handle_sigterm(self, *args: Any, **kwargs: Any) -> None:
|
364
378
|
"""
|
365
379
|
Gracefully shuts down the runner when a SIGTERM is received.
|
366
380
|
"""
|
@@ -411,7 +425,8 @@ class Runner:
|
|
411
425
|
"""
|
412
426
|
from prefect.runner.server import start_webserver
|
413
427
|
|
414
|
-
|
428
|
+
if threading.current_thread() is threading.main_thread():
|
429
|
+
signal.signal(signal.SIGTERM, self.handle_sigterm)
|
415
430
|
|
416
431
|
webserver = webserver if webserver is not None else self.webserver
|
417
432
|
|
@@ -478,7 +493,7 @@ class Runner:
|
|
478
493
|
|
479
494
|
def execute_in_background(
|
480
495
|
self, func: Callable[..., Any], *args: Any, **kwargs: Any
|
481
|
-
):
|
496
|
+
) -> "concurrent.futures.Future[Any]":
|
482
497
|
"""
|
483
498
|
Executes a function in the background.
|
484
499
|
"""
|
@@ -487,8 +502,8 @@ class Runner:
|
|
487
502
|
|
488
503
|
return asyncio.run_coroutine_threadsafe(func(*args, **kwargs), self._loop)
|
489
504
|
|
490
|
-
async def cancel_all(self):
|
491
|
-
runs_to_cancel = []
|
505
|
+
async def cancel_all(self) -> None:
|
506
|
+
runs_to_cancel: list[FlowRun] = []
|
492
507
|
|
493
508
|
# done to avoid dictionary size changing during iteration
|
494
509
|
for info in self._flow_run_process_map.values():
|
@@ -524,7 +539,7 @@ class Runner:
|
|
524
539
|
|
525
540
|
async def execute_flow_run(
|
526
541
|
self, flow_run_id: UUID, entrypoint: Optional[str] = None
|
527
|
-
):
|
542
|
+
) -> None:
|
528
543
|
"""
|
529
544
|
Executes a single flow run with the given ID.
|
530
545
|
|
@@ -551,7 +566,7 @@ class Runner:
|
|
551
566
|
),
|
552
567
|
)
|
553
568
|
|
554
|
-
self._flow_run_process_map[flow_run.id] =
|
569
|
+
self._flow_run_process_map[flow_run.id] = ProcessMapEntry(
|
555
570
|
pid=pid, flow_run=flow_run
|
556
571
|
)
|
557
572
|
|
@@ -582,7 +597,7 @@ class Runner:
|
|
582
597
|
)
|
583
598
|
)
|
584
599
|
|
585
|
-
def _get_flow_run_logger(self, flow_run: "FlowRun") -> PrefectLogAdapter:
|
600
|
+
def _get_flow_run_logger(self, flow_run: "FlowRun | FlowRun") -> PrefectLogAdapter:
|
586
601
|
return flow_run_logger(flow_run=flow_run).getChild(
|
587
602
|
"runner",
|
588
603
|
extra={
|
@@ -593,9 +608,9 @@ class Runner:
|
|
593
608
|
async def _run_process(
|
594
609
|
self,
|
595
610
|
flow_run: "FlowRun",
|
596
|
-
task_status: Optional[anyio.abc.TaskStatus[
|
611
|
+
task_status: Optional[anyio.abc.TaskStatus[int]] = None,
|
597
612
|
entrypoint: Optional[str] = None,
|
598
|
-
):
|
613
|
+
) -> int:
|
599
614
|
"""
|
600
615
|
Runs the given flow run in a subprocess.
|
601
616
|
|
@@ -633,7 +648,11 @@ class Runner:
|
|
633
648
|
)
|
634
649
|
env.update(**os.environ) # is this really necessary??
|
635
650
|
|
636
|
-
storage =
|
651
|
+
storage = (
|
652
|
+
self._deployment_storage_map.get(flow_run.deployment_id)
|
653
|
+
if flow_run.deployment_id
|
654
|
+
else None
|
655
|
+
)
|
637
656
|
if storage and storage.pull_interval:
|
638
657
|
# perform an adhoc pull of code before running the flow if an
|
639
658
|
# adhoc pull hasn't been performed in the last pull_interval
|
@@ -657,12 +676,14 @@ class Runner:
|
|
657
676
|
command=command,
|
658
677
|
stream_output=True,
|
659
678
|
task_status=task_status,
|
679
|
+
task_status_handler=None,
|
660
680
|
env=env,
|
661
|
-
**kwargs,
|
662
681
|
cwd=storage.destination if storage else None,
|
682
|
+
**kwargs,
|
663
683
|
)
|
664
684
|
|
665
|
-
|
685
|
+
if process.returncode is None:
|
686
|
+
raise RuntimeError("Process exited with None return code")
|
666
687
|
|
667
688
|
if process.returncode:
|
668
689
|
help_message = None
|
@@ -776,7 +797,7 @@ class Runner:
|
|
776
797
|
if self.stopping:
|
777
798
|
return
|
778
799
|
runs_response = await self._get_scheduled_flow_runs()
|
779
|
-
self.last_polled = pendulum.now("UTC")
|
800
|
+
self.last_polled: pendulum.DateTime = pendulum.now("UTC")
|
780
801
|
return await self._submit_scheduled_flow_runs(flow_run_response=runs_response)
|
781
802
|
|
782
803
|
async def _check_for_cancelled_flow_runs(
|
@@ -858,9 +879,12 @@ class Runner:
|
|
858
879
|
async def _cancel_run(self, flow_run: "FlowRun", state_msg: Optional[str] = None):
|
859
880
|
run_logger = self._get_flow_run_logger(flow_run)
|
860
881
|
|
861
|
-
|
882
|
+
process_map_entry = self._flow_run_process_map.get(flow_run.id)
|
883
|
+
|
884
|
+
pid = process_map_entry.get("pid") if process_map_entry else None
|
862
885
|
if not pid:
|
863
|
-
|
886
|
+
if flow_run.state:
|
887
|
+
await self._run_on_cancellation_hooks(flow_run, flow_run.state)
|
864
888
|
await self._mark_flow_run_as_cancelled(
|
865
889
|
flow_run,
|
866
890
|
state_updates={
|
@@ -876,7 +900,8 @@ class Runner:
|
|
876
900
|
await self._kill_process(pid)
|
877
901
|
except RuntimeError as exc:
|
878
902
|
self._logger.warning(f"{exc} Marking flow run as cancelled.")
|
879
|
-
|
903
|
+
if flow_run.state:
|
904
|
+
await self._run_on_cancellation_hooks(flow_run, flow_run.state)
|
880
905
|
await self._mark_flow_run_as_cancelled(flow_run)
|
881
906
|
except Exception:
|
882
907
|
run_logger.exception(
|
@@ -886,7 +911,8 @@ class Runner:
|
|
886
911
|
# We will try again on generic exceptions
|
887
912
|
self._cancelling_flow_run_ids.remove(flow_run.id)
|
888
913
|
else:
|
889
|
-
|
914
|
+
if flow_run.state:
|
915
|
+
await self._run_on_cancellation_hooks(flow_run, flow_run.state)
|
890
916
|
await self._mark_flow_run_as_cancelled(
|
891
917
|
flow_run,
|
892
918
|
state_updates={
|
@@ -1033,7 +1059,7 @@ class Runner:
|
|
1033
1059
|
|
1034
1060
|
async def _get_scheduled_flow_runs(
|
1035
1061
|
self,
|
1036
|
-
) ->
|
1062
|
+
) -> list["FlowRun"]:
|
1037
1063
|
"""
|
1038
1064
|
Retrieve scheduled flow runs for this runner.
|
1039
1065
|
"""
|
@@ -1058,9 +1084,11 @@ class Runner:
|
|
1058
1084
|
Returns:
|
1059
1085
|
- bool: True if the limit has not been reached, False otherwise.
|
1060
1086
|
"""
|
1087
|
+
if not self._limiter:
|
1088
|
+
return False
|
1061
1089
|
return self._limiter.available_tokens > 0
|
1062
1090
|
|
1063
|
-
def _acquire_limit_slot(self, flow_run_id:
|
1091
|
+
def _acquire_limit_slot(self, flow_run_id: UUID) -> bool:
|
1064
1092
|
"""
|
1065
1093
|
Enforces flow run limit set on runner.
|
1066
1094
|
|
@@ -1085,6 +1113,8 @@ class Runner:
|
|
1085
1113
|
else:
|
1086
1114
|
raise
|
1087
1115
|
except anyio.WouldBlock:
|
1116
|
+
if TYPE_CHECKING:
|
1117
|
+
assert self._limiter is not None
|
1088
1118
|
self._logger.info(
|
1089
1119
|
f"Flow run limit reached; {self._limiter.borrowed_tokens} flow runs"
|
1090
1120
|
" in progress. You can control this limit by passing a `limit` value"
|
@@ -1092,7 +1122,7 @@ class Runner:
|
|
1092
1122
|
)
|
1093
1123
|
return False
|
1094
1124
|
|
1095
|
-
def _release_limit_slot(self, flow_run_id:
|
1125
|
+
def _release_limit_slot(self, flow_run_id: UUID) -> None:
|
1096
1126
|
"""
|
1097
1127
|
Frees up a slot taken by the given flow run id.
|
1098
1128
|
"""
|
@@ -1102,15 +1132,17 @@ class Runner:
|
|
1102
1132
|
|
1103
1133
|
async def _submit_scheduled_flow_runs(
|
1104
1134
|
self,
|
1105
|
-
flow_run_response:
|
1106
|
-
entrypoints:
|
1107
|
-
) ->
|
1135
|
+
flow_run_response: list["FlowRun"],
|
1136
|
+
entrypoints: list[str] | None = None,
|
1137
|
+
) -> list["FlowRun"]:
|
1108
1138
|
"""
|
1109
1139
|
Takes a list of FlowRuns and submits the referenced flow runs
|
1110
1140
|
for execution by the runner.
|
1111
1141
|
"""
|
1112
|
-
submittable_flow_runs =
|
1113
|
-
|
1142
|
+
submittable_flow_runs = sorted(
|
1143
|
+
flow_run_response,
|
1144
|
+
key=lambda run: run.next_scheduled_start_time or datetime.datetime.max,
|
1145
|
+
)
|
1114
1146
|
|
1115
1147
|
for i, flow_run in enumerate(submittable_flow_runs):
|
1116
1148
|
if flow_run.id in self._submitting_flow_run_ids:
|
@@ -1159,7 +1191,7 @@ class Runner:
|
|
1159
1191
|
)
|
1160
1192
|
|
1161
1193
|
if readiness_result and not isinstance(readiness_result, Exception):
|
1162
|
-
self._flow_run_process_map[flow_run.id] =
|
1194
|
+
self._flow_run_process_map[flow_run.id] = ProcessMapEntry(
|
1163
1195
|
pid=readiness_result, flow_run=flow_run
|
1164
1196
|
)
|
1165
1197
|
# Heartbeats are opt-in and only emitted if a heartbeat frequency is set
|
@@ -1176,7 +1208,7 @@ class Runner:
|
|
1176
1208
|
async def _submit_run_and_capture_errors(
|
1177
1209
|
self,
|
1178
1210
|
flow_run: "FlowRun",
|
1179
|
-
task_status:
|
1211
|
+
task_status: anyio.abc.TaskStatus[int | Exception],
|
1180
1212
|
entrypoint: Optional[str] = None,
|
1181
1213
|
) -> Union[Optional[int], Exception]:
|
1182
1214
|
run_logger = self._get_flow_run_logger(flow_run)
|
@@ -1187,24 +1219,8 @@ class Runner:
|
|
1187
1219
|
task_status=task_status,
|
1188
1220
|
entrypoint=entrypoint,
|
1189
1221
|
)
|
1190
|
-
except (
|
1191
|
-
AcquireConcurrencySlotTimeoutError,
|
1192
|
-
ConcurrencySlotAcquisitionError,
|
1193
|
-
) as exc:
|
1194
|
-
self._logger.info(
|
1195
|
-
(
|
1196
|
-
"Deployment %s reached its concurrency limit when attempting to execute flow run %s. Will attempt to execute later."
|
1197
|
-
),
|
1198
|
-
flow_run.deployment_id,
|
1199
|
-
flow_run.name,
|
1200
|
-
)
|
1201
|
-
await self._propose_scheduled_state(flow_run)
|
1202
|
-
|
1203
|
-
if not task_status._future.done():
|
1204
|
-
task_status.started(exc)
|
1205
|
-
return exc
|
1206
1222
|
except Exception as exc:
|
1207
|
-
if
|
1223
|
+
if task_status:
|
1208
1224
|
# This flow run was being submitted and did not start successfully
|
1209
1225
|
run_logger.exception(
|
1210
1226
|
f"Failed to start process for flow run '{flow_run.id}'."
|
@@ -1232,7 +1248,7 @@ class Runner:
|
|
1232
1248
|
|
1233
1249
|
api_flow_run = await self._client.read_flow_run(flow_run_id=flow_run.id)
|
1234
1250
|
terminal_state = api_flow_run.state
|
1235
|
-
if terminal_state.is_crashed():
|
1251
|
+
if terminal_state and terminal_state.is_crashed():
|
1236
1252
|
await self._run_on_crashed_hooks(flow_run=flow_run, state=terminal_state)
|
1237
1253
|
|
1238
1254
|
return status_code
|
@@ -1307,12 +1323,19 @@ class Runner:
|
|
1307
1323
|
)
|
1308
1324
|
|
1309
1325
|
async def _mark_flow_run_as_cancelled(
|
1310
|
-
self, flow_run: "FlowRun", state_updates: Optional[dict] = None
|
1326
|
+
self, flow_run: "FlowRun", state_updates: Optional[dict[str, Any]] = None
|
1311
1327
|
) -> None:
|
1312
1328
|
state_updates = state_updates or {}
|
1313
1329
|
state_updates.setdefault("name", "Cancelled")
|
1314
1330
|
state_updates.setdefault("type", StateType.CANCELLED)
|
1315
|
-
state =
|
1331
|
+
state = (
|
1332
|
+
flow_run.state.model_copy(update=state_updates) if flow_run.state else None
|
1333
|
+
)
|
1334
|
+
if not state:
|
1335
|
+
self._logger.warning(
|
1336
|
+
f"Could not find state for flow run {flow_run.id} and cancellation cannot be guaranteed."
|
1337
|
+
)
|
1338
|
+
return
|
1316
1339
|
|
1317
1340
|
await self._client.set_flow_run_state(flow_run.id, state, force=True)
|
1318
1341
|
|
@@ -1323,7 +1346,9 @@ class Runner:
|
|
1323
1346
|
60 * 10, self._cancelling_flow_run_ids.remove, flow_run.id
|
1324
1347
|
)
|
1325
1348
|
|
1326
|
-
async def _schedule_task(
|
1349
|
+
async def _schedule_task(
|
1350
|
+
self, __in_seconds: int, fn: Callable[..., Any], *args: Any, **kwargs: Any
|
1351
|
+
) -> None:
|
1327
1352
|
"""
|
1328
1353
|
Schedule a background task to start after some time.
|
1329
1354
|
|
@@ -1332,7 +1357,7 @@ class Runner:
|
|
1332
1357
|
The function may be async or sync. Async functions will be awaited.
|
1333
1358
|
"""
|
1334
1359
|
|
1335
|
-
async def wrapper(task_status):
|
1360
|
+
async def wrapper(task_status: anyio.abc.TaskStatus[None]) -> None:
|
1336
1361
|
# If we are shutting down, do not sleep; otherwise sleep until the scheduled
|
1337
1362
|
# time or shutdown
|
1338
1363
|
if self.started:
|
@@ -1389,12 +1414,12 @@ class Runner:
|
|
1389
1414
|
|
1390
1415
|
await _run_hooks(hooks, flow_run, flow, state)
|
1391
1416
|
|
1392
|
-
async def __aenter__(self):
|
1417
|
+
async def __aenter__(self) -> Self:
|
1393
1418
|
self._logger.debug("Starting runner...")
|
1394
1419
|
self._client = get_client()
|
1395
1420
|
self._tmp_dir.mkdir(parents=True)
|
1396
1421
|
|
1397
|
-
self._limiter = anyio.CapacityLimiter(self.limit)
|
1422
|
+
self._limiter = anyio.CapacityLimiter(self.limit) if self.limit else None
|
1398
1423
|
|
1399
1424
|
if not hasattr(self, "_loop") or not self._loop:
|
1400
1425
|
self._loop = asyncio.get_event_loop()
|
@@ -1411,7 +1436,7 @@ class Runner:
|
|
1411
1436
|
self.started = True
|
1412
1437
|
return self
|
1413
1438
|
|
1414
|
-
async def __aexit__(self, *exc_info):
|
1439
|
+
async def __aexit__(self, *exc_info: Any) -> None:
|
1415
1440
|
self._logger.debug("Stopping runner...")
|
1416
1441
|
if self.pause_on_shutdown:
|
1417
1442
|
await self._pause_schedules()
|
@@ -1429,7 +1454,7 @@ class Runner:
|
|
1429
1454
|
shutil.rmtree(str(self._tmp_dir))
|
1430
1455
|
del self._runs_task_group, self._loops_task_group
|
1431
1456
|
|
1432
|
-
def __repr__(self):
|
1457
|
+
def __repr__(self) -> str:
|
1433
1458
|
return f"Runner(name={self.name!r})"
|
1434
1459
|
|
1435
1460
|
|
@@ -1439,7 +1464,10 @@ if sys.platform == "win32":
|
|
1439
1464
|
|
1440
1465
|
|
1441
1466
|
async def _run_hooks(
|
1442
|
-
hooks:
|
1467
|
+
hooks: list[FlowStateHook[Any, Any]],
|
1468
|
+
flow_run: "FlowRun",
|
1469
|
+
flow: "Flow[..., Any]",
|
1470
|
+
state: State,
|
1443
1471
|
):
|
1444
1472
|
logger = flow_run_logger(flow_run, flow)
|
1445
1473
|
for hook in hooks:
|
prefect/runner/server.py
CHANGED
@@ -26,12 +26,14 @@ from prefect.utilities.asyncutils import run_coro_as_sync
|
|
26
26
|
from prefect.utilities.importtools import load_script_as_module
|
27
27
|
|
28
28
|
if TYPE_CHECKING:
|
29
|
+
import logging
|
30
|
+
|
29
31
|
from prefect.client.schemas.responses import DeploymentResponse
|
30
32
|
from prefect.runner import Runner
|
31
33
|
|
32
34
|
from pydantic import BaseModel
|
33
35
|
|
34
|
-
logger = get_logger("webserver")
|
36
|
+
logger: "logging.Logger" = get_logger("webserver")
|
35
37
|
|
36
38
|
RunnableEndpoint = Literal["deployment", "flow", "task"]
|
37
39
|
|
prefect/runner/storage.py
CHANGED
@@ -33,7 +33,7 @@ class RunnerStorage(Protocol):
|
|
33
33
|
remotely stored flow code.
|
34
34
|
"""
|
35
35
|
|
36
|
-
def set_base_path(self, path: Path):
|
36
|
+
def set_base_path(self, path: Path) -> None:
|
37
37
|
"""
|
38
38
|
Sets the base path to use when pulling contents from remote storage to
|
39
39
|
local storage.
|
@@ -55,7 +55,7 @@ class RunnerStorage(Protocol):
|
|
55
55
|
"""
|
56
56
|
...
|
57
57
|
|
58
|
-
async def pull_code(self):
|
58
|
+
async def pull_code(self) -> None:
|
59
59
|
"""
|
60
60
|
Pulls contents from remote storage to the local filesystem.
|
61
61
|
"""
|
@@ -150,7 +150,7 @@ class GitRepository:
|
|
150
150
|
def destination(self) -> Path:
|
151
151
|
return self._storage_base_path / self._name
|
152
152
|
|
153
|
-
def set_base_path(self, path: Path):
|
153
|
+
def set_base_path(self, path: Path) -> None:
|
154
154
|
self._storage_base_path = path
|
155
155
|
|
156
156
|
@property
|
@@ -221,7 +221,7 @@ class GitRepository:
|
|
221
221
|
except Exception:
|
222
222
|
return False
|
223
223
|
|
224
|
-
async def pull_code(self):
|
224
|
+
async def pull_code(self) -> None:
|
225
225
|
"""
|
226
226
|
Pulls the contents of the configured repository to the local filesystem.
|
227
227
|
"""
|
@@ -324,7 +324,7 @@ class GitRepository:
|
|
324
324
|
cwd=self.destination,
|
325
325
|
)
|
326
326
|
|
327
|
-
def __eq__(self, __value) -> bool:
|
327
|
+
def __eq__(self, __value: Any) -> bool:
|
328
328
|
if isinstance(__value, GitRepository):
|
329
329
|
return (
|
330
330
|
self._url == __value._url
|
@@ -339,7 +339,7 @@ class GitRepository:
|
|
339
339
|
f" branch={self._branch!r})"
|
340
340
|
)
|
341
341
|
|
342
|
-
def to_pull_step(self) ->
|
342
|
+
def to_pull_step(self) -> dict[str, Any]:
|
343
343
|
pull_step = {
|
344
344
|
"prefect.deployments.steps.git_clone": {
|
345
345
|
"repository": self._url,
|
@@ -466,7 +466,7 @@ class RemoteStorage:
|
|
466
466
|
|
467
467
|
return fsspec.filesystem(scheme, **settings_with_block_values)
|
468
468
|
|
469
|
-
def set_base_path(self, path: Path):
|
469
|
+
def set_base_path(self, path: Path) -> None:
|
470
470
|
self._storage_base_path = path
|
471
471
|
|
472
472
|
@property
|
@@ -492,7 +492,7 @@ class RemoteStorage:
|
|
492
492
|
_, netloc, urlpath, _, _ = urlsplit(self._url)
|
493
493
|
return Path(netloc) / Path(urlpath.lstrip("/"))
|
494
494
|
|
495
|
-
async def pull_code(self):
|
495
|
+
async def pull_code(self) -> None:
|
496
496
|
"""
|
497
497
|
Pulls contents from remote storage to the local filesystem.
|
498
498
|
"""
|
@@ -522,7 +522,7 @@ class RemoteStorage:
|
|
522
522
|
f" {self.destination!r}"
|
523
523
|
) from exc
|
524
524
|
|
525
|
-
def to_pull_step(self) -> dict:
|
525
|
+
def to_pull_step(self) -> dict[str, Any]:
|
526
526
|
"""
|
527
527
|
Returns a dictionary representation of the storage object that can be
|
528
528
|
used as a deployment pull step.
|
@@ -551,7 +551,7 @@ class RemoteStorage:
|
|
551
551
|
] = required_package
|
552
552
|
return step
|
553
553
|
|
554
|
-
def __eq__(self, __value) -> bool:
|
554
|
+
def __eq__(self, __value: Any) -> bool:
|
555
555
|
"""
|
556
556
|
Equality check for runner storage objects.
|
557
557
|
"""
|
@@ -590,7 +590,7 @@ class BlockStorageAdapter:
|
|
590
590
|
else str(uuid4())
|
591
591
|
)
|
592
592
|
|
593
|
-
def set_base_path(self, path: Path):
|
593
|
+
def set_base_path(self, path: Path) -> None:
|
594
594
|
self._storage_base_path = path
|
595
595
|
|
596
596
|
@property
|
@@ -601,12 +601,12 @@ class BlockStorageAdapter:
|
|
601
601
|
def destination(self) -> Path:
|
602
602
|
return self._storage_base_path / self._name
|
603
603
|
|
604
|
-
async def pull_code(self):
|
604
|
+
async def pull_code(self) -> None:
|
605
605
|
if not self.destination.exists():
|
606
606
|
self.destination.mkdir(parents=True, exist_ok=True)
|
607
607
|
await self._block.get_directory(local_path=str(self.destination))
|
608
608
|
|
609
|
-
def to_pull_step(self) -> dict:
|
609
|
+
def to_pull_step(self) -> dict[str, Any]:
|
610
610
|
# Give blocks the change to implement their own pull step
|
611
611
|
if hasattr(self._block, "get_pull_step"):
|
612
612
|
return self._block.get_pull_step()
|
@@ -623,7 +623,7 @@ class BlockStorageAdapter:
|
|
623
623
|
}
|
624
624
|
}
|
625
625
|
|
626
|
-
def __eq__(self, __value) -> bool:
|
626
|
+
def __eq__(self, __value: Any) -> bool:
|
627
627
|
if isinstance(__value, BlockStorageAdapter):
|
628
628
|
return self._block == __value._block
|
629
629
|
return False
|
@@ -658,19 +658,19 @@ class LocalStorage:
|
|
658
658
|
def destination(self) -> Path:
|
659
659
|
return self._path
|
660
660
|
|
661
|
-
def set_base_path(self, path: Path):
|
661
|
+
def set_base_path(self, path: Path) -> None:
|
662
662
|
self._storage_base_path = path
|
663
663
|
|
664
664
|
@property
|
665
665
|
def pull_interval(self) -> Optional[int]:
|
666
666
|
return self._pull_interval
|
667
667
|
|
668
|
-
async def pull_code(self):
|
668
|
+
async def pull_code(self) -> None:
|
669
669
|
# Local storage assumes the code already exists on the local filesystem
|
670
670
|
# and does not need to be pulled from a remote location
|
671
671
|
pass
|
672
672
|
|
673
|
-
def to_pull_step(self) -> dict:
|
673
|
+
def to_pull_step(self) -> dict[str, Any]:
|
674
674
|
"""
|
675
675
|
Returns a dictionary representation of the storage object that can be
|
676
676
|
used as a deployment pull step.
|
@@ -682,7 +682,7 @@ class LocalStorage:
|
|
682
682
|
}
|
683
683
|
return step
|
684
684
|
|
685
|
-
def __eq__(self, __value) -> bool:
|
685
|
+
def __eq__(self, __value: Any) -> bool:
|
686
686
|
if isinstance(__value, LocalStorage):
|
687
687
|
return self._path == __value._path
|
688
688
|
return False
|