prefect 3.9.1.dev7__py3-none-any.whl → 3.9.2.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/AGENTS.md +1 -1
- prefect/_build_info.py +3 -3
- prefect/cli/AGENTS.md +1 -1
- prefect/cli/block.py +7 -7
- prefect/cli/deployment.py +2 -2
- prefect/cli/flow_run.py +78 -62
- prefect/cli/task_run.py +1 -1
- prefect/concurrency/AGENTS.md +1 -1
- prefect/concurrency/_leases.py +64 -8
- prefect/deployments/runner.py +4 -4
- prefect/flow_engine.py +8 -2
- prefect/flow_runs.py +14 -1
- prefect/flows.py +5 -5
- prefect/runner/_flow_run_executor.py +42 -16
- prefect/server/AGENTS.md +2 -2
- prefect/server/api/deployments.py +8 -2
- prefect/server/api/flow_runs.py +18 -8
- prefect/server/api/work_queues.py +19 -5
- prefect/server/api/workers.py +52 -8
- prefect/server/collection_blocks_data.json +1 -1
- prefect/server/models/deployments.py +31 -0
- prefect/server/models/work_queues.py +36 -10
- prefect/server/orchestration/core_policy.py +34 -1
- prefect/server/schemas/schedules.py +19 -73
- prefect/server/services/task_run_recorder.py +56 -6
- prefect/server/ui-v2/assets/{artifact-data-view-BYdudVyj.js → artifact-data-view-BvNVCoeh.js} +1 -1
- prefect/server/ui-v2/assets/{artifact._id-omPWdjY2.js → artifact._id-kn4FfYeS.js} +1 -1
- prefect/server/ui-v2/assets/{artifacts-COnfVAQz.js → artifacts-B-0bc-yH.js} +1 -1
- prefect/server/ui-v2/assets/{automation-wizard-xTKRgvQw.js → automation-wizard-DIHGfhzY.js} +1 -1
- prefect/server/ui-v2/assets/{automation._id-Tfvc3N-g.js → automation._id-CMdNKW9_.js} +1 -1
- prefect/server/ui-v2/assets/{automation_._id.edit-CwQiDjzj.js → automation_._id.edit-MWO3JC4p.js} +1 -1
- prefect/server/ui-v2/assets/{automations-DqlFt2OB.js → automations-ClVYZZl7.js} +1 -1
- prefect/server/ui-v2/assets/{automations-header-DCAk3wBk.js → automations-header-DI4iaAAk.js} +1 -1
- prefect/server/ui-v2/assets/{automations-ClZN30AA.js → automations-j-O0F0nW.js} +1 -1
- prefect/server/ui-v2/assets/{automations-wizard-C7PG85D_.js → automations-wizard-BHzU9JfF.js} +1 -1
- prefect/server/ui-v2/assets/{base-job-template-form-section-ni0htvTs.js → base-job-template-form-section-DnDhjRMt.js} +1 -1
- prefect/server/ui-v2/assets/{block-type-details-ClCYZgyd.js → block-type-details-D8j70s2E.js} +1 -1
- prefect/server/ui-v2/assets/{block._id-CJ4CfovF.js → block._id-DK288xHT.js} +1 -1
- prefect/server/ui-v2/assets/{block_._id.edit-CRsOHWgo.js → block_._id.edit-Bb0eDqFe.js} +1 -1
- prefect/server/ui-v2/assets/{blocks-BYPYGpqm.js → blocks-C7124FVg.js} +1 -1
- prefect/server/ui-v2/assets/{catalog-CS4OWhXf.js → catalog-BVYORobl.js} +1 -1
- prefect/server/ui-v2/assets/{catalog_._slug-BsatNhpH.js → catalog_._slug-C_W0sjwY.js} +1 -1
- prefect/server/ui-v2/assets/{catalog_._slug_.create-C71KaxdV.js → catalog_._slug_.create-CJUaVKJr.js} +1 -1
- prefect/server/ui-v2/assets/{concurrency-limit._id-D4qwBcOY.js → concurrency-limit._id-DCb7SW9T.js} +1 -1
- prefect/server/ui-v2/assets/{concurrency-limits-D1OqVLA3.js → concurrency-limits-C71LVKx7.js} +1 -1
- prefect/server/ui-v2/assets/{create-Npdsar9N.js → create-BDj28gbP.js} +1 -1
- prefect/server/ui-v2/assets/{create-DoAQI2Al.js → create-DeELn4zG.js} +1 -1
- prefect/server/ui-v2/assets/data-table-CcNAeSEr.js +1 -0
- prefect/server/ui-v2/assets/{date-range-select-kKoaxRGF.js → date-range-select-BRqKS7Gl.js} +1 -1
- prefect/server/ui-v2/assets/{date-time-picker-B_sYB-Y8.js → date-time-picker-DjiPatkZ.js} +1 -1
- prefect/server/ui-v2/assets/{deployment-form-CgndAeGv.js → deployment-form-C-vkE_74.js} +2 -2
- prefect/server/ui-v2/assets/{deployment._id-D4vkk2vg.js → deployment._id-CeNC-xE1.js} +1 -1
- prefect/server/ui-v2/assets/{deployment_._id.duplicate-Cg-iM7YP.js → deployment_._id.duplicate-Chqr81Za.js} +1 -1
- prefect/server/ui-v2/assets/{deployment_._id.edit-EIMrX2Um.js → deployment_._id.edit-DPVT7rtt.js} +1 -1
- prefect/server/ui-v2/assets/{deployment_._id.run-vQWG65Th.js → deployment_._id.run-spYhS_hx.js} +1 -1
- prefect/server/ui-v2/assets/{deployments-D_Os-oC6.js → deployments-Dju5jBTi.js} +1 -1
- prefect/server/ui-v2/assets/{event._eventDate._eventId-C94W3Sz9.js → event._eventDate._eventId-sa-x9y0k.js} +1 -1
- prefect/server/ui-v2/assets/{events-qXndbbBd.js → events-C_QVbm6d.js} +1 -1
- prefect/server/ui-v2/assets/{events-timeline-uYyG5D41.js → events-timeline-DRyir5tS.js} +1 -1
- prefect/server/ui-v2/assets/{flow-run-action-odjzsqmU.js → flow-run-action-BwPVELPM.js} +1 -1
- prefect/server/ui-v2/assets/{flow-run-graph-BOjYRKot.js → flow-run-graph-CPbBOYK4.js} +1 -1
- prefect/server/ui-v2/assets/{flow-run._id-C6BUl6UV.js → flow-run._id-CX03eOAY.js} +2 -2
- prefect/server/ui-v2/assets/{flow-run._id-BYxKw47V.js → flow-run._id-iSlFOk9P.js} +1 -1
- prefect/server/ui-v2/assets/{flow-runs-list-DceS9aVL.js → flow-runs-list-6qJo_VIJ.js} +1 -1
- prefect/server/ui-v2/assets/{flow._id-ffg7n-2k.js → flow._id-BB-aZLlC.js} +1 -1
- prefect/server/ui-v2/assets/flow._id-CC1C-_N9.js +1 -0
- prefect/server/ui-v2/assets/{flows-KKB0FVu3.js → flows-UpoTVXra.js} +1 -1
- prefect/server/ui-v2/assets/{index-BA7RQHw0.js → index-C_iMHDKH.js} +3 -3
- prefect/server/ui-v2/assets/{json-input-B5Lhpd3G.js → json-input-BZOHUDyS.js} +1 -1
- prefect/server/ui-v2/assets/{json-input-lazy-DOfc36m8.js → json-input-lazy-FkvpiPOK.js} +2 -2
- prefect/server/ui-v2/assets/{key._key-BWDZyEWp.js → key._key-UalRHWYB.js} +1 -1
- prefect/server/ui-v2/assets/{lazy-markdown-Bzvip61d.js → lazy-markdown-CWdfKZzn.js} +1 -1
- prefect/server/ui-v2/assets/{login-Cdn2jwN-.js → login-D4fS5A4k.js} +1 -1
- prefect/server/ui-v2/assets/{markdown-input-BFo7pXGt.js → markdown-input--eUdC7G2.js} +1 -1
- prefect/server/ui-v2/assets/{python-example-snippet-BPJIVp3a.js → python-example-snippet-nroYZ7FN.js} +2 -2
- prefect/server/ui-v2/assets/{python-input-ib14hqtr.js → python-input-CZM186ZE.js} +1 -1
- prefect/server/ui-v2/assets/{run-flow-button-CDg8tAeY.js → run-flow-button-CB4td9UG.js} +1 -1
- prefect/server/ui-v2/assets/{runs-BhO6T6Xw.js → runs-CJP9oOX1.js} +1 -1
- prefect/server/ui-v2/assets/{schedule-badge-DvqpRoDs.js → schedule-badge-zgVE3_3K.js} +1 -1
- prefect/server/ui-v2/assets/schema-form-DJ8a5LW4.js +5 -0
- prefect/server/ui-v2/assets/{settings-BY8F7Uda.js → settings-un3IDXs2.js} +1 -1
- prefect/server/ui-v2/assets/{task-run._id-Cnw9Cxng.js → task-run._id-Df0ucR8e.js} +1 -1
- prefect/server/ui-v2/assets/{toggle-group-CNC1tMBY.js → toggle-group-D6LezfEX.js} +1 -1
- prefect/server/ui-v2/assets/{use-delete-automation-confirmation-dialog-KibJb7_o.js → use-delete-automation-confirmation-dialog-DUM4epVY.js} +1 -1
- prefect/server/ui-v2/assets/{use-page-size-preference-9vzx_msI.js → use-page-size-preference-BX_JyRGQ.js} +1 -1
- prefect/server/ui-v2/assets/{use-quick-run-D8vXClAb.js → use-quick-run-CSM2zUxG.js} +1 -1
- prefect/server/ui-v2/assets/{variables-CeBM7epa.js → variables-BLHSN7_z.js} +1 -1
- prefect/server/ui-v2/assets/{work-pool-queue-form-DQ2YM6qE.js → work-pool-queue-form-CTUC8f6C.js} +1 -1
- prefect/server/ui-v2/assets/{work-pool-queue-toggle-CfDkBFnv.js → work-pool-queue-toggle-DTbSZYAW.js} +1 -1
- prefect/server/ui-v2/assets/{work-pool._workPoolName-CQLfCi8X.js → work-pool._workPoolName-9RLJ8uaa.js} +1 -1
- prefect/server/ui-v2/assets/{work-pool_._workPoolName.edit-JynH9YIG.js → work-pool_._workPoolName.edit-o01ZNu2Q.js} +1 -1
- prefect/server/ui-v2/assets/{work-pool_._workPoolName.queue._workQueueName-B71k_xPj.js → work-pool_._workPoolName.queue._workQueueName-lTAu2KMh.js} +1 -1
- prefect/server/ui-v2/assets/{work-pool_._workPoolName.queue.create-BysGlQqM.js → work-pool_._workPoolName.queue.create-BmLm0MQK.js} +1 -1
- prefect/server/ui-v2/assets/{work-pool_._workPoolName.queue_._workQueueName.edit-BIiwTidl.js → work-pool_._workPoolName.queue_._workQueueName.edit-2G8kwDGC.js} +1 -1
- prefect/server/ui-v2/assets/{work-pools-IJ2EpfUW.js → work-pools-Dr4PX4eK.js} +1 -1
- prefect/server/ui-v2/index.html +5 -5
- prefect/types/_datetime.py +4 -1
- {prefect-3.9.1.dev7.dist-info → prefect-3.9.2.dev1.dist-info}/METADATA +2 -2
- {prefect-3.9.1.dev7.dist-info → prefect-3.9.2.dev1.dist-info}/RECORD +102 -102
- prefect/server/ui-v2/assets/data-table-b7DPuz9_.js +0 -1
- prefect/server/ui-v2/assets/flow._id-C_FeZCL5.js +0 -1
- prefect/server/ui-v2/assets/schema-form-CLwLzabq.js +0 -5
- {prefect-3.9.1.dev7.dist-info → prefect-3.9.2.dev1.dist-info}/WHEEL +0 -0
- {prefect-3.9.1.dev7.dist-info → prefect-3.9.2.dev1.dist-info}/entry_points.txt +0 -0
- {prefect-3.9.1.dev7.dist-info → prefect-3.9.2.dev1.dist-info}/licenses/LICENSE +0 -0
prefect/AGENTS.md
CHANGED
|
@@ -29,7 +29,7 @@ Runtime flow/task parameter values must come from `utilities.callables.get_call_
|
|
|
29
29
|
- **Sync and async must stay in sync.** Both `flow_engine.py` and `task_engine.py` have sync and async paths. Any behavior change must be applied to both.
|
|
30
30
|
- **Event and log WebSocket subscribers are parallel implementations.** `events/clients.py::PrefectEventSubscriber` and `logging/clients.py::PrefectLogsSubscriber` duplicate the same reconnect/backoff/dedup logic, including the `reconnect_on_clean_close` option (used by `events/subscribers.py::FlowRunSubscriber` so a clean disconnect doesn't hide a terminal event). Mirror any reconnection-behavior change across both.
|
|
31
31
|
- **Flow run observers own their stream recovery.** `_internal/observers.py` observers create subscribers with `reconnection_attempts=0` and delegate polling fallback, reconnect backoff, and setup deadlines to `_internal/_observation_delivery.py::ObservationDelivery`. Subscribers close any socket left by a failed or cancelled `__aenter__`, since callers never exit a context that failed to enter.
|
|
32
|
-
- **Use `whenever` compat helpers for all datetime conversions on Python 3.13+.** `prefect-client` depends on `whenever` only on Python 3.13+, so modules it ships must keep `whenever` imports behind `sys.version_info >= (3, 13)`; only
|
|
32
|
+
- **Use `whenever` compat helpers for all datetime conversions on Python 3.13+.** `prefect-client` depends on `whenever` only on Python 3.13+, so modules it ships must keep `whenever` imports behind `sys.version_info >= (3, 13)`; only code that the client build strips may import it unconditionally (see the repo-root `client/AGENTS.md`). `types/_datetime.py` provides `now()`, `_whenever_to_stdlib()`, `_whenever_zdt_from_py()`, and `_whenever_pdt_from_py()` to abstract over API differences across supported whenever releases. Version-specific code is guarded by feature-detection flags (`_WHENEVER_NEW_API` for ≥ 0.10.0, `_SYSTEM_TZ` for ≥ 0.11), not version checks. These flags are only probed on Python 3.13+ (`_WHENEVER_NEW_API` is `False` and `_SYSTEM_TZ` is undefined on 3.11/3.12), so a helper that server-only code calls on every Python must feature-detect on the `whenever` class itself. Never call `DateTime.now()` or `pendulum.now()` directly — use `now()` from `types/_datetime.py` instead. Avoid no-argument `datetime.datetime.now()` because it returns a tz-naive datetime that raises `TypeError` when compared to the tz-aware datetimes produced by `now()`. Explicitly timezone-aware stdlib calls such as `datetime.now(ZoneInfo("UTC"))` are allowed when needed to preserve timezone information. Never call `ZonedDateTime.from_py_datetime()`, `PlainDateTime.from_py_datetime()`, or `.py_datetime()` directly — use the helpers instead. Likewise avoid `.assume_system_tz()`/`.to_system_tz()` (deprecated in 0.11; the warning fails tests under `filterwarnings = error`) — use `in_local_tz()` — and direct `DateTimeDelta` imports (removed in 0.11) — use the `Interval` alias. Violations will silently break on whichever whenever version the helpers weren't written for.
|
|
33
33
|
- **Call Pendulum-only members via `types/_datetime.py::_pendulum_call` on Python 3.11/3.12.** Prefect's `DateTime`/`Date`/`Duration` wrap Pendulum-only members (and some shared stdlib methods like `replace`) with user-facing deprecation warnings; `_pendulum_call` bypasses them so users only see warnings for their own calls. `pyproject.toml` ignores these warnings in pytest, so direct calls from Prefect code pass tests silently unless covered by the guards in `tests/test_datetime.py`.
|
|
34
34
|
- **Flow and task engines advance state differently.** The flow engine makes blocking API calls to the server to propose and advance states. The task engine emits `prefect.task_run.*` events (delivered via WebSockets) but advances state locally through `set_state` calls with polling/backoff — do not assume the two engines work the same way.
|
|
35
35
|
- **Flow state transitions go through the server.** The flow engine proposes states to the server, which accepts or rejects them via orchestration rules. The task engine, by contrast, manages state transitions locally via `set_state` and emits `prefect.task_run.*` events — it does not propose states to the server.
|
prefect/_build_info.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Generated by versioningit
|
|
2
|
-
__version__ = "3.9.
|
|
3
|
-
__build_date__ = "2026-
|
|
4
|
-
__git_commit__ = "
|
|
2
|
+
__version__ = "3.9.2.dev1"
|
|
3
|
+
__build_date__ = "2026-11-01 08:12:52.684608+00:00"
|
|
4
|
+
__git_commit__ = "2b76fb3da359c20fc5fa4e4db615a6fc76330b1e"
|
|
5
5
|
__dirty__ = False
|
prefect/cli/AGENTS.md
CHANGED
|
@@ -7,7 +7,7 @@ Prefect command-line interface, powered by **cyclopts**.
|
|
|
7
7
|
- **Use `rich` for all output.** Console output via `rich.console`, tables via `rich.table`, progress bars via `rich.progress`. Always use `exit_with_error` for error exits.
|
|
8
8
|
- **Support JSON output whenever possible.** Commands should have a `--json` flag or equivalent for machine-readable output. Any diagnostic or informational `print()` calls must be suppressed when JSON output is active — mixing human-readable text with the JSON payload breaks machine consumers.
|
|
9
9
|
- **Print JSON and other verbatim user data with `markup=False, emoji=False`.** `console.print` otherwise parses `[...]` as Rich markup and `:name:` as emoji aliases, silently rewriting payloads (or raising `MarkupError` on stray closing tags). `print_json` and `Pretty` are not affected.
|
|
10
|
-
- **Page through "all records" reads.** Client `read_*` list calls without `limit` return at most `PREFECT_SERVER_API_DEFAULT_LIMIT` (200) rows (server `LimitBody`)
|
|
10
|
+
- **Page through "all records" reads.** Client `read_*` list calls without `limit` return at most `PREFECT_SERVER_API_DEFAULT_LIMIT` (200) rows (server `LimitBody`) — or fewer when the client method has its own default `limit` (e.g. `read_global_concurrency_limits` defaults to 10) — so `--all`/bulk commands must loop on `offset`. When deleting while paging, re-read from offset 0 each pass — advancing `offset` skips rows that shifted into earlier pages.
|
|
11
11
|
- **Excluded from `prefect-client`** — the entire `cli/` directory is stripped during the client package build.
|
|
12
12
|
|
|
13
13
|
## Structure
|
prefect/cli/block.py
CHANGED
|
@@ -334,7 +334,7 @@ async def block_delete(
|
|
|
334
334
|
if slug is None and block_id is not None:
|
|
335
335
|
try:
|
|
336
336
|
if _cli.is_interactive() and not confirm(
|
|
337
|
-
f"Are you sure you want to delete block with id {block_id
|
|
337
|
+
f"Are you sure you want to delete block with id '{block_id}'?",
|
|
338
338
|
default=False,
|
|
339
339
|
console=_cli.console,
|
|
340
340
|
):
|
|
@@ -342,11 +342,11 @@ async def block_delete(
|
|
|
342
342
|
await client.delete_block_document(block_id)
|
|
343
343
|
exit_with_success(f"Deleted Block '{block_id}'.")
|
|
344
344
|
except ObjectNotFound:
|
|
345
|
-
exit_with_error(f"Block {block_id
|
|
345
|
+
exit_with_error(f"Block '{block_id}' not found!")
|
|
346
346
|
elif slug is not None:
|
|
347
|
-
if "/"
|
|
347
|
+
if slug.count("/") != 1:
|
|
348
348
|
exit_with_error(
|
|
349
|
-
f"{slug!r} is not valid. Slug must contain
|
|
349
|
+
f"{slug!r} is not valid. Slug must contain exactly one '/', e.g. 'json/my-json-block'"
|
|
350
350
|
)
|
|
351
351
|
block_type_slug, block_document_name = slug.split("/")
|
|
352
352
|
try:
|
|
@@ -445,11 +445,11 @@ async def block_inspect(
|
|
|
445
445
|
block_id, include_secrets=False
|
|
446
446
|
)
|
|
447
447
|
except ObjectNotFound:
|
|
448
|
-
exit_with_error(f"Block {block_id
|
|
448
|
+
exit_with_error(f"Block '{block_id}' not found!")
|
|
449
449
|
elif slug is not None:
|
|
450
|
-
if "/"
|
|
450
|
+
if slug.count("/") != 1:
|
|
451
451
|
exit_with_error(
|
|
452
|
-
f"{slug!r} is not valid. Slug must contain
|
|
452
|
+
f"{slug!r} is not valid. Slug must contain exactly one '/', e.g. 'json/my-json-block'"
|
|
453
453
|
)
|
|
454
454
|
block_type_slug, block_document_name = slug.split("/")
|
|
455
455
|
try:
|
prefect/cli/deployment.py
CHANGED
|
@@ -674,14 +674,14 @@ async def delete(
|
|
|
674
674
|
if name is None and deployment_id is not None:
|
|
675
675
|
try:
|
|
676
676
|
if _cli.is_interactive() and not confirm(
|
|
677
|
-
f"Are you sure you want to delete deployment with id {deployment_id
|
|
677
|
+
f"Are you sure you want to delete deployment with id '{deployment_id}'?",
|
|
678
678
|
default=False,
|
|
679
679
|
):
|
|
680
680
|
exit_with_error("Deletion aborted.")
|
|
681
681
|
await client.delete_deployment(deployment_id)
|
|
682
682
|
exit_with_success(f"Deleted deployment '{deployment_id}'.")
|
|
683
683
|
except ObjectNotFound:
|
|
684
|
-
exit_with_error(f"Deployment {deployment_id
|
|
684
|
+
exit_with_error(f"Deployment '{deployment_id}' not found!")
|
|
685
685
|
elif name is not None:
|
|
686
686
|
try:
|
|
687
687
|
deployment = await client.read_deployment_by_name(name)
|
prefect/cli/flow_run.py
CHANGED
|
@@ -42,7 +42,10 @@ from prefect.client.schemas.sorting import FlowRunSort, LogSort
|
|
|
42
42
|
from prefect.exceptions import Abort, FlowRunWatchError, ObjectNotFound, Pause
|
|
43
43
|
from prefect.logging import get_logger
|
|
44
44
|
from prefect.runner._control_channel import ControlSignalStatus
|
|
45
|
-
from prefect.runner._flow_run_executor import
|
|
45
|
+
from prefect.runner._flow_run_executor import (
|
|
46
|
+
FlowRunExecutionResult,
|
|
47
|
+
FlowRunExecutorContext,
|
|
48
|
+
)
|
|
46
49
|
from prefect.runner._workspace_starter import WorkspaceResolvingEngineCommandStarter
|
|
47
50
|
from prefect.states import AwaitingRetry, State, exception_to_crashed_state
|
|
48
51
|
from prefect.types._datetime import human_friendly_diff
|
|
@@ -188,7 +191,7 @@ async def inspect(
|
|
|
188
191
|
)
|
|
189
192
|
|
|
190
193
|
await run_sync_in_worker_thread(webbrowser.open_new_tab, flow_run_url)
|
|
191
|
-
exit_with_success(f"Opened flow run {id
|
|
194
|
+
exit_with_success(f"Opened flow run '{id}' in browser.")
|
|
192
195
|
else:
|
|
193
196
|
if output and output.lower() == "json":
|
|
194
197
|
flow_run_json = flow_run.model_dump(mode="json")
|
|
@@ -345,7 +348,7 @@ async def delete(id: UUID):
|
|
|
345
348
|
async with get_client() as client:
|
|
346
349
|
try:
|
|
347
350
|
if _cli.is_interactive() and not confirm(
|
|
348
|
-
f"Are you sure you want to delete flow run with id {id
|
|
351
|
+
f"Are you sure you want to delete flow run with id '{id}'?",
|
|
349
352
|
default=False,
|
|
350
353
|
):
|
|
351
354
|
exit_with_error("Deletion aborted.")
|
|
@@ -779,6 +782,7 @@ async def execute(
|
|
|
779
782
|
intent = _termination_intent()
|
|
780
783
|
terminated = False
|
|
781
784
|
rescheduled = False
|
|
785
|
+
result: FlowRunExecutionResult | None = None
|
|
782
786
|
|
|
783
787
|
with tempfile.TemporaryDirectory(prefix="prefect-flow-run-") as workspace_root:
|
|
784
788
|
async with FlowRunExecutorContext() as ctx:
|
|
@@ -798,68 +802,67 @@ async def execute(
|
|
|
798
802
|
|
|
799
803
|
terminating = anyio.Event()
|
|
800
804
|
if intent is None or not _install_termination_handler(terminating.set):
|
|
801
|
-
await executor.submit()
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
if signal_status is ControlSignalStatus.ALREADY_CONCLUDED:
|
|
816
|
-
logger.info(
|
|
817
|
-
"Engine already reported a final state; leaving it unchanged."
|
|
818
|
-
)
|
|
819
|
-
return
|
|
820
|
-
|
|
821
|
-
acknowledged = signal_status is ControlSignalStatus.ACKNOWLEDGED
|
|
822
|
-
if not acknowledged:
|
|
823
|
-
logger.warning(
|
|
824
|
-
"Engine did not acknowledge %s intent; stopping it without"
|
|
825
|
-
" a graceful signal.",
|
|
826
|
-
intent,
|
|
827
|
-
)
|
|
828
|
-
|
|
829
|
-
if intent == "reschedule":
|
|
830
|
-
already_final = await _final_state_blocking_reschedule(
|
|
831
|
-
ctx.client, id
|
|
832
|
-
)
|
|
833
|
-
if already_final is not None:
|
|
805
|
+
result = await executor.submit()
|
|
806
|
+
else:
|
|
807
|
+
async with anyio.create_task_group() as tg:
|
|
808
|
+
|
|
809
|
+
async def _terminate_on_signal() -> None:
|
|
810
|
+
nonlocal terminated, rescheduled
|
|
811
|
+
await terminating.wait()
|
|
812
|
+
logger.info("SIGTERM received, initiating graceful shutdown...")
|
|
813
|
+
|
|
814
|
+
# The engine must learn the intent before it sees a SIGTERM so it
|
|
815
|
+
# exits without proposing a state we then override; one that never
|
|
816
|
+
# acknowledged gets no chance to propose at all.
|
|
817
|
+
signal_status = await ctx.control_channel.signal(id, intent)
|
|
818
|
+
if signal_status is ControlSignalStatus.ALREADY_CONCLUDED:
|
|
834
819
|
logger.info(
|
|
835
|
-
"
|
|
836
|
-
|
|
837
|
-
|
|
820
|
+
"Engine already reported a final state; leaving it unchanged."
|
|
821
|
+
)
|
|
822
|
+
return
|
|
823
|
+
|
|
824
|
+
acknowledged = signal_status is ControlSignalStatus.ACKNOWLEDGED
|
|
825
|
+
if not acknowledged:
|
|
826
|
+
logger.warning(
|
|
827
|
+
"Engine did not acknowledge %s intent; stopping it without"
|
|
828
|
+
" a graceful signal.",
|
|
829
|
+
intent,
|
|
838
830
|
)
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
831
|
+
|
|
832
|
+
if intent == "reschedule":
|
|
833
|
+
already_final = await _final_state_blocking_reschedule(
|
|
834
|
+
ctx.client, id
|
|
835
|
+
)
|
|
836
|
+
if already_final is not None:
|
|
837
|
+
logger.info(
|
|
838
|
+
"Flow run is already in final state %r; not"
|
|
839
|
+
" rescheduling.",
|
|
840
|
+
already_final.name,
|
|
843
841
|
)
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
842
|
+
else:
|
|
843
|
+
try:
|
|
844
|
+
state = await propose_state(
|
|
845
|
+
ctx.client, AwaitingRetry(), flow_run_id=id
|
|
846
|
+
)
|
|
847
|
+
rescheduled = state.is_scheduled()
|
|
848
|
+
except (Abort, Pause):
|
|
849
|
+
pass
|
|
850
|
+
except Exception:
|
|
851
|
+
logger.exception("Failed to reschedule flow run")
|
|
852
|
+
|
|
853
|
+
await ctx.process_manager.kill(id, force=not acknowledged)
|
|
854
|
+
terminated = True
|
|
855
|
+
# We own the run's next state now, so stop the executor before it
|
|
856
|
+
# can start a process or propose a state of its own. Do not await
|
|
857
|
+
# between the kill and this cancel, or the executor will observe
|
|
858
|
+
# the exit code and propose `Crashed` from it.
|
|
859
|
+
tg.cancel_scope.cancel()
|
|
860
|
+
|
|
861
|
+
tg.start_soon(_terminate_on_signal)
|
|
862
|
+
result = await executor.submit()
|
|
863
|
+
# Leave a shutdown already in flight alone; it cancels the group itself.
|
|
864
|
+
if not terminating.is_set():
|
|
865
|
+
tg.cancel_scope.cancel()
|
|
863
866
|
|
|
864
867
|
# Exits go here, not inside the context: a `SystemExit` unwinding through an
|
|
865
868
|
# anyio task group gets wrapped in an exception group, losing the exit code.
|
|
@@ -872,3 +875,16 @@ async def execute(
|
|
|
872
875
|
)
|
|
873
876
|
# Non-zero so the terminating infrastructure retries this attempt.
|
|
874
877
|
exit_with_error("Flow run relinquished to infrastructure retry.")
|
|
878
|
+
|
|
879
|
+
if result is not None and result.crash_unreported:
|
|
880
|
+
# Surface the crash through the exit code so infrastructure-level crash
|
|
881
|
+
# detection can record it instead. Negative codes are signals; report them
|
|
882
|
+
# the way a shell would.
|
|
883
|
+
status_code = result.status_code
|
|
884
|
+
exit_with_error(
|
|
885
|
+
f"Flow run process crashed (status code {status_code}) and its crash"
|
|
886
|
+
" could not be reported.",
|
|
887
|
+
code=(128 - status_code if status_code < 0 else status_code)
|
|
888
|
+
if status_code
|
|
889
|
+
else 1,
|
|
890
|
+
)
|
prefect/cli/task_run.py
CHANGED
|
@@ -74,7 +74,7 @@ async def inspect(
|
|
|
74
74
|
)
|
|
75
75
|
|
|
76
76
|
await run_sync_in_worker_thread(webbrowser.open_new_tab, task_run_url)
|
|
77
|
-
exit_with_success(f"Opened task run {id
|
|
77
|
+
exit_with_success(f"Opened task run '{id}' in browser.")
|
|
78
78
|
else:
|
|
79
79
|
if output and output.lower() == "json":
|
|
80
80
|
task_run_json = task_run.model_dump(mode="json")
|
prefect/concurrency/AGENTS.md
CHANGED
|
@@ -18,7 +18,7 @@ Does NOT define concurrency limits (server-side in `server/`). Does NOT handle t
|
|
|
18
18
|
|
|
19
19
|
**`raise_on_lease_renewal_failure` (public parameter):** Controls lease renewal failure behavior independently from `strict`. When `None` (default), falls back to the value of `strict` for backward compatibility. Set to `False` to let long-running tasks continue even if a transient renewal error occurs; set to `True` to terminate immediately on renewal failure regardless of `strict`. This means `strict=True, raise_on_lease_renewal_failure=False` gives strict slot acquisition but non-fatal renewal failures, and vice versa.
|
|
20
20
|
|
|
21
|
-
**Lease renewal:** `_leases.py` starts a daemon thread that renews immediately on entry, then waits for 75% of `lease_duration` between renewals. The renewal thread uses the standard `get_client(sync_client=True)` path and retries each renewal up to 3 times with exponential backoff. If all attempts fail, a shared failure handler either cancels execution with the existing sync/async cancel scope or logs a warning, depending on `raise_on_lease_renewal_failure`.
|
|
21
|
+
**Lease renewal:** `_leases.py` starts a daemon thread that renews immediately on entry, then waits for 75% of `lease_duration` between renewals. The renewal thread uses the standard `get_client(sync_client=True)` path and retries each renewal up to 3 times with exponential backoff. If all attempts fail, a shared failure handler either cancels execution with the existing sync/async cancel scope or logs a warning, depending on `raise_on_lease_renewal_failure`. Exception: when a `flow_run_id` is given and renewal fails with `410 Gone`, the thread reads the flow run and stops without cancelling if its server state is final, because the terminal transition releases the lease before the engine receives the proposal response.
|
|
22
22
|
|
|
23
23
|
**Sync/async lockstep invariant:** `asyncio.py`/`sync.py` and `_asyncio.py`/`_sync.py` are parallel implementations. Any behavior change to one must be mirrored in the other.
|
|
24
24
|
|
prefect/concurrency/_leases.py
CHANGED
|
@@ -4,6 +4,9 @@ from contextlib import asynccontextmanager, contextmanager
|
|
|
4
4
|
from typing import TYPE_CHECKING, AsyncGenerator, Generator
|
|
5
5
|
from uuid import UUID
|
|
6
6
|
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from prefect._internal.compatibility.starlette import status
|
|
7
10
|
from prefect._internal.concurrency.cancellation import (
|
|
8
11
|
AsyncCancelScope,
|
|
9
12
|
WatcherThreadCancelScope,
|
|
@@ -90,6 +93,45 @@ def _lease_renewal_loop(
|
|
|
90
93
|
stop_event.wait(lease_duration * _RENEWAL_FRACTION)
|
|
91
94
|
|
|
92
95
|
|
|
96
|
+
def _lease_released_by_final_flow_run(
|
|
97
|
+
exc: BaseException, lease_id: UUID, flow_run_id: UUID | None
|
|
98
|
+
) -> bool:
|
|
99
|
+
"""
|
|
100
|
+
Check whether a lease renewal failure is explained by the flow run having
|
|
101
|
+
reached a final state on the server, which releases the lease before the
|
|
102
|
+
engine has received the response to its terminal state proposal.
|
|
103
|
+
"""
|
|
104
|
+
if flow_run_id is None or not (
|
|
105
|
+
isinstance(exc, httpx.HTTPStatusError)
|
|
106
|
+
and exc.response.status_code == status.HTTP_410_GONE
|
|
107
|
+
):
|
|
108
|
+
return False
|
|
109
|
+
|
|
110
|
+
logger = get_logger("concurrency")
|
|
111
|
+
try:
|
|
112
|
+
with get_client(sync_client=True) as client:
|
|
113
|
+
flow_run = client.read_flow_run(flow_run_id)
|
|
114
|
+
except Exception:
|
|
115
|
+
logger.debug(
|
|
116
|
+
"Failed to read flow run %s after its concurrency lease was not found",
|
|
117
|
+
flow_run_id,
|
|
118
|
+
exc_info=True,
|
|
119
|
+
)
|
|
120
|
+
return False
|
|
121
|
+
|
|
122
|
+
if flow_run.state is None or not flow_run.state.is_final():
|
|
123
|
+
return False
|
|
124
|
+
|
|
125
|
+
logger.debug(
|
|
126
|
+
"Concurrency lease %s was released because flow run %s reached final "
|
|
127
|
+
"state %s; stopping lease renewal",
|
|
128
|
+
lease_id,
|
|
129
|
+
flow_run_id,
|
|
130
|
+
flow_run.state.type.value,
|
|
131
|
+
)
|
|
132
|
+
return True
|
|
133
|
+
|
|
134
|
+
|
|
93
135
|
def _handle_lease_renewal_failure(
|
|
94
136
|
exc: BaseException,
|
|
95
137
|
lease_id: UUID,
|
|
@@ -139,6 +181,7 @@ def _start_lease_renewal_thread(
|
|
|
139
181
|
raise_on_lease_renewal_failure: bool,
|
|
140
182
|
suppress_warnings: bool,
|
|
141
183
|
cancel_scope: AsyncCancelScope | WatcherThreadCancelScope,
|
|
184
|
+
flow_run_id: UUID | None = None,
|
|
142
185
|
) -> tuple[threading.Event, threading.Thread]:
|
|
143
186
|
stop_event = threading.Event()
|
|
144
187
|
renewal_ctx = contextvars.copy_context()
|
|
@@ -152,14 +195,17 @@ def _start_lease_renewal_thread(
|
|
|
152
195
|
try:
|
|
153
196
|
_lease_renewal_loop(lease_id, lease_duration, stop_event)
|
|
154
197
|
except Exception as exc:
|
|
155
|
-
if
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
198
|
+
if stop_event.is_set() or _lease_released_by_final_flow_run(
|
|
199
|
+
exc, lease_id, flow_run_id
|
|
200
|
+
):
|
|
201
|
+
return
|
|
202
|
+
_handle_lease_renewal_failure(
|
|
203
|
+
exc,
|
|
204
|
+
lease_id,
|
|
205
|
+
raise_on_lease_renewal_failure,
|
|
206
|
+
suppress_warnings,
|
|
207
|
+
cancel_scope,
|
|
208
|
+
)
|
|
163
209
|
|
|
164
210
|
def renewal_loop() -> None:
|
|
165
211
|
renewal_ctx.run(_run_with_failure_handler)
|
|
@@ -179,6 +225,7 @@ def maintain_concurrency_lease(
|
|
|
179
225
|
lease_duration: float,
|
|
180
226
|
raise_on_lease_renewal_failure: bool = False,
|
|
181
227
|
suppress_warnings: bool = False,
|
|
228
|
+
flow_run_id: UUID | None = None,
|
|
182
229
|
) -> Generator[None, None, None]:
|
|
183
230
|
"""
|
|
184
231
|
Maintain a concurrency lease for the given lease ID.
|
|
@@ -187,6 +234,9 @@ def maintain_concurrency_lease(
|
|
|
187
234
|
lease_id: The ID of the lease to maintain.
|
|
188
235
|
lease_duration: The duration of the lease in seconds.
|
|
189
236
|
raise_on_lease_renewal_failure: A boolean specifying whether to raise an error if the lease renewal fails.
|
|
237
|
+
flow_run_id: The ID of the flow run holding the lease. When the lease is
|
|
238
|
+
not found and this flow run is final on the server, the lease was
|
|
239
|
+
released by the terminal transition and execution is not terminated.
|
|
190
240
|
"""
|
|
191
241
|
with WatcherThreadCancelScope() as cancel_scope:
|
|
192
242
|
stop_event: threading.Event | None = None
|
|
@@ -198,6 +248,7 @@ def maintain_concurrency_lease(
|
|
|
198
248
|
raise_on_lease_renewal_failure,
|
|
199
249
|
suppress_warnings,
|
|
200
250
|
cancel_scope,
|
|
251
|
+
flow_run_id,
|
|
201
252
|
)
|
|
202
253
|
yield
|
|
203
254
|
finally:
|
|
@@ -213,6 +264,7 @@ async def amaintain_concurrency_lease(
|
|
|
213
264
|
lease_duration: float,
|
|
214
265
|
raise_on_lease_renewal_failure: bool = False,
|
|
215
266
|
suppress_warnings: bool = False,
|
|
267
|
+
flow_run_id: UUID | None = None,
|
|
216
268
|
) -> AsyncGenerator[None, None]:
|
|
217
269
|
"""
|
|
218
270
|
Maintain a concurrency lease for the given lease ID.
|
|
@@ -221,6 +273,9 @@ async def amaintain_concurrency_lease(
|
|
|
221
273
|
lease_id: The ID of the lease to maintain.
|
|
222
274
|
lease_duration: The duration of the lease in seconds.
|
|
223
275
|
raise_on_lease_renewal_failure: A boolean specifying whether to raise an error if the lease renewal fails.
|
|
276
|
+
flow_run_id: The ID of the flow run holding the lease. When the lease is
|
|
277
|
+
not found and this flow run is final on the server, the lease was
|
|
278
|
+
released by the terminal transition and execution is not terminated.
|
|
224
279
|
"""
|
|
225
280
|
with AsyncCancelScope() as cancel_scope:
|
|
226
281
|
stop_event: threading.Event | None = None
|
|
@@ -231,6 +286,7 @@ async def amaintain_concurrency_lease(
|
|
|
231
286
|
raise_on_lease_renewal_failure,
|
|
232
287
|
suppress_warnings,
|
|
233
288
|
cancel_scope,
|
|
289
|
+
flow_run_id,
|
|
234
290
|
)
|
|
235
291
|
yield
|
|
236
292
|
finally:
|
prefect/deployments/runner.py
CHANGED
|
@@ -1671,14 +1671,14 @@ async def adeploy(
|
|
|
1671
1671
|
" or specify a remote storage location for the flow with `.from_source`."
|
|
1672
1672
|
" If you are attempting to deploy a flow to a local process work pool,"
|
|
1673
1673
|
" consider using `flow.serve` instead. See the documentation for more"
|
|
1674
|
-
" information: https://www.prefect.io/docs/
|
|
1674
|
+
" information: https://www.prefect.io/docs/v3/how-to-guides/deployment_infra/run-flows-in-local-processes"
|
|
1675
1675
|
)
|
|
1676
1676
|
elif work_pool.type == "process" and not ignore_warnings:
|
|
1677
1677
|
console.print(
|
|
1678
1678
|
"Looks like you're deploying to a process work pool. If you're creating a"
|
|
1679
1679
|
" deployment for local development, calling `.serve` on your flow is a great"
|
|
1680
1680
|
" way to get started. See the documentation for more information:"
|
|
1681
|
-
" https://www.prefect.io/docs/
|
|
1681
|
+
" https://www.prefect.io/docs/v3/how-to-guides/deployment_infra/run-flows-in-local-processes "
|
|
1682
1682
|
" Set `ignore_warnings=True` to suppress this message.",
|
|
1683
1683
|
style="yellow",
|
|
1684
1684
|
)
|
|
@@ -1912,14 +1912,14 @@ def deploy(
|
|
|
1912
1912
|
" or specify a remote storage location for the flow with `.from_source`."
|
|
1913
1913
|
" If you are attempting to deploy a flow to a local process work pool,"
|
|
1914
1914
|
" consider using `flow.serve` instead. See the documentation for more"
|
|
1915
|
-
" information: https://www.prefect.io/docs/
|
|
1915
|
+
" information: https://www.prefect.io/docs/v3/how-to-guides/deployment_infra/run-flows-in-local-processes"
|
|
1916
1916
|
)
|
|
1917
1917
|
elif work_pool.type == "process" and not ignore_warnings:
|
|
1918
1918
|
console.print(
|
|
1919
1919
|
"Looks like you're deploying to a process work pool. If you're creating a"
|
|
1920
1920
|
" deployment for local development, calling `.serve` on your flow is a great"
|
|
1921
1921
|
" way to get started. See the documentation for more information:"
|
|
1922
|
-
" https://www.prefect.io/docs/
|
|
1922
|
+
" https://www.prefect.io/docs/v3/how-to-guides/deployment_infra/run-flows-in-local-processes "
|
|
1923
1923
|
" Set `ignore_warnings=True` to suppress this message.",
|
|
1924
1924
|
style="yellow",
|
|
1925
1925
|
)
|
prefect/flow_engine.py
CHANGED
|
@@ -1137,7 +1137,10 @@ class FlowRunEngine(BaseFlowRunEngine[P, R]):
|
|
|
1137
1137
|
if lease_id := self.state.state_details.deployment_concurrency_lease_id:
|
|
1138
1138
|
stack.enter_context(
|
|
1139
1139
|
maintain_concurrency_lease(
|
|
1140
|
-
lease_id,
|
|
1140
|
+
lease_id,
|
|
1141
|
+
300,
|
|
1142
|
+
raise_on_lease_renewal_failure=True,
|
|
1143
|
+
flow_run_id=self.flow_run.id,
|
|
1141
1144
|
)
|
|
1142
1145
|
)
|
|
1143
1146
|
|
|
@@ -1865,7 +1868,10 @@ class AsyncFlowRunEngine(BaseFlowRunEngine[P, R]):
|
|
|
1865
1868
|
if lease_id := self.state.state_details.deployment_concurrency_lease_id:
|
|
1866
1869
|
await stack.enter_async_context(
|
|
1867
1870
|
amaintain_concurrency_lease(
|
|
1868
|
-
lease_id,
|
|
1871
|
+
lease_id,
|
|
1872
|
+
300,
|
|
1873
|
+
raise_on_lease_renewal_failure=True,
|
|
1874
|
+
flow_run_id=self.flow_run.id,
|
|
1869
1875
|
)
|
|
1870
1876
|
)
|
|
1871
1877
|
|
prefect/flow_runs.py
CHANGED
|
@@ -133,6 +133,7 @@ async def wait_for_flow_run(
|
|
|
133
133
|
)
|
|
134
134
|
|
|
135
135
|
with anyio.move_on_after(timeout):
|
|
136
|
+
received_final_event = False
|
|
136
137
|
async with get_events_subscriber(filter=event_filter) as subscriber:
|
|
137
138
|
flow_run = await client.read_flow_run(flow_run_id)
|
|
138
139
|
if flow_run.state and flow_run.state.is_final():
|
|
@@ -151,7 +152,19 @@ async def wait_for_flow_run(
|
|
|
151
152
|
logger.info(f"Flow run is in state {state.name!r}")
|
|
152
153
|
|
|
153
154
|
if state.is_final():
|
|
154
|
-
|
|
155
|
+
received_final_event = True
|
|
156
|
+
break
|
|
157
|
+
|
|
158
|
+
if received_final_event:
|
|
159
|
+
# A final state event can be delivered before the state change is
|
|
160
|
+
# visible to API reads, so read until the run reports a final state
|
|
161
|
+
poll_delay = 0.1
|
|
162
|
+
while True:
|
|
163
|
+
flow_run = await client.read_flow_run(flow_run_id)
|
|
164
|
+
if flow_run.state and flow_run.state.is_final():
|
|
165
|
+
return flow_run
|
|
166
|
+
await anyio.sleep(poll_delay)
|
|
167
|
+
poll_delay = min(poll_delay * 2, 5)
|
|
155
168
|
|
|
156
169
|
raise FlowRunWaitTimeout(
|
|
157
170
|
f"Flow run with ID {flow_run_id} exceeded watch timeout of {timeout} seconds"
|
prefect/flows.py
CHANGED
|
@@ -706,11 +706,11 @@ class Flow(Generic[P, R]):
|
|
|
706
706
|
from fastapi.encoders import jsonable_encoder
|
|
707
707
|
|
|
708
708
|
serialized_parameters[key] = jsonable_encoder(value)
|
|
709
|
-
except
|
|
710
|
-
# `jsonable_encoder`
|
|
711
|
-
#
|
|
712
|
-
#
|
|
713
|
-
#
|
|
709
|
+
except Exception:
|
|
710
|
+
# `jsonable_encoder` can raise arbitrary errors for values it
|
|
711
|
+
# cannot encode (e.g. `RecursionError` for self-referential
|
|
712
|
+
# objects, or any error from `__reduce__` while deep-copying a
|
|
713
|
+
# dataclass). Fall back to the placeholder below.
|
|
714
714
|
logger.debug(
|
|
715
715
|
f"Parameter {key!r} for flow {self.name!r} is unserializable. "
|
|
716
716
|
f"Type {type(value).__name__!r} and will not be stored "
|
|
@@ -51,10 +51,16 @@ class ProcessStarter(Protocol):
|
|
|
51
51
|
|
|
52
52
|
@dataclass(frozen=True)
|
|
53
53
|
class FlowRunExecutionResult:
|
|
54
|
-
"""Process handle plus the infrastructure status for a completed attempt.
|
|
54
|
+
"""Process handle plus the infrastructure status for a completed attempt.
|
|
55
|
+
|
|
56
|
+
`crash_unreported` is `True` when the process exited non-zero or failed
|
|
57
|
+
after starting, and the runner could not record a final state for the flow
|
|
58
|
+
run in its place.
|
|
59
|
+
"""
|
|
55
60
|
|
|
56
61
|
handle: ProcessHandle
|
|
57
62
|
status_code: int | None
|
|
63
|
+
crash_unreported: bool = False
|
|
58
64
|
|
|
59
65
|
|
|
60
66
|
class FlowRunExecutor:
|
|
@@ -124,6 +130,7 @@ class FlowRunExecutor:
|
|
|
124
130
|
"""
|
|
125
131
|
handle: ProcessHandle | None = None
|
|
126
132
|
attempt_conclusion: AttemptConclusion | None = None
|
|
133
|
+
process_error: Exception | None = None
|
|
127
134
|
try:
|
|
128
135
|
# Step 1a: already-cancelling precheck (runs regardless of propose_submitting)
|
|
129
136
|
if self._flow_run.state and self._flow_run.state.is_cancelling():
|
|
@@ -165,16 +172,26 @@ class FlowRunExecutor:
|
|
|
165
172
|
handle = await self._start_process(task_status)
|
|
166
173
|
|
|
167
174
|
except Exception as exc:
|
|
175
|
+
# A failure after the process exited (e.g. its output stream broke)
|
|
176
|
+
# still leaves an exit to account for. A live process stays
|
|
177
|
+
# registered so the process manager can still stop it.
|
|
178
|
+
started_handle = self._process_manager.get(self._flow_run.id)
|
|
179
|
+
if started_handle is None or started_handle.returncode is None:
|
|
180
|
+
self._logger.exception(
|
|
181
|
+
"Flow run '%s' could not start: %s",
|
|
182
|
+
self._flow_run.id,
|
|
183
|
+
exc,
|
|
184
|
+
)
|
|
185
|
+
await self._state_proposer.propose_crashed(
|
|
186
|
+
self._flow_run,
|
|
187
|
+
message=f"Flow run could not start: {exc}",
|
|
188
|
+
)
|
|
189
|
+
return
|
|
168
190
|
self._logger.exception(
|
|
169
|
-
"Flow run '%s'
|
|
170
|
-
self._flow_run.id,
|
|
171
|
-
exc,
|
|
172
|
-
)
|
|
173
|
-
await self._state_proposer.propose_crashed(
|
|
174
|
-
self._flow_run,
|
|
175
|
-
message=f"Flow run could not start: {exc}",
|
|
191
|
+
"Flow run '%s' process failed: %s", self._flow_run.id, exc
|
|
176
192
|
)
|
|
177
|
-
|
|
193
|
+
handle = started_handle
|
|
194
|
+
process_error = exc
|
|
178
195
|
finally:
|
|
179
196
|
# Step 6: snapshot terminal evidence before process cleanup unregisters
|
|
180
197
|
# the attempt control session, then remove the process handle.
|
|
@@ -199,14 +216,23 @@ class FlowRunExecutor:
|
|
|
199
216
|
info.resolution,
|
|
200
217
|
extra={"flow_run_id": self._flow_run.id},
|
|
201
218
|
)
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
if crashed_state is not None:
|
|
207
|
-
await self._hook_runner.run_crashed_hooks(self._flow_run, crashed_state)
|
|
219
|
+
elif process_error is not None:
|
|
220
|
+
msg = f"Flow run process failed: {process_error}"
|
|
221
|
+
else:
|
|
222
|
+
return FlowRunExecutionResult(handle=handle, status_code=exit_code)
|
|
208
223
|
|
|
209
|
-
|
|
224
|
+
crashed_state = await self._state_proposer.propose_crashed(
|
|
225
|
+
self._flow_run, message=msg
|
|
226
|
+
)
|
|
227
|
+
# Step 8: run crashed hooks
|
|
228
|
+
if crashed_state is not None:
|
|
229
|
+
await self._hook_runner.run_crashed_hooks(self._flow_run, crashed_state)
|
|
230
|
+
|
|
231
|
+
return FlowRunExecutionResult(
|
|
232
|
+
handle=handle,
|
|
233
|
+
status_code=exit_code,
|
|
234
|
+
crash_unreported=crashed_state is None or not crashed_state.is_final(),
|
|
235
|
+
)
|
|
210
236
|
|
|
211
237
|
async def _start_process(
|
|
212
238
|
self,
|