prefect-client 3.7.5.dev3__tar.gz → 3.7.5.dev5__tar.gz

{prefect_client-3.7.5.dev3 → prefect_client-3.7.5.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect-client
-Version: 3.7.5.dev3
+Version: 3.7.5.dev5
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io

prefect_client-3.7.5.dev5/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.7.5.dev5"
+__build_date__ = "2026-06-17 09:45:07.057987+00:00"
+__git_commit__ = "7107dafe23e062b1f8eb4239e85a0a9ef756e79e"
+__dirty__ = False

{prefect_client-3.7.5.dev3 → prefect_client-3.7.5.dev5}/src/prefect/_internal/concurrency/threads.py

@@ -38,9 +38,18 @@ def _reset_after_fork_in_child():
 
     This handler is called by os.register_at_fork() in the child process after fork().
     """
+    global _global_loop, _run_sync_loop
+
     for instance in list(_active_instances):
         instance.reset_for_fork()
 
+    # Clear global references so get_global_loop() / get_run_sync_loop() create
+    # fresh instances.  After fork the old EventLoopThread objects have a stale
+    # Thread whose is_alive() still returns True (the dead thread's tstate lock
+    # is held), so the "is the thread alive?" guard in the getters never fires.
+    _global_loop = None
+    _run_sync_loop = None
+
 
 # Register fork handler if supported (POSIX systems)
 if hasattr(os, "register_at_fork"):

{prefect_client-3.7.5.dev3 → prefect_client-3.7.5.dev5}/src/prefect/runner/AGENTS.md

@@ -15,7 +15,7 @@ Thin facade over single-responsibility extracted classes. New behavior belongs i
 | CancelFinalizer | _cancel_finalizer.py | Persist Cancelled state after kill; fall back to Crashed if state cannot be confirmed |
 | ControlChannel | _control_channel.py | Runner-side TCP loopback IPC for delivering cancel intent to child processes before kill |
 | HookRunner | _hook_runner.py | on_cancellation / on_crashed hook execution |
-| EventEmitter | _event_emitter.py | Event emission via EventsClient; degrades to NullEventsClient on WebSocket rejection |
+| EventEmitter | _event_emitter.py | Event emission via EventsClient; degrades to NullEventsClient on any WebSocket connection failure |
 | LimitManager | _limit_manager.py | Concurrency limiting |
 | DeploymentRegistry | _deployment_registry.py | Deployment/flow/storage/bundle maps |
 | ScheduledRunPoller | _scheduled_run_poller.py | Poll loop, run discovery, scheduling |
@@ -49,7 +49,7 @@ These will be removed once internal callers (notably ProcessWorker) are migrated
 
 ## EventEmitter WebSocket Degradation
 
-`EventEmitter.__aenter__` catches `websockets.exceptions.InvalidStatus` (HTTP 4xx rejections) and silently swaps the failed client for a `NullEventsClient`. This handles old clients (≤3.6.13) connecting to servers ≥3.6.14 with `PREFECT_SERVER_API_AUTH_STRING` configured — the server rejects the WebSocket handshake, but events are non-critical telemetry so the flow run must not crash. A `WARNING` is logged. If `__aenter__` raises, `__aexit__` is **not** called on the original client (it was never successfully entered); the replacement `NullEventsClient` is entered instead.
+`EventEmitter.__aenter__` catches connection failures and silently swaps the failed client for a `NullEventsClient` — events are non-critical telemetry (the runner emits only `prefect.runner.cancelled-flow-run`), so the flow run must not crash when the events WebSocket is unreachable. Two failure classes are caught (`_NONFATAL_CONNECTION_EXCEPTIONS`): permanent rejections (`websockets.exceptions.InvalidStatus`, i.e. HTTP 4xx — e.g. a client ≤3.6.13 connecting to a server ≥3.6.14 with `PREFECT_SERVER_API_AUTH_STRING` configured) and transient failures (`RETRYABLE_EXCEPTIONS` from `events/clients.py`: `ConnectionClosed`, `TimeoutError`, `OSError`) that outlive the events client's own reconnection attempts. A `WARNING` is logged. If `__aenter__` raises, `__aexit__` is **not** called on the original client (it was never successfully entered); the replacement `NullEventsClient` is entered instead.
 
 ## AsyncExitStack LIFO Ordering
 

{prefect_client-3.7.5.dev3 → prefect_client-3.7.5.dev5}/src/prefect/runner/_event_emitter.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Callable
 from cachetools import TTLCache
 from websockets.exceptions import InvalidStatus as _WsInvalidStatus
 
-from prefect.events.clients import NullEventsClient
+from prefect.events.clients import RETRYABLE_EXCEPTIONS, NullEventsClient
 from prefect.events.related import tags_as_related_resources
 from prefect.events.schemas.events import Event, RelatedResource, Resource
 from prefect.logging import get_logger
@@ -18,12 +18,19 @@ if TYPE_CHECKING:
     from prefect.client.schemas.responses import DeploymentResponse
     from prefect.events.clients import EventsClient
 
-# Exceptions that indicate a permanent connection failure (e.g. auth rejected).
-# On these, EventEmitter degrades gracefully to NullEventsClient rather than
-# crashing the flow run.  This preserves backwards-compatibility with clients
-# running against a server that has PREFECT_SERVER_API_AUTH_STRING configured
-# (server >=3.6.14) where the WS handshake is rejected with HTTP 401/403.
-_NONFATAL_CONNECTION_EXCEPTIONS: tuple[type[Exception], ...] = (_WsInvalidStatus,)
+# Connection failures that must not crash the flow run.  The events WebSocket
+# carries non-critical telemetry, so a failure to connect degrades to
+# NullEventsClient rather than taking the flow run down with it.
+# - _WsInvalidStatus: the server rejected the handshake (HTTP 401/403), e.g. a
+#   client <=3.6.13 connecting to a server >=3.6.14 that has
+#   PREFECT_SERVER_API_AUTH_STRING configured.  A permanent failure.
+# - RETRYABLE_EXCEPTIONS: transient failures (handshake timeout, dropped
+#   connection, network errors) that the events client already retries; once
+#   those retries are exhausted the error must still not be fatal here.
+_NONFATAL_CONNECTION_EXCEPTIONS: tuple[type[Exception], ...] = (
+    _WsInvalidStatus,
+    *RETRYABLE_EXCEPTIONS,
+)
 
 
 def _default_get_events_client() -> "EventsClient":
@@ -60,17 +67,18 @@ class EventEmitter:
         try:
             await self._events_client.__aenter__()
         except _NONFATAL_CONNECTION_EXCEPTIONS as exc:
-            # The events WebSocket was rejected by the server (e.g. HTTP 401/403).
-            # This happens when an old client (<=3.6.13) connects to a server
-            # >=3.6.14 that has PREFECT_SERVER_API_AUTH_STRING configured: the
-            # server now requires the "prefect" subprotocol + auth handshake, but
-            # old clients omit it.  Events are non-critical telemetry, so we
-            # degrade gracefully instead of crashing the flow run.
+            # The events WebSocket could not be established -- either rejected by
+            # the server (HTTP 401/403, e.g. a client <=3.6.13 connecting to a
+            # server >=3.6.14 with PREFECT_SERVER_API_AUTH_STRING configured) or a
+            # transient connection failure that outlived the events client's own
+            # retries.  Events are non-critical telemetry, so degrade gracefully
+            # instead of crashing the flow run.
             self._logger.warning(
                 "Unable to connect to the events WebSocket (%s). "
-                "Event data will not be emitted for this runner. "
-                "Upgrade the Prefect client to >=3.6.14 to restore event support "
-                "when the server has authentication configured.",
+                "Event data will not be emitted for this runner; the flow run "
+                "will continue. Upgrade the Prefect client to >=3.6.14 to "
+                "restore event support if the server has authentication "
+                "configured.",
                 exc,
             )
             # __aenter__ failed, so __aexit__ must NOT be called on the original

{prefect_client-3.7.5.dev3 → prefect_client-3.7.5.dev5}/src/prefect/server/api/background_workers.py

@@ -14,6 +14,7 @@ from prefect.server.models.work_queues import mark_work_queues_ready
 from prefect.server.services.cancellation_cleanup import (
     cancel_child_task_runs,
     cancel_subflow_run,
+    handle_cancelling_timeout,
 )
 from prefect.server.services.db_vacuum import (
     vacuum_events_with_retention_overrides,
@@ -40,6 +41,7 @@ task_functions: list[Callable[..., Any]] = [
     delete_task_run_logs,
     delete_flow_run_logs,
     # Find-and-flood pattern tasks used by perpetual services
+    handle_cancelling_timeout,
     cancel_child_task_runs,
     cancel_subflow_run,
     fail_expired_pause,

{prefect_client-3.7.5.dev3 → prefect_client-3.7.5.dev5}/src/prefect/server/api/flow_runs.py

@@ -13,13 +13,14 @@ from uuid import UUID
 import orjson
 import sqlalchemy as sa
 from docket import Depends as DocketDepends
-from docket import Retry
+from docket import Docket, Retry
 from fastapi import (
     Body,
     Depends,
     HTTPException,
     Path,
     Query,
+    Request,
     Response,
 )
 from fastapi.encoders import jsonable_encoder
@@ -53,6 +54,9 @@ from prefect.server.schemas.responses import (
     FlowRunPaginationResponse,
     OrchestrationResult,
 )
+from prefect.server.services.cancellation_cleanup import (
+    maybe_schedule_cancelling_timeout_check_for_state,
+)
 from prefect.server.utilities.server import PrefectRouter
 from prefect.types import DateTime
 from prefect.types._datetime import earliest_possible_datetime, now
@@ -66,9 +70,41 @@ logger: "logging.Logger" = get_logger("server.api")
 router: PrefectRouter = PrefectRouter(prefix="/flow_runs", tags=["Flow Runs"])
 
 
+def _get_request_docket(request: Request) -> Docket | None:
+    return getattr(request.app.state, "docket", None)
+
+
+async def _maybe_schedule_cancelling_timeout_check_for_state(
+    *,
+    request: Request,
+    flow_run_id: UUID,
+    state: schemas.states.State | None,
+) -> None:
+    docket = _get_request_docket(request)
+    if docket is None:
+        return
+
+    try:
+        await maybe_schedule_cancelling_timeout_check_for_state(
+            docket=docket,
+            flow_run_id=flow_run_id,
+            state=state,
+        )
+    except Exception:
+        logger.exception(
+            "Failed to schedule CANCELLING timeout check; allowing accepted "
+            "state transition to proceed",
+            extra={
+                "flow_run_id": str(flow_run_id),
+                "flow_run_state_id": str(state.id) if state and state.id else None,
+            },
+        )
+
+
 @router.post("/")
 async def create_flow_run(
     flow_run: schemas.actions.FlowRunCreate,
+    request: Request,
     db: PrefectDBInterface = Depends(provide_database_interface),
     response: Response = None,  # type: ignore
     created_by: Optional[schemas.core.CreatedBy] = Depends(dependencies.get_created_by),
@@ -124,13 +160,27 @@ async def create_flow_run(
             flow_run=flow_run_object,
             orchestration_parameters=orchestration_parameters,
         )
-        if model.created >= right_now:
-            response.status_code = status.HTTP_201_CREATED
-
-        return schemas.responses.FlowRunResponse.model_validate(
+        created = model.created >= right_now
+        timeout_check_state = (
+            schemas.states.State.from_orm_without_result(model.state)
+            if created and model.state
+            else None
+        )
+        flow_run_id = model.id
+        flow_run_response = schemas.responses.FlowRunResponse.model_validate(
             model, from_attributes=True
         )
 
+    await _maybe_schedule_cancelling_timeout_check_for_state(
+        request=request,
+        flow_run_id=flow_run_id,
+        state=timeout_check_state,
+    )
+    if created:
+        response.status_code = status.HTTP_201_CREATED
+
+    return flow_run_response
+
 
 @router.patch("/{id:uuid}", status_code=status.HTTP_204_NO_CONTENT)
 async def update_flow_run(
@@ -669,6 +719,7 @@ async def bulk_delete_flow_runs(
 
 @router.post("/bulk_set_state")
 async def bulk_set_flow_run_state(
+    request: Request,
     flow_runs: Optional[schemas.filters.FlowRunFilter] = Body(
         None, description="Filter criteria for flow runs to update"
     ),
@@ -718,6 +769,7 @@ async def bulk_set_flow_run_state(
 
     # Process flow runs sequentially to avoid session conflicts
     for flow_run in db_flow_runs:
+        state_to_schedule: schemas.states.State | None = None
         async with db.session_context(
             begin_transaction=True, with_for_update=True
         ) as session:
@@ -739,6 +791,11 @@ async def bulk_set_flow_run_state(
                         details=orchestration_result.details,
                     )
                 )
+                if (
+                    orchestration_result.status
+                    == schemas.responses.SetStateStatus.ACCEPT
+                ):
+                    state_to_schedule = orchestration_result.state
             except Exception as e:
                 results.append(
                     FlowRunOrchestrationResult(
@@ -748,6 +805,14 @@ async def bulk_set_flow_run_state(
                         details=schemas.responses.StateAbortDetails(reason=str(e)),
                     )
                 )
+                continue
+
+        if state_to_schedule is not None:
+            await _maybe_schedule_cancelling_timeout_check_for_state(
+                request=request,
+                flow_run_id=flow_run.id,
+                state=state_to_schedule,
+            )
 
     return FlowRunBulkSetStateResponse(results=results)
 
@@ -755,6 +820,7 @@ async def bulk_set_flow_run_state(
 @router.post("/{id:uuid}/set_state")
 async def set_flow_run_state(
     response: Response,
+    request: Request,
     flow_run_id: UUID = Path(..., description="The flow run id", alias="id"),
     state: schemas.actions.StateCreate = Body(..., description="The intended state."),
     force: bool = Body(
@@ -796,6 +862,13 @@ async def set_flow_run_state(
             client_version=client_version,
         )
 
+    if orchestration_result.status == schemas.responses.SetStateStatus.ACCEPT:
+        await _maybe_schedule_cancelling_timeout_check_for_state(
+            request=request,
+            flow_run_id=flow_run_id,
+            state=orchestration_result.state,
+        )
+
     # set the 201 if a new state was created
     if orchestration_result.state and orchestration_result.state.timestamp >= right_now:
         response.status_code = status.HTTP_201_CREATED

{prefect_client-3.7.5.dev3 → prefect_client-3.7.5.dev5}/src/prefect/server/api/server.py

@@ -10,6 +10,7 @@ import base64
 import contextlib
 import gc
 import hmac
+import importlib.metadata
 import logging
 import mimetypes
 import os
@@ -41,6 +42,7 @@ from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.openapi.utils import get_openapi
 from fastapi.responses import JSONResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
+from packaging.version import Version
 from starlette.exceptions import HTTPException
 from typing_extensions import Self
 
@@ -73,6 +75,12 @@ from prefect.utilities.hashing import hash_objects
 
 logfire: Any | None = configure_logfire()
 
+# FastAPI < 0.137 copies routes when including a router; 0.137+ keeps a
+# reference to the original via _IncludedRouter.
+_FASTAPI_COPIES_ROUTES_ON_INCLUDE: bool = Version(
+    importlib.metadata.version("fastapi")
+) < Version("0.137.0")
+
 TITLE = "Prefect Server"
 API_TITLE = "Prefect REST API"
 UI_TITLE = "Prefect REST API UI"
@@ -491,21 +499,16 @@ def create_api_app(
 
     for router in API_ROUTERS:
         api_app.include_router(router, dependencies=dependencies)
-        if final:
-            # Important note about how FastAPI works:
-            #
-            # When including a router, FastAPI copies the routes and builds entirely new
-            # Pydantic models to represent the request bodies of the routes in the
-            # router.  This is because the dependencies may change if the same router is
-            # included multiple times, but it also means that we are holding onto an
-            # entire set of Pydantic models on the original routers for the duration of
-            # the server process that will never be used.
+        if final and _FASTAPI_COPIES_ROUTES_ON_INCLUDE:
+            # When including a router, older versions of FastAPI (< 0.137) copy
+            # the routes and build entirely new Pydantic models.  Since Prefect
+            # does not reuse routers, we can delete the originals to reclaim
+            # ~50-55 MB of memory.
             #
-            # Because Prefect does not reuse routers, we are free to clean up the routes
-            # because we know they won't be used again.  Thus, if we have the hint that
-            # this is the final instance we will create in this process, we can clean up
-            # the routes on the original source routers to conserve memory (~50-55MB as
-            # of introducing this change).
+            # FastAPI 0.137+ wraps included routers in an _IncludedRouter that
+            # references the original router for request matching, so the
+            # originals must be kept.  The duplication no longer occurs in this
+            # case either, so the optimisation is unnecessary.
             del router.routes
 
     if final:

{prefect_client-3.7.5.dev3 → prefect_client-3.7.5.dev5}/src/prefect/settings/_types.py

@@ -138,6 +138,7 @@ SettingAccessor: TypeAlias = Literal[
     "server.memoize_block_auto_registration",
     "server.metrics_enabled",
     "server.register_blocks_on_start",
+    "server.services.cancellation_cleanup.cancelling_timeout_seconds",
     "server.services.cancellation_cleanup.enabled",
     "server.services.cancellation_cleanup.loop_seconds",
     "server.services.cleanup_reconciler.batch_size",

{prefect_client-3.7.5.dev3 → prefect_client-3.7.5.dev5}/src/prefect/settings/models/server/services.py

@@ -44,6 +44,12 @@ class ServerServicesCancellationCleanupSettings(ServicesBaseSetting):
         ),
     )
 
+    cancelling_timeout_seconds: float = Field(
+        default=300,
+        gt=0,
+        description="The cancellation cleanup service will enqueue worker cleanup for flow runs that remain in CANCELLING longer than this many seconds. Defaults to `300`.",
+    )
+
 
 _VALID_VACUUM_TYPES = frozenset({"events", "flow_runs"})
 

prefect_client-3.7.5.dev3/src/prefect/_build_info.py

@@ -1,5 +0,0 @@
-# Generated by versioningit
-__version__ = "3.7.5.dev3"
-__build_date__ = "2026-06-13 09:10:40.924494+00:00"
-__git_commit__ = "7d7cd722bc2d696ba5b2cb79135f69e8b67811a8"
-__dirty__ = False