prefect-client 3.8.1.dev2__tar.gz → 3.8.1.dev4__tar.gz

{prefect_client-3.8.1.dev2 → prefect_client-3.8.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect-client
-Version: 3.8.1.dev2
+Version: 3.8.1.dev4
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io

prefect_client-3.8.1.dev4/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.8.1.dev4"
+__build_date__ = "2026-07-29 09:01:47.239051+00:00"
+__git_commit__ = "9fef82c1bf275f325220516421790f47b837100a"
+__dirty__ = False

{prefect_client-3.8.1.dev2 → prefect_client-3.8.1.dev4}/src/prefect/futures.py

@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any, Callable, Generic
 from typing_extensions import NamedTuple, Self, TypeVar
 
 from prefect._flow_run_suspension import raise_if_flow_run_suspension_requested
+from prefect._internal.concurrency.cancellation import CancelledError
 from prefect._internal.waiters import FlowRunWaiter
 from prefect.client.orchestration import get_client
 from prefect.exceptions import ObjectNotFound
@@ -34,6 +35,14 @@ if TYPE_CHECKING:
 logger: "logging.Logger" = get_logger(__name__)
 
 
+class _WaitTimeoutError(TimeoutError):
+    """Raised when Prefect's own waiting on futures times out.
+
+    Distinguishes a timeout produced by Prefect from a `TimeoutError` raised by
+    user code inside a task, which must be surfaced unchanged.
+    """
+
+
 class PrefectFuture(abc.ABC, Generic[R]):
     """
     Abstract base class for Prefect futures. A Prefect future is a handle to the
@@ -570,25 +579,38 @@ class PrefectFutureList(list[PrefectFuture[R]], Iterator[PrefectFuture[R]]):
             future_to_indices.setdefault(future, []).append(idx)
 
         results: list[R] = [None] * len(self)  # type: ignore[list-item]
+        timeout_message = (
+            f"Timed out waiting for all futures to complete within {timeout} seconds"
+        )
+        deadline = time.monotonic() + timeout if timeout is not None else None
 
         try:
-            # Wrap the entire loop in timeout_context so that both the wait
-            # for futures *and* any slow result retrieval (e.g. large data
-            # deserialization) are bounded by the caller's timeout.
-            with timeout_context(timeout):
-                # as_completed de-duplicates internally; each unique future
-                # is yielded exactly once, in completion order.
-                for future in as_completed(list(self), timeout=timeout):
-                    result = future.result(raise_on_failure=raise_on_failure)
-                    for i in future_to_indices[future]:
-                        results[i] = result
-        except TimeoutError as exc:
-            # timeout came from inside the task
-            if "Scope timed out after {timeout} second(s)." not in str(exc):
+            # `as_completed` de-duplicates internally; each unique future is
+            # yielded exactly once, in completion order. Its timeout scope stays
+            # entered while the loop body runs, so slow result retrieval (e.g.
+            # large data deserialization) is bounded by the caller's timeout too.
+            for future in as_completed(list(self), timeout=timeout):
+                result = future.result(raise_on_failure=raise_on_failure)
+                for i in future_to_indices[future]:
+                    results[i] = result
+                # Retrieval can overrun the deadline without cancellation
+                # arriving, e.g. on Windows where no sync cancel scope arms.
+                if deadline is not None and time.monotonic() >= deadline:
+                    raise TimeoutError(timeout_message)
+        # A `TimeoutError` from inside a task is not a `_WaitTimeoutError` and
+        # propagates unchanged.
+        except _WaitTimeoutError as exc:
+            raise TimeoutError(timeout_message) from exc
+        except CancelledError as exc:
+            # Cancel scopes deliver cancellation to the frame the supervised
+            # thread is currently in. While `as_completed` is suspended at its
+            # `yield`, that frame is this one, so its `timeout_context` never
+            # sees the error and cannot convert it. Only claim the cancellation
+            # if our own deadline has passed; otherwise it belongs to an
+            # enclosing scope, such as a flow run timeout.
+            if deadline is None or time.monotonic() < deadline:
                 raise
-            raise TimeoutError(
-                f"Timed out waiting for all futures to complete within {timeout} seconds"
-            ) from exc
+            raise TimeoutError(timeout_message) from exc
 
         return results
 
@@ -612,7 +634,7 @@ def as_completed(
     pending = unique_futures
     deadline: float | None = None
     try:
-        with timeout_context(timeout):
+        with timeout_context(timeout, timeout_exc_type=_WaitTimeoutError):
             done = {f for f in unique_futures if f._final_state}  # type: ignore[privateUsage]
             pending = unique_futures - done
             yield from done
@@ -638,7 +660,7 @@ def as_completed(
                     assert deadline is not None
                     remaining = deadline - time.monotonic()
                     if not finished_event.wait(timeout=max(remaining, 0.0)):
-                        raise TimeoutError
+                        raise _WaitTimeoutError
 
                 with finished_lock:
                     done = finished_futures
@@ -649,8 +671,8 @@ def as_completed(
                     pending.remove(future)
                     yield future
 
-    except TimeoutError:
-        raise TimeoutError(
+    except _WaitTimeoutError:
+        raise _WaitTimeoutError(
             "%d (of %d) futures unfinished" % (len(pending), total_futures)
         )
 

{prefect_client-3.8.1.dev2 → prefect_client-3.8.1.dev4}/src/prefect/workers/base.py

@@ -26,6 +26,7 @@ from zoneinfo import ZoneInfo
 
 import anyio
 import anyio.abc
+import httpx
 from exceptiongroup import BaseExceptionGroup, ExceptionGroup
 from pydantic import BaseModel, Field, PrivateAttr, field_validator
 from pydantic.json_schema import GenerateJsonSchema
@@ -118,6 +119,13 @@ if TYPE_CHECKING:
     from prefect.flows import Flow
 
 
+def _is_transient_api_error(exc: httpx.HTTPError) -> bool:
+    """Whether an API error is likely to resolve itself if retried."""
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code >= 500
+    return isinstance(exc, httpx.TransportError)
+
+
 class BaseJobConfiguration(BaseModel):
     command: Optional[str] = Field(
         default=None,
@@ -681,6 +689,7 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
         self._submitting_flow_run_ids: set[UUID] = set()
         self._scheduled_task_scopes: set[anyio.CancelScope] = set()
         self._worker_channel: Optional[WorkerChannel] = None
+        self._has_successfully_synced = False
 
         # Cancellation handling
         self._cancelling_observer: Optional[FlowRunCancellingObserver] = None
@@ -836,29 +845,36 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
         healthcheck_thread = None
         try:
             async with self as worker:
-                # schedule the scheduled flow run polling loop
+                polling_service = partial(
+                    critical_service_loop,
+                    workload=self.get_and_submit_flow_runs,
+                    interval=PREFECT_WORKER_QUERY_SECONDS.value(),
+                    run_once=run_once,
+                    jitter_range=0.3,
+                    backoff=4,  # Up to ~1 minute interval during backoff
+                )
+                sync_service = partial(
+                    critical_service_loop,
+                    workload=self._sync_and_initialize,
+                    interval=self.heartbeat_interval_seconds,
+                    run_once=run_once,
+                    jitter_range=0.3,
+                    backoff=4,
+                )
+
                 async with anyio.create_task_group() as loops_task_group:
-                    loops_task_group.start_soon(
-                        partial(
-                            critical_service_loop,
-                            workload=self.get_and_submit_flow_runs,
-                            interval=PREFECT_WORKER_QUERY_SECONDS.value(),
-                            run_once=run_once,
-                            jitter_range=0.3,
-                            backoff=4,  # Up to ~1 minute interval during backoff
-                        )
-                    )
-                    # schedule the sync loop
-                    loops_task_group.start_soon(
-                        partial(
-                            critical_service_loop,
-                            workload=self.sync_with_backend,
-                            interval=self.heartbeat_interval_seconds,
-                            run_once=run_once,
-                            jitter_range=0.3,
-                            backoff=4,
-                        )
-                    )
+                    if run_once:
+
+                        async def run_once_services() -> None:
+                            # A one-shot poll needs the work-pool configuration
+                            # produced by the one-shot synchronization attempt.
+                            await sync_service()
+                            await polling_service()
+
+                        loops_task_group.start_soon(run_once_services)
+                    else:
+                        loops_task_group.start_soon(polling_service)
+                        loops_task_group.start_soon(sync_service)
 
                     self._started_event = await self._emit_worker_started_event()
 
@@ -1193,6 +1209,7 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
     async def setup(self) -> None:
         """Prepares the worker to run."""
         self._logger.debug("Setting up worker...")
+        self._has_successfully_synced = False
         self._runs_task_group = anyio.create_task_group()
         self._limiter = (
             anyio.CapacityLimiter(self._limit) if self._limit is not None else None
@@ -1210,31 +1227,16 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
         # it in all API requests made by this worker process.
         os.environ["PREFECT__WORKER_NAME"] = self.name
 
-        await self.sync_with_backend()
-
-        # Initialize cancellation handling if enabled
-        if get_current_settings().worker.enable_cancellation and self._work_pool:
-            try:
-                self._cancelling_observer = await self._exit_stack.enter_async_context(
-                    FlowRunCancellingObserver(
-                        on_cancelling=lambda flow_run_id: (
-                            self._runs_task_group.start_soon(
-                                self._cancel_run, flow_run_id
-                            )
-                        ),
-                        polling_interval=get_current_settings().worker.cancellation_poll_seconds,
-                        event_filter=EventFilter(
-                            event=EventNameFilter(name=["prefect.flow-run.Cancelling"]),
-                            any_resource=EventAnyResourceFilter(
-                                id=[f"prefect.work-pool.{self._work_pool.id}"]
-                            ),
-                        ),
-                    )
-                )
-            except Exception as exc:
-                self._logger.warning(
-                    "Failed to setup cancellation handling: %s", exc, exc_info=True
-                )
+        try:
+            await self._sync_and_initialize()
+        except httpx.HTTPError as exc:
+            if not _is_transient_api_error(exc):
+                raise
+            self._logger.warning(
+                "Unable to reach the Prefect API while starting up; the worker will "
+                "keep attempting to connect: %s",
+                exc,
+            )
 
         self.is_setup = True
 
@@ -1266,6 +1268,8 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
         self._runs_task_group = None
         self._client = None
         self._worker_channel = None
+        self._has_successfully_synced = False
+        self._cancelling_observer = None
 
     def is_worker_still_polling(self, query_interval_seconds: float) -> bool:
         """
@@ -1297,6 +1301,14 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
         return is_still_polling
 
     async def get_and_submit_flow_runs(self) -> list["FlowRun"]:
+        if not self._has_successfully_synced:
+            self._logger.debug(
+                "Worker has not yet synced with the Prefect API; "
+                "skipping flow run submission."
+            )
+            self._last_polled_time = prefect.types._datetime.now("UTC")
+            return []
+
         runs_response = await self._get_scheduled_flow_runs()
 
         self._last_polled_time = prefect.types._datetime.now("UTC")
@@ -1341,6 +1353,15 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
             return WorkerMetadata(integrations=integration_versions)
         return None
 
+    async def _sync_and_initialize(self) -> None:
+        """Synchronize the worker and initialize sync-dependent services."""
+        await self.sync_with_backend()
+
+        # Preserve startup behavior for custom workers that override the public
+        # synchronization method without calling `super()`.
+        if not self._has_successfully_synced:
+            await self._initialize_after_sync()
+
     async def sync_with_backend(self) -> None:
         """
         Updates the worker's local information about it's current work pool and
@@ -1353,12 +1374,55 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
             return
 
         await self._worker_channel.sync(self._runs_task_group)
+        await self._initialize_after_sync()
 
         self._logger.debug(
             "Worker synchronized with the Prefect API server. "
             + (f"Remote ID: {self.backend_id}" if self.backend_id else "")
         )
 
+    async def _initialize_after_sync(self) -> None:
+        """Initialize services that depend on a completed backend sync."""
+        # Synchronization can succeed without a work pool when automatic work-pool
+        # creation is disabled. The worker cannot submit runs without its template.
+        if self._work_pool is None:
+            return
+
+        await self._setup_cancellation_observer()
+        self._has_successfully_synced = True
+
+    async def _setup_cancellation_observer(self) -> None:
+        """Start observing flow run cancellations if not already observing.
+
+        Requires a work pool, which is only available once the worker has
+        synced with the API.
+        """
+        if self._cancelling_observer is not None or self._work_pool is None:
+            return
+
+        if not get_current_settings().worker.enable_cancellation:
+            return
+
+        try:
+            self._cancelling_observer = await self._exit_stack.enter_async_context(
+                FlowRunCancellingObserver(
+                    on_cancelling=lambda flow_run_id: self._runs_task_group.start_soon(
+                        self._cancel_run, flow_run_id
+                    ),
+                    polling_interval=get_current_settings().worker.cancellation_poll_seconds,
+                    event_filter=EventFilter(
+                        event=EventNameFilter(name=["prefect.flow-run.Cancelling"]),
+                        any_resource=EventAnyResourceFilter(
+                            id=[f"prefect.work-pool.{self._work_pool.id}"]
+                        ),
+                    ),
+                )
+            )
+        except Exception as exc:
+            self._logger.warning(
+                "Failed to setup cancellation handling: %s", exc, exc_info=True
+            )
+
     def _ensure_worker_channel(self) -> None:
         if self._client is None:
             return

{prefect_client-3.8.1.dev2 → prefect_client-3.8.1.dev4}/src/prefect/workers/process.py

@@ -19,11 +19,9 @@ from __future__ import annotations
 import contextlib
 import os
 import tempfile
-import threading
 import warnings
-from functools import partial
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar
+from typing import TYPE_CHECKING, Any, Optional, TypeVar
 
 import anyio
 import anyio.abc
@@ -32,14 +30,8 @@ from pydantic import Field, field_validator
 from prefect._internal.schemas.validators import validate_working_dir
 from prefect.client.schemas.objects import Flow as APIFlow
 from prefect.runner.runner import Runner
-from prefect.settings import PREFECT_WORKER_QUERY_SECONDS
 from prefect.states import Pending
 from prefect.utilities.processutils import command_to_string, get_sys_executable
-from prefect.utilities.services import (
-    critical_service_loop,
-    start_client_metrics_server,
-    stop_client_metrics_server,
-)
 from prefect.workers.base import (
     BaseJobConfiguration,
     BaseVariables,
@@ -141,99 +133,6 @@ class ProcessWorker(
     _documentation_url = "https://docs.prefect.io/latest/get-started/quickstart"
     _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/356e6766a91baf20e1d08bbe16e8b5aaef4d8643-48x48.png"
 
-    async def start(
-        self,
-        run_once: bool = False,
-        with_healthcheck: bool = False,
-        printer: Callable[..., None] = print,
-    ) -> None:
-        """
-        Starts the worker and runs the main worker loops.
-
-        By default, the worker will run loops to poll for scheduled/cancelled flow
-        runs and sync with the Prefect API server.
-
-        If `run_once` is set, the worker will only run each loop once and then return.
-
-        If `with_healthcheck` is set, the worker will start a healthcheck server which
-        can be used to determine if the worker is still polling for flow runs and restart
-        the worker if necessary.
-
-        Args:
-            run_once: If set, the worker will only run each loop once then return.
-            with_healthcheck: If set, the worker will start a healthcheck server.
-            printer: A `print`-like function where logs will be reported.
-        """
-        healthcheck_server = None
-        healthcheck_thread = None
-        try:
-            async with self as worker:
-                # wait for an initial heartbeat to configure the worker
-                await worker.sync_with_backend()
-                # schedule the scheduled flow run polling loop
-                async with anyio.create_task_group() as loops_task_group:
-                    loops_task_group.start_soon(
-                        partial(
-                            critical_service_loop,
-                            workload=self.get_and_submit_flow_runs,
-                            interval=PREFECT_WORKER_QUERY_SECONDS.value(),
-                            run_once=run_once,
-                            jitter_range=0.3,
-                            backoff=4,  # Up to ~1 minute interval during backoff
-                        )
-                    )
-                    # schedule the sync loop
-                    loops_task_group.start_soon(
-                        partial(
-                            critical_service_loop,
-                            workload=self.sync_with_backend,
-                            interval=self.heartbeat_interval_seconds,
-                            run_once=run_once,
-                            jitter_range=0.3,
-                            backoff=4,
-                        )
-                    )
-
-                    self._started_event = await self._emit_worker_started_event()
-
-                    start_client_metrics_server()
-
-                    if with_healthcheck:
-                        from prefect.workers.server import build_healthcheck_server
-
-                        # we'll start the ASGI server in a separate thread so that
-                        # uvicorn does not block the main thread
-                        healthcheck_server = build_healthcheck_server(
-                            worker=worker,
-                            query_interval_seconds=PREFECT_WORKER_QUERY_SECONDS.value(),
-                        )
-                        healthcheck_thread = threading.Thread(
-                            name="healthcheck-server-thread",
-                            target=healthcheck_server.run,
-                            daemon=True,
-                        )
-                        healthcheck_thread.start()
-                    printer(f"Worker {worker.name!r} started!")
-
-                # If running once, wait for active runs to complete before exiting
-                if run_once and self._limiter:
-                    while self.limiter.borrowed_tokens > 0:
-                        self._logger.debug(
-                            "Waiting for %s active run(s) to finish before shutdown...",
-                            self.limiter.borrowed_tokens,
-                        )
-                        await anyio.sleep(0.1)
-        finally:
-            stop_client_metrics_server()
-
-            if healthcheck_server and healthcheck_thread:
-                self._logger.debug("Stopping healthcheck server...")
-                healthcheck_server.should_exit = True
-                healthcheck_thread.join()
-                self._logger.debug("Healthcheck server stopped.")
-
-        printer(f"Worker {worker.name!r} stopped!")
-
     async def run(
         self,
         flow_run: "FlowRun",

prefect_client-3.8.1.dev2/src/prefect/_build_info.py

@@ -1,5 +0,0 @@
-# Generated by versioningit
-__version__ = "3.8.1.dev2"
-__build_date__ = "2026-07-27 09:12:40.986268+00:00"
-__git_commit__ = "d630146acb690f2967e89c3f429ec3425644d87a"
-__dirty__ = False