prefect 3.6.6__py3-none-any.whl → 3.6.7.dev3__py3-none-any.whl

prefect/_build_info.py

@@ -1,5 +1,5 @@
 # Generated by versioningit
-__version__ = "3.6.6"
-__build_date__ = "2025-12-11 20:20:23.556858+00:00"
-__git_commit__ = "f7d4baf4e04fc31b7a7949b9bc6f2e59d848272b"
+__version__ = "3.6.7.dev3"
+__build_date__ = "2025-12-14 08:09:01.617063+00:00"
+__git_commit__ = "96ab33fd33df46b6a16e3015b5bd1131e61f3ee1"
 __dirty__ = False

prefect/_experimental/plugins/spec.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import logging
 from dataclasses import dataclass
-from typing import Callable, Mapping, Optional
+from typing import Any, Callable, Mapping, Optional
 
 import pluggy
 
@@ -83,3 +83,22 @@ class HookSpec:
             - May be async or sync
             - Exceptions are caught and logged unless required=True in strict mode
         """
+
+    @hookspec
+    def set_database_connection_params(
+        self, connection_url: str, settings: Any
+    ) -> Mapping[str, Any]:
+        """
+        Set additional database connection parameters.
+
+        This hook is called when creating a database engine. It allows plugins
+        to provide additional connection parameters, such as authentication
+        tokens or SSL configuration.
+
+        Args:
+            connection_url: The database connection URL
+            settings: The current Prefect settings
+
+        Returns:
+            Dictionary of connection parameters to merge into connect_args
+        """

prefect/server/api/background_workers.py

@@ -14,10 +14,12 @@ from prefect.server.services.cancellation_cleanup import (
     cancel_child_task_runs,
     cancel_subflow_run,
 )
+from prefect.server.services.late_runs import mark_flow_run_late
 from prefect.server.services.pause_expirations import fail_expired_pause
 from prefect.server.services.perpetual_services import (
     register_and_schedule_perpetual_services,
 )
+from prefect.server.services.repossessor import revoke_expired_lease
 
 # Task functions to register with docket for background processing
 task_functions: list[Callable[..., Any]] = [
@@ -30,6 +32,8 @@ task_functions: list[Callable[..., Any]] = [
     cancel_child_task_runs,
     cancel_subflow_run,
     fail_expired_pause,
+    mark_flow_run_late,
+    revoke_expired_lease,
 ]
 
 

prefect/server/api/server.py

@@ -156,6 +156,7 @@ def _install_sqlite_locked_log_filter() -> None:
 
     filter_ = _SQLiteLockedOperationalErrorFilter()
     logging.getLogger("uvicorn.error").addFilter(filter_)
+    logging.getLogger("docket.worker").addFilter(filter_)
     _SQLITE_LOCKED_LOG_FILTER = filter_
 
 

prefect/server/database/configurations.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import sqlite3
 import ssl
 import traceback
@@ -25,6 +26,12 @@ from sqlalchemy.ext.asyncio import (
 from sqlalchemy.pool import ConnectionPoolEntry
 from typing_extensions import TypeAlias
 
+from prefect._experimental.plugins import (
+    HookSpec,
+    build_manager,
+    call_async_hook,
+    load_entry_point_plugins,
+)
 from prefect._internal.observability import configure_logfire
 from prefect.settings import (
     PREFECT_API_DATABASE_CONNECTION_TIMEOUT,
@@ -279,6 +286,33 @@ class AsyncPostgresConfiguration(BaseDatabaseConfiguration):
                 pg_ctx.verify_mode = ssl.CERT_REQUIRED
                 connect_args["ssl"] = pg_ctx
 
+            # Initialize plugin manager
+            if get_current_settings().experiments.plugins.enabled:
+                pm = build_manager(HookSpec)
+                load_entry_point_plugins(
+                    pm,
+                    allow=get_current_settings().experiments.plugins.allow,
+                    deny=get_current_settings().experiments.plugins.deny,
+                    logger=logging.getLogger("prefect.plugins"),
+                )
+
+                # Call set_database_connection_params hook
+                results = await call_async_hook(
+                    pm,
+                    "set_database_connection_params",
+                    connection_url=self.connection_url,
+                    settings=get_current_settings(),
+                )
+
+                for _, params, error in results:
+                    if error:
+                        # Log error but don't fail, other plugins might succeed
+                        logging.getLogger("prefect.server.database").warning(
+                            "Plugin failed to set database connection params: %s", error
+                        )
+                    elif params:
+                        connect_args.update(params)
+
             if connect_args:
                 kwargs["connect_args"] = connect_args
 

prefect/server/services/base.py

@@ -38,22 +38,14 @@ def _known_service_modules() -> list[ModuleType]:
     )
     from prefect.server.logs import stream as logs_stream
     from prefect.server.services import (
-        foreman,
-        late_runs,
-        repossessor,
         scheduler,
         task_run_recorder,
-        telemetry,
     )
 
     return [
         # Orchestration services
-        foreman,
-        late_runs,
-        repossessor,
         scheduler,
         task_run_recorder,
-        telemetry,
         # Events services
         event_logger,
         event_persister,

prefect/server/services/foreman.py

@@ -1,14 +1,18 @@
 """
-Foreman is a loop service designed to monitor workers.
+The Foreman service. Monitors workers and marks stale resources as offline/not ready.
 """
 
+from __future__ import annotations
+
+import logging
 from datetime import timedelta
-from typing import Any, Optional
 
 import sqlalchemy as sa
+from docket import Perpetual
 
+from prefect.logging import get_logger
 from prefect.server import models
-from prefect.server.database import PrefectDBInterface, db_injector
+from prefect.server.database import PrefectDBInterface, provide_database_interface
 from prefect.server.models.deployments import mark_deployments_not_ready
 from prefect.server.models.work_queues import mark_work_queues_not_ready
 from prefect.server.models.workers import emit_work_pool_status_event
@@ -18,224 +22,194 @@ from prefect.server.schemas.statuses import (
     WorkerStatus,
     WorkPoolStatus,
 )
-from prefect.server.services.base import LoopService
-from prefect.settings import (
-    PREFECT_API_SERVICES_FOREMAN_DEPLOYMENT_LAST_POLLED_TIMEOUT_SECONDS,
-    PREFECT_API_SERVICES_FOREMAN_FALLBACK_HEARTBEAT_INTERVAL_SECONDS,
-    PREFECT_API_SERVICES_FOREMAN_INACTIVITY_HEARTBEAT_MULTIPLE,
-    PREFECT_API_SERVICES_FOREMAN_LOOP_SECONDS,
-    PREFECT_API_SERVICES_FOREMAN_WORK_QUEUE_LAST_POLLED_TIMEOUT_SECONDS,
-    get_current_settings,
-)
-from prefect.settings.models.server.services import ServicesBaseSetting
+from prefect.server.services.perpetual_services import perpetual_service
+from prefect.settings.context import get_current_settings
 from prefect.types._datetime import now
 
+logger: logging.Logger = get_logger(__name__)
+
+
+@perpetual_service(
+    enabled_getter=lambda: get_current_settings().server.services.foreman.enabled,
+)
+async def monitor_worker_health(
+    perpetual: Perpetual = Perpetual(
+        automatic=False,
+        every=timedelta(
+            seconds=get_current_settings().server.services.foreman.loop_seconds
+        ),
+    ),
+) -> None:
+    """
+    Monitor workers and mark stale resources as offline/not ready.
 
-class Foreman(LoopService):
+    Iterates over workers currently marked as online. Marks workers as offline
+    if they have an old last_heartbeat_time. Marks work pools as not ready
+    if they do not have any online workers and are currently marked as ready.
+    Marks deployments as not ready if they have a last_polled time that is
+    older than the configured deployment last polled timeout.
     """
-    Monitors the status of workers and their associated work pools
+    settings = get_current_settings().server.services.foreman
+    db = provide_database_interface()
+
+    await _mark_online_workers_without_recent_heartbeat_as_offline(
+        db=db,
+        inactivity_heartbeat_multiple=settings.inactivity_heartbeat_multiple,
+        fallback_heartbeat_interval_seconds=settings.fallback_heartbeat_interval_seconds,
+    )
+    await _mark_work_pools_as_not_ready(db=db)
+    await _mark_deployments_as_not_ready(
+        db=db,
+        deployment_last_polled_timeout_seconds=settings.deployment_last_polled_timeout_seconds,
+    )
+    await _mark_work_queues_as_not_ready(
+        db=db,
+        work_queue_last_polled_timeout_seconds=settings.work_queue_last_polled_timeout_seconds,
+    )
+
+
+async def _mark_online_workers_without_recent_heartbeat_as_offline(
+    db: PrefectDBInterface,
+    inactivity_heartbeat_multiple: int,
+    fallback_heartbeat_interval_seconds: int,
+) -> None:
     """
+    Updates the status of workers that have an old last heartbeat time to OFFLINE.
 
-    @classmethod
-    def service_settings(cls) -> ServicesBaseSetting:
-        return get_current_settings().server.services.foreman
-
-    def __init__(
-        self,
-        loop_seconds: Optional[float] = None,
-        inactivity_heartbeat_multiple: Optional[int] = None,
-        fallback_heartbeat_interval_seconds: Optional[int] = None,
-        deployment_last_polled_timeout_seconds: Optional[int] = None,
-        work_queue_last_polled_timeout_seconds: Optional[int] = None,
-        **kwargs: Any,
-    ):
-        super().__init__(
-            loop_seconds=loop_seconds
-            or PREFECT_API_SERVICES_FOREMAN_LOOP_SECONDS.value(),
-            **kwargs,
-        )
-        self._inactivity_heartbeat_multiple = (
-            PREFECT_API_SERVICES_FOREMAN_INACTIVITY_HEARTBEAT_MULTIPLE.value()
-            if inactivity_heartbeat_multiple is None
-            else inactivity_heartbeat_multiple
-        )
-        self._fallback_heartbeat_interval_seconds = (
-            PREFECT_API_SERVICES_FOREMAN_FALLBACK_HEARTBEAT_INTERVAL_SECONDS.value()
-            if fallback_heartbeat_interval_seconds is None
-            else fallback_heartbeat_interval_seconds
-        )
-        self._deployment_last_polled_timeout_seconds = (
-            PREFECT_API_SERVICES_FOREMAN_DEPLOYMENT_LAST_POLLED_TIMEOUT_SECONDS.value()
-            if deployment_last_polled_timeout_seconds is None
-            else deployment_last_polled_timeout_seconds
+    An old heartbeat is one that is more than the worker's heartbeat interval
+    multiplied by the inactivity_heartbeat_multiple seconds ago.
+    """
+    async with db.session_context(begin_transaction=True) as session:
+        worker_update_stmt = (
+            sa.update(db.Worker)
+            .values(status=WorkerStatus.OFFLINE)
+            .where(
+                sa.func.date_diff_seconds(db.Worker.last_heartbeat_time)
+                > (
+                    sa.func.coalesce(
+                        db.Worker.heartbeat_interval_seconds,
+                        sa.bindparam("default_interval", sa.Integer),
+                    )
+                    * sa.bindparam("multiplier", sa.Integer)
+                ),
+                db.Worker.status == WorkerStatus.ONLINE,
+            )
         )
-        self._work_queue_last_polled_timeout_seconds = (
-            PREFECT_API_SERVICES_FOREMAN_WORK_QUEUE_LAST_POLLED_TIMEOUT_SECONDS.value()
-            if work_queue_last_polled_timeout_seconds is None
-            else work_queue_last_polled_timeout_seconds
+
+        result = await session.execute(
+            worker_update_stmt,
+            {
+                "multiplier": inactivity_heartbeat_multiple,
+                "default_interval": fallback_heartbeat_interval_seconds,
+            },
         )
 
-    @db_injector
-    async def run_once(self, db: PrefectDBInterface) -> None:
-        """
-        Iterate over workers current marked as online. Mark workers as offline
-        if they have an old last_heartbeat_time. Marks work pools as not ready
-        if they do not have any online workers and are currently marked as ready.
-        Mark deployments as not ready if they have a last_polled time that is
-        older than the configured deployment last polled timeout.
-        """
-        await self._mark_online_workers_without_a_recent_heartbeat_as_offline()
-        await self._mark_work_pools_as_not_ready()
-        await self._mark_deployments_as_not_ready()
-        await self._mark_work_queues_as_not_ready()
-
-    @db_injector
-    async def _mark_online_workers_without_a_recent_heartbeat_as_offline(
-        self, db: PrefectDBInterface
-    ) -> None:
-        """
-        Updates the status of workers that have an old last heartbeat time
-        to OFFLINE.
-
-        An old heartbeat last heartbeat that is one more than
-        their heartbeat interval multiplied by the
-        INACTIVITY_HEARTBEAT_MULTIPLE seconds ago.
-
-        Args:
-            session (AsyncSession): The session to use for the database operation.
-        """
-        async with db.session_context(begin_transaction=True) as session:
-            worker_update_stmt = (
-                sa.update(db.Worker)
-                .values(status=WorkerStatus.OFFLINE)
-                .where(
-                    sa.func.date_diff_seconds(db.Worker.last_heartbeat_time)
-                    > (
-                        sa.func.coalesce(
-                            db.Worker.heartbeat_interval_seconds,
-                            sa.bindparam("default_interval", sa.Integer),
-                        )
-                        * sa.bindparam("multiplier", sa.Integer)
-                    ),
-                    db.Worker.status == WorkerStatus.ONLINE,
-                )
-            )
+    if result.rowcount:
+        logger.info(f"Marked {result.rowcount} workers as offline.")
+
 
-            result = await session.execute(
-                worker_update_stmt,
-                {
-                    "multiplier": self._inactivity_heartbeat_multiple,
-                    "default_interval": self._fallback_heartbeat_interval_seconds,
-                },
+async def _mark_work_pools_as_not_ready(db: PrefectDBInterface) -> None:
+    """
+    Marks work pools as not ready if they have no online workers.
+
+    Emits an event and updates any bookkeeping fields on the work pool.
+    """
+    async with db.session_context(begin_transaction=True) as session:
+        work_pools_select_stmt = (
+            sa.select(db.WorkPool)
+            .filter(db.WorkPool.status == "READY")
+            .outerjoin(
+                db.Worker,
+                sa.and_(
+                    db.Worker.work_pool_id == db.WorkPool.id,
+                    db.Worker.status == "ONLINE",
+                ),
             )
+            .group_by(db.WorkPool.id)
+            .having(sa.func.count(db.Worker.id) == 0)
+        )
 
-        if result.rowcount:
-            self.logger.info(f"Marked {result.rowcount} workers as offline.")
-
-    @db_injector
-    async def _mark_work_pools_as_not_ready(self, db: PrefectDBInterface):
-        """
-        Marks a work pool as not ready.
-
-        Emits and event and updates any bookkeeping fields on the work pool.
-
-        Args:
-            work_pool (db.WorkPool): The work pool to mark as not ready.
-        """
-        async with db.session_context(begin_transaction=True) as session:
-            work_pools_select_stmt = (
-                sa.select(db.WorkPool)
-                .filter(db.WorkPool.status == "READY")
-                .outerjoin(
-                    db.Worker,
-                    sa.and_(
-                        db.Worker.work_pool_id == db.WorkPool.id,
-                        db.Worker.status == "ONLINE",
-                    ),
-                )
-                .group_by(db.WorkPool.id)
-                .having(sa.func.count(db.Worker.id) == 0)
+        result = await session.execute(work_pools_select_stmt)
+        work_pools = result.scalars().all()
+
+        for work_pool in work_pools:
+            await models.workers.update_work_pool(
+                session=session,
+                work_pool_id=work_pool.id,
+                work_pool=InternalWorkPoolUpdate(status=WorkPoolStatus.NOT_READY),
+                emit_status_change=emit_work_pool_status_event,
             )
 
-            result = await session.execute(work_pools_select_stmt)
-            work_pools = result.scalars().all()
+            logger.info(f"Marked work pool {work_pool.id} as NOT_READY.")
 
-            for work_pool in work_pools:
-                await models.workers.update_work_pool(
-                    session=session,
-                    work_pool_id=work_pool.id,
-                    work_pool=InternalWorkPoolUpdate(status=WorkPoolStatus.NOT_READY),
-                    emit_status_change=emit_work_pool_status_event,
-                )
 
-                self.logger.info(f"Marked work pool {work_pool.id} as NOT_READY.")
-
-    @db_injector
-    async def _mark_deployments_as_not_ready(self, db: PrefectDBInterface) -> None:
-        """
-        Marks a deployment as NOT_READY and emits a deployment status event.
-        Emits an event and updates any bookkeeping fields on the deployment.
-        Args:
-            session (AsyncSession): The session to use for the database operation.
-        """
-        async with db.session_context(begin_transaction=True) as session:
-            status_timeout_threshold = now("UTC") - timedelta(
-                seconds=self._deployment_last_polled_timeout_seconds
-            )
-            deployment_id_select_stmt = (
-                sa.select(db.Deployment.id)
-                .outerjoin(db.WorkQueue, db.WorkQueue.id == db.Deployment.work_queue_id)
-                .filter(db.Deployment.status == DeploymentStatus.READY)
-                .filter(db.Deployment.last_polled.isnot(None))
-                .filter(
-                    sa.or_(
-                        # if work_queue.last_polled doesn't exist, use only deployment's
-                        # last_polled
-                        sa.and_(
-                            db.WorkQueue.last_polled.is_(None),
-                            db.Deployment.last_polled < status_timeout_threshold,
-                        ),
-                        # if work_queue.last_polled exists, both times should be less than
-                        # the threshold
-                        sa.and_(
-                            db.WorkQueue.last_polled.isnot(None),
-                            db.Deployment.last_polled < status_timeout_threshold,
-                            db.WorkQueue.last_polled < status_timeout_threshold,
-                        ),
-                    )
+async def _mark_deployments_as_not_ready(
+    db: PrefectDBInterface,
+    deployment_last_polled_timeout_seconds: int,
+) -> None:
+    """
+    Marks deployments as NOT_READY based on their last_polled field.
+
+    Emits an event and updates any bookkeeping fields on the deployment.
+    """
+    async with db.session_context(begin_transaction=True) as session:
+        status_timeout_threshold = now("UTC") - timedelta(
+            seconds=deployment_last_polled_timeout_seconds
+        )
+        deployment_id_select_stmt = (
+            sa.select(db.Deployment.id)
+            .outerjoin(db.WorkQueue, db.WorkQueue.id == db.Deployment.work_queue_id)
+            .filter(db.Deployment.status == DeploymentStatus.READY)
+            .filter(db.Deployment.last_polled.isnot(None))
+            .filter(
+                sa.or_(
+                    # if work_queue.last_polled doesn't exist, use only deployment's
+                    # last_polled
+                    sa.and_(
+                        db.WorkQueue.last_polled.is_(None),
+                        db.Deployment.last_polled < status_timeout_threshold,
+                    ),
+                    # if work_queue.last_polled exists, both times should be less than
+                    # the threshold
+                    sa.and_(
+                        db.WorkQueue.last_polled.isnot(None),
+                        db.Deployment.last_polled < status_timeout_threshold,
+                        db.WorkQueue.last_polled < status_timeout_threshold,
+                    ),
                 )
             )
-            result = await session.execute(deployment_id_select_stmt)
+        )
+        result = await session.execute(deployment_id_select_stmt)
 
-            deployment_ids_to_mark_unready = result.scalars().all()
+        deployment_ids_to_mark_unready = result.scalars().all()
 
-        await mark_deployments_not_ready(
-            deployment_ids=deployment_ids_to_mark_unready,
-        )
+    await mark_deployments_not_ready(
+        deployment_ids=deployment_ids_to_mark_unready,
+    )
 
-    @db_injector
-    async def _mark_work_queues_as_not_ready(self, db: PrefectDBInterface):
-        """
-        Marks work queues as NOT_READY based on their last_polled field.
-
-        Args:
-            session (AsyncSession): The session to use for the database operation.
-        """
-        async with db.session_context(begin_transaction=True) as session:
-            status_timeout_threshold = now("UTC") - timedelta(
-                seconds=self._work_queue_last_polled_timeout_seconds
-            )
-            id_select_stmt = (
-                sa.select(db.WorkQueue.id)
-                .outerjoin(db.WorkPool, db.WorkPool.id == db.WorkQueue.work_pool_id)
-                .filter(db.WorkQueue.status == "READY")
-                .filter(db.WorkQueue.last_polled.isnot(None))
-                .filter(db.WorkQueue.last_polled < status_timeout_threshold)
-                .order_by(db.WorkQueue.last_polled.asc())
-            )
-            result = await session.execute(id_select_stmt)
-            unready_work_queue_ids = result.scalars().all()
 
-        await mark_work_queues_not_ready(
-            work_queue_ids=unready_work_queue_ids,
+async def _mark_work_queues_as_not_ready(
+    db: PrefectDBInterface,
+    work_queue_last_polled_timeout_seconds: int,
+) -> None:
+    """
+    Marks work queues as NOT_READY based on their last_polled field.
+    """
+    async with db.session_context(begin_transaction=True) as session:
+        status_timeout_threshold = now("UTC") - timedelta(
+            seconds=work_queue_last_polled_timeout_seconds
         )
+        id_select_stmt = (
+            sa.select(db.WorkQueue.id)
+            .outerjoin(db.WorkPool, db.WorkPool.id == db.WorkQueue.work_pool_id)
+            .filter(db.WorkQueue.status == "READY")
+            .filter(db.WorkQueue.last_polled.isnot(None))
+            .filter(db.WorkQueue.last_polled < status_timeout_threshold)
+            .order_by(db.WorkQueue.last_polled.asc())
+        )
+        result = await session.execute(id_select_stmt)
+        unready_work_queue_ids = result.scalars().all()
+
+    await mark_work_queues_not_ready(
+        work_queue_ids=unready_work_queue_ids,
+    )