prefect 3.4.19.dev5__tar.gz → 3.4.20.dev2__tar.gz

{prefect-3.4.19.dev5 → prefect-3.4.20.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect
-Version: 3.4.19.dev5
+Version: 3.4.20.dev2
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io

{prefect-3.4.19.dev5 → prefect-3.4.20.dev2}/pyproject.toml

@@ -159,7 +159,7 @@ dev = [
     "watchfiles>=1.0.4",
     "respx>=0.22.0",
     "codespell==2.4.1",
-    "ruff==0.13.0",
+    "ruff==0.13.1",
     "mypy==1.15.0",
     "types-cachetools>=5.3.1",
     "types-PyYAML>=6.0.12",

prefect-3.4.20.dev2/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.4.20.dev2"
+__build_date__ = "2025-09-25 08:09:36.522054+00:00"
+__git_commit__ = "11a83d5bc22024732ffc982a591f0ab14aec92e6"
+__dirty__ = False

{prefect-3.4.19.dev5 → prefect-3.4.20.dev2}/src/prefect/cli/server.py

@@ -43,6 +43,7 @@ from prefect.settings import (
     PREFECT_SERVER_LOGGING_LEVEL,
     PREFECT_UI_ENABLED,
     Profile,
+    get_current_settings,
     load_current_profile,
     load_profiles,
     save_profiles,
@@ -222,6 +223,84 @@ def prestart_check(base_url: str) -> None:
             )
 
 
+def _validate_multi_worker(workers: int) -> None:
+    """
+    Validates the configuration for running multiple Prefect server workers.
+
+    Multi-worker mode requires specific infrastructure components to ensure proper
+    coordination and data consistency across worker processes:
+
+    - **Database**: PostgreSQL is required (SQLite causes database locking issues)
+    - **Messaging**: Redis is required for event messaging (in-memory messaging
+      doesn't work across processes)
+
+    Args:
+        workers: The number of worker processes to run. Must be >= 1.
+
+    Raises:
+        exit_with_error: If the configuration is invalid.
+    """
+    from prefect.server.utilities.database import get_dialect
+
+    if workers == 1:
+        return
+
+    if workers < 1:
+        exit_with_error("Number of workers must be >= 1")
+
+    settings = get_current_settings()
+
+    try:
+        dialect = get_dialect(
+            settings.server.database.connection_url.get_secret_value()
+        )
+    except Exception as e:
+        exit_with_error(f"Unable to validate database configuration: {e}")
+
+    if dialect.name != "postgresql":
+        exit_with_error(
+            "Multi-worker mode (--workers > 1) is not supported with SQLite database."
+        )
+
+    try:
+        messaging_cache = settings.server.events.messaging_cache
+        messaging_broker = settings.server.events.messaging_broker
+        causal_ordering = settings.server.events.causal_ordering
+        lease_storage = settings.server.concurrency.lease_storage
+    except Exception as e:
+        exit_with_error(f"Unable to validate messaging configuration: {e}")
+
+    if (
+        messaging_cache == "prefect.server.utilities.messaging.memory"
+        or messaging_broker == "prefect.server.utilities.messaging.memory"
+        or causal_ordering == "prefect.server.events.ordering.memory"
+        or lease_storage == "prefect.server.concurrency.lease_storage.memory"
+    ):
+        error_message = textwrap.dedent(
+            """
+            Multi-worker mode (--workers > 1) requires Redis for messaging and lease storage.
+            
+            Please configure the following settings to use Redis:
+            
+                prefect config set PREFECT_MESSAGING_BROKER="prefect_redis.messaging"
+                prefect config set PREFECT_MESSAGING_CACHE="prefect_redis.messaging"
+                prefect config set PREFECT_SERVER_EVENTS_CAUSAL_ORDERING="prefect_redis.ordering"
+                prefect config set PREFECT_SERVER_CONCURRENCY_LEASE_STORAGE="prefect_redis.lease_storage"
+            
+            You'll also need to configure your Redis connection:
+            
+                export PREFECT_REDIS_MESSAGING_HOST="your-redis-host"
+                export PREFECT_REDIS_MESSAGING_PORT="6379"
+                export PREFECT_REDIS_MESSAGING_DB="0"
+            
+            For complete setup instructions, see:
+            https://docs.prefect.io/v3/how-to-guides/self-hosted/server-cli#multi-worker-api-server
+            https://docs.prefect.io/v3/advanced/self-hosted#redis-setup
+            """
+        ).strip()
+        exit_with_error(error_message)
+
+
 @server_app.command()
 def start(
     host: str = SettingsOption(PREFECT_SERVER_API_HOST),
@@ -240,6 +319,11 @@ def start(
     background: bool = typer.Option(
         False, "--background", "-b", help="Run the server in the background"
     ),
+    workers: int = typer.Option(
+        1,
+        "--workers",
+        help="Number of worker processes to run. Only runs the webserver API and UI",
+    ),
 ):
     """
     Start a Prefect server instance
@@ -251,6 +335,10 @@ def start(
         except Exception:
             pass
 
+    if workers > 1:
+        no_services = True
+    _validate_multi_worker(workers)
+
     server_settings = {
         "PREFECT_API_SERVICES_SCHEDULER_ENABLED": str(scheduler),
         "PREFECT_SERVER_ANALYTICS_ENABLED": str(analytics),
@@ -297,12 +385,30 @@ def start(
     app.console.print(generate_welcome_blurb(base_url, ui_enabled=ui))
     app.console.print("\n")
 
+    if workers > 1:
+        app.console.print(
+            f"Starting server with {workers} worker processes.\n", style="blue"
+        )
+
     if background:
         _run_in_background(
-            pid_file, server_settings, host, port, keep_alive_timeout, no_services
+            pid_file,
+            server_settings,
+            host,
+            port,
+            keep_alive_timeout,
+            no_services,
+            workers,
         )
     else:
-        _run_in_foreground(server_settings, host, port, keep_alive_timeout, no_services)
+        _run_in_foreground(
+            server_settings,
+            host,
+            port,
+            keep_alive_timeout,
+            no_services,
+            workers,
+        )
 
 
 def _run_in_background(
@@ -312,6 +418,7 @@ def _run_in_background(
     port: int,
     keep_alive_timeout: int,
     no_services: bool,
+    workers: int,
 ) -> None:
     command = [
         sys.executable,
@@ -327,6 +434,8 @@ def _run_in_background(
         str(port),
         "--timeout-keep-alive",
         str(keep_alive_timeout),
+        "--workers",
+        str(workers),
     ]
     logger.debug("Opening server process with command: %s", shlex.join(command))
 
@@ -356,6 +465,7 @@ def _run_in_foreground(
     port: int,
     keep_alive_timeout: int,
     no_services: bool,
+    workers: int,
 ) -> None:
     from prefect.server.api.server import create_app
 
@@ -363,16 +473,34 @@ def _run_in_foreground(
         with temporary_settings(
             {getattr(prefect.settings, k): v for k, v in server_settings.items()}
         ):
-            uvicorn.run(
-                app=create_app(final=True, webserver_only=no_services),
-                app_dir=str(prefect.__module_path__.parent),
-                host=host,
-                port=port,
-                timeout_keep_alive=keep_alive_timeout,
-                log_level=server_settings.get(
-                    "PREFECT_SERVER_LOGGING_LEVEL", "info"
-                ).lower(),
-            )
+            if workers == 1:
+                uvicorn.run(
+                    app=create_app(final=True, webserver_only=no_services),
+                    app_dir=str(prefect.__module_path__.parent),
+                    host=host,
+                    port=port,
+                    timeout_keep_alive=keep_alive_timeout,
+                    log_level=server_settings.get(
+                        "PREFECT_SERVER_LOGGING_LEVEL", "info"
+                    ).lower(),
+                )
+
+            else:
+                os.environ["PREFECT__SERVER_FINAL"] = "1"
+                os.environ["PREFECT__SERVER_WEBSERVER_ONLY"] = "1"
+
+                uvicorn.run(
+                    app="prefect.server.api.server:create_app",
+                    factory=True,
+                    host=host,
+                    port=port,
+                    timeout_keep_alive=keep_alive_timeout,
+                    log_level=server_settings.get(
+                        "PREFECT_SERVER_LOGGING_LEVEL", "info"
+                    ).lower(),
+                    workers=workers,
+                )
+
     finally:
         app.console.print("Server stopped!")
 

{prefect-3.4.19.dev5 → prefect-3.4.20.dev2}/src/prefect/client/schemas/filters.py

@@ -612,6 +612,21 @@ class LogFilterTaskRunId(PrefectBaseModel):
     )
 
 
+class LogFilterTextSearch(PrefectBaseModel):
+    """Filter by text search across log content."""
+
+    query: str = Field(
+        description="Text search query string",
+        examples=[
+            "error",
+            "error -debug",
+            '"connection timeout"',
+            "+required -excluded",
+        ],
+        max_length=200,
+    )
+
+
 class LogFilter(PrefectBaseModel, OperatorMixin):
     """Filter logs. Only logs matching all criteria will be returned"""
 
@@ -627,6 +642,9 @@ class LogFilter(PrefectBaseModel, OperatorMixin):
     task_run_id: Optional[LogFilterTaskRunId] = Field(
         default=None, description="Filter criteria for `Log.task_run_id`"
     )
+    text: Optional[LogFilterTextSearch] = Field(
+        default=None, description="Filter criteria for text search across log content"
+    )
 
 
 class FilterSet(PrefectBaseModel):

{prefect-3.4.19.dev5 → prefect-3.4.20.dev2}/src/prefect/events/filters.py

@@ -231,6 +231,21 @@ class EventIDFilter(EventDataFilter):
         return True
 
 
+class EventTextFilter(EventDataFilter):
+    """Filter by text search across event content."""
+
+    query: str = Field(
+        description="Text search query string",
+        examples=[
+            "error",
+            "error -debug",
+            '"connection timeout"',
+            "+required -excluded",
+        ],
+        max_length=200,
+    )
+
+
 class EventOrder(AutoEnum):
     ASC = "ASC"
     DESC = "DESC"
@@ -263,6 +278,10 @@ class EventFilter(EventDataFilter):
         default_factory=lambda: EventIDFilter(id=[]),
         description="Filter criteria for the events' ID",
     )
+    text: Optional[EventTextFilter] = Field(
+        default=None,
+        description="Filter criteria for text search across event content",
+    )
 
     order: EventOrder = Field(
         default=EventOrder.DESC,

prefect-3.4.20.dev2/src/prefect/runner/_observers.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import asyncio
+import uuid
+from contextlib import AsyncExitStack
+from typing import Any, Protocol
+
+from prefect.client.orchestration import PrefectClient, get_client
+from prefect.client.schemas.filters import (
+    FlowRunFilter,
+    FlowRunFilterId,
+    FlowRunFilterState,
+    FlowRunFilterStateName,
+    FlowRunFilterStateType,
+)
+from prefect.client.schemas.objects import StateType
+from prefect.events.clients import PrefectEventSubscriber, get_events_subscriber
+from prefect.events.filters import EventFilter, EventNameFilter
+from prefect.logging.loggers import get_logger
+from prefect.utilities.services import critical_service_loop
+
+
+class OnCancellingCallback(Protocol):
+    def __call__(self, flow_run_id: uuid.UUID) -> None: ...
+
+
+class FlowRunCancellingObserver:
+    def __init__(
+        self, on_cancelling: OnCancellingCallback, polling_interval: float = 10
+    ):
+        """
+        Observer that cancels flow runs when they are marked as cancelling.
+
+        Will use a websocket connection to listen for cancelling flow run events by default with a fallback
+        to polling when the websocket connection is lost.
+
+        Args:
+            on_cancelling: Callback to call when a flow run is marked as cancelling.
+            flow_run_ids: List of flow run IDs to poll when websocket connection is lost.
+            polling_interval: Interval in seconds to poll for cancelling flow runs when websocket connection is lost.
+        """
+        self.logger = get_logger("FlowRunCancellingObserver")
+        self.on_cancelling = on_cancelling
+        self.polling_interval = polling_interval
+        self._in_flight_flow_run_ids: set[uuid.UUID] = set()
+        self._events_subscriber: PrefectEventSubscriber | None
+        self._exit_stack = AsyncExitStack()
+        self._consumer_task: asyncio.Task[None] | None = None
+        self._polling_task: asyncio.Task[None] | None = None
+        self._is_shutting_down = False
+        self._client: PrefectClient | None = None
+        self._cancelling_flow_run_ids: set[uuid.UUID] = set()
+
+    def add_in_flight_flow_run_id(self, flow_run_id: uuid.UUID):
+        self.logger.debug("Adding in-flight flow run ID: %s", flow_run_id)
+        self._in_flight_flow_run_ids.add(flow_run_id)
+
+    def remove_in_flight_flow_run_id(self, flow_run_id: uuid.UUID):
+        self.logger.debug("Removing in-flight flow run ID: %s", flow_run_id)
+        self._in_flight_flow_run_ids.discard(flow_run_id)
+
+    async def _consume_events(self):
+        if self._events_subscriber is None:
+            raise RuntimeError(
+                "Events subscriber not initialized. Please use `async with` to initialize the observer."
+            )
+        async for event in self._events_subscriber:
+            try:
+                flow_run_id = uuid.UUID(
+                    event.resource["prefect.resource.id"].replace(
+                        "prefect.flow-run.", ""
+                    )
+                )
+                self.on_cancelling(flow_run_id)
+            except ValueError:
+                self.logger.debug(
+                    "Received event with invalid flow run ID: %s",
+                    event.resource["prefect.resource.id"],
+                )
+
+    def _start_polling_task(self, task: asyncio.Task[None]):
+        if task.cancelled():
+            # If the consumer task was cancelled, the observer is shutting down
+            # and we don't need to start the polling task
+            return
+        if exc := task.exception():
+            self.logger.debug(
+                "The FlowRunCancellingObserver websocket failed with an exception. Switching to polling mode.",
+                exc_info=exc,
+            )
+            self._polling_task = asyncio.create_task(
+                critical_service_loop(
+                    workload=self._check_for_cancelled_flow_runs,
+                    interval=self.polling_interval,
+                    jitter_range=0.3,
+                )
+            )
+            self._polling_task.add_done_callback(
+                lambda task: self.logger.error(
+                    "Cancellation polling task failed. Execution will continue, but flow run cancellation will fail.",
+                    exc_info=task.exception(),
+                )
+                if task.exception()
+                else self.logger.debug("Polling task completed")
+            )
+
+    async def _check_for_cancelled_flow_runs(self):
+        if self._is_shutting_down:
+            return
+        if self._client is None:
+            raise RuntimeError(
+                "Client not initialized. Please use `async with` to initialize the observer."
+            )
+
+        self.logger.debug("Checking for cancelled flow runs")
+        named_cancelling_flow_runs = await self._client.read_flow_runs(
+            flow_run_filter=FlowRunFilter(
+                state=FlowRunFilterState(
+                    type=FlowRunFilterStateType(any_=[StateType.CANCELLED]),
+                    name=FlowRunFilterStateName(any_=["Cancelling"]),
+                ),
+                # Avoid duplicate cancellation calls
+                id=FlowRunFilterId(
+                    any_=list(
+                        self._in_flight_flow_run_ids - self._cancelling_flow_run_ids
+                    )
+                ),
+            ),
+        )
+
+        typed_cancelling_flow_runs = await self._client.read_flow_runs(
+            flow_run_filter=FlowRunFilter(
+                state=FlowRunFilterState(
+                    type=FlowRunFilterStateType(any_=[StateType.CANCELLING]),
+                ),
+                # Avoid duplicate cancellation calls
+                id=FlowRunFilterId(
+                    any_=list(
+                        self._in_flight_flow_run_ids - self._cancelling_flow_run_ids
+                    )
+                ),
+            ),
+        )
+
+        cancelling_flow_runs = named_cancelling_flow_runs + typed_cancelling_flow_runs
+
+        if cancelling_flow_runs:
+            self.logger.info(
+                "Found %s flow runs awaiting cancellation.", len(cancelling_flow_runs)
+            )
+
+        for flow_run in cancelling_flow_runs:
+            self._cancelling_flow_run_ids.add(flow_run.id)
+            self.on_cancelling(flow_run.id)
+
+    async def __aenter__(self):
+        self._events_subscriber = await self._exit_stack.enter_async_context(
+            get_events_subscriber(
+                filter=EventFilter(
+                    event=EventNameFilter(name=["prefect.flow-run.Cancelling"])
+                ),
+            )
+        )
+        self._client = await self._exit_stack.enter_async_context(get_client())
+        self._consumer_task = asyncio.create_task(self._consume_events())
+        self._consumer_task.add_done_callback(self._start_polling_task)
+        return self
+
+    async def __aexit__(self, *exc_info: Any):
+        self.logger.debug("Shutting down FlowRunCancellingObserver")
+        self._is_shutting_down = True
+        await self._exit_stack.__aexit__(*exc_info)
+        if self._consumer_task is not None:
+            self._consumer_task.cancel()
+            try:
+                await self._consumer_task
+            except asyncio.CancelledError:
+                pass
+            except Exception:
+                self.logger.warning(
+                    "Consumer task exited with exception", exc_info=True
+                )
+                pass
+
+        if self._polling_task is not None:
+            self._polling_task.cancel()
+            try:
+                await self._polling_task
+            except asyncio.CancelledError:
+                pass
+            except Exception:
+                self.logger.warning("Polling task exited with exception", exc_info=True)
+                pass

{prefect-3.4.19.dev5 → prefect-3.4.20.dev2}/src/prefect/runner/runner.py

@@ -228,6 +228,7 @@ class Runner:
 
         self._exit_stack = AsyncExitStack()
         self._limiter: anyio.CapacityLimiter | None = None
+        self._cancelling_observer: FlowRunCancellingObserver | None = None
         self._client: PrefectClient = get_client()
         self._submitting_flow_run_ids: set[UUID] = set()
         self._cancelling_flow_run_ids: set[UUID] = set()
@@ -259,6 +260,24 @@ class Runner:
             self.__flow_run_process_map_lock = asyncio.Lock()
         return self.__flow_run_process_map_lock
 
+    async def _add_flow_run_process_map_entry(
+        self, flow_run_id: UUID, process_map_entry: ProcessMapEntry
+    ):
+        async with self._flow_run_process_map_lock:
+            self._flow_run_process_map[flow_run_id] = process_map_entry
+
+            if TYPE_CHECKING:
+                assert self._cancelling_observer is not None
+            self._cancelling_observer.add_in_flight_flow_run_id(flow_run_id)
+
+    async def _remove_flow_run_process_map_entry(self, flow_run_id: UUID):
+        async with self._flow_run_process_map_lock:
+            self._flow_run_process_map.pop(flow_run_id, None)
+
+            if TYPE_CHECKING:
+                assert self._cancelling_observer is not None
+            self._cancelling_observer.remove_in_flight_flow_run_id(flow_run_id)
+
     @sync_compatible
     async def add_deployment(
         self,
@@ -594,12 +613,11 @@ class Runner:
             if self.heartbeat_seconds is not None:
                 await self._emit_flow_run_heartbeat(flow_run)
 
-            async with self._flow_run_process_map_lock:
-                # Only add the process to the map if it is still running
-                if process.returncode is None:
-                    self._flow_run_process_map[flow_run.id] = ProcessMapEntry(
-                        pid=process.pid, flow_run=flow_run
-                    )
+            # Only add the process to the map if it is still running
+            if process.returncode is None:
+                await self._add_flow_run_process_map_entry(
+                    flow_run.id, ProcessMapEntry(pid=process.pid, flow_run=flow_run)
+                )
 
             while True:
                 # Wait until flow run execution is complete and the process has been removed from the map
@@ -641,14 +659,14 @@ class Runner:
             if self.heartbeat_seconds is not None:
                 await self._emit_flow_run_heartbeat(flow_run)
 
-            self._flow_run_process_map[flow_run.id] = ProcessMapEntry(
-                pid=process.pid, flow_run=flow_run
+            await self._add_flow_run_process_map_entry(
+                flow_run.id, ProcessMapEntry(pid=process.pid, flow_run=flow_run)
             )
             self._flow_run_bundle_map[flow_run.id] = bundle
 
             await anyio.to_thread.run_sync(process.join)
 
-            self._flow_run_process_map.pop(flow_run.id)
+            await self._remove_flow_run_process_map_entry(flow_run.id)
 
             flow_run_logger = self._get_flow_run_logger(flow_run)
             if process.exitcode is None:
@@ -1263,10 +1281,10 @@ class Runner:
             )
 
             if readiness_result and not isinstance(readiness_result, Exception):
-                async with self._flow_run_process_map_lock:
-                    self._flow_run_process_map[flow_run.id] = ProcessMapEntry(
-                        pid=readiness_result.pid, flow_run=flow_run
-                    )
+                await self._add_flow_run_process_map_entry(
+                    flow_run.id,
+                    ProcessMapEntry(pid=readiness_result.pid, flow_run=flow_run),
+                )
             # Heartbeats are opt-in and only emitted if a heartbeat frequency is set
             if self.heartbeat_seconds is not None:
                 await self._emit_flow_run_heartbeat(flow_run)
@@ -1321,8 +1339,7 @@ class Runner:
         finally:
             self._release_limit_slot(flow_run.id)
 
-            async with self._flow_run_process_map_lock:
-                self._flow_run_process_map.pop(flow_run.id, None)
+            await self._remove_flow_run_process_map_entry(flow_run.id)
 
         if status_code != 0 and not self._rescheduling:
             await self._propose_crashed_state(
@@ -1496,11 +1513,12 @@ class Runner:
         if not hasattr(self, "_loop") or not self._loop:
             self._loop = asyncio.get_event_loop()
 
-        await self._exit_stack.enter_async_context(
+        self._cancelling_observer = await self._exit_stack.enter_async_context(
             FlowRunCancellingObserver(
                 on_cancelling=lambda flow_run_id: self._runs_task_group.start_soon(
                     self._cancel_run, flow_run_id
-                )
+                ),
+                polling_interval=self.query_seconds,
             )
         )
         await self._exit_stack.enter_async_context(self._client)

{prefect-3.4.19.dev5 → prefect-3.4.20.dev2}/src/prefect/server/api/concurrency_limits.py

@@ -8,6 +8,7 @@ continues to work for backward compatibility.
 
 from __future__ import annotations
 
+import logging
 from datetime import timedelta
 from typing import List, Optional, Sequence
 from uuid import UUID
@@ -17,6 +18,7 @@ from fastapi import Body, Depends, HTTPException, Path, Response, status
 import prefect.server.api.dependencies as dependencies
 import prefect.server.models as models
 import prefect.server.schemas as schemas
+from prefect.logging.loggers import get_logger
 from prefect.server.api.concurrency_limits_v2 import MinimalConcurrencyLimitResponse
 from prefect.server.concurrency.lease_storage import (
     ConcurrencyLimitLeaseMetadata,
@@ -25,7 +27,6 @@ from prefect.server.concurrency.lease_storage import (
 from prefect.server.database import PrefectDBInterface, provide_database_interface
 from prefect.server.models import concurrency_limits
 from prefect.server.models import concurrency_limits_v2 as cl_v2_models
-from prefect.server.utilities.leasing import ResourceLease
 from prefect.server.utilities.server import PrefectRouter
 from prefect.settings import PREFECT_TASK_RUN_TAG_CONCURRENCY_SLOT_WAIT_SECONDS
 from prefect.types._concurrency import ConcurrencyLeaseHolder
@@ -33,7 +34,7 @@ from prefect.types._concurrency import ConcurrencyLeaseHolder
 router: PrefectRouter = PrefectRouter(
     prefix="/concurrency_limits", tags=["Concurrency Limits"]
 )
-
+logger: logging.Logger = get_logger(__name__)
 # V1 clients cannot renew leases; use a long TTL
 V1_LEASE_TTL = timedelta(days=100 * 365)  # ~100 years
 
@@ -597,7 +598,7 @@ async def decrement_concurrency_limits_v1(
 
         # Find and revoke lease for V2 limits
         if v2_limits:
-            leases_to_revoke: list[ResourceLease[ConcurrencyLimitLeaseMetadata]] = []
+            leases_ids_to_revoke: set[UUID] = set()
 
             for limit in v2_limits:
                 holders = await lease_storage.list_holders_for_limit(limit.id)
@@ -605,15 +606,19 @@ async def decrement_concurrency_limits_v1(
                     if holder.id == task_run_id:
                         lease = await lease_storage.read_lease(lease_id)
                         if lease:
-                            leases_to_revoke.append(lease)
+                            leases_ids_to_revoke.add(lease.id)
 
-            for lease in leases_to_revoke:
-                await cl_v2_models.bulk_decrement_active_slots(
-                    session=session,
-                    concurrency_limit_ids=lease.resource_ids,
-                    slots=lease.metadata.slots if lease.metadata else 0,
-                )
-                await lease_storage.revoke_lease(lease.id)
+            for lease_id in leases_ids_to_revoke:
+                lease = await lease_storage.read_lease(lease_id)
+                if lease:
+                    await cl_v2_models.bulk_decrement_active_slots(
+                        session=session,
+                        concurrency_limit_ids=lease.resource_ids,
+                        slots=lease.metadata.slots if lease.metadata else 0,
+                    )
+                    await lease_storage.revoke_lease(lease.id)
+                else:
+                    logger.warning(f"Lease {lease_id} not found during decrement")
 
             results.extend(
                 [