prefect 3.4.9.dev3__tar.gz → 3.4.9.dev5__tar.gz

{prefect-3.4.9.dev3 → prefect-3.4.9.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect
-Version: 3.4.9.dev3
+Version: 3.4.9.dev5
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io

prefect-3.4.9.dev5/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.4.9.dev5"
+__build_date__ = "2025-07-17 08:09:55.197660+00:00"
+__git_commit__ = "723bddc79cdcfb7d9c234189308cfcf3340caf84"
+__dirty__ = False

{prefect-3.4.9.dev3 → prefect-3.4.9.dev5}/src/prefect/cli/deployment.py

@@ -873,6 +873,16 @@ async def run(
                 f"deployment: {listrepr(unknown_keys, sep=', ')}"
                 f"\n{available_parameters}"
             )
+        templating_parameters = {**(deployment.parameters or {}), **(parameters or {})}
+        if flow_run_name:
+            try:
+                flow_run_name = flow_run_name.format(**templating_parameters)
+            except KeyError as e:
+                exit_with_error(
+                    f"Missing parameter for flow run name: '{e.args[0]}' is undefined"
+                )
+            except Exception as e:
+                exit_with_error(f"Failed to format flow run name: {e}")
 
         app.console.print(
             f"Creating flow run for deployment '{flow.name}/{deployment.name}'...",

{prefect-3.4.9.dev3 → prefect-3.4.9.dev5}/src/prefect/server/api/concurrency_limits_v2.py

@@ -1,11 +1,17 @@
+from datetime import timedelta
 from typing import List, Literal, Optional, Union
 from uuid import UUID
 
 from fastapi import Body, Depends, HTTPException, Path, status
+from sqlalchemy.ext.asyncio import AsyncSession
 
 import prefect.server.models as models
 import prefect.server.schemas as schemas
 from prefect.server.api.dependencies import LimitBody
+from prefect.server.concurrency.lease_storage import (
+    ConcurrencyLimitLeaseMetadata,
+    get_concurrency_lease_storage,
+)
 from prefect.server.database import PrefectDBInterface, provide_database_interface
 from prefect.server.schemas import actions
 from prefect.server.utilities.schemas import PrefectBaseModel
@@ -151,6 +157,97 @@ class MinimalConcurrencyLimitResponse(PrefectBaseModel):
     limit: int
 
 
+class ConcurrencyLimitWithLeaseResponse(PrefectBaseModel):
+    lease_id: UUID
+    limits: list[MinimalConcurrencyLimitResponse]
+
+
+async def _acquire_concurrency_slots(
+    session: AsyncSession,
+    names: List[str],
+    slots: int,
+    mode: Literal["concurrency", "rate_limit"],
+) -> tuple[list[schemas.core.ConcurrencyLimitV2], bool]:
+    limits = [
+        schemas.core.ConcurrencyLimitV2.model_validate(limit)
+        for limit in (
+            await models.concurrency_limits_v2.bulk_read_concurrency_limits(
+                session=session, names=names
+            )
+        )
+    ]
+
+    active_limits = [limit for limit in limits if bool(limit.active)]
+
+    if any(limit.limit < slots for limit in active_limits):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Slots requested is greater than the limit",
+        )
+
+    non_decaying = [
+        str(limit.name) for limit in active_limits if limit.slot_decay_per_second == 0.0
+    ]
+
+    if mode == "rate_limit" and non_decaying:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Only concurrency limits with slot decay can be used for "
+                "rate limiting. The following limits do not have a decay "
+                f"configured: {','.join(non_decaying)!r}"
+            ),
+        )
+    acquired = await models.concurrency_limits_v2.bulk_increment_active_slots(
+        session=session,
+        concurrency_limit_ids=[limit.id for limit in active_limits],
+        slots=slots,
+    )
+
+    if not acquired:
+        await session.rollback()
+
+    return limits, acquired
+
+
+async def _generate_concurrency_locked_response(
+    session: AsyncSession,
+    limits: list[schemas.core.ConcurrencyLimitV2],
+    slots: int,
+) -> HTTPException:
+    active_limits = [limit for limit in limits if bool(limit.active)]
+
+    await models.concurrency_limits_v2.bulk_update_denied_slots(
+        session=session,
+        concurrency_limit_ids=[limit.id for limit in active_limits],
+        slots=slots,
+    )
+
+    def num_blocking_slots(limit: schemas.core.ConcurrencyLimitV2) -> float:
+        if limit.slot_decay_per_second > 0:
+            return slots + limit.denied_slots
+        else:
+            return (slots + limit.denied_slots) / limit.limit
+
+    blocking_limit = max((limit for limit in active_limits), key=num_blocking_slots)
+    blocking_slots = num_blocking_slots(blocking_limit)
+
+    wait_time_per_slot = (
+        blocking_limit.avg_slot_occupancy_seconds
+        if blocking_limit.slot_decay_per_second == 0.0
+        else (1.0 / blocking_limit.slot_decay_per_second)
+    )
+
+    retry_after = wait_time_per_slot * blocking_slots
+
+    return HTTPException(
+        status_code=status.HTTP_423_LOCKED,
+        headers={
+            "Retry-After": str(retry_after),
+        },
+    )
+
+
 @router.post("/increment", status_code=status.HTTP_200_OK)
 async def bulk_increment_active_slots(
     slots: int = Body(..., gt=0),
@@ -163,86 +260,75 @@ async def bulk_increment_active_slots(
     db: PrefectDBInterface = Depends(provide_database_interface),
 ) -> List[MinimalConcurrencyLimitResponse]:
     async with db.session_context(begin_transaction=True) as session:
-        limits = [
-            schemas.core.ConcurrencyLimitV2.model_validate(limit)
-            for limit in (
-                await models.concurrency_limits_v2.bulk_read_concurrency_limits(
-                    session=session, names=names
-                )
-            )
-        ]
-
-        active_limits = [limit for limit in limits if bool(limit.active)]
-
-        if any(limit.limit < slots for limit in active_limits):
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail="Slots requested is greater than the limit",
-            )
-
-        non_decaying = [
-            str(limit.name)
-            for limit in active_limits
-            if limit.slot_decay_per_second == 0.0
-        ]
-
-        if mode == "rate_limit" and non_decaying:
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=(
-                    "Only concurrency limits with slot decay can be used for "
-                    "rate limiting. The following limits do not have a decay "
-                    f"configured: {','.join(non_decaying)!r}"
-                ),
-            )
-        acquired = await models.concurrency_limits_v2.bulk_increment_active_slots(
+        acquired_limits, acquired = await _acquire_concurrency_slots(
             session=session,
-            concurrency_limit_ids=[limit.id for limit in active_limits],
+            names=names,
             slots=slots,
+            mode=mode,
         )
 
-        if not acquired:
-            await session.rollback()
-
     if acquired:
         return [
             MinimalConcurrencyLimitResponse(
                 id=limit.id, name=str(limit.name), limit=limit.limit
             )
-            for limit in limits
+            for limit in acquired_limits
         ]
     else:
         async with db.session_context(begin_transaction=True) as session:
-            await models.concurrency_limits_v2.bulk_update_denied_slots(
+            raise await _generate_concurrency_locked_response(
                 session=session,
-                concurrency_limit_ids=[limit.id for limit in active_limits],
+                limits=acquired_limits,
                 slots=slots,
             )
 
-        def num_blocking_slots(limit: schemas.core.ConcurrencyLimitV2) -> float:
-            if limit.slot_decay_per_second > 0.0:
-                return slots + limit.denied_slots
-            else:
-                return (slots + limit.denied_slots) / limit.limit
 
-        blocking_limit = max((limit for limit in active_limits), key=num_blocking_slots)
-        blocking_slots = num_blocking_slots(blocking_limit)
-
-        wait_time_per_slot = (
-            blocking_limit.avg_slot_occupancy_seconds
-            if blocking_limit.slot_decay_per_second == 0.0
-            else (1.0 / blocking_limit.slot_decay_per_second)
+@router.post("/increment-with-lease", status_code=status.HTTP_200_OK)
+async def bulk_increment_active_slots_with_lease(
+    slots: int = Body(..., gt=0),
+    names: List[str] = Body(..., min_items=1),
+    mode: Literal["concurrency", "rate_limit"] = Body("concurrency"),
+    lease_duration: float = Body(
+        300,  # 5 minutes
+        ge=60,  # 1 minute
+        le=60 * 60 * 24,  # 1 day
+        description="The duration of the lease in seconds.",
+    ),
+    db: PrefectDBInterface = Depends(provide_database_interface),
+) -> ConcurrencyLimitWithLeaseResponse:
+    async with db.session_context(begin_transaction=True) as session:
+        acquired_limits, acquired = await _acquire_concurrency_slots(
+            session=session,
+            names=names,
+            slots=slots,
+            mode=mode,
         )
 
-        retry_after = wait_time_per_slot * blocking_slots
-
-        raise HTTPException(
-            status_code=status.HTTP_423_LOCKED,
-            headers={
-                "Retry-After": str(retry_after),
-            },
+    if acquired:
+        lease_storage = get_concurrency_lease_storage()
+        lease = await lease_storage.create_lease(
+            resource_ids=[limit.id for limit in acquired_limits],
+            ttl=timedelta(seconds=lease_duration),
+            metadata=ConcurrencyLimitLeaseMetadata(slots=slots),
+        )
+        return ConcurrencyLimitWithLeaseResponse(
+            lease_id=lease.id,
+            limits=[
+                MinimalConcurrencyLimitResponse(
+                    id=limit.id, name=str(limit.name), limit=limit.limit
+                )
+                for limit in acquired_limits
+            ],
         )
 
+    else:
+        async with db.session_context(begin_transaction=True) as session:
+            raise await _generate_concurrency_locked_response(
+                session=session,
+                limits=acquired_limits,
+                slots=slots,
+            )
+
 
 @router.post("/decrement", status_code=status.HTTP_200_OK)
 async def bulk_decrement_active_slots(
@@ -276,3 +362,51 @@ async def bulk_decrement_active_slots(
         )
         for limit in limits
     ]
+
+
+@router.post("/decrement-with-lease", status_code=status.HTTP_204_NO_CONTENT)
+async def bulk_decrement_active_slots_with_lease(
+    lease_id: UUID = Body(
+        ...,
+        description="The ID of the lease corresponding to the concurrency limits to decrement.",
+    ),
+    occupancy_seconds: Optional[float] = Body(None, gt=0.0),
+    db: PrefectDBInterface = Depends(provide_database_interface),
+) -> None:
+    lease_storage = get_concurrency_lease_storage()
+    lease = await lease_storage.read_lease(lease_id)
+    if not lease:
+        return
+
+    async with db.session_context(begin_transaction=True) as session:
+        await models.concurrency_limits_v2.bulk_decrement_active_slots(
+            session=session,
+            concurrency_limit_ids=lease.resource_ids,
+            slots=lease.metadata.slots if lease.metadata else 0,
+            occupancy_seconds=occupancy_seconds,
+        )
+    await lease_storage.revoke_lease(lease_id)
+
+
+@router.post("/leases/{lease_id}/renew", status_code=status.HTTP_204_NO_CONTENT)
+async def renew_concurrency_lease(
+    lease_id: UUID = Path(..., description="The ID of the lease to renew"),
+    lease_duration: float = Body(
+        300,  # 5 minutes
+        ge=60,  # 1 minute
+        le=60 * 60 * 24,  # 1 day
+        description="The duration of the lease in seconds.",
+        embed=True,
+    ),
+) -> None:
+    lease_storage = get_concurrency_lease_storage()
+    lease = await lease_storage.read_lease(lease_id)
+    if not lease:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail="Lease not found"
+        )
+
+    await lease_storage.renew_lease(
+        lease_id=lease_id,
+        ttl=timedelta(seconds=lease_duration),
+    )

{prefect-3.4.9.dev3 → prefect-3.4.9.dev5}/src/prefect/server/api/flow_runs.py

@@ -11,6 +11,7 @@ from uuid import UUID
 import orjson
 import sqlalchemy as sa
 from fastapi import (
+    BackgroundTasks,
     Body,
     Depends,
     HTTPException,
@@ -570,6 +571,7 @@ async def read_flow_runs(
 
 @router.delete("/{id:uuid}", status_code=status.HTTP_204_NO_CONTENT)
 async def delete_flow_run(
+    background_tasks: BackgroundTasks,
     flow_run_id: UUID = Path(..., description="The flow run id", alias="id"),
     db: PrefectDBInterface = Depends(provide_database_interface),
 ) -> None:
@@ -584,6 +586,17 @@ async def delete_flow_run(
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND, detail="Flow run not found"
         )
+    background_tasks.add_task(delete_flow_run_logs, db, flow_run_id)
+
+
+async def delete_flow_run_logs(db: PrefectDBInterface, flow_run_id: UUID) -> None:
+    async with db.session_context(begin_transaction=True) as session:
+        await models.logs.delete_logs(
+            session=session,
+            log_filter=schemas.filters.LogFilter(
+                flow_run_id=schemas.filters.LogFilterFlowRunId(any_=[flow_run_id])
+            ),
+        )
 
 
 @router.post("/{id:uuid}/set_state")

{prefect-3.4.9.dev3 → prefect-3.4.9.dev5}/src/prefect/server/api/task_runs.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional
 from uuid import UUID
 
 from fastapi import (
+    BackgroundTasks,
     Body,
     Depends,
     HTTPException,
@@ -267,6 +268,7 @@ async def paginate_task_runs(
 
 @router.delete("/{id:uuid}", status_code=status.HTTP_204_NO_CONTENT)
 async def delete_task_run(
+    background_tasks: BackgroundTasks,
     task_run_id: UUID = Path(..., description="The task run id", alias="id"),
     db: PrefectDBInterface = Depends(provide_database_interface),
 ) -> None:
@@ -279,6 +281,17 @@ async def delete_task_run(
         )
     if not result:
         raise HTTPException(status.HTTP_404_NOT_FOUND, detail="Task not found")
+    background_tasks.add_task(delete_task_run_logs, db, task_run_id)
+
+
+async def delete_task_run_logs(db: PrefectDBInterface, task_run_id: UUID) -> None:
+    async with db.session_context(begin_transaction=True) as session:
+        await models.logs.delete_logs(
+            session=session,
+            log_filter=schemas.filters.LogFilter(
+                task_run_id=schemas.filters.LogFilterTaskRunId(any_=[task_run_id])
+            ),
+        )
 
 
 @router.post("/{id:uuid}/set_state")

prefect-3.4.9.dev5/src/prefect/server/concurrency/lease_storage/__init__.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import timedelta
+import importlib
+from typing import Protocol, runtime_checkable
+from uuid import UUID
+
+from prefect.server.utilities.leasing import LeaseStorage, ResourceLease
+from prefect.settings.context import get_current_settings
+
+
+@runtime_checkable
+class ConcurrencyLeaseStorageModule(Protocol):
+    ConcurrencyLeaseStorage: type[ConcurrencyLeaseStorage]
+
+
+@dataclass
+class ConcurrencyLimitLeaseMetadata:
+    slots: int
+
+
+class ConcurrencyLeaseStorage(LeaseStorage[ConcurrencyLimitLeaseMetadata]):
+    async def create_lease(
+        self,
+        resource_ids: list[UUID],
+        ttl: timedelta,
+        metadata: ConcurrencyLimitLeaseMetadata | None = None,
+    ) -> ResourceLease[ConcurrencyLimitLeaseMetadata]: ...
+
+    async def read_lease(
+        self, lease_id: UUID
+    ) -> ResourceLease[ConcurrencyLimitLeaseMetadata] | None: ...
+
+    async def renew_lease(self, lease_id: UUID, ttl: timedelta) -> None: ...
+
+    async def revoke_lease(self, lease_id: UUID) -> None: ...
+
+    async def read_expired_lease_ids(self, limit: int = 100) -> list[UUID]: ...
+
+
+def get_concurrency_lease_storage() -> ConcurrencyLeaseStorage:
+    """
+    Returns a ConcurrencyLeaseStorage instance based on the configured lease storage module.
+
+    Will raise a ValueError if the configured module does not pass a type check.
+    """
+    concurrency_lease_storage_module = importlib.import_module(
+        get_current_settings().server.concurrency.lease_storage
+    )
+    if not isinstance(concurrency_lease_storage_module, ConcurrencyLeaseStorageModule):
+        raise ValueError(
+            f"The module {get_current_settings().server.concurrency.lease_storage} does not contain a ConcurrencyLeaseStorage class"
+        )
+    return concurrency_lease_storage_module.ConcurrencyLeaseStorage()

prefect-3.4.9.dev5/src/prefect/server/concurrency/lease_storage/filesystem.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, TypedDict
+from uuid import UUID
+
+import anyio
+
+from prefect.server.concurrency.lease_storage import (
+    ConcurrencyLeaseStorage as _ConcurrencyLeaseStorage,
+)
+from prefect.server.concurrency.lease_storage import (
+    ConcurrencyLimitLeaseMetadata,
+)
+from prefect.server.utilities.leasing import ResourceLease
+from prefect.settings.context import get_current_settings
+
+
+class _LeaseFile(TypedDict):
+    resource_ids: list[str]
+    metadata: dict[str, Any] | None
+    expiration: str
+
+
+class ConcurrencyLeaseStorage(_ConcurrencyLeaseStorage):
+    """
+    A file-based concurrency lease storage implementation that stores leases on disk.
+    """
+
+    def __init__(self, storage_path: Path | None = None):
+        prefect_home = get_current_settings().home
+        self.storage_path: Path = Path(
+            storage_path or prefect_home / "concurrency_leases"
+        )
+
+    def _ensure_storage_path(self) -> None:
+        """Ensure the storage path exists, creating it if necessary."""
+        self.storage_path.mkdir(parents=True, exist_ok=True)
+
+    def _lease_file_path(self, lease_id: UUID) -> Path:
+        return self.storage_path / f"{lease_id}.json"
+
+    def _expiration_index_path(self) -> anyio.Path:
+        return anyio.Path(self.storage_path / "expirations.json")
+
+    async def _load_expiration_index(self) -> dict[str, str]:
+        """Load the expiration index from disk."""
+        expiration_file = self._expiration_index_path()
+        if not await expiration_file.exists():
+            return {}
+
+        try:
+            return json.loads(await expiration_file.read_text())
+        except (json.JSONDecodeError, KeyError, ValueError):
+            return {}
+
+    def _save_expiration_index(self, index: dict[str, str]) -> None:
+        """Save the expiration index to disk."""
+        self._ensure_storage_path()
+        expiration_file = self._expiration_index_path()
+
+        with open(expiration_file, "w") as f:
+            json.dump(index, f)
+
+    async def _update_expiration_index(
+        self, lease_id: UUID, expiration: datetime
+    ) -> None:
+        """Update a single lease's expiration in the index."""
+        index = await self._load_expiration_index()
+        index[str(lease_id)] = expiration.isoformat()
+        self._save_expiration_index(index)
+
+    async def _remove_from_expiration_index(self, lease_id: UUID) -> None:
+        """Remove a lease from the expiration index."""
+        index = await self._load_expiration_index()
+        index.pop(str(lease_id), None)
+        self._save_expiration_index(index)
+
+    def _serialize_lease(
+        self, lease: ResourceLease[ConcurrencyLimitLeaseMetadata], expiration: datetime
+    ) -> _LeaseFile:
+        return {
+            "resource_ids": [str(rid) for rid in lease.resource_ids],
+            "metadata": {"slots": lease.metadata.slots} if lease.metadata else None,
+            "expiration": expiration.isoformat(),
+        }
+
+    def _deserialize_lease(
+        self, data: _LeaseFile
+    ) -> ResourceLease[ConcurrencyLimitLeaseMetadata]:
+        resource_ids = [UUID(rid) for rid in data["resource_ids"]]
+        metadata = (
+            ConcurrencyLimitLeaseMetadata(slots=data["metadata"]["slots"])
+            if data["metadata"]
+            else None
+        )
+        expiration = datetime.fromisoformat(data["expiration"])
+        lease = ResourceLease(
+            resource_ids=resource_ids, metadata=metadata, expiration=expiration
+        )
+        return lease
+
+    async def create_lease(
+        self,
+        resource_ids: list[UUID],
+        ttl: timedelta,
+        metadata: ConcurrencyLimitLeaseMetadata | None = None,
+    ) -> ResourceLease[ConcurrencyLimitLeaseMetadata]:
+        expiration = datetime.now(timezone.utc) + ttl
+        lease = ResourceLease(
+            resource_ids=resource_ids, metadata=metadata, expiration=expiration
+        )
+
+        self._ensure_storage_path()
+        lease_file = self._lease_file_path(lease.id)
+        lease_data = self._serialize_lease(lease, expiration)
+
+        with open(lease_file, "w") as f:
+            json.dump(lease_data, f)
+
+        # Update expiration index
+        await self._update_expiration_index(lease.id, expiration)
+
+        return lease
+
+    async def read_lease(
+        self, lease_id: UUID
+    ) -> ResourceLease[ConcurrencyLimitLeaseMetadata] | None:
+        lease_file = self._lease_file_path(lease_id)
+
+        if not lease_file.exists():
+            return None
+
+        try:
+            with open(lease_file, "r") as f:
+                lease_data = json.load(f)
+
+            lease = self._deserialize_lease(lease_data)
+
+            # Check if lease is expired
+            if lease.expiration < datetime.now(timezone.utc):
+                # Clean up expired lease
+                lease_file.unlink(missing_ok=True)
+                await self._remove_from_expiration_index(lease_id)
+                return None
+
+            return lease
+        except (json.JSONDecodeError, KeyError, ValueError):
+            # Clean up corrupted lease file
+            lease_file.unlink(missing_ok=True)
+            await self._remove_from_expiration_index(lease_id)
+            return None
+
+    async def renew_lease(self, lease_id: UUID, ttl: timedelta) -> None:
+        lease_file = self._lease_file_path(lease_id)
+
+        if not lease_file.exists():
+            return
+
+        try:
+            with open(lease_file, "r") as f:
+                lease_data = json.load(f)
+
+            # Update expiration time
+            new_expiration = datetime.now(timezone.utc) + ttl
+            lease_data["expiration"] = new_expiration.isoformat()
+
+            self._ensure_storage_path()
+            with open(lease_file, "w") as f:
+                json.dump(lease_data, f)
+
+            # Update expiration index
+            await self._update_expiration_index(lease_id, new_expiration)
+        except (json.JSONDecodeError, KeyError, ValueError):
+            # Clean up corrupted lease file
+            lease_file.unlink(missing_ok=True)
+            await self._remove_from_expiration_index(lease_id)
+
+    async def revoke_lease(self, lease_id: UUID) -> None:
+        lease_file = self._lease_file_path(lease_id)
+        lease_file.unlink(missing_ok=True)
+
+        # Remove from expiration index
+        await self._remove_from_expiration_index(lease_id)
+
+    async def read_expired_lease_ids(self, limit: int = 100) -> list[UUID]:
+        expired_leases: list[UUID] = []
+        now = datetime.now(timezone.utc)
+
+        # Load expiration index
+        expiration_index = await self._load_expiration_index()
+
+        for lease_id_str, expiration_str in expiration_index.items():
+            if len(expired_leases) >= limit:
+                break
+
+            try:
+                lease_id = UUID(lease_id_str)
+                expiration = datetime.fromisoformat(expiration_str)
+
+                if expiration < now:
+                    expired_leases.append(lease_id)
+            except (ValueError, TypeError):
+                # Clean up corrupted index entry
+                continue
+
+        return expired_leases