prefect 3.4.9.dev4__tar.gz → 3.4.10.dev1__tar.gz

{prefect-3.4.9.dev4 → prefect-3.4.10.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect
-Version: 3.4.9.dev4
+Version: 3.4.10.dev1
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io
@@ -53,7 +53,7 @@ Requires-Dist: packaging<25.1,>=21.3
 Requires-Dist: pathspec>=0.8.0
 Requires-Dist: pendulum<4,>=3.0.0; python_version < '3.13'
 Requires-Dist: prometheus-client>=0.20.0
-Requires-Dist: pydantic!=2.10.0,!=2.11.0,!=2.11.1,!=2.11.2,!=2.11.3,!=2.11.4,<3.0.0,>=2.9
+Requires-Dist: pydantic!=2.11.0,!=2.11.1,!=2.11.2,!=2.11.3,!=2.11.4,<3.0.0,>=2.10.1
 Requires-Dist: pydantic-core<3.0.0,>=2.12.0
 Requires-Dist: pydantic-extra-types<3.0.0,>=2.8.2
 Requires-Dist: pydantic-settings!=2.9.0,<3.0.0,>2.2.1

{prefect-3.4.9.dev4 → prefect-3.4.10.dev1}/pyproject.toml

@@ -64,7 +64,7 @@ dependencies = [
     "pathspec>=0.8.0",
     "pendulum>=3.0.0,<4; python_version<'3.13'",
     "prometheus-client>=0.20.0",
-    "pydantic>=2.9,<3.0.0,!=2.10.0,!=2.11.0,!=2.11.1,!=2.11.2,!=2.11.3,!=2.11.4",
+    "pydantic>=2.10.1,<3.0.0,!=2.11.0,!=2.11.1,!=2.11.2,!=2.11.3,!=2.11.4",
     "pydantic_core>=2.12.0,<3.0.0",
     "pydantic_extra_types>=2.8.2,<3.0.0",
     "pydantic_settings>2.2.1,<3.0.0,!=2.9.0",

prefect-3.4.10.dev1/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.4.10.dev1"
+__build_date__ = "2025-07-22 08:09:58.892327+00:00"
+__git_commit__ = "855e2d9504627b56bc34dea45f4b2578706d7ab7"
+__dirty__ = False

{prefect-3.4.9.dev4 → prefect-3.4.10.dev1}/src/prefect/_experimental/bundles/__init__.py

@@ -3,14 +3,18 @@ from __future__ import annotations
 import asyncio
 import base64
 import gzip
+import json
+import logging
 import multiprocessing
 import multiprocessing.context
 import os
 import subprocess
 import sys
 from pathlib import Path
+import tempfile
 from typing import Any, TypedDict
 
+import anyio
 import cloudpickle  # pyright: ignore[reportMissingTypeStubs]
 
 from prefect.client.schemas.objects import FlowRun
@@ -18,12 +22,15 @@ from prefect.context import SettingsContext, get_settings_context, serialize_con
 from prefect.engine import handle_engine_signals
 from prefect.flow_engine import run_flow
 from prefect.flows import Flow
+from prefect.logging import get_logger
 from prefect.settings.context import get_current_settings
 from prefect.settings.models.root import Settings
 from prefect.utilities.slugify import slugify
 
 from .execute import execute_bundle_from_file
 
+logger: logging.Logger = get_logger(__name__)
+
 
 def _get_uv_path() -> str:
     try:
@@ -246,6 +253,64 @@ def convert_step_to_command(
     return command
 
 
+def upload_bundle_to_storage(
+    bundle: SerializedBundle, key: str, upload_command: list[str]
+) -> None:
+    """
+    Uploads a bundle to storage.
+
+    Args:
+        bundle: The serialized bundle to upload.
+        key: The key to use for the remote file when uploading.
+        upload_command: The command to use to upload the bundle as a list of strings.
+    """
+    # Write the bundle to a temporary directory so it can be uploaded to the bundle storage
+    # via the upload command
+    with tempfile.TemporaryDirectory() as temp_dir:
+        Path(temp_dir).joinpath(key).write_bytes(json.dumps(bundle).encode("utf-8"))
+
+        try:
+            full_command = upload_command + [key]
+            logger.debug("Uploading execution bundle with command: %s", full_command)
+            subprocess.check_call(
+                full_command,
+                cwd=temp_dir,
+            )
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(e.stderr.decode("utf-8")) from e
+
+
+async def aupload_bundle_to_storage(
+    bundle: SerializedBundle, key: str, upload_command: list[str]
+) -> None:
+    """
+    Asynchronously uploads a bundle to storage.
+
+    Args:
+        bundle: The serialized bundle to upload.
+        key: The key to use for the remote file when uploading.
+        upload_command: The command to use to upload the bundle as a list of strings.
+    """
+    # Write the bundle to a temporary directory so it can be uploaded to the bundle storage
+    # via the upload command
+    with tempfile.TemporaryDirectory() as temp_dir:
+        await (
+            anyio.Path(temp_dir)
+            .joinpath(key)
+            .write_bytes(json.dumps(bundle).encode("utf-8"))
+        )
+
+        try:
+            full_command = upload_command + [key]
+            logger.debug("Uploading execution bundle with command: %s", full_command)
+            await anyio.run_process(
+                full_command,
+                cwd=temp_dir,
+            )
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(e.stderr.decode("utf-8")) from e
+
+
 __all__ = [
     "execute_bundle_from_file",
     "convert_step_to_command",

{prefect-3.4.9.dev4 → prefect-3.4.10.dev1}/src/prefect/assets/core.py

@@ -7,6 +7,8 @@ from pydantic import ConfigDict, Field
 from prefect._internal.schemas.bases import PrefectBaseModel
 from prefect.types import ValidAssetKey
 
+MAX_ASSET_DESCRIPTION_LENGTH = 2500
+
 
 class AssetProperties(PrefectBaseModel):
     """
@@ -22,7 +24,9 @@ class AssetProperties(PrefectBaseModel):
         default=None, description="Visitable url to view the Asset."
     )
     description: Optional[str] = Field(
-        default=None, description="Description of the Asset."
+        default=None,
+        description="Description of the Asset.",
+        max_length=MAX_ASSET_DESCRIPTION_LENGTH,
     )
     owners: Optional[list[str]] = Field(
         default=None, description="Owners of the Asset."

{prefect-3.4.9.dev4 → prefect-3.4.10.dev1}/src/prefect/cli/deployment.py

@@ -873,6 +873,16 @@ async def run(
                 f"deployment: {listrepr(unknown_keys, sep=', ')}"
                 f"\n{available_parameters}"
             )
+        templating_parameters = {**(deployment.parameters or {}), **(parameters or {})}
+        if flow_run_name:
+            try:
+                flow_run_name = flow_run_name.format(**templating_parameters)
+            except KeyError as e:
+                exit_with_error(
+                    f"Missing parameter for flow run name: '{e.args[0]}' is undefined"
+                )
+            except Exception as e:
+                exit_with_error(f"Failed to format flow run name: {e}")
 
         app.console.print(
             f"Creating flow run for deployment '{flow.name}/{deployment.name}'...",

{prefect-3.4.9.dev4 → prefect-3.4.10.dev1}/src/prefect/client/orchestration/_concurrency_limits/client.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
 
 from httpx import HTTPStatusError, RequestError
 
@@ -245,6 +245,14 @@ class ConcurrencyLimitClient(BaseClient):
         slots: int,
         mode: str,
     ) -> "Response":
+        """
+        Increment concurrency slots for the specified limits.
+
+        Args:
+            names: A list of limit names for which to occupy slots.
+            slots: The number of concurrency slots to occupy.
+            mode: The mode of the concurrency limits.
+        """
         return self.request(
             "POST",
             "/v2/concurrency_limits/increment",
@@ -255,6 +263,52 @@ class ConcurrencyLimitClient(BaseClient):
             },
         )
 
+    def increment_concurrency_slots_with_lease(
+        self,
+        names: list[str],
+        slots: int,
+        mode: Literal["concurrency", "rate_limit"],
+        lease_duration: float,
+    ) -> "Response":
+        """
+        Increment concurrency slots for the specified limits with a lease.
+
+        Args:
+            names: A list of limit names for which to occupy slots.
+            slots: The number of concurrency slots to occupy.
+            mode: The mode of the concurrency limits.
+            lease_duration: The duration of the lease in seconds.
+        """
+        return self.request(
+            "POST",
+            "/v2/concurrency_limits/increment-with-lease",
+            json={
+                "names": names,
+                "slots": slots,
+                "mode": mode,
+                "lease_duration": lease_duration,
+            },
+        )
+
+    def renew_concurrency_lease(
+        self,
+        lease_id: "UUID",
+        lease_duration: float,
+    ) -> "Response":
+        """
+        Renew a concurrency lease.
+
+        Args:
+            lease_id: The ID of the lease to renew.
+            lease_duration: The new lease duration in seconds.
+        """
+        return self.request(
+            "POST",
+            "/v2/concurrency_limits/leases/{lease_id}/renew",
+            path_params={"lease_id": lease_id},
+            json={"lease_duration": lease_duration},
+        )
+
     def release_concurrency_slots(
         self, names: list[str], slots: int, occupancy_seconds: float
     ) -> "Response":
@@ -281,6 +335,24 @@ class ConcurrencyLimitClient(BaseClient):
             },
         )
 
+    def release_concurrency_slots_with_lease(
+        self,
+        lease_id: "UUID",
+    ) -> "Response":
+        """
+        Release concurrency slots for the specified lease.
+
+        Args:
+            lease_id: The ID of the lease corresponding to the concurrency limits to release.
+        """
+        return self.request(
+            "POST",
+            "/v2/concurrency_limits/decrement-with-lease",
+            json={
+                "lease_id": str(lease_id),
+            },
+        )
+
     def create_global_concurrency_limit(
         self, concurrency_limit: "GlobalConcurrencyLimitCreate"
     ) -> "UUID":
@@ -611,8 +683,16 @@ class ConcurrencyLimitAsyncClient(BaseAsyncClient):
         self,
         names: list[str],
         slots: int,
-        mode: str,
+        mode: Literal["concurrency", "rate_limit"],
     ) -> "Response":
+        """
+        Increment concurrency slots for the specified limits.
+
+        Args:
+            names: A list of limit names for which to occupy slots.
+            slots: The number of concurrency slots to occupy.
+            mode: The mode of the concurrency limits.
+        """
         return await self.request(
             "POST",
             "/v2/concurrency_limits/increment",
@@ -623,6 +703,52 @@ class ConcurrencyLimitAsyncClient(BaseAsyncClient):
             },
         )
 
+    async def increment_concurrency_slots_with_lease(
+        self,
+        names: list[str],
+        slots: int,
+        mode: Literal["concurrency", "rate_limit"],
+        lease_duration: float,
+    ) -> "Response":
+        """
+        Increment concurrency slots for the specified limits with a lease.
+
+        Args:
+            names: A list of limit names for which to occupy slots.
+            slots: The number of concurrency slots to occupy.
+            mode: The mode of the concurrency limits.
+            lease_duration: The duration of the lease in seconds.
+        """
+        return await self.request(
+            "POST",
+            "/v2/concurrency_limits/increment-with-lease",
+            json={
+                "names": names,
+                "slots": slots,
+                "mode": mode,
+                "lease_duration": lease_duration,
+            },
+        )
+
+    async def renew_concurrency_lease(
+        self,
+        lease_id: "UUID",
+        lease_duration: float,
+    ) -> "Response":
+        """
+        Renew a concurrency lease.
+
+        Args:
+            lease_id: The ID of the lease to renew.
+            lease_duration: The new lease duration in seconds.
+        """
+        return await self.request(
+            "POST",
+            "/v2/concurrency_limits/leases/{lease_id}/renew",
+            path_params={"lease_id": lease_id},
+            json={"lease_duration": lease_duration},
+        )
+
     async def release_concurrency_slots(
         self, names: list[str], slots: int, occupancy_seconds: float
     ) -> "Response":
@@ -649,6 +775,24 @@ class ConcurrencyLimitAsyncClient(BaseAsyncClient):
             },
         )
 
+    async def release_concurrency_slots_with_lease(
+        self,
+        lease_id: "UUID",
+    ) -> "Response":
+        """
+        Release concurrency slots for the specified lease.
+
+        Args:
+            lease_id: The ID of the lease corresponding to the concurrency limits to release.
+        """
+        return await self.request(
+            "POST",
+            "/v2/concurrency_limits/decrement-with-lease",
+            json={
+                "lease_id": str(lease_id),
+            },
+        )
+
     async def create_global_concurrency_limit(
         self, concurrency_limit: "GlobalConcurrencyLimitCreate"
     ) -> "UUID":

{prefect-3.4.9.dev4 → prefect-3.4.10.dev1}/src/prefect/client/orchestration/routes.py

@@ -115,8 +115,11 @@ ServerRoutes = Literal[
     "/v2/concurrency_limits/",
     "/v2/concurrency_limits/{id_or_name}",
     "/v2/concurrency_limits/decrement",
+    "/v2/concurrency_limits/decrement-with-lease",
     "/v2/concurrency_limits/filter",
     "/v2/concurrency_limits/increment",
+    "/v2/concurrency_limits/increment-with-lease",
+    "/v2/concurrency_limits/leases/{lease_id}/renew",
     "/variables/",
     "/variables/{id}",
     "/variables/count",

{prefect-3.4.9.dev4 → prefect-3.4.10.dev1}/src/prefect/client/schemas/responses.py

@@ -477,6 +477,13 @@ class MinimalConcurrencyLimitResponse(PrefectBaseModel):
     limit: int
 
 
+class ConcurrencyLimitWithLeaseResponse(PrefectBaseModel):
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore")
+
+    lease_id: UUID
+    limits: list[MinimalConcurrencyLimitResponse]
+
+
 class GlobalConcurrencyLimitResponse(ObjectBaseModel):
     """
     A response object for global concurrency limits.

prefect-3.4.10.dev1/src/prefect/concurrency/_asyncio.py

@@ -0,0 +1,177 @@
+import asyncio
+import logging
+from typing import Literal, Optional
+from uuid import UUID
+
+import httpx
+
+from prefect.client.orchestration import get_client
+from prefect.client.schemas.responses import (
+    ConcurrencyLimitWithLeaseResponse,
+    MinimalConcurrencyLimitResponse,
+)
+from prefect.logging import get_logger
+from prefect.logging.loggers import get_run_logger
+from prefect.utilities.timeout import timeout_async
+
+from .services import ConcurrencySlotAcquisitionService
+
+
+class ConcurrencySlotAcquisitionError(Exception):
+    """Raised when an unhandlable occurs while acquiring concurrency slots."""
+
+
+class AcquireConcurrencySlotTimeoutError(TimeoutError):
+    """Raised when acquiring a concurrency slot times out."""
+
+
+logger: logging.Logger = get_logger("concurrency")
+
+
+async def aacquire_concurrency_slots(
+    names: list[str],
+    slots: int,
+    mode: Literal["concurrency", "rate_limit"] = "concurrency",
+    timeout_seconds: Optional[float] = None,
+    max_retries: Optional[int] = None,
+    strict: bool = False,
+) -> list[MinimalConcurrencyLimitResponse]:
+    service = ConcurrencySlotAcquisitionService.instance(frozenset(names))
+    future = service.send((slots, mode, timeout_seconds, max_retries))
+    try:
+        response = await asyncio.wrap_future(future)
+    except TimeoutError as timeout:
+        raise AcquireConcurrencySlotTimeoutError(
+            f"Attempt to acquire concurrency slots timed out after {timeout_seconds} second(s)"
+        ) from timeout
+    except Exception as exc:
+        raise ConcurrencySlotAcquisitionError(
+            f"Unable to acquire concurrency slots on {names!r}"
+        ) from exc
+
+    retval = _response_to_minimal_concurrency_limit_response(response)
+
+    if not retval:
+        if strict:
+            raise ConcurrencySlotAcquisitionError(
+                f"Concurrency limits {names!r} must be created before acquiring slots"
+            )
+        try:
+            logger = get_run_logger()
+        except Exception:
+            pass
+        else:
+            logger.warning(
+                f"Concurrency limits {names!r} do not exist - skipping acquisition."
+            )
+
+    return retval
+
+
+async def aacquire_concurrency_slots_with_lease(
+    names: list[str],
+    slots: int,
+    mode: Literal["concurrency", "rate_limit"] = "concurrency",
+    timeout_seconds: Optional[float] = None,
+    max_retries: Optional[int] = None,
+    lease_duration: float = 300,
+    strict: bool = False,
+) -> ConcurrencyLimitWithLeaseResponse:
+    try:
+        # Use a run logger if available
+        logger = get_run_logger()
+    except Exception:
+        logger = get_logger("concurrency")
+
+    try:
+        with timeout_async(seconds=timeout_seconds):
+            async with get_client() as client:
+                while True:
+                    try:
+                        response = await client.increment_concurrency_slots_with_lease(
+                            names=names,
+                            slots=slots,
+                            mode=mode,
+                            lease_duration=lease_duration,
+                        )
+                        retval = ConcurrencyLimitWithLeaseResponse.model_validate(
+                            response.json()
+                        )
+                        if not retval.limits:
+                            if strict:
+                                raise ConcurrencySlotAcquisitionError(
+                                    f"Concurrency limits {names!r} must be created before acquiring slots"
+                                )
+                            else:
+                                logger.warning(
+                                    f"Concurrency limits {names!r} do not exist - skipping acquisition."
+                                )
+
+                        return retval
+                    except httpx.HTTPStatusError as exc:
+                        if not exc.response.status_code == 423:  # HTTP_423_LOCKED
+                            raise
+
+                        if max_retries is not None and max_retries <= 0:
+                            raise exc
+                        retry_after = float(exc.response.headers["Retry-After"])
+                        logger.debug(
+                            f"Unable to acquire concurrency slot. Retrying in {retry_after} second(s)."
+                        )
+                        await asyncio.sleep(retry_after)
+                        if max_retries is not None:
+                            max_retries -= 1
+    except TimeoutError as timeout:
+        raise AcquireConcurrencySlotTimeoutError(
+            f"Attempt to acquire concurrency slots timed out after {timeout_seconds} second(s)"
+        ) from timeout
+    except Exception as exc:
+        raise ConcurrencySlotAcquisitionError(
+            f"Unable to acquire concurrency slots on {names!r}"
+        ) from exc
+
+
+async def amaintain_concurrency_lease(
+    lease_id: UUID,
+    lease_duration: float,
+) -> None:
+    """
+    Maintain a concurrency lease by renewing it after the given interval.
+
+    Args:
+        lease_id: The ID of the lease to maintain.
+        lease_duration: The duration of the lease in seconds.
+    """
+    async with get_client() as client:
+        while True:
+            await asyncio.sleep(  # Renew the lease 3/4 of the way through the lease duration
+                lease_duration * 0.75
+            )
+            await client.renew_concurrency_lease(
+                lease_id=lease_id, lease_duration=lease_duration
+            )
+
+
+async def arelease_concurrency_slots(
+    names: list[str], slots: int, occupancy_seconds: float
+) -> list[MinimalConcurrencyLimitResponse]:
+    async with get_client() as client:
+        response = await client.release_concurrency_slots(
+            names=names, slots=slots, occupancy_seconds=occupancy_seconds
+        )
+        return _response_to_minimal_concurrency_limit_response(response)
+
+
+async def arelease_concurrency_slots_with_lease(
+    lease_id: UUID,
+) -> None:
+    async with get_client() as client:
+        await client.release_concurrency_slots_with_lease(lease_id=lease_id)
+
+
+def _response_to_minimal_concurrency_limit_response(
+    response: httpx.Response,
+) -> list[MinimalConcurrencyLimitResponse]:
+    return [
+        MinimalConcurrencyLimitResponse.model_validate(obj_) for obj_ in response.json()
+    ]