prefect 3.2.7.dev1__tar.gz → 3.2.8.dev2__tar.gz

{prefect-3.2.7.dev1 → prefect-3.2.8.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect
-Version: 3.2.7.dev1
+Version: 3.2.8.dev2
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io
@@ -71,6 +71,7 @@ Requires-Dist: toml>=0.10.0
 Requires-Dist: typer!=0.12.2,<0.16.0,>=0.12.0
 Requires-Dist: typing-extensions<5.0.0,>=4.5.0
 Requires-Dist: ujson<6.0.0,>=5.8.0
+Requires-Dist: uv>=0.6.0
 Requires-Dist: uvicorn!=0.29.0,>=0.14.0
 Requires-Dist: websockets<14.0,>=10.4
 Provides-Extra: aws

{prefect-3.2.7.dev1 → prefect-3.2.8.dev2}/pyproject.toml

@@ -80,6 +80,7 @@ dependencies = [
     "ujson>=5.8.0,<6.0.0",
     "uvicorn>=0.14.0,!=0.29.0",
     "websockets>=10.4,<14.0",
+    "uv>=0.6.0",
 ]
 [project.urls]
 Changelog = "https://github.com/PrefectHQ/prefect/releases"
@@ -298,6 +299,38 @@ extend-select = ["I"]
 [tool.ruff.isort]
 known-third-party = []
 
+
+[tool.codespell]
+skip = [
+    ".git",
+    "*.pdf",
+    "*.svg",
+    "package-lock.json",
+    "_vendor",
+    "*.css",
+    "tests/utilities/test_text.py",
+    "pyproject.toml",
+]
+# from https://github.com/PrefectHQ/prefect/pull/10813#issuecomment-1732676130
+ignore-regex = ".*lazy=\"selectin\"|.*e import Bloc$|America/Nome"
+
+ignore-words-list = [
+    "selectin",
+    "aci",
+    "wqs",
+    "aks",
+    "ines",
+    "dependant",
+    "fsspec",
+    "automations",
+    "nmme",
+    "afterall",
+    "medias",
+]
+
+check-hidden = true
+
+
 [tool.uv.sources]
 prefect-aws = { path = "src/integrations/prefect-aws" }
 prefect-azure = { path = "src/integrations/prefect-azure" }

prefect-3.2.8.dev2/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.2.8.dev2"
+__build_date__ = "2025-02-25 08:09:51.795333+00:00"
+__git_commit__ = "9f97ceee40727350e347b83db7c2dbf5dfbc0189"
+__dirty__ = False

{prefect-3.2.7.dev1 → prefect-3.2.8.dev2}/src/prefect/_experimental/bundles.py

@@ -6,9 +6,11 @@ import gzip
 import multiprocessing
 import multiprocessing.context
 import os
+import subprocess
 from typing import Any, TypedDict
 
 import cloudpickle
+import uv
 
 from prefect.client.schemas.objects import FlowRun
 from prefect.context import SettingsContext, get_settings_context, serialize_context
@@ -28,6 +30,7 @@ class SerializedBundle(TypedDict):
     function: str
     context: str
     flow_run: dict[str, Any]
+    dependencies: str
 
 
 def _serialize_bundle_object(obj: Any) -> str:
@@ -66,6 +69,17 @@ def create_bundle_for_flow_run(
         "function": _serialize_bundle_object(flow),
         "context": _serialize_bundle_object(context),
         "flow_run": flow_run.model_dump(mode="json"),
+        "dependencies": subprocess.check_output(
+            [
+                uv.find_uv_bin(),
+                "pip",
+                "freeze",
+                # Exclude editable installs because we won't be able to install them in the execution environment
+                "--exclude-editable",
+            ]
+        )
+        .decode()
+        .strip(),
     }
 
 
@@ -129,6 +143,12 @@ def execute_bundle_in_subprocess(
 
     ctx = multiprocessing.get_context("spawn")
 
+    # Install dependencies if necessary
+    if dependencies := bundle.get("dependencies"):
+        subprocess.check_call(
+            [uv.find_uv_bin(), "pip", "install", *dependencies.split("\n")],
+        )
+
     process = ctx.Process(
         target=_extract_and_run_flow,
         kwargs={

prefect-3.2.8.dev2/src/prefect/_waiters.py

@@ -0,0 +1,254 @@
+from __future__ import annotations
+
+import asyncio
+import atexit
+import threading
+import uuid
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+)
+
+import anyio
+from cachetools import TTLCache
+from typing_extensions import Self
+
+from prefect._internal.concurrency.api import create_call, from_async, from_sync
+from prefect._internal.concurrency.threads import get_global_loop
+from prefect.client.schemas.objects import (
+    TERMINAL_STATES,
+)
+from prefect.events.clients import get_events_subscriber
+from prefect.events.filters import EventFilter, EventNameFilter
+from prefect.logging import get_logger
+
+if TYPE_CHECKING:
+    import logging
+
+
+class FlowRunWaiter:
+    """
+    A service used for waiting for a flow run to finish.
+
+    This service listens for flow run events and provides a way to wait for a specific
+    flow run to finish. This is useful for waiting for a flow run to finish before
+    continuing execution.
+
+    The service is a singleton and must be started before use. The service will
+    automatically start when the first instance is created. A single websocket
+    connection is used to listen for flow run events.
+
+    The service can be used to wait for a flow run to finish by calling
+    `FlowRunWaiter.wait_for_flow_run` with the flow run ID to wait for. The method
+    will return when the flow run has finished or the timeout has elapsed.
+
+    The service will automatically stop when the Python process exits or when the
+    global loop thread is stopped.
+
+    Example:
+    ```python
+    import asyncio
+    from uuid import uuid4
+
+    from prefect import flow
+    from prefect.flow_engine import run_flow_async
+    from prefect.flow_runs import FlowRunWaiter
+
+
+    @flow
+    async def test_flow():
+        await asyncio.sleep(5)
+        print("Done!")
+
+
+    async def main():
+        flow_run_id = uuid4()
+        asyncio.create_flow(run_flow_async(flow=test_flow, flow_run_id=flow_run_id))
+
+        await FlowRunWaiter.wait_for_flow_run(flow_run_id)
+        print("Flow run finished")
+
+
+    if __name__ == "__main__":
+        asyncio.run(main())
+    ```
+    """
+
+    _instance: Self | None = None
+    _instance_lock = threading.Lock()
+
+    def __init__(self):
+        self.logger: "logging.Logger" = get_logger("FlowRunWaiter")
+        self._consumer_task: asyncio.Task[None] | None = None
+        self._observed_completed_flow_runs: TTLCache[uuid.UUID, bool] = TTLCache(
+            maxsize=10000, ttl=600
+        )
+        self._completion_events: dict[uuid.UUID, asyncio.Event] = {}
+        self._completion_callbacks: dict[uuid.UUID, Callable[[], None]] = {}
+        self._loop: asyncio.AbstractEventLoop | None = None
+        self._observed_completed_flow_runs_lock = threading.Lock()
+        self._completion_events_lock = threading.Lock()
+        self._started = False
+
+    def start(self) -> None:
+        """
+        Start the FlowRunWaiter service.
+        """
+        if self._started:
+            return
+        self.logger.debug("Starting FlowRunWaiter")
+        loop_thread = get_global_loop()
+
+        if not asyncio.get_running_loop() == loop_thread.loop:
+            raise RuntimeError("FlowRunWaiter must run on the global loop thread.")
+
+        self._loop = loop_thread.loop
+        if TYPE_CHECKING:
+            assert self._loop is not None
+
+        consumer_started = asyncio.Event()
+        self._consumer_task = self._loop.create_task(
+            self._consume_events(consumer_started)
+        )
+        asyncio.run_coroutine_threadsafe(consumer_started.wait(), self._loop)
+
+        loop_thread.add_shutdown_call(create_call(self.stop))
+        atexit.register(self.stop)
+        self._started = True
+
+    async def _consume_events(self, consumer_started: asyncio.Event):
+        async with get_events_subscriber(
+            filter=EventFilter(
+                event=EventNameFilter(
+                    name=[
+                        f"prefect.flow-run.{state.name.title()}"
+                        for state in TERMINAL_STATES
+                    ],
+                )
+            )
+        ) as subscriber:
+            consumer_started.set()
+            async for event in subscriber:
+                try:
+                    self.logger.debug(
+                        f"Received event: {event.resource['prefect.resource.id']}"
+                    )
+                    flow_run_id = uuid.UUID(
+                        event.resource["prefect.resource.id"].replace(
+                            "prefect.flow-run.", ""
+                        )
+                    )
+
+                    with self._observed_completed_flow_runs_lock:
+                        # Cache the flow run ID for a short period of time to avoid
+                        # unnecessary waits
+                        self._observed_completed_flow_runs[flow_run_id] = True
+                    with self._completion_events_lock:
+                        # Set the event for the flow run ID if it is in the cache
+                        # so the waiter can wake up the waiting coroutine
+                        if flow_run_id in self._completion_events:
+                            self._completion_events[flow_run_id].set()
+                        if flow_run_id in self._completion_callbacks:
+                            self._completion_callbacks[flow_run_id]()
+                except Exception as exc:
+                    self.logger.error(f"Error processing event: {exc}")
+
+    def stop(self) -> None:
+        """
+        Stop the FlowRunWaiter service.
+        """
+        self.logger.debug("Stopping FlowRunWaiter")
+        if self._consumer_task:
+            self._consumer_task.cancel()
+            self._consumer_task = None
+        self.__class__._instance = None
+        self._started = False
+
+    @classmethod
+    async def wait_for_flow_run(
+        cls, flow_run_id: uuid.UUID, timeout: float | None = None
+    ) -> None:
+        """
+        Wait for a flow run to finish.
+
+        Note this relies on a websocket connection to receive events from the server
+        and will not work with an ephemeral server.
+
+        Args:
+            flow_run_id: The ID of the flow run to wait for.
+            timeout: The maximum time to wait for the flow run to
+                finish. Defaults to None.
+        """
+        instance = cls.instance()
+        with instance._observed_completed_flow_runs_lock:
+            if flow_run_id in instance._observed_completed_flow_runs:
+                return
+
+        # Need to create event in loop thread to ensure it can be set
+        # from the loop thread
+        finished_event = await from_async.wait_for_call_in_loop_thread(
+            create_call(asyncio.Event)
+        )
+        with instance._completion_events_lock:
+            # Cache the event for the flow run ID so the consumer can set it
+            # when the event is received
+            instance._completion_events[flow_run_id] = finished_event
+
+        try:
+            # Now check one more time whether the flow run arrived before we start to
+            # wait on it, in case it came in while we were setting up the event above.
+            with instance._observed_completed_flow_runs_lock:
+                if flow_run_id in instance._observed_completed_flow_runs:
+                    return
+
+            with anyio.move_on_after(delay=timeout):
+                await from_async.wait_for_call_in_loop_thread(
+                    create_call(finished_event.wait)
+                )
+        finally:
+            with instance._completion_events_lock:
+                # Remove the event from the cache after it has been waited on
+                instance._completion_events.pop(flow_run_id, None)
+
+    @classmethod
+    def add_done_callback(
+        cls, flow_run_id: uuid.UUID, callback: Callable[[], None]
+    ) -> None:
+        """
+        Add a callback to be called when a flow run finishes.
+
+        Args:
+            flow_run_id: The ID of the flow run to wait for.
+            callback: The callback to call when the flow run finishes.
+        """
+        instance = cls.instance()
+        with instance._observed_completed_flow_runs_lock:
+            if flow_run_id in instance._observed_completed_flow_runs:
+                callback()
+                return
+
+        with instance._completion_events_lock:
+            # Cache the event for the flow run ID so the consumer can set it
+            # when the event is received
+            instance._completion_callbacks[flow_run_id] = callback
+
+    @classmethod
+    def instance(cls) -> Self:
+        """
+        Get the singleton instance of FlowRunWaiter.
+        """
+        with cls._instance_lock:
+            if cls._instance is None:
+                cls._instance = cls._new_instance()
+            return cls._instance
+
+    @classmethod
+    def _new_instance(cls):
+        instance = cls()
+
+        if threading.get_ident() == get_global_loop().thread.ident:
+            instance.start()
+        else:
+            from_sync.call_soon_in_loop_thread(create_call(instance.start)).result()
+
+        return instance

{prefect-3.2.7.dev1 → prefect-3.2.8.dev2}/src/prefect/cli/deploy.py

@@ -746,7 +746,7 @@ async def _run_single_deploy(
 
     _schedules = deploy_config.pop("schedules")
 
-    deploy_config = apply_values(deploy_config, step_outputs)
+    deploy_config = apply_values(deploy_config, step_outputs, warn_on_notset=True)
     deploy_config["parameter_openapi_schema"] = _parameter_schema
     deploy_config["schedules"] = _schedules
 

{prefect-3.2.7.dev1 → prefect-3.2.8.dev2}/src/prefect/events/schemas/automations.py

@@ -84,7 +84,7 @@ class Trigger(PrefectBaseModel, abc.ABC, extra="ignore"):  # type: ignore[call-a
                 getattr(self, "name", None)
                 or f"Automation for deployment {self._deployment_id}"
             ),
-            description="",
+            description=getattr(self, "description", ""),
             enabled=getattr(self, "enabled", True),
             trigger=trigger,
             actions=self.actions(),

{prefect-3.2.7.dev1 → prefect-3.2.8.dev2}/src/prefect/flow_runs.py

@@ -1,7 +1,8 @@
+from __future__ import annotations
+
 from typing import (
     TYPE_CHECKING,
     Any,
-    Optional,
     Type,
     TypeVar,
     overload,
@@ -52,9 +53,9 @@ if TYPE_CHECKING:
 @inject_client
 async def wait_for_flow_run(
     flow_run_id: UUID,
-    timeout: Optional[int] = 10800,
+    timeout: int | None = 10800,
     poll_interval: int = 5,
-    client: Optional["PrefectClient"] = None,
+    client: "PrefectClient | None" = None,
     log_states: bool = False,
 ) -> FlowRun:
     """
@@ -119,8 +120,8 @@ async def wait_for_flow_run(
         while True:
             flow_run = await client.read_flow_run(flow_run_id)
             flow_state = flow_run.state
-            if log_states:
-                logger.info(f"Flow run is in state {flow_run.state.name!r}")
+            if log_states and flow_state:
+                logger.info(f"Flow run is in state {flow_state.name!r}")
             if flow_state and flow_state.is_final():
                 return flow_run
             await anyio.sleep(poll_interval)
@@ -138,7 +139,7 @@ async def pause_flow_run(
     wait_for_input: None = None,
     timeout: int = 3600,
     poll_interval: int = 10,
-    key: Optional[str] = None,
+    key: str | None = None,
 ) -> None: ...
 
 
@@ -147,17 +148,17 @@ async def pause_flow_run(
     wait_for_input: Type[T],
     timeout: int = 3600,
     poll_interval: int = 10,
-    key: Optional[str] = None,
+    key: str | None = None,
 ) -> T: ...
 
 
 @sync_compatible
 async def pause_flow_run(
-    wait_for_input: Optional[Type[T]] = None,
+    wait_for_input: Type[T] | None = None,
     timeout: int = 3600,
     poll_interval: int = 10,
-    key: Optional[str] = None,
-) -> Optional[T]:
+    key: str | None = None,
+) -> T | None:
     """
     Pauses the current flow run by blocking execution until resumed.
 
@@ -213,10 +214,13 @@ async def pause_flow_run(
 async def _in_process_pause(
     timeout: int = 3600,
     poll_interval: int = 10,
-    key: Optional[str] = None,
-    client=None,
-    wait_for_input: Optional[T] = None,
-) -> Optional[T]:
+    key: str | None = None,
+    client: "PrefectClient | None" = None,
+    wait_for_input: Type[T] | None = None,
+) -> T | None:
+    if TYPE_CHECKING:
+        assert client is not None
+
     if TaskRunContext.get():
         raise RuntimeError("Cannot pause task runs.")
 
@@ -302,32 +306,32 @@ async def _in_process_pause(
 @overload
 async def suspend_flow_run(
     wait_for_input: None = None,
-    flow_run_id: Optional[UUID] = None,
-    timeout: Optional[int] = 3600,
-    key: Optional[str] = None,
-    client: Optional[PrefectClient] = None,
+    flow_run_id: UUID | None = None,
+    timeout: int | None = 3600,
+    key: str | None = None,
+    client: "PrefectClient | None" = None,
 ) -> None: ...
 
 
 @overload
 async def suspend_flow_run(
     wait_for_input: Type[T],
-    flow_run_id: Optional[UUID] = None,
-    timeout: Optional[int] = 3600,
-    key: Optional[str] = None,
-    client: Optional[PrefectClient] = None,
+    flow_run_id: UUID | None = None,
+    timeout: int | None = 3600,
+    key: str | None = None,
+    client: "PrefectClient | None" = None,
 ) -> T: ...
 
 
 @sync_compatible
 @inject_client
 async def suspend_flow_run(
-    wait_for_input: Optional[Type[T]] = None,
-    flow_run_id: Optional[UUID] = None,
-    timeout: Optional[int] = 3600,
-    key: Optional[str] = None,
-    client: Optional[PrefectClient] = None,
-) -> Optional[T]:
+    wait_for_input: Type[T] | None = None,
+    flow_run_id: UUID | None = None,
+    timeout: int | None = 3600,
+    key: str | None = None,
+    client: "PrefectClient | None" = None,
+) -> T | None:
     """
     Suspends a flow run by stopping code execution until resumed.
 
@@ -357,6 +361,9 @@ async def suspend_flow_run(
             resumed with the input, the flow will resume and the input will be
             loaded and returned from this function.
     """
+    if TYPE_CHECKING:
+        assert client is not None
+
     context = FlowRunContext.get()
 
     if flow_run_id is None:
@@ -427,7 +434,7 @@ async def suspend_flow_run(
 
 @sync_compatible
 async def resume_flow_run(
-    flow_run_id: UUID, run_input: Optional[dict[str, Any]] = None
+    flow_run_id: UUID, run_input: dict[str, Any] | None = None
 ) -> None:
     """
     Resumes a paused flow.