prefect-client 3.0.0rc10__py3-none-any.whl → 3.0.0rc11__py3-none-any.whl

prefect/flows.py

@@ -5,6 +5,7 @@ Module containing the base workflow class and decorator - for most use cases, us
 # This file requires type-checking with pyright because mypy does not yet support PEP612
 # See https://github.com/python/mypy/issues/8645
 import ast
+import asyncio
 import datetime
 import importlib.util
 import inspect
@@ -28,6 +29,8 @@ from typing import (
     List,
     NoReturn,
     Optional,
+    Set,
+    Tuple,
     Type,
     TypeVar,
     Union,
@@ -44,7 +47,9 @@ from pydantic.v1.errors import ConfigError  # TODO
 from rich.console import Console
 from typing_extensions import Literal, ParamSpec, Self
 
-from prefect._internal.compatibility.deprecated import deprecated_parameter
+from prefect._internal.compatibility.deprecated import (
+    deprecated_parameter,
+)
 from prefect._internal.concurrency.api import create_call, from_async
 from prefect.blocks.core import Block
 from prefect.client.orchestration import get_client
@@ -60,6 +65,7 @@ from prefect.exceptions import (
     MissingFlowError,
     ObjectNotFound,
     ParameterTypeError,
+    ScriptError,
     UnspecifiedFlowError,
 )
 from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
@@ -778,8 +784,7 @@ class Flow(Generic[P, R]):
         self.on_failure_hooks.append(fn)
         return fn
 
-    @sync_compatible
-    async def serve(
+    def serve(
         self,
         name: Optional[str] = None,
         interval: Optional[
@@ -884,7 +889,7 @@ class Flow(Generic[P, R]):
             name = Path(name).stem
 
         runner = Runner(name=name, pause_on_shutdown=pause_on_shutdown, limit=limit)
-        deployment_id = await runner.add_flow(
+        deployment_id = runner.add_flow(
             self,
             name=name,
             triggers=triggers,
@@ -917,15 +922,27 @@ class Flow(Generic[P, R]):
 
             console = Console()
             console.print(help_message, soft_wrap=True)
-        await runner.start(webserver=webserver)
+
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError as exc:
+            if "no running event loop" in str(exc):
+                loop = None
+            else:
+                raise
+
+        if loop is not None:
+            loop.run_until_complete(runner.start(webserver=webserver))
+        else:
+            asyncio.run(runner.start(webserver=webserver))
 
     @classmethod
     @sync_compatible
     async def from_source(
-        cls: Type[F],
+        cls: Type["Flow[P, R]"],
         source: Union[str, "RunnerStorage", ReadableDeploymentStorage],
         entrypoint: str,
-    ) -> F:
+    ) -> "Flow[P, R]":
         """
         Loads a flow from a remote source.
 
@@ -1003,7 +1020,9 @@ class Flow(Generic[P, R]):
             create_storage_from_source,
         )
 
-        if isinstance(source, str):
+        if isinstance(source, (Path, str)):
+            if isinstance(source, Path):
+                source = str(source)
             storage = create_storage_from_source(source)
         elif isinstance(source, RunnerStorage):
             storage = source
@@ -1186,9 +1205,9 @@ class Flow(Generic[P, R]):
             entrypoint_type=entrypoint_type,
         )
 
-        from prefect.deployments import runner
+        from prefect.deployments.runner import deploy
 
-        deployment_ids = await runner.deploy(
+        deployment_ids = await deploy(
             deployment,
             work_pool_name=work_pool_name,
             image=image,
@@ -1712,6 +1731,14 @@ def load_flow_from_entrypoint(
         raise MissingFlowError(
             f"Flow function with name {func_name!r} not found in {path!r}. "
         ) from exc
+    except ScriptError as exc:
+        # If the flow has dependencies that are not installed in the current
+        # environment, fallback to loading the flow via AST parsing. The
+        # drawback of this approach is that we're unable to actually load the
+        # function, so we create a placeholder flow that will re-raise this
+        # exception when called.
+
+        flow = load_placeholder_flow(entrypoint=entrypoint, raises=exc)
 
     if not isinstance(flow, Flow):
         raise MissingFlowError(
@@ -1722,14 +1749,13 @@ def load_flow_from_entrypoint(
     return flow
 
 
-@sync_compatible
-async def serve(
+def serve(
     *args: "RunnerDeployment",
     pause_on_shutdown: bool = True,
     print_starting_message: bool = True,
     limit: Optional[int] = None,
     **kwargs,
-) -> NoReturn:
+):
     """
     Serve the provided list of deployments.
 
@@ -1779,7 +1805,7 @@ async def serve(
 
     runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
     for deployment in args:
-        await runner.add_deployment(deployment)
+        runner.add_deployment(deployment)
 
     if print_starting_message:
         help_message_top = (
@@ -1810,7 +1836,18 @@ async def serve(
             Group(help_message_top, table, help_message_bottom), soft_wrap=True
         )
 
-    await runner.start()
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError as exc:
+        if "no running event loop" in str(exc):
+            loop = None
+        else:
+            raise
+
+    if loop is not None:
+        loop.run_until_complete(runner.start())
+    else:
+        asyncio.run(runner.start())
 
 
 @client_injector
@@ -1892,24 +1929,138 @@ async def load_flow_from_flow_run(
     return flow
 
 
-def load_flow_argument_from_entrypoint(
-    entrypoint: str, arg: str = "name"
-) -> Optional[str]:
+def load_placeholder_flow(entrypoint: str, raises: Exception):
     """
-    Extract a flow argument from an entrypoint string.
+    Load a placeholder flow that is initialized with the same arguments as the
+    flow specified in the entrypoint. If called the flow will raise `raises`.
 
-    Loads the source code of the entrypoint and extracts the flow argument from the
-    `flow` decorator.
+    This is useful when a flow can't be loaded due to missing dependencies or
+    other issues but the base metadata defining the flow is still needed.
 
     Args:
-        entrypoint: a string in the format `<path_to_script>:<flow_func_name>` or a module path
-            to a flow function
+        entrypoint: a string in the format `<path_to_script>:<flow_func_name>`
+          or a module path to a flow function
+        raises: an exception to raise when the flow is called
+    """
+
+    def _base_placeholder():
+        raise raises
+
+    def sync_placeholder_flow(*args, **kwargs):
+        _base_placeholder()
+
+    async def async_placeholder_flow(*args, **kwargs):
+        _base_placeholder()
+
+    placeholder_flow = (
+        async_placeholder_flow
+        if is_entrypoint_async(entrypoint)
+        else sync_placeholder_flow
+    )
+
+    arguments = load_flow_arguments_from_entrypoint(entrypoint)
+    arguments["fn"] = placeholder_flow
+
+    return Flow(**arguments)
+
+
+def load_flow_arguments_from_entrypoint(
+    entrypoint: str, arguments: Optional[Union[List[str], Set[str]]] = None
+) -> dict[str, Any]:
+    """
+    Extract flow arguments from an entrypoint string.
+
+    Loads the source code of the entrypoint and extracts the flow arguments
+    from the `flow` decorator.
+
+    Args:
+        entrypoint: a string in the format `<path_to_script>:<flow_func_name>`
+          or a module path to a flow function
+    """
+
+    func_def, source_code = _entrypoint_definition_and_source(entrypoint)
+
+    if arguments is None:
+        # If no arguments are provided default to known arguments that are of
+        # built-in types.
+        arguments = {
+            "name",
+            "version",
+            "retries",
+            "retry_delay_seconds",
+            "description",
+            "timeout_seconds",
+            "validate_parameters",
+            "persist_result",
+            "cache_result_in_memory",
+            "log_prints",
+        }
+
+    result = {}
+
+    for decorator in func_def.decorator_list:
+        if (
+            isinstance(decorator, ast.Call)
+            and getattr(decorator.func, "id", "") == "flow"
+        ):
+            for keyword in decorator.keywords:
+                if keyword.arg not in arguments:
+                    continue
+
+                if isinstance(keyword.value, ast.Constant):
+                    # Use the string value of the argument
+                    result[keyword.arg] = str(keyword.value.value)
+                    continue
+
+                # if the arg value is not a raw str (i.e. a variable or expression),
+                # then attempt to evaluate it
+                namespace = safe_load_namespace(source_code)
+                literal_arg_value = ast.get_source_segment(source_code, keyword.value)
+                cleaned_value = (
+                    literal_arg_value.replace("\n", "") if literal_arg_value else ""
+                )
+
+                try:
+                    evaluated_value = eval(cleaned_value, namespace)  # type: ignore
+                    result[keyword.arg] = str(evaluated_value)
+                except Exception as e:
+                    logger.info(
+                        "Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",
+                        keyword.arg,
+                        literal_arg_value,
+                        exc_info=e,
+                    )
+                    # ignore the decorator arg and fallback to default behavior
+                    continue
+
+    if "name" in arguments and "name" not in result:
+        # If no matching decorator or keyword argument for `name' is found
+        # fallback to the function name.
+        result["name"] = func_def.name.replace("_", "-")
+
+    return result
+
+
+def is_entrypoint_async(entrypoint: str) -> bool:
+    """
+    Determine if the function specified in the entrypoint is asynchronous.
+
+    Args:
+        entrypoint: A string in the format `<path_to_script>:<func_name>` or
+          a module path to a function.
 
     Returns:
-        The flow argument value
+        True if the function is asynchronous, False otherwise.
     """
+    func_def, _ = _entrypoint_definition_and_source(entrypoint)
+    return isinstance(func_def, ast.AsyncFunctionDef)
+
+
+def _entrypoint_definition_and_source(
+    entrypoint: str,
+) -> Tuple[Union[ast.FunctionDef, ast.AsyncFunctionDef], str]:
     if ":" in entrypoint:
-        # split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
+        # Split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
         path, func_name = entrypoint.rsplit(":", maxsplit=1)
         source_code = Path(path).read_text()
     else:
@@ -1918,6 +2069,7 @@ def load_flow_argument_from_entrypoint(
         if not spec or not spec.origin:
             raise ValueError(f"Could not find module {path!r}")
         source_code = Path(spec.origin).read_text()
+
     parsed_code = ast.parse(source_code)
     func_def = next(
         (
@@ -1934,46 +2086,8 @@ def load_flow_argument_from_entrypoint(
         ),
         None,
     )
+
     if not func_def:
         raise ValueError(f"Could not find flow {func_name!r} in {path!r}")
-    for decorator in func_def.decorator_list:
-        if (
-            isinstance(decorator, ast.Call)
-            and getattr(decorator.func, "id", "") == "flow"
-        ):
-            for keyword in decorator.keywords:
-                if keyword.arg == arg:
-                    if isinstance(keyword.value, ast.Constant):
-                        return (
-                            keyword.value.value
-                        )  # Return the string value of the argument
-
-                    # if the arg value is not a raw str (i.e. a variable or expression),
-                    # then attempt to evaluate it
-                    namespace = safe_load_namespace(source_code)
-                    literal_arg_value = ast.get_source_segment(
-                        source_code, keyword.value
-                    )
-                    cleaned_value = (
-                        literal_arg_value.replace("\n", "") if literal_arg_value else ""
-                    )
-
-                    try:
-                        evaluated_value = eval(cleaned_value, namespace)  # type: ignore
-                    except Exception as e:
-                        logger.info(
-                            "Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",
-                            arg,
-                            literal_arg_value,
-                            exc_info=e,
-                        )
-                        # ignore the decorator arg and fallback to default behavior
-                        break
-                    return str(evaluated_value)
-
-    if arg == "name":
-        return func_name.replace(
-            "_", "-"
-        )  # If no matching decorator or keyword argument is found
 
-    return None
+    return func_def, source_code

prefect/futures.py

@@ -1,4 +1,5 @@
 import abc
+import collections
 import concurrent.futures
 import inspect
 import uuid
@@ -256,13 +257,7 @@ class PrefectFutureList(list, Iterator, Generic[F]):
             timeout: The maximum number of seconds to wait for all futures to
                 complete. This method will not raise if the timeout is reached.
         """
-        try:
-            with timeout_context(timeout):
-                for future in self:
-                    future.wait()
-        except TimeoutError:
-            logger.debug("Timed out waiting for all futures to complete.")
-            return
+        wait(self, timeout=timeout)
 
     def result(
         self,
@@ -297,6 +292,57 @@ class PrefectFutureList(list, Iterator, Generic[F]):
             ) from exc
 
 
+DoneAndNotDoneFutures = collections.namedtuple("DoneAndNotDoneFutures", "done not_done")
+
+
+def wait(futures: List[PrefectFuture], timeout=None) -> DoneAndNotDoneFutures:
+    """
+    Wait for the futures in the given sequence to complete.
+
+    Args:
+        futures: The sequence of Futures to wait upon.
+        timeout: The maximum number of seconds to wait. If None, then there
+            is no limit on the wait time.
+
+    Returns:
+        A named 2-tuple of sets. The first set, named 'done', contains the
+        futures that completed (is finished or cancelled) before the wait
+        completed. The second set, named 'not_done', contains uncompleted
+        futures. Duplicate futures given to *futures* are removed and will be
+        returned only once.
+
+    Examples:
+        ```python
+        @task
+        def sleep_task(seconds):
+            sleep(seconds)
+            return 42
+
+        @flow
+        def flow():
+            futures = random_task.map(range(10))
+            done, not_done = wait(futures, timeout=5)
+            print(f"Done: {len(done)}")
+            print(f"Not Done: {len(not_done)}")
+        ```
+    """
+    futures = set(futures)
+    done = {f for f in futures if f._final_state}
+    not_done = futures - done
+    if len(done) == len(futures):
+        return DoneAndNotDoneFutures(done, not_done)
+    try:
+        with timeout_context(timeout):
+            for future in not_done.copy():
+                future.wait()
+                done.add(future)
+                not_done.remove(future)
+            return DoneAndNotDoneFutures(done, not_done)
+    except TimeoutError:
+        logger.debug("Timed out waiting for all futures to complete.")
+        return DoneAndNotDoneFutures(done, not_done)
+
+
 def resolve_futures_to_states(
     expr: Union[PrefectFuture, Any],
 ) -> Union[State, Any]:

prefect/logging/loggers.py

@@ -97,7 +97,7 @@ def get_logger(name: Optional[str] = None) -> logging.Logger:
 
 
 def get_run_logger(
-    context: "RunContext" = None, **kwargs: str
+    context: Optional["RunContext"] = None, **kwargs: str
 ) -> Union[logging.Logger, logging.LoggerAdapter]:
     """
     Get a Prefect logger for the current task run or flow run.

prefect/runner/runner.py

@@ -65,17 +65,13 @@ from prefect.client.schemas.filters import (
     FlowRunFilterStateName,
     FlowRunFilterStateType,
 )
-from prefect.client.schemas.objects import (
-    FlowRun,
-    State,
-    StateType,
-)
+from prefect.client.schemas.objects import Flow as APIFlow
+from prefect.client.schemas.objects import FlowRun, State, StateType
 from prefect.client.schemas.schedules import SCHEDULE_TYPES
-from prefect.deployments.runner import (
-    EntrypointType,
-    RunnerDeployment,
-)
 from prefect.events import DeploymentTriggerTypes, TriggerTypes
+from prefect.events.related import tags_as_related_resources
+from prefect.events.schemas.events import RelatedResource
+from prefect.events.utilities import emit_event
 from prefect.exceptions import Abort, ObjectNotFound
 from prefect.flows import Flow, load_flow_from_flow_run
 from prefect.logging.loggers import PrefectLogAdapter, flow_run_logger, get_logger
@@ -88,6 +84,7 @@ from prefect.settings import (
     get_current_settings,
 )
 from prefect.states import Crashed, Pending, exception_to_failed_state
+from prefect.types.entrypoint import EntrypointType
 from prefect.utilities.asyncutils import (
     asyncnullcontext,
     is_async_fn,
@@ -96,9 +93,12 @@ from prefect.utilities.asyncutils import (
 from prefect.utilities.engine import propose_state
 from prefect.utilities.processutils import _register_signal, run_process
 from prefect.utilities.services import critical_service_loop
+from prefect.utilities.slugify import slugify
 
 if TYPE_CHECKING:
+    from prefect.client.schemas.objects import Deployment
     from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
+    from prefect.deployments.runner import RunnerDeployment
 
 __all__ = ["Runner"]
 
@@ -130,6 +130,7 @@ class Runner:
         Examples:
             Set up a Runner to manage the execute of scheduled flow runs for two flows:
                 ```python
+                import asyncio
                 from prefect import flow, Runner
 
                 @flow
@@ -149,7 +150,7 @@ class Runner:
                     # Run on a cron schedule
                     runner.add_flow(goodbye_flow, schedule={"cron": "0 * * * *"})
 
-                    runner.start()
+                    asyncio.run(runner.start())
                 ```
         """
         if name and ("/" in name or "%" in name):
@@ -166,9 +167,6 @@ class Runner:
         self.query_seconds = query_seconds or PREFECT_RUNNER_POLL_FREQUENCY.value()
         self._prefetch_seconds = prefetch_seconds
 
-        self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
-        self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
-
         self._limiter: Optional[anyio.CapacityLimiter] = anyio.CapacityLimiter(
             self.limit
         )
@@ -177,19 +175,20 @@ class Runner:
         self._cancelling_flow_run_ids = set()
         self._scheduled_task_scopes = set()
         self._deployment_ids: Set[UUID] = set()
-        self._flow_run_process_map = dict()
+        self._flow_run_process_map: Dict[UUID, Dict] = dict()
 
         self._tmp_dir: Path = (
             Path(tempfile.gettempdir()) / "runner_storage" / str(uuid4())
         )
         self._storage_objs: List[RunnerStorage] = []
         self._deployment_storage_map: Dict[UUID, RunnerStorage] = {}
-        self._loop = asyncio.get_event_loop()
+
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
 
     @sync_compatible
     async def add_deployment(
         self,
-        deployment: RunnerDeployment,
+        deployment: "RunnerDeployment",
     ) -> UUID:
         """
         Registers the deployment with the Prefect API and will monitor for work once
@@ -324,7 +323,6 @@ class Runner:
 
         sys.exit(0)
 
-    @sync_compatible
     async def start(
         self, run_once: bool = False, webserver: Optional[bool] = None
     ) -> None:
@@ -342,6 +340,7 @@ class Runner:
             Initialize a Runner, add two flows, and serve them by starting the Runner:
 
             ```python
+            import asyncio
             from prefect import flow, Runner
 
             @flow
@@ -361,7 +360,7 @@ class Runner:
                 # Run on a cron schedule
                 runner.add_flow(goodbye_flow, schedule={"cron": "0 * * * *"})
 
-                runner.start()
+                asyncio.run(runner.start())
             ```
         """
         from prefect.runner.server import start_webserver
@@ -695,8 +694,9 @@ class Runner:
         """
         self._logger.info("Pausing all deployments...")
         for deployment_id in self._deployment_ids:
-            self._logger.debug(f"Pausing deployment '{deployment_id}'")
             await self._client.set_deployment_paused_state(deployment_id, True)
+            self._logger.debug(f"Paused deployment '{deployment_id}'")
+
         self._logger.info("All deployments have been paused!")
 
     async def _get_and_submit_flow_runs(self):
@@ -818,8 +818,71 @@ class Runner:
                     "message": state_msg or "Flow run was cancelled successfully."
                 },
             )
+            try:
+                deployment = await self._client.read_deployment(flow_run.deployment_id)
+            except ObjectNotFound:
+                deployment = None
+            try:
+                flow = await self._client.read_flow(flow_run.flow_id)
+            except ObjectNotFound:
+                flow = None
+            self._emit_flow_run_cancelled_event(
+                flow_run=flow_run, flow=flow, deployment=deployment
+            )
             run_logger.info(f"Cancelled flow run '{flow_run.name}'!")
 
+    def _event_resource(self):
+        from prefect import __version__
+
+        return {
+            "prefect.resource.id": f"prefect.runner.{slugify(self.name)}",
+            "prefect.resource.name": self.name,
+            "prefect.version": __version__,
+        }
+
+    def _emit_flow_run_cancelled_event(
+        self,
+        flow_run: "FlowRun",
+        flow: "Optional[APIFlow]",
+        deployment: "Optional[Deployment]",
+    ):
+        related = []
+        tags = []
+        if deployment:
+            related.append(
+                {
+                    "prefect.resource.id": f"prefect.deployment.{deployment.id}",
+                    "prefect.resource.role": "deployment",
+                    "prefect.resource.name": deployment.name,
+                }
+            )
+            tags.extend(deployment.tags)
+        if flow:
+            related.append(
+                {
+                    "prefect.resource.id": f"prefect.flow.{flow.id}",
+                    "prefect.resource.role": "flow",
+                    "prefect.resource.name": flow.name,
+                }
+            )
+        related.append(
+            {
+                "prefect.resource.id": f"prefect.flow-run.{flow_run.id}",
+                "prefect.resource.role": "flow-run",
+                "prefect.resource.name": flow_run.name,
+            }
+        )
+        tags.extend(flow_run.tags)
+
+        related = [RelatedResource.model_validate(r) for r in related]
+        related += tags_as_related_resources(set(tags))
+
+        emit_event(
+            event="prefect.runner.cancelled-flow-run",
+            resource=self._event_resource(),
+            related=related,
+        )
+
     async def _get_scheduled_flow_runs(
         self,
     ) -> List["FlowRun"]:
@@ -956,7 +1019,7 @@ class Runner:
             # If the run is not ready to submit, release the concurrency slot
             self._release_limit_slot(flow_run.id)
 
-        self._submitting_flow_run_ids.remove(flow_run.id)
+        self._submitting_flow_run_ids.discard(flow_run.id)
 
     async def _submit_run_and_capture_errors(
         self,
@@ -1163,6 +1226,16 @@ class Runner:
         self._logger.debug("Starting runner...")
         self._client = get_client()
         self._tmp_dir.mkdir(parents=True)
+
+        if not hasattr(self, "_loop") or not self._loop:
+            self._loop = asyncio.get_event_loop()
+
+        if not hasattr(self, "_runs_task_group") or not self._runs_task_group:
+            self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
+
+        if not hasattr(self, "_loops_task_group") or not self._loops_task_group:
+            self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
+
         await self._client.__aenter__()
         await self._runs_task_group.__aenter__()