prefect-client 3.0.0rc10__py3-none-any.whl → 3.0.0rc12__py3-none-any.whl

prefect/flows.py

@@ -5,6 +5,7 @@ Module containing the base workflow class and decorator - for most use cases, us
 # This file requires type-checking with pyright because mypy does not yet support PEP612
 # See https://github.com/python/mypy/issues/8645
 import ast
+import asyncio
 import datetime
 import importlib.util
 import inspect
@@ -28,6 +29,8 @@ from typing import (
     List,
     NoReturn,
     Optional,
+    Set,
+    Tuple,
     Type,
     TypeVar,
     Union,
@@ -44,7 +47,9 @@ from pydantic.v1.errors import ConfigError  # TODO
 from rich.console import Console
 from typing_extensions import Literal, ParamSpec, Self
 
-from prefect._internal.compatibility.deprecated import deprecated_parameter
+from prefect._internal.compatibility.deprecated import (
+    deprecated_parameter,
+)
 from prefect._internal.concurrency.api import create_call, from_async
 from prefect.blocks.core import Block
 from prefect.client.orchestration import get_client
@@ -60,6 +65,7 @@ from prefect.exceptions import (
     MissingFlowError,
     ObjectNotFound,
     ParameterTypeError,
+    ScriptError,
     UnspecifiedFlowError,
 )
 from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
@@ -778,8 +784,7 @@ class Flow(Generic[P, R]):
         self.on_failure_hooks.append(fn)
         return fn
 
-    @sync_compatible
-    async def serve(
+    def serve(
         self,
         name: Optional[str] = None,
         interval: Optional[
@@ -793,7 +798,7 @@ class Flow(Generic[P, R]):
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[List["FlexibleScheduleList"]] = None,
+        schedules: Optional["FlexibleScheduleList"] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
         is_schedule_active: Optional[bool] = None,
         triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
@@ -884,7 +889,7 @@ class Flow(Generic[P, R]):
             name = Path(name).stem
 
         runner = Runner(name=name, pause_on_shutdown=pause_on_shutdown, limit=limit)
-        deployment_id = await runner.add_flow(
+        deployment_id = runner.add_flow(
             self,
             name=name,
             triggers=triggers,
@@ -917,15 +922,27 @@ class Flow(Generic[P, R]):
 
             console = Console()
             console.print(help_message, soft_wrap=True)
-        await runner.start(webserver=webserver)
+
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError as exc:
+            if "no running event loop" in str(exc):
+                loop = None
+            else:
+                raise
+
+        if loop is not None:
+            loop.run_until_complete(runner.start(webserver=webserver))
+        else:
+            asyncio.run(runner.start(webserver=webserver))
 
     @classmethod
     @sync_compatible
     async def from_source(
-        cls: Type[F],
+        cls: Type["Flow[P, R]"],
         source: Union[str, "RunnerStorage", ReadableDeploymentStorage],
         entrypoint: str,
-    ) -> F:
+    ) -> "Flow[P, R]":
         """
         Loads a flow from a remote source.
 
@@ -1003,7 +1020,9 @@ class Flow(Generic[P, R]):
             create_storage_from_source,
         )
 
-        if isinstance(source, str):
+        if isinstance(source, (Path, str)):
+            if isinstance(source, Path):
+                source = str(source)
             storage = create_storage_from_source(source)
         elif isinstance(source, RunnerStorage):
             storage = source
@@ -1186,9 +1205,9 @@ class Flow(Generic[P, R]):
             entrypoint_type=entrypoint_type,
         )
 
-        from prefect.deployments import runner
+        from prefect.deployments.runner import deploy
 
-        deployment_ids = await runner.deploy(
+        deployment_ids = await deploy(
             deployment,
             work_pool_name=work_pool_name,
             image=image,
@@ -1685,6 +1704,7 @@ def select_flow(
 
 def load_flow_from_entrypoint(
     entrypoint: str,
+    use_placeholder_flow: bool = True,
 ) -> Flow:
     """
     Extract a flow object from a script at an entrypoint by running all of the code in the file.
@@ -1692,6 +1712,8 @@ def load_flow_from_entrypoint(
     Args:
         entrypoint: a string in the format `<path_to_script>:<flow_func_name>` or a module path
             to a flow function
+        use_placeholder_flow: if True, use a placeholder Flow object if the actual flow object
+            cannot be loaded from the entrypoint (e.g. dependencies are missing)
 
     Returns:
         The flow object from the script
@@ -1712,6 +1734,16 @@ def load_flow_from_entrypoint(
         raise MissingFlowError(
             f"Flow function with name {func_name!r} not found in {path!r}. "
         ) from exc
+    except ScriptError as exc:
+        # If the flow has dependencies that are not installed in the current
+        # environment, fallback to loading the flow via AST parsing. The
+        # drawback of this approach is that we're unable to actually load the
+        # function, so we create a placeholder flow that will re-raise this
+        # exception when called.
+        if use_placeholder_flow:
+            flow = load_placeholder_flow(entrypoint=entrypoint, raises=exc)
+        else:
+            raise
 
     if not isinstance(flow, Flow):
         raise MissingFlowError(
@@ -1722,14 +1754,13 @@ def load_flow_from_entrypoint(
     return flow
 
 
-@sync_compatible
-async def serve(
+def serve(
     *args: "RunnerDeployment",
     pause_on_shutdown: bool = True,
     print_starting_message: bool = True,
     limit: Optional[int] = None,
     **kwargs,
-) -> NoReturn:
+):
     """
     Serve the provided list of deployments.
 
@@ -1779,7 +1810,7 @@ async def serve(
 
     runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
     for deployment in args:
-        await runner.add_deployment(deployment)
+        runner.add_deployment(deployment)
 
     if print_starting_message:
         help_message_top = (
@@ -1810,7 +1841,18 @@ async def serve(
             Group(help_message_top, table, help_message_bottom), soft_wrap=True
         )
 
-    await runner.start()
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError as exc:
+        if "no running event loop" in str(exc):
+            loop = None
+        else:
+            raise
+
+    if loop is not None:
+        loop.run_until_complete(runner.start())
+    else:
+        asyncio.run(runner.start())
 
 
 @client_injector
@@ -1819,6 +1861,7 @@ async def load_flow_from_flow_run(
     flow_run: "FlowRun",
     ignore_storage: bool = False,
     storage_base_path: Optional[str] = None,
+    use_placeholder_flow: bool = True,
 ) -> Flow:
     """
     Load a flow from the location/script provided in a deployment's storage document.
@@ -1845,7 +1888,9 @@ async def load_flow_from_flow_run(
             f"Importing flow code from module path {deployment.entrypoint}"
         )
         flow = await run_sync_in_worker_thread(
-            load_flow_from_entrypoint, deployment.entrypoint
+            load_flow_from_entrypoint,
+            deployment.entrypoint,
+            use_placeholder_flow=use_placeholder_flow,
         )
         return flow
 
@@ -1887,29 +1932,147 @@ async def load_flow_from_flow_run(
     import_path = relative_path_to_current_platform(deployment.entrypoint)
     run_logger.debug(f"Importing flow code from '{import_path}'")
 
-    flow = await run_sync_in_worker_thread(load_flow_from_entrypoint, str(import_path))
+    flow = await run_sync_in_worker_thread(
+        load_flow_from_entrypoint,
+        str(import_path),
+        use_placeholder_flow=use_placeholder_flow,
+    )
 
     return flow
 
 
-def load_flow_argument_from_entrypoint(
-    entrypoint: str, arg: str = "name"
-) -> Optional[str]:
+def load_placeholder_flow(entrypoint: str, raises: Exception):
     """
-    Extract a flow argument from an entrypoint string.
+    Load a placeholder flow that is initialized with the same arguments as the
+    flow specified in the entrypoint. If called the flow will raise `raises`.
 
-    Loads the source code of the entrypoint and extracts the flow argument from the
-    `flow` decorator.
+    This is useful when a flow can't be loaded due to missing dependencies or
+    other issues but the base metadata defining the flow is still needed.
 
     Args:
-        entrypoint: a string in the format `<path_to_script>:<flow_func_name>` or a module path
-            to a flow function
+        entrypoint: a string in the format `<path_to_script>:<flow_func_name>`
+          or a module path to a flow function
+        raises: an exception to raise when the flow is called
+    """
+
+    def _base_placeholder():
+        raise raises
+
+    def sync_placeholder_flow(*args, **kwargs):
+        _base_placeholder()
+
+    async def async_placeholder_flow(*args, **kwargs):
+        _base_placeholder()
+
+    placeholder_flow = (
+        async_placeholder_flow
+        if is_entrypoint_async(entrypoint)
+        else sync_placeholder_flow
+    )
+
+    arguments = load_flow_arguments_from_entrypoint(entrypoint)
+    arguments["fn"] = placeholder_flow
+
+    return Flow(**arguments)
+
+
+def load_flow_arguments_from_entrypoint(
+    entrypoint: str, arguments: Optional[Union[List[str], Set[str]]] = None
+) -> dict[str, Any]:
+    """
+    Extract flow arguments from an entrypoint string.
+
+    Loads the source code of the entrypoint and extracts the flow arguments
+    from the `flow` decorator.
+
+    Args:
+        entrypoint: a string in the format `<path_to_script>:<flow_func_name>`
+          or a module path to a flow function
+    """
+
+    func_def, source_code = _entrypoint_definition_and_source(entrypoint)
+
+    if arguments is None:
+        # If no arguments are provided default to known arguments that are of
+        # built-in types.
+        arguments = {
+            "name",
+            "version",
+            "retries",
+            "retry_delay_seconds",
+            "description",
+            "timeout_seconds",
+            "validate_parameters",
+            "persist_result",
+            "cache_result_in_memory",
+            "log_prints",
+        }
+
+    result = {}
+
+    for decorator in func_def.decorator_list:
+        if (
+            isinstance(decorator, ast.Call)
+            and getattr(decorator.func, "id", "") == "flow"
+        ):
+            for keyword in decorator.keywords:
+                if keyword.arg not in arguments:
+                    continue
+
+                if isinstance(keyword.value, ast.Constant):
+                    # Use the string value of the argument
+                    result[keyword.arg] = str(keyword.value.value)
+                    continue
+
+                # if the arg value is not a raw str (i.e. a variable or expression),
+                # then attempt to evaluate it
+                namespace = safe_load_namespace(source_code)
+                literal_arg_value = ast.get_source_segment(source_code, keyword.value)
+                cleaned_value = (
+                    literal_arg_value.replace("\n", "") if literal_arg_value else ""
+                )
+
+                try:
+                    evaluated_value = eval(cleaned_value, namespace)  # type: ignore
+                    result[keyword.arg] = str(evaluated_value)
+                except Exception as e:
+                    logger.info(
+                        "Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",
+                        keyword.arg,
+                        literal_arg_value,
+                        exc_info=e,
+                    )
+                    # ignore the decorator arg and fallback to default behavior
+                    continue
+
+    if "name" in arguments and "name" not in result:
+        # If no matching decorator or keyword argument for `name' is found
+        # fallback to the function name.
+        result["name"] = func_def.name.replace("_", "-")
+
+    return result
+
+
+def is_entrypoint_async(entrypoint: str) -> bool:
+    """
+    Determine if the function specified in the entrypoint is asynchronous.
+
+    Args:
+        entrypoint: A string in the format `<path_to_script>:<func_name>` or
+          a module path to a function.
 
     Returns:
-        The flow argument value
+        True if the function is asynchronous, False otherwise.
     """
+    func_def, _ = _entrypoint_definition_and_source(entrypoint)
+    return isinstance(func_def, ast.AsyncFunctionDef)
+
+
+def _entrypoint_definition_and_source(
+    entrypoint: str,
+) -> Tuple[Union[ast.FunctionDef, ast.AsyncFunctionDef], str]:
     if ":" in entrypoint:
-        # split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
+        # Split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
         path, func_name = entrypoint.rsplit(":", maxsplit=1)
         source_code = Path(path).read_text()
     else:
@@ -1918,6 +2081,7 @@ def load_flow_argument_from_entrypoint(
         if not spec or not spec.origin:
             raise ValueError(f"Could not find module {path!r}")
         source_code = Path(spec.origin).read_text()
+
     parsed_code = ast.parse(source_code)
     func_def = next(
         (
@@ -1934,46 +2098,8 @@ def load_flow_argument_from_entrypoint(
         ),
         None,
     )
+
     if not func_def:
         raise ValueError(f"Could not find flow {func_name!r} in {path!r}")
-    for decorator in func_def.decorator_list:
-        if (
-            isinstance(decorator, ast.Call)
-            and getattr(decorator.func, "id", "") == "flow"
-        ):
-            for keyword in decorator.keywords:
-                if keyword.arg == arg:
-                    if isinstance(keyword.value, ast.Constant):
-                        return (
-                            keyword.value.value
-                        )  # Return the string value of the argument
-
-                    # if the arg value is not a raw str (i.e. a variable or expression),
-                    # then attempt to evaluate it
-                    namespace = safe_load_namespace(source_code)
-                    literal_arg_value = ast.get_source_segment(
-                        source_code, keyword.value
-                    )
-                    cleaned_value = (
-                        literal_arg_value.replace("\n", "") if literal_arg_value else ""
-                    )
-
-                    try:
-                        evaluated_value = eval(cleaned_value, namespace)  # type: ignore
-                    except Exception as e:
-                        logger.info(
-                            "Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",
-                            arg,
-                            literal_arg_value,
-                            exc_info=e,
-                        )
-                        # ignore the decorator arg and fallback to default behavior
-                        break
-                    return str(evaluated_value)
-
-    if arg == "name":
-        return func_name.replace(
-            "_", "-"
-        )  # If no matching decorator or keyword argument is found
 
-    return None
+    return func_def, source_code

prefect/futures.py

@@ -1,4 +1,5 @@
 import abc
+import collections
 import concurrent.futures
 import inspect
 import uuid
@@ -256,13 +257,7 @@ class PrefectFutureList(list, Iterator, Generic[F]):
             timeout: The maximum number of seconds to wait for all futures to
                 complete. This method will not raise if the timeout is reached.
         """
-        try:
-            with timeout_context(timeout):
-                for future in self:
-                    future.wait()
-        except TimeoutError:
-            logger.debug("Timed out waiting for all futures to complete.")
-            return
+        wait(self, timeout=timeout)
 
     def result(
         self,
@@ -297,6 +292,57 @@ class PrefectFutureList(list, Iterator, Generic[F]):
             ) from exc
 
 
+DoneAndNotDoneFutures = collections.namedtuple("DoneAndNotDoneFutures", "done not_done")
+
+
+def wait(futures: List[PrefectFuture], timeout=None) -> DoneAndNotDoneFutures:
+    """
+    Wait for the futures in the given sequence to complete.
+
+    Args:
+        futures: The sequence of Futures to wait upon.
+        timeout: The maximum number of seconds to wait. If None, then there
+            is no limit on the wait time.
+
+    Returns:
+        A named 2-tuple of sets. The first set, named 'done', contains the
+        futures that completed (is finished or cancelled) before the wait
+        completed. The second set, named 'not_done', contains uncompleted
+        futures. Duplicate futures given to *futures* are removed and will be
+        returned only once.
+
+    Examples:
+        ```python
+        @task
+        def sleep_task(seconds):
+            sleep(seconds)
+            return 42
+
+        @flow
+        def flow():
+            futures = random_task.map(range(10))
+            done, not_done = wait(futures, timeout=5)
+            print(f"Done: {len(done)}")
+            print(f"Not Done: {len(not_done)}")
+        ```
+    """
+    futures = set(futures)
+    done = {f for f in futures if f._final_state}
+    not_done = futures - done
+    if len(done) == len(futures):
+        return DoneAndNotDoneFutures(done, not_done)
+    try:
+        with timeout_context(timeout):
+            for future in not_done.copy():
+                future.wait()
+                done.add(future)
+                not_done.remove(future)
+            return DoneAndNotDoneFutures(done, not_done)
+    except TimeoutError:
+        logger.debug("Timed out waiting for all futures to complete.")
+        return DoneAndNotDoneFutures(done, not_done)
+
+
 def resolve_futures_to_states(
     expr: Union[PrefectFuture, Any],
 ) -> Union[State, Any]:

prefect/logging/loggers.py

@@ -97,7 +97,7 @@ def get_logger(name: Optional[str] = None) -> logging.Logger:
 
 
 def get_run_logger(
-    context: "RunContext" = None, **kwargs: str
+    context: Optional["RunContext"] = None, **kwargs: str
 ) -> Union[logging.Logger, logging.LoggerAdapter]:
     """
     Get a Prefect logger for the current task run or flow run.

prefect/results.py

@@ -25,7 +25,7 @@ from typing_extensions import ParamSpec, Self
 import prefect
 from prefect.blocks.core import Block
 from prefect.client.utilities import inject_client
-from prefect.exceptions import MissingResult, ObjectAlreadyExists
+from prefect.exceptions import MissingResult
 from prefect.filesystems import (
     LocalFileSystem,
     WritableFileSystem,
@@ -64,51 +64,6 @@ R = TypeVar("R")
 _default_storages: Dict[Tuple[str, str], WritableFileSystem] = {}
 
 
-async def _get_or_create_default_storage(block_document_slug: str) -> ResultStorage:
-    """
-    Generate a default file system for storage.
-    """
-    default_storage_name, storage_path = cache_key = (
-        block_document_slug,
-        PREFECT_LOCAL_STORAGE_PATH.value(),
-    )
-
-    async def get_storage() -> WritableFileSystem:
-        try:
-            return await Block.load(default_storage_name)
-        except ValueError as e:
-            if "Unable to find" not in str(e):
-                raise e
-
-        block_type_slug, name = default_storage_name.split("/")
-        if block_type_slug == "local-file-system":
-            block = LocalFileSystem(basepath=storage_path)
-        else:
-            raise ValueError(
-                "The default storage block does not exist, but it is of type "
-                f"'{block_type_slug}' which cannot be created implicitly.  Please create "
-                "the block manually."
-            )
-
-        try:
-            await block.save(name, overwrite=False)
-        except ValueError as e:
-            if "already in use" not in str(e):
-                raise e
-        except ObjectAlreadyExists:
-            # Another client created the block before we reached this line
-            block = await Block.load(default_storage_name)
-
-        return block
-
-    try:
-        return _default_storages[cache_key]
-    except KeyError:
-        storage = await get_storage()
-        _default_storages[cache_key] = storage
-        return storage
-
-
 @sync_compatible
 async def get_default_result_storage() -> ResultStorage:
     """