prefect-client 3.0.0rc7__py3-none-any.whl → 3.0.0rc9__py3-none-any.whl

prefect/futures.py

@@ -2,11 +2,13 @@ import abc
 import concurrent.futures
 import inspect
 import uuid
+from collections.abc import Iterator
 from functools import partial
-from typing import Any, Generic, Optional, Set, Union, cast
+from typing import Any, Generic, List, Optional, Set, Union, cast
 
 from typing_extensions import TypeVar
 
+from prefect._internal.compatibility.deprecated import deprecated_async_method
 from prefect.client.orchestration import get_client
 from prefect.client.schemas.objects import TaskRun
 from prefect.exceptions import ObjectNotFound
@@ -16,13 +18,15 @@ from prefect.task_runs import TaskRunWaiter
 from prefect.utilities.annotations import quote
 from prefect.utilities.asyncutils import run_coro_as_sync
 from prefect.utilities.collections import StopVisiting, visit_collection
+from prefect.utilities.timeout import timeout as timeout_context
 
 F = TypeVar("F")
+R = TypeVar("R")
 
 logger = get_logger(__name__)
 
 
-class PrefectFuture(abc.ABC):
+class PrefectFuture(abc.ABC, Generic[R]):
     """
     Abstract base class for Prefect futures. A Prefect future is a handle to the
     asynchronous execution of a task run. It provides methods to wait for the task
@@ -31,7 +35,7 @@ class PrefectFuture(abc.ABC):
 
     def __init__(self, task_run_id: uuid.UUID):
         self._task_run_id = task_run_id
-        self._final_state = None
+        self._final_state: Optional[State[R]] = None
 
     @property
     def task_run_id(self) -> uuid.UUID:
@@ -61,7 +65,7 @@ class PrefectFuture(abc.ABC):
         If the task run has already completed, this method will return immediately.
 
         Args:
-            - timeout: The maximum number of seconds to wait for the task run to complete.
+            timeout: The maximum number of seconds to wait for the task run to complete.
               If the task run has not completed after the timeout has elapsed, this method will return.
         """
 
@@ -70,7 +74,7 @@ class PrefectFuture(abc.ABC):
         self,
         timeout: Optional[float] = None,
         raise_on_failure: bool = True,
-    ) -> Any:
+    ) -> R:
         ...
         """
         Get the result of the task run associated with this future.
@@ -78,16 +82,16 @@ class PrefectFuture(abc.ABC):
         If the task run has not completed, this method will wait for the task run to complete.
 
         Args:
-            - timeout: The maximum number of seconds to wait for the task run to complete.
+            timeout: The maximum number of seconds to wait for the task run to complete.
             If the task run has not completed after the timeout has elapsed, this method will return.
-            - raise_on_failure: If `True`, an exception will be raised if the task run fails.
+            raise_on_failure: If `True`, an exception will be raised if the task run fails.
 
         Returns:
             The result of the task run.
         """
 
 
-class PrefectWrappedFuture(PrefectFuture, abc.ABC, Generic[F]):
+class PrefectWrappedFuture(PrefectFuture, abc.ABC, Generic[R, F]):
     """
     A Prefect future that wraps another future object.
     """
@@ -102,12 +106,13 @@ class PrefectWrappedFuture(PrefectFuture, abc.ABC, Generic[F]):
         return self._wrapped_future
 
 
-class PrefectConcurrentFuture(PrefectWrappedFuture[concurrent.futures.Future]):
+class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]):
     """
     A Prefect future that wraps a concurrent.futures.Future. This future is used
     when the task run is submitted to a ThreadPoolExecutor.
     """
 
+    @deprecated_async_method
     def wait(self, timeout: Optional[float] = None) -> None:
         try:
             result = self._wrapped_future.result(timeout=timeout)
@@ -116,11 +121,12 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[concurrent.futures.Future]):
         if isinstance(result, State):
             self._final_state = result
 
+    @deprecated_async_method
     def result(
         self,
         timeout: Optional[float] = None,
         raise_on_failure: bool = True,
-    ) -> Any:
+    ) -> R:
         if not self._final_state:
             try:
                 future_result = self._wrapped_future.result(timeout=timeout)
@@ -156,7 +162,7 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[concurrent.futures.Future]):
         )
 
 
-class PrefectDistributedFuture(PrefectFuture):
+class PrefectDistributedFuture(PrefectFuture[R]):
     """
     Represents the result of a computation happening anywhere.
 
@@ -165,6 +171,7 @@ class PrefectDistributedFuture(PrefectFuture):
     any task run scheduled in Prefect's API.
     """
 
+    @deprecated_async_method
     def wait(self, timeout: Optional[float] = None) -> None:
         return run_coro_as_sync(self.wait_async(timeout=timeout))
 
@@ -201,11 +208,12 @@ class PrefectDistributedFuture(PrefectFuture):
                 self._final_state = task_run.state
             return
 
+    @deprecated_async_method
     def result(
         self,
         timeout: Optional[float] = None,
         raise_on_failure: bool = True,
-    ) -> Any:
+    ) -> R:
         return run_coro_as_sync(
             self.result_async(timeout=timeout, raise_on_failure=raise_on_failure)
         )
@@ -214,7 +222,7 @@ class PrefectDistributedFuture(PrefectFuture):
         self,
         timeout: Optional[float] = None,
         raise_on_failure: bool = True,
-    ):
+    ) -> R:
         if not self._final_state:
             await self.wait_async(timeout=timeout)
             if not self._final_state:
@@ -232,6 +240,63 @@ class PrefectDistributedFuture(PrefectFuture):
         return self.task_run_id == other.task_run_id
 
 
+class PrefectFutureList(list, Iterator, Generic[F]):
+    """
+    A list of Prefect futures.
+
+    This class provides methods to wait for all futures
+    in the list to complete and to retrieve the results of all task runs.
+    """
+
+    def wait(self, timeout: Optional[float] = None) -> None:
+        """
+        Wait for all futures in the list to complete.
+
+        Args:
+            timeout: The maximum number of seconds to wait for all futures to
+                complete. This method will not raise if the timeout is reached.
+        """
+        try:
+            with timeout_context(timeout):
+                for future in self:
+                    future.wait()
+        except TimeoutError:
+            logger.debug("Timed out waiting for all futures to complete.")
+            return
+
+    def result(
+        self,
+        timeout: Optional[float] = None,
+        raise_on_failure: bool = True,
+    ) -> List:
+        """
+        Get the results of all task runs associated with the futures in the list.
+
+        Args:
+            timeout: The maximum number of seconds to wait for all futures to
+                complete.
+            raise_on_failure: If `True`, an exception will be raised if any task run fails.
+
+        Returns:
+            A list of results of the task runs.
+
+        Raises:
+            TimeoutError: If the timeout is reached before all futures complete.
+        """
+        try:
+            with timeout_context(timeout):
+                return [
+                    future.result(raise_on_failure=raise_on_failure) for future in self
+                ]
+        except TimeoutError as exc:
+            # timeout came from inside the task
+            if "Scope timed out after {timeout} second(s)." not in str(exc):
+                raise
+            raise TimeoutError(
+                f"Timed out waiting for all futures to complete within {timeout} seconds"
+            ) from exc
+
+
 def resolve_futures_to_states(
     expr: Union[PrefectFuture, Any],
 ) -> Union[State, Any]:

prefect/logging/loggers.py

@@ -115,7 +115,7 @@ def get_run_logger(
             addition to the run metadata
 
     Raises:
-        RuntimeError: If no context can be found
+        MissingContextError: If no context can be found
     """
     # Check for existing contexts
     task_run_context = prefect.context.TaskRunContext.get()

prefect/main.py

@@ -0,0 +1,70 @@
+# Import user-facing API
+from prefect.deployments import deploy
+from prefect.states import State
+from prefect.logging import get_run_logger
+from prefect.flows import flow, Flow, serve
+from prefect.transactions import Transaction
+from prefect.tasks import task, Task
+from prefect.context import tags
+from prefect.manifests import Manifest
+from prefect.utilities.annotations import unmapped, allow_failure
+from prefect.results import BaseResult
+from prefect.flow_runs import pause_flow_run, resume_flow_run, suspend_flow_run
+from prefect.client.orchestration import get_client, PrefectClient
+from prefect.client.cloud import get_cloud_client, CloudClient
+import prefect.variables
+import prefect.runtime
+
+# Import modules that register types
+import prefect.serializers
+import prefect.blocks.notifications
+import prefect.blocks.system
+
+# Initialize the process-wide profile and registry at import time
+import prefect.context
+
+# Perform any forward-ref updates needed for Pydantic models
+import prefect.client.schemas
+
+prefect.context.FlowRunContext.model_rebuild()
+prefect.context.TaskRunContext.model_rebuild()
+prefect.client.schemas.State.model_rebuild()
+prefect.client.schemas.StateCreate.model_rebuild()
+Transaction.model_rebuild()
+
+# Configure logging
+import prefect.logging.configuration
+
+prefect.logging.configuration.setup_logging()
+prefect.logging.get_logger("profiles").debug(
+    f"Using profile {prefect.context.get_settings_context().profile.name!r}"
+)
+
+
+from prefect._internal.compatibility.deprecated import (
+    inject_renamed_module_alias_finder,
+)
+
+inject_renamed_module_alias_finder()
+
+
+# Declare API for type-checkers
+__all__ = [
+    "allow_failure",
+    "flow",
+    "Flow",
+    "get_client",
+    "get_run_logger",
+    "Manifest",
+    "State",
+    "tags",
+    "task",
+    "Task",
+    "Transaction",
+    "unmapped",
+    "serve",
+    "deploy",
+    "pause_flow_run",
+    "resume_flow_run",
+    "suspend_flow_run",
+]

prefect/plugins.py

@@ -8,12 +8,11 @@ Currently supported entrypoints:
         should be imported when Prefect is imported.
 """
 
-import sys
 from types import ModuleType
 from typing import Any, Dict, Union
 
 import prefect.settings
-from prefect.utilities.compat import EntryPoint, EntryPoints, entry_points
+from prefect.utilities.compat import EntryPoints, entry_points
 
 
 def safe_load_entrypoints(entrypoints: EntryPoints) -> Dict[str, Union[Exception, Any]]:
@@ -39,68 +38,6 @@ def safe_load_entrypoints(entrypoints: EntryPoints) -> Dict[str, Union[Exception
     return results
 
 
-def load_extra_entrypoints() -> Dict[str, Union[Exception, Any]]:
-    # Note: Return values are only exposed for testing.
-    results = {}
-
-    if not prefect.settings.PREFECT_EXTRA_ENTRYPOINTS.value():
-        return results
-
-    values = {
-        value.strip()
-        for value in prefect.settings.PREFECT_EXTRA_ENTRYPOINTS.value().split(",")
-    }
-
-    entrypoints = []
-    for value in values:
-        try:
-            entrypoint = EntryPoint(name=None, value=value, group="prefect-extra")
-        except Exception as exc:
-            print(
-                (
-                    f"Warning! Failed to parse extra entrypoint {value!r}:"
-                    f" {type(exc).__name__}: {exc}"
-                ),
-                file=sys.stderr,
-            )
-            results[value] = exc
-        else:
-            entrypoints.append(entrypoint)
-
-    for value, result in zip(
-        values, safe_load_entrypoints(EntryPoints(entrypoints)).values()
-    ):
-        results[value] = result
-
-        if isinstance(result, Exception):
-            print(
-                (
-                    f"Warning! Failed to load extra entrypoint {value!r}:"
-                    f" {type(result).__name__}: {result}"
-                ),
-                file=sys.stderr,
-            )
-        elif callable(result):
-            try:
-                results[value] = result()
-            except Exception as exc:
-                print(
-                    (
-                        f"Warning! Failed to run callable entrypoint {value!r}:"
-                        f" {type(exc).__name__}: {exc}"
-                    ),
-                    file=sys.stderr,
-                )
-                results[value] = exc
-        else:
-            if prefect.settings.PREFECT_DEBUG_MODE:
-                print(
-                    "Loaded extra entrypoint {value!r} successfully.", file=sys.stderr
-                )
-
-    return results
-
-
 def load_prefect_collections() -> Dict[str, ModuleType]:
     """
     Load all Prefect collections that define an entrypoint in the group

prefect/results.py

@@ -1,4 +1,5 @@
 import abc
+import inspect
 import uuid
 from functools import partial
 from typing import (
@@ -142,38 +143,6 @@ def get_default_persist_setting() -> bool:
     return PREFECT_RESULTS_PERSIST_BY_DEFAULT.value()
 
 
-def flow_features_require_result_persistence(flow: "Flow") -> bool:
-    """
-    Returns `True` if the given flow uses features that require its result to be
-    persisted.
-    """
-    if not flow.cache_result_in_memory:
-        return True
-    return False
-
-
-def flow_features_require_child_result_persistence(flow: "Flow") -> bool:
-    """
-    Returns `True` if the given flow uses features that require child flow and task
-    runs to persist their results.
-    """
-    if flow and flow.retries:
-        return True
-    return False
-
-
-def task_features_require_result_persistence(task: "Task") -> bool:
-    """
-    Returns `True` if the given task uses features that require its result to be
-    persisted.
-    """
-    if task.cache_key_fn:
-        return True
-    if not task.cache_result_in_memory:
-        return True
-    return False
-
-
 def _format_user_supplied_storage_key(key: str) -> str:
     # Note here we are pinning to task runs since flow runs do not support storage keys
     # yet; we'll need to split logic in the future or have two separate functions
@@ -235,17 +204,7 @@ class ResultFactory(BaseModel):
                 result_storage=flow.result_storage or ctx.result_factory.storage_block,
                 result_serializer=flow.result_serializer
                 or ctx.result_factory.serializer,
-                persist_result=(
-                    flow.persist_result
-                    if flow.persist_result is not None
-                    # !! Child flows persist their result by default if the it or the
-                    #    parent flow uses a feature that requires it
-                    else (
-                        flow_features_require_result_persistence(flow)
-                        or flow_features_require_child_result_persistence(ctx.flow)
-                        or get_default_persist_setting()
-                    )
-                ),
+                persist_result=flow.persist_result,
                 cache_result_in_memory=flow.cache_result_in_memory,
                 storage_key_fn=DEFAULT_STORAGE_KEY_FN,
                 client=client,
@@ -258,16 +217,7 @@ class ResultFactory(BaseModel):
                 client=client,
                 result_storage=flow.result_storage,
                 result_serializer=flow.result_serializer,
-                persist_result=(
-                    flow.persist_result
-                    if flow.persist_result is not None
-                    # !! Flows persist their result by default if uses a feature that
-                    #    requires it
-                    else (
-                        flow_features_require_result_persistence(flow)
-                        or get_default_persist_setting()
-                    )
-                ),
+                persist_result=flow.persist_result,
                 cache_result_in_memory=flow.cache_result_in_memory,
                 storage_key_fn=DEFAULT_STORAGE_KEY_FN,
             )
@@ -318,21 +268,7 @@ class ResultFactory(BaseModel):
             if ctx and ctx.result_factory
             else get_default_result_serializer()
         )
-        persist_result = (
-            task.persist_result
-            if task.persist_result is not None
-            # !! Tasks persist their result by default if their parent flow uses a
-            #    feature that requires it or the task uses a feature that requires it
-            else (
-                (
-                    flow_features_require_child_result_persistence(ctx.flow)
-                    if ctx
-                    else False
-                )
-                or task_features_require_result_persistence(task)
-                or get_default_persist_setting()
-            )
-        )
+        persist_result = task.persist_result
 
         cache_result_in_memory = task.cache_result_in_memory
 
@@ -355,11 +291,14 @@ class ResultFactory(BaseModel):
         cls: Type[Self],
         result_storage: ResultStorage,
         result_serializer: ResultSerializer,
-        persist_result: bool,
+        persist_result: Optional[bool],
         cache_result_in_memory: bool,
         storage_key_fn: Callable[[], str],
         client: "PrefectClient",
     ) -> Self:
+        if persist_result is None:
+            persist_result = get_default_persist_setting()
+
         storage_block_id, storage_block = await cls.resolve_storage_block(
             result_storage, client=client, persist_result=persist_result
         )
@@ -614,7 +553,6 @@ class PersistedResult(BaseResult):
         """
         Retrieve the data and deserialize it into the original object.
         """
-
         if self.has_cached_object():
             return self._cache
 
@@ -680,7 +618,46 @@ class PersistedResult(BaseResult):
             # this could error if the serializer requires kwargs
             serializer = Serializer(type=self.serializer_type)
 
-        data = serializer.dumps(obj)
+        try:
+            data = serializer.dumps(obj)
+        except Exception as exc:
+            extra_info = (
+                'You can try a different serializer (e.g. result_serializer="json") '
+                "or disabling persistence (persist_result=False) for this flow or task."
+            )
+            # check if this is a known issue with cloudpickle and pydantic
+            # and add extra information to help the user recover
+
+            if (
+                isinstance(exc, TypeError)
+                and isinstance(obj, BaseModel)
+                and str(exc).startswith("cannot pickle")
+            ):
+                try:
+                    from IPython import get_ipython
+
+                    if get_ipython() is not None:
+                        extra_info = inspect.cleandoc(
+                            """
+                            This is a known issue in Pydantic that prevents
+                            locally-defined (non-imported) models from being
+                            serialized by cloudpickle in IPython/Jupyter
+                            environments. Please see
+                            https://github.com/pydantic/pydantic/issues/8232 for
+                            more information. To fix the issue, either: (1) move
+                            your Pydantic class definition to an importable
+                            location, (2) use the JSON serializer for your flow
+                            or task (`result_serializer="json"`), or (3)
+                            disable result persistence for your flow or task
+                            (`persist_result=False`).
+                            """
+                        ).replace("\n", " ")
+                except ImportError:
+                    pass
+            raise ValueError(
+                f"Failed to serialize object of type {type(obj).__name__!r} with "
+                f"serializer {serializer.type!r}. {extra_info}"
+            ) from exc
         blob = PersistedResultBlob(
             serializer=serializer, data=data, expiration=self.expiration
         )

prefect/runner/storage.py

@@ -559,8 +559,70 @@ class BlockStorageAdapter:
         return False
 
 
-def create_storage_from_url(
-    url: str, pull_interval: Optional[int] = 60
+class LocalStorage:
+    """
+    Sets the working directory in the local filesystem.
+    Parameters:
+        Path: Local file path to set the working directory for the flow
+    Examples:
+        Sets the working directory for the local path to the flow:
+        ```python
+        from prefect.runner.storage import Localstorage
+        storage = LocalStorage(
+            path="/path/to/local/flow_directory",
+        )
+        ```
+    """
+
+    def __init__(
+        self,
+        path: str,
+        pull_interval: Optional[int] = None,
+    ):
+        self._path = Path(path).resolve()
+        self._logger = get_logger("runner.storage.local-storage")
+        self._storage_base_path = Path.cwd()
+        self._pull_interval = pull_interval
+
+    @property
+    def destination(self) -> Path:
+        return self._path
+
+    def set_base_path(self, path: Path):
+        self._storage_base_path = path
+
+    @property
+    def pull_interval(self) -> Optional[int]:
+        return self._pull_interval
+
+    async def pull_code(self):
+        # Local storage assumes the code already exists on the local filesystem
+        # and does not need to be pulled from a remote location
+        pass
+
+    def to_pull_step(self) -> dict:
+        """
+        Returns a dictionary representation of the storage object that can be
+        used as a deployment pull step.
+        """
+        step = {
+            "prefect.deployments.steps.set_working_directory": {
+                "directory": str(self.destination)
+            }
+        }
+        return step
+
+    def __eq__(self, __value) -> bool:
+        if isinstance(__value, LocalStorage):
+            return self._path == __value._path
+        return False
+
+    def __repr__(self) -> str:
+        return f"LocalStorage(path={self._path!r})"
+
+
+def create_storage_from_source(
+    source: str, pull_interval: Optional[int] = 60
 ) -> RunnerStorage:
     """
     Creates a storage object from a URL.
@@ -574,11 +636,18 @@ def create_storage_from_url(
     Returns:
         RunnerStorage: A runner storage compatible object
     """
-    parsed_url = urlparse(url)
-    if parsed_url.scheme == "git" or parsed_url.path.endswith(".git"):
-        return GitRepository(url=url, pull_interval=pull_interval)
+    logger = get_logger("runner.storage")
+    parsed_source = urlparse(source)
+    if parsed_source.scheme == "git" or parsed_source.path.endswith(".git"):
+        return GitRepository(url=source, pull_interval=pull_interval)
+    elif parsed_source.scheme in ("file", "local"):
+        source_path = source.split("://", 1)[-1]
+        return LocalStorage(path=source_path, pull_interval=pull_interval)
+    elif parsed_source.scheme in fsspec.available_protocols():
+        return RemoteStorage(url=source, pull_interval=pull_interval)
     else:
-        return RemoteStorage(url=url, pull_interval=pull_interval)
+        logger.debug("No valid fsspec protocol found for URL, assuming local storage.")
+        return LocalStorage(path=source, pull_interval=pull_interval)
 
 
 def _format_token_from_credentials(netloc: str, credentials: dict) -> str: