prefect-client 3.0.0rc9__tar.gz → 3.0.0rc10__tar.gz

{prefect-client-3.0.0rc9/src/prefect_client.egg-info → prefect-client-3.0.0rc10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 3.0.0rc9
+Version: 3.0.0rc10
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/requirements-client.txt

@@ -33,6 +33,6 @@ sniffio >=1.3.0, < 2.0.0
 toml >= 0.10.0
 typing_extensions >= 4.5.0, < 5.0.0
 ujson >= 5.8.0, < 6.0.0
-uvicorn >= 0.14.0, < 0.29.0
+uvicorn >=0.14.0, !=0.29.0
 websockets >= 10.4, < 13.0
 wrapt >= 1.16.0

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/requirements-dev.txt

@@ -21,6 +21,7 @@ pytest-xdist >= 3.6.1
 pyyaml
 redis>=5.0.1
 setuptools
+vale
 vermin
 virtualenv
 watchfiles

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/src/prefect/_internal/compatibility/migration.py

@@ -5,11 +5,41 @@ The `getattr_migration` function is used to handle imports for moved or removed
 It is used in the `__getattr__` attribute of modules that have moved or removed objects.
 
 Usage:
-```python
-from prefect._internal.compatibility.migration import getattr_migration
 
-__getattr__ = getattr_migration(__name__)
-```
+Moved objects:
+1. Add the old and new path to the `MOVED_IN_V3` dictionary, e.g. `MOVED_IN_V3 = {"old_path": "new_path"}`
+2. In the module where the object was moved from, add the following lines:
+    ```python
+    # at top
+    from prefect._internal.compatibility.migration import getattr_migration
+
+    # at bottom
+    __getattr__ = getattr_migration(__name__)
+    ```
+
+    Example at src/prefect/engine.py
+
+Removed objects:
+1. Add the old path and error message to the `REMOVED_IN_V3` dictionary, e.g. `REMOVED_IN_V3 = {"old_path": "error_message"}`
+2. In the module where the object was removed, add the following lines:
+    ```python
+    # at top
+    from prefect._internal.compatibility.migration import getattr_migration
+
+    # at bottom
+    __getattr__ = getattr_migration(__name__)
+
+    ```
+    If the entire old module was removed, add a stub for the module with the following lines:
+    ```python
+    # at top
+    from prefect._internal.compatibility.migration import getattr_migration
+    
+    # at bottom
+    __getattr__ = getattr_migration(__name__)
+    ```
+
+    Example at src/prefect/infrastructure/base.py
 """
 
 import sys
@@ -30,14 +60,24 @@ MOVED_IN_V3 = {
     "prefect.client:get_client": "prefect.client.orchestration:get_client",
 }
 
+upgrade_guide_msg = "Refer to the upgrade guide for more information: https://docs.prefect.io/latest/guides/upgrade-guide-agents-to-workers/."
+
 REMOVED_IN_V3 = {
     "prefect.client.schemas.objects:MinimalDeploymentSchedule": "Use `prefect.client.schemas.actions.DeploymentScheduleCreate` instead.",
+    "prefect.context:PrefectObjectRegistry": upgrade_guide_msg,
     "prefect.deployments.deployments:Deployment": "Use `flow.serve()`, `flow.deploy()`, or `prefect deploy` instead.",
     "prefect.deployments:Deployment": "Use `flow.serve()`, `flow.deploy()`, or `prefect deploy` instead.",
-    "prefect.filesystems:GCS": "Use `prefect_gcp` instead.",
-    "prefect.filesystems:Azure": "Use `prefect_azure` instead.",
-    "prefect.filesystems:S3": "Use `prefect_aws` instead.",
+    "prefect.filesystems:GCS": "Use `prefect_gcp.GcsBucket` instead.",
+    "prefect.filesystems:Azure": "Use `prefect_azure.AzureBlobStorageContainer` instead.",
+    "prefect.filesystems:S3": "Use `prefect_aws.S3Bucket` instead.",
+    "prefect.filesystems:GitHub": "Use `prefect_github.GitHubRepository` instead.",
     "prefect.engine:_out_of_process_pause": "Use `prefect.flow_runs.pause_flow_run` instead.",
+    "prefect.agent:PrefectAgent": "Use workers instead. " + upgrade_guide_msg,
+    "prefect.infrastructure:KubernetesJob": "Use workers instead. " + upgrade_guide_msg,
+    "prefect.infrastructure.base:Infrastructure": "Use the `BaseWorker` class to create custom infrastructure integrations instead. "
+    + upgrade_guide_msg,
+    "prefect.workers.block:BlockWorkerJobConfiguration": upgrade_guide_msg,
+    "prefect.workers.cloud:BlockWorker": upgrade_guide_msg,
 }
 
 # IMPORTANT FOR USAGE: When adding new modules to MOVED_IN_V3 or REMOVED_IN_V3, include the following lines at the bottom of that module:
@@ -114,7 +154,7 @@ def getattr_migration(module_name: str) -> Callable[[str], Any]:
         if import_path in REMOVED_IN_V3.keys():
             error_message = REMOVED_IN_V3[import_path]
             raise PrefectImportError(
-                f"{import_path!r} has been removed. {error_message}"
+                f"`{import_path}` has been removed. {error_message}"
             )
 
         globals: Dict[str, Any] = sys.modules[module_name].__dict__

prefect-client-3.0.0rc10/src/prefect/agent.py

@@ -0,0 +1,6 @@
+"""
+2024-06-27: This surfaces an actionable error message for moved or removed objects in Prefect 3.0 upgrade.
+"""
+from prefect._internal.compatibility.migration import getattr_migration
+
+__getattr__ = getattr_migration(__name__)

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/src/prefect/client/schemas/objects.py

@@ -8,7 +8,6 @@ from typing import (
     Generic,
     List,
     Optional,
-    TypeVar,
     Union,
     overload,
 )
@@ -26,7 +25,7 @@ from pydantic import (
     model_validator,
 )
 from pydantic_extra_types.pendulum_dt import DateTime
-from typing_extensions import Literal, Self
+from typing_extensions import Literal, Self, TypeVar
 
 from prefect._internal.compatibility.migration import getattr_migration
 from prefect._internal.schemas.bases import ObjectBaseModel, PrefectBaseModel
@@ -61,7 +60,7 @@ if TYPE_CHECKING:
     from prefect.results import BaseResult
 
 
-R = TypeVar("R")
+R = TypeVar("R", default=Any)
 
 
 DEFAULT_BLOCK_SCHEMA_VERSION = "non-versioned"

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/src/prefect/context.py

@@ -32,6 +32,7 @@ from typing_extensions import Self
 import prefect.logging
 import prefect.logging.configuration
 import prefect.settings
+from prefect._internal.compatibility.migration import getattr_migration
 from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_client
 from prefect.client.schemas import FlowRun, TaskRun
 from prefect.events.worker import EventsWorker
@@ -608,3 +609,8 @@ def root_settings_context():
 
 
 GLOBAL_SETTINGS_CONTEXT: SettingsContext = root_settings_context()
+
+
+# 2024-07-02: This surfaces an actionable error message for removed objects
+# in Prefect 3.0 upgrade.
+__getattr__ = getattr_migration(__name__)

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/src/prefect/deployments/schedules.py

@@ -1,11 +1,14 @@
-from typing import TYPE_CHECKING, Any, List, Optional
+from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Union
 
 from prefect.client.schemas.actions import DeploymentScheduleCreate
 from prefect.client.schemas.schedules import is_schedule_type
 
 if TYPE_CHECKING:
     from prefect.client.schemas.schedules import SCHEDULE_TYPES
-    from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
+
+FlexibleScheduleList = Sequence[
+    Union[DeploymentScheduleCreate, dict[str, Any], "SCHEDULE_TYPES"]
+]
 
 
 def create_deployment_schedule_create(

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/src/prefect/events/schemas/automations.py

@@ -187,18 +187,18 @@ class EventTrigger(ResourceTrigger):
         within: Optional[timedelta] = data.get("within")
 
         if isinstance(within, (int, float)):
-            data["within"] = within = timedelta(seconds=within)
+            within = timedelta(seconds=within)
 
         if posture == Posture.Proactive:
             if not within or within == timedelta(0):
-                data["within"] = timedelta(seconds=10.0)
+                within = timedelta(seconds=10.0)
             elif within < timedelta(seconds=10.0):
                 raise ValueError(
                     "`within` for Proactive triggers must be greater than or equal to "
                     "10 seconds"
                 )
 
-        return data
+        return data | {"within": within} if within else data
 
     def describe_for_cli(self, indent: int = 0) -> str:
         """Return a human-readable description of this trigger for the CLI"""

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/src/prefect/exceptions.py

@@ -178,7 +178,10 @@ class ParameterTypeError(PrefectException):
 
     @classmethod
     def from_validation_error(cls, exc: ValidationError) -> Self:
-        bad_params = [f'{".".join(err["loc"])}: {err["msg"]}' for err in exc.errors()]
+        bad_params = [
+            f'{".".join(str(item) for item in err["loc"])}: {err["msg"]}'
+            for err in exc.errors()
+        ]
         msg = "Flow run received invalid parameters:\n - " + "\n - ".join(bad_params)
         return cls(msg)
 

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/src/prefect/filesystems.py

@@ -95,7 +95,7 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
     def cast_pathlib(cls, value):
         return stringify_path(value)
 
-    def _resolve_path(self, path: str) -> Path:
+    def _resolve_path(self, path: str, validate: bool = False) -> Path:
         # Only resolve the base path at runtime, default to the current directory
         basepath = (
             Path(self.basepath).expanduser().resolve()
@@ -114,11 +114,12 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
             resolved_path = basepath / resolved_path
         else:
             resolved_path = resolved_path.resolve()
+
+        if validate:
             if basepath not in resolved_path.parents and (basepath != resolved_path):
                 raise ValueError(
                     f"Provided path {resolved_path} is outside of the base path {basepath}."
                 )
-
         return resolved_path
 
     @sync_compatible
@@ -184,7 +185,7 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
         Defaults to copying the entire contents of the current working directory to the block's basepath.
         An `ignore_file` path may be provided that can include gitignore style expressions for filepaths to ignore.
         """
-        destination_path = self._resolve_path(to_path)
+        destination_path = self._resolve_path(to_path, validate=True)
 
         if not local_path:
             local_path = Path(".").absolute()

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/src/prefect/flow_engine.py

@@ -16,6 +16,7 @@ from typing import (
     Literal,
     Optional,
     Tuple,
+    Type,
     TypeVar,
     Union,
     cast,
@@ -30,7 +31,13 @@ from prefect.client.schemas import FlowRun, TaskRun
 from prefect.client.schemas.filters import FlowRunFilter
 from prefect.client.schemas.sorting import FlowRunSort
 from prefect.context import ClientContext, FlowRunContext, TagsContext
-from prefect.exceptions import Abort, Pause, PrefectException, UpstreamTaskError
+from prefect.exceptions import (
+    Abort,
+    Pause,
+    PrefectException,
+    TerminationSignal,
+    UpstreamTaskError,
+)
 from prefect.flows import Flow, load_flow_from_entrypoint, load_flow_from_flow_run
 from prefect.futures import PrefectFuture, resolve_futures_to_states
 from prefect.logging.loggers import (
@@ -39,7 +46,7 @@ from prefect.logging.loggers import (
     get_run_logger,
     patch_print,
 )
-from prefect.results import ResultFactory
+from prefect.results import BaseResult, ResultFactory
 from prefect.settings import PREFECT_DEBUG_MODE
 from prefect.states import (
     Failed,
@@ -50,6 +57,7 @@ from prefect.states import (
     exception_to_failed_state,
     return_value_to_state,
 )
+from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import run_coro_as_sync
 from prefect.utilities.callables import (
     call_with_parameters,
@@ -72,6 +80,10 @@ P = ParamSpec("P")
 R = TypeVar("R")
 
 
+class FlowRunTimeoutError(TimeoutError):
+    """Raised when a flow run exceeds its defined timeout."""
+
+
 def load_flow_and_flow_run(flow_run_id: UUID) -> Tuple[FlowRun, Flow]:
     ## TODO: add error handling to update state and log tracebacks
     entrypoint = os.environ.get("PREFECT__FLOW_ENTRYPOINT")
@@ -95,6 +107,10 @@ class FlowRunEngine(Generic[P, R]):
     flow_run_id: Optional[UUID] = None
     logger: logging.Logger = field(default_factory=lambda: get_logger("engine"))
     wait_for: Optional[Iterable[PrefectFuture]] = None
+    # holds the return value from the user code
+    _return_value: Union[R, Type[NotSet]] = NotSet
+    # holds the exception raised by the user code, if any
+    _raised: Union[Exception, Type[NotSet]] = NotSet
     _is_started: bool = False
     _client: Optional[SyncPrefectClient] = None
     short_circuit: bool = False
@@ -208,6 +224,30 @@ class FlowRunEngine(Generic[P, R]):
         return state
 
     def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
+        if self._return_value is not NotSet and not isinstance(
+            self._return_value, State
+        ):
+            if isinstance(self._return_value, BaseResult):
+                _result = self._return_value.get()
+            else:
+                _result = self._return_value
+
+            if inspect.isawaitable(_result):
+                # getting the value for a BaseResult may return an awaitable
+                # depending on whether the parent frame is sync or not
+                _result = run_coro_as_sync(_result)
+            return _result
+
+        if self._raised is not NotSet:
+            if raise_on_failure:
+                raise self._raised
+            return self._raised
+
+        # This is a fall through case which leans on the existing state result mechanics to get the
+        # return value. This is necessary because we currently will return a State object if the
+        # the State was Prefect-created.
+        # TODO: Remove the need to get the result from a State except in cases where the return value
+        # is a State object.
         _result = self.state.result(raise_on_failure=raise_on_failure, fetch=True)  # type: ignore
         # state.result is a `sync_compatible` function that may or may not return an awaitable
         # depending on whether the parent frame is sync or not
@@ -219,13 +259,15 @@ class FlowRunEngine(Generic[P, R]):
         result_factory = getattr(FlowRunContext.get(), "result_factory", None)
         if result_factory is None:
             raise ValueError("Result factory is not set")
+        resolved_result = resolve_futures_to_states(result)
         terminal_state = run_coro_as_sync(
             return_value_to_state(
-                resolve_futures_to_states(result),
+                resolved_result,
                 result_factory=result_factory,
             )
         )
         self.set_state(terminal_state)
+        self._return_value = resolved_result
         return result
 
     def handle_exception(
@@ -252,10 +294,16 @@ class FlowRunEngine(Generic[P, R]):
                 ),
             )
             state = self.set_state(Running())
+        self._raised = exc
         return state
 
     def handle_timeout(self, exc: TimeoutError) -> None:
-        message = f"Flow run exceeded timeout of {self.flow.timeout_seconds} seconds"
+        if isinstance(exc, FlowRunTimeoutError):
+            message = (
+                f"Flow run exceeded timeout of {self.flow.timeout_seconds} second(s)"
+            )
+        else:
+            message = f"Flow run failed due to timeout: {exc!r}"
         self.logger.error(message)
         state = Failed(
             data=exc,
@@ -263,12 +311,14 @@ class FlowRunEngine(Generic[P, R]):
             name="TimedOut",
         )
         self.set_state(state)
+        self._raised = exc
 
     def handle_crash(self, exc: BaseException) -> None:
         state = run_coro_as_sync(exception_to_crashed_state(exc))
         self.logger.error(f"Crash detected! {state.message}")
         self.logger.debug("Crash details:", exc_info=exc)
         self.set_state(state, force=True)
+        self._raised = exc
 
     def load_subflow_run(
         self,
@@ -315,7 +365,9 @@ class FlowRunEngine(Generic[P, R]):
                 limit=1,
             )
             if flow_runs:
-                return flow_runs[-1]
+                loaded_flow_run = flow_runs[-1]
+                self._return_value = loaded_flow_run.state
+                return loaded_flow_run
 
     def create_flow_run(self, client: SyncPrefectClient) -> FlowRun:
         flow_run_ctx = FlowRunContext.get()
@@ -363,7 +415,7 @@ class FlowRunEngine(Generic[P, R]):
 
         return flow_run
 
-    def call_hooks(self, state: State = None) -> Iterable[Callable]:
+    def call_hooks(self, state: Optional[State] = None) -> Iterable[Callable]:
         if state is None:
             state = self.state
         flow = self.flow
@@ -504,6 +556,11 @@ class FlowRunEngine(Generic[P, R]):
                 )
             try:
                 yield self
+
+            except TerminationSignal as exc:
+                self.cancel_all_tasks()
+                self.handle_crash(exc)
+                raise
             except Exception:
                 # regular exceptions are caught and re-raised to the user
                 raise
@@ -539,6 +596,10 @@ class FlowRunEngine(Generic[P, R]):
             return False  # TODO: handle this differently?
         return getattr(self, "flow_run").state.is_pending()
 
+    def cancel_all_tasks(self):
+        if hasattr(self.flow.task_runner, "cancel_all"):
+            self.flow.task_runner.cancel_all()  # type: ignore
+
     # --------------------------
     #
     # The following methods compose the main task run loop
@@ -564,7 +625,10 @@ class FlowRunEngine(Generic[P, R]):
         # reenter the run context to ensure it is up to date for every run
         with self.setup_run_context():
             try:
-                with timeout_context(seconds=self.flow.timeout_seconds):
+                with timeout_context(
+                    seconds=self.flow.timeout_seconds,
+                    timeout_exc_type=FlowRunTimeoutError,
+                ):
                     self.logger.debug(
                         f"Executing flow {self.flow.name!r} for flow run {self.flow_run.name!r}..."
                     )
@@ -572,7 +636,7 @@ class FlowRunEngine(Generic[P, R]):
             except TimeoutError as exc:
                 self.handle_timeout(exc)
             except Exception as exc:
-                self.logger.exception(f"Encountered exception during execution: {exc}")
+                self.logger.exception("Encountered exception during execution: %r", exc)
                 self.handle_exception(exc)
 
     def call_flow_fn(self) -> Union[R, Coroutine[Any, Any, R]]:

{prefect-client-3.0.0rc9 → prefect-client-3.0.0rc10}/src/prefect/flows.py

@@ -187,7 +187,7 @@ class Flow(Generic[P, R]):
         timeout_seconds: Union[int, float, None] = None,
         validate_parameters: bool = True,
         persist_result: Optional[bool] = None,
-        result_storage: Optional[ResultStorage] = None,
+        result_storage: Optional[Union[ResultStorage, str]] = None,
         result_serializer: Optional[ResultSerializer] = None,
         cache_result_in_memory: bool = True,
         log_prints: Optional[bool] = None,
@@ -335,7 +335,18 @@ class Flow(Generic[P, R]):
                     "Disable validation or change the argument names."
                 ) from exc
 
+        # result persistence settings
+        if persist_result is None:
+            if result_storage is not None or result_serializer is not None:
+                persist_result = True
+
         self.persist_result = persist_result
+        if result_storage and not isinstance(result_storage, str):
+            if getattr(result_storage, "_block_document_id", None) is None:
+                raise TypeError(
+                    "Result storage configuration must be persisted server-side."
+                    " Please call `.save()` on your block before passing it in."
+                )
         self.result_storage = result_storage
         self.result_serializer = result_serializer
         self.cache_result_in_memory = cache_result_in_memory
@@ -960,6 +971,29 @@ class Flow(Generic[P, R]):
 
             my_flow()
             ```
+
+            Load a flow from a local directory:
+
+            ``` python
+            # from_local_source.py
+
+            from pathlib import Path
+            from prefect import flow
+
+            @flow(log_prints=True)
+            def my_flow(name: str = "world"):
+                print(f"Hello {name}! I'm a flow from a Python script!")
+
+            if __name__ == "__main__":
+                my_flow.from_source(
+                    source=str(Path(__file__).parent),
+                    entrypoint="from_local_source.py:my_flow",
+                ).deploy(
+                    name="my-deployment",
+                    parameters=dict(name="Marvin"),
+                    work_pool_name="local",
+                )
+            ```
         """
 
         from prefect.runner.storage import (
@@ -988,7 +1022,7 @@ class Flow(Generic[P, R]):
             await storage.pull_code()
 
             full_entrypoint = str(storage.destination / entrypoint)
-            flow: "Flow" = await from_async.wait_for_call_in_new_thread(
+            flow: Flow = await from_async.wait_for_call_in_new_thread(
                 create_call(load_flow_from_entrypoint, full_entrypoint)
             )
             flow._storage = storage
@@ -1115,7 +1149,13 @@ class Flow(Generic[P, R]):
                 )
             ```
         """
-        work_pool_name = work_pool_name or PREFECT_DEFAULT_WORK_POOL_NAME.value()
+        if not (
+            work_pool_name := work_pool_name or PREFECT_DEFAULT_WORK_POOL_NAME.value()
+        ):
+            raise ValueError(
+                "No work pool name provided. Please provide a `work_pool_name` or set the"
+                " `PREFECT_DEFAULT_WORK_POOL_NAME` environment variable."
+            )
 
         try:
             async with get_client() as client:
@@ -1914,8 +1954,12 @@ def load_flow_argument_from_entrypoint(
                     literal_arg_value = ast.get_source_segment(
                         source_code, keyword.value
                     )
+                    cleaned_value = (
+                        literal_arg_value.replace("\n", "") if literal_arg_value else ""
+                    )
+
                     try:
-                        evaluated_value = eval(literal_arg_value, namespace)  # type: ignore
+                        evaluated_value = eval(cleaned_value, namespace)  # type: ignore
                     except Exception as e:
                         logger.info(
                             "Failed to parse @flow argument: `%s=%s` due to the following error. Ignoring and falling back to default behavior.",

prefect-client-3.0.0rc10/src/prefect/infrastructure/__init__.py

@@ -0,0 +1,6 @@
+"""
+2024-06-27: This surfaces an actionable error message for moved or removed objects in Prefect 3.0 upgrade.
+"""
+from prefect._internal.compatibility.migration import getattr_migration
+
+__getattr__ = getattr_migration(__name__)

prefect-client-3.0.0rc10/src/prefect/infrastructure/base.py

@@ -0,0 +1,6 @@
+"""
+2024-06-27: This surfaces an actionable error message for moved or removed objects in Prefect 3.0 upgrade.
+"""
+from prefect._internal.compatibility.migration import getattr_migration
+
+__getattr__ = getattr_migration(__name__)