prefect-client 3.0.0rc9__py3-none-any.whl → 3.0.0rc11__py3-none-any.whl

prefect/_internal/compatibility/migration.py

@@ -5,11 +5,41 @@ The `getattr_migration` function is used to handle imports for moved or removed
 It is used in the `__getattr__` attribute of modules that have moved or removed objects.
 
 Usage:
-```python
-from prefect._internal.compatibility.migration import getattr_migration
 
-__getattr__ = getattr_migration(__name__)
-```
+Moved objects:
+1. Add the old and new path to the `MOVED_IN_V3` dictionary, e.g. `MOVED_IN_V3 = {"old_path": "new_path"}`
+2. In the module where the object was moved from, add the following lines:
+    ```python
+    # at top
+    from prefect._internal.compatibility.migration import getattr_migration
+
+    # at bottom
+    __getattr__ = getattr_migration(__name__)
+    ```
+
+    Example at src/prefect/engine.py
+
+Removed objects:
+1. Add the old path and error message to the `REMOVED_IN_V3` dictionary, e.g. `REMOVED_IN_V3 = {"old_path": "error_message"}`
+2. In the module where the object was removed, add the following lines:
+    ```python
+    # at top
+    from prefect._internal.compatibility.migration import getattr_migration
+
+    # at bottom
+    __getattr__ = getattr_migration(__name__)
+
+    ```
+    If the entire old module was removed, add a stub for the module with the following lines:
+    ```python
+    # at top
+    from prefect._internal.compatibility.migration import getattr_migration
+    
+    # at bottom
+    __getattr__ = getattr_migration(__name__)
+    ```
+
+    Example at src/prefect/infrastructure/base.py
 """
 
 import sys
@@ -30,14 +60,24 @@ MOVED_IN_V3 = {
     "prefect.client:get_client": "prefect.client.orchestration:get_client",
 }
 
+upgrade_guide_msg = "Refer to the upgrade guide for more information: https://docs.prefect.io/latest/guides/upgrade-guide-agents-to-workers/."
+
 REMOVED_IN_V3 = {
     "prefect.client.schemas.objects:MinimalDeploymentSchedule": "Use `prefect.client.schemas.actions.DeploymentScheduleCreate` instead.",
+    "prefect.context:PrefectObjectRegistry": upgrade_guide_msg,
     "prefect.deployments.deployments:Deployment": "Use `flow.serve()`, `flow.deploy()`, or `prefect deploy` instead.",
     "prefect.deployments:Deployment": "Use `flow.serve()`, `flow.deploy()`, or `prefect deploy` instead.",
-    "prefect.filesystems:GCS": "Use `prefect_gcp` instead.",
-    "prefect.filesystems:Azure": "Use `prefect_azure` instead.",
-    "prefect.filesystems:S3": "Use `prefect_aws` instead.",
+    "prefect.filesystems:GCS": "Use `prefect_gcp.GcsBucket` instead.",
+    "prefect.filesystems:Azure": "Use `prefect_azure.AzureBlobStorageContainer` instead.",
+    "prefect.filesystems:S3": "Use `prefect_aws.S3Bucket` instead.",
+    "prefect.filesystems:GitHub": "Use `prefect_github.GitHubRepository` instead.",
     "prefect.engine:_out_of_process_pause": "Use `prefect.flow_runs.pause_flow_run` instead.",
+    "prefect.agent:PrefectAgent": "Use workers instead. " + upgrade_guide_msg,
+    "prefect.infrastructure:KubernetesJob": "Use workers instead. " + upgrade_guide_msg,
+    "prefect.infrastructure.base:Infrastructure": "Use the `BaseWorker` class to create custom infrastructure integrations instead. "
+    + upgrade_guide_msg,
+    "prefect.workers.block:BlockWorkerJobConfiguration": upgrade_guide_msg,
+    "prefect.workers.cloud:BlockWorker": upgrade_guide_msg,
 }
 
 # IMPORTANT FOR USAGE: When adding new modules to MOVED_IN_V3 or REMOVED_IN_V3, include the following lines at the bottom of that module:
@@ -114,7 +154,7 @@ def getattr_migration(module_name: str) -> Callable[[str], Any]:
         if import_path in REMOVED_IN_V3.keys():
             error_message = REMOVED_IN_V3[import_path]
             raise PrefectImportError(
-                f"{import_path!r} has been removed. {error_message}"
+                f"`{import_path}` has been removed. {error_message}"
             )
 
         globals: Dict[str, Any] = sys.modules[module_name].__dict__

prefect/_internal/concurrency/api.py

@@ -151,7 +151,7 @@ class from_async(_base):
         __call: Union[Callable[[], T], Call[T]],
         timeout: Optional[float] = None,
         done_callbacks: Optional[Iterable[Call]] = None,
-    ) -> Call[T]:
+    ) -> T:
         call = _cast_to_call(__call)
         waiter = AsyncWaiter(call=call)
         for callback in done_callbacks or []:

prefect/_internal/retries.py

@@ -0,0 +1,61 @@
+import asyncio
+from functools import wraps
+from typing import Any, Callable, Tuple, Type
+
+from prefect.logging.loggers import get_logger
+from prefect.utilities.math import clamped_poisson_interval
+
+logger = get_logger("retries")
+
+
+def exponential_backoff_with_jitter(
+    attempt: int, base_delay: float, max_delay: float
+) -> float:
+    average_interval = min(base_delay * (2**attempt), max_delay)
+    return clamped_poisson_interval(average_interval, clamping_factor=0.3)
+
+
+def retry_async_fn(
+    max_attempts: int = 3,
+    backoff_strategy: Callable[
+        [int, float, float], float
+    ] = exponential_backoff_with_jitter,
+    base_delay: float = 1,
+    max_delay: float = 10,
+    retry_on_exceptions: Tuple[Type[Exception], ...] = (Exception,),
+):
+    """A decorator for retrying an async function.
+
+    Args:
+        max_attempts: The maximum number of times to retry the function.
+        backoff_strategy: A function that takes in the number of attempts, the base
+            delay, and the maximum delay, and returns the delay to use for the next
+            attempt. Defaults to an exponential backoff with jitter.
+        base_delay: The base delay to use for the first attempt.
+        max_delay: The maximum delay to use for the last attempt.
+        retry_on_exceptions: A tuple of exception types to retry on. Defaults to
+            retrying on all exceptions.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args: Any, **kwargs: Any) -> Any:
+            for attempt in range(max_attempts):
+                try:
+                    return await func(*args, **kwargs)
+                except retry_on_exceptions as e:
+                    if attempt == max_attempts - 1:
+                        logger.exception(
+                            f"Function {func.__name__!r} failed after {max_attempts} attempts"
+                        )
+                        raise
+                    delay = backoff_strategy(attempt, base_delay, max_delay)
+                    logger.warning(
+                        f"Attempt {attempt + 1} of function {func.__name__!r} failed with {type(e).__name__}. "
+                        f"Retrying in {delay:.2f} seconds..."
+                    )
+                    await asyncio.sleep(delay)
+
+        return wrapper
+
+    return decorator

prefect/agent.py

@@ -0,0 +1,6 @@
+"""
+2024-06-27: This surfaces an actionable error message for moved or removed objects in Prefect 3.0 upgrade.
+"""
+from prefect._internal.compatibility.migration import getattr_migration
+
+__getattr__ = getattr_migration(__name__)

prefect/client/cloud.py

@@ -9,7 +9,7 @@ from starlette import status
 import prefect.context
 import prefect.settings
 from prefect.client.base import PrefectHttpxAsyncClient
-from prefect.client.schemas import Workspace
+from prefect.client.schemas.objects import Workspace
 from prefect.exceptions import ObjectNotFound, PrefectException
 from prefect.settings import (
     PREFECT_API_KEY,

prefect/client/schemas/objects.py

@@ -8,7 +8,6 @@ from typing import (
     Generic,
     List,
     Optional,
-    TypeVar,
     Union,
     overload,
 )
@@ -26,7 +25,7 @@ from pydantic import (
     model_validator,
 )
 from pydantic_extra_types.pendulum_dt import DateTime
-from typing_extensions import Literal, Self
+from typing_extensions import Literal, Self, TypeVar
 
 from prefect._internal.compatibility.migration import getattr_migration
 from prefect._internal.schemas.bases import ObjectBaseModel, PrefectBaseModel
@@ -61,7 +60,7 @@ if TYPE_CHECKING:
     from prefect.results import BaseResult
 
 
-R = TypeVar("R")
+R = TypeVar("R", default=Any)
 
 
 DEFAULT_BLOCK_SCHEMA_VERSION = "non-versioned"
@@ -792,7 +791,7 @@ class TaskRun(ObjectBaseModel):
 
     state: Optional[State] = Field(
         default=None,
-        description="The state of the flow run.",
+        description="The state of the task run.",
         examples=["State(type=StateType.COMPLETED)"],
     )
 

prefect/concurrency/asyncio.py

@@ -1,6 +1,6 @@
 import asyncio
 from contextlib import asynccontextmanager
-from typing import List, Literal, Optional, Union, cast
+from typing import AsyncGenerator, List, Literal, Optional, Union, cast
 
 import httpx
 import pendulum
@@ -34,7 +34,7 @@ async def concurrency(
     names: Union[str, List[str]],
     occupy: int = 1,
     timeout_seconds: Optional[float] = None,
-):
+) -> AsyncGenerator[None, None]:
     """A context manager that acquires and releases concurrency slots from the
     given concurrency limits.
 
@@ -77,7 +77,7 @@ async def concurrency(
         _emit_concurrency_release_events(limits, occupy, emitted_events)
 
 
-async def rate_limit(names: Union[str, List[str]], occupy: int = 1):
+async def rate_limit(names: Union[str, List[str]], occupy: int = 1) -> None:
     """Block execution until an `occupy` number of slots of the concurrency
     limits given in `names` are acquired. Requires that all given concurrency
     limits have a slot decay.

prefect/concurrency/events.py

@@ -54,6 +54,6 @@ def _emit_concurrency_release_events(
     limits: List[MinimalConcurrencyLimitResponse],
     occupy: int,
     events: Dict[UUID, Optional[Event]],
-):
+) -> None:
     for limit in limits:
         _emit_concurrency_event("released", limit, limits, occupy, events[limit.id])

prefect/concurrency/services.py

@@ -3,6 +3,7 @@ import concurrent.futures
 from contextlib import asynccontextmanager
 from typing import (
     TYPE_CHECKING,
+    AsyncGenerator,
     FrozenSet,
     Optional,
     Tuple,
@@ -27,14 +28,14 @@ class ConcurrencySlotAcquisitionService(QueueService):
         self.concurrency_limit_names = sorted(list(concurrency_limit_names))
 
     @asynccontextmanager
-    async def _lifespan(self):
+    async def _lifespan(self) -> AsyncGenerator[None, None]:
         async with get_client() as client:
             self._client = client
             yield
 
     async def _handle(
         self, item: Tuple[int, str, Optional[float], concurrent.futures.Future]
-    ):
+    ) -> None:
         occupy, mode, timeout_seconds, future = item
         try:
             response = await self.acquire_slots(occupy, mode, timeout_seconds)

prefect/concurrency/sync.py

@@ -1,5 +1,15 @@
 from contextlib import contextmanager
-from typing import List, Optional, Union, cast
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Generator,
+    List,
+    Optional,
+    TypeVar,
+    Union,
+    cast,
+)
 
 import pendulum
 
@@ -22,13 +32,15 @@ from .events import (
     _emit_concurrency_release_events,
 )
 
+T = TypeVar("T")
+
 
 @contextmanager
 def concurrency(
     names: Union[str, List[str]],
     occupy: int = 1,
     timeout_seconds: Optional[float] = None,
-):
+) -> Generator[None, None, None]:
     """A context manager that acquires and releases concurrency slots from the
     given concurrency limits.
 
@@ -75,7 +87,7 @@ def concurrency(
         _emit_concurrency_release_events(limits, occupy, emitted_events)
 
 
-def rate_limit(names: Union[str, List[str]], occupy: int = 1):
+def rate_limit(names: Union[str, List[str]], occupy: int = 1) -> None:
     """Block execution until an `occupy` number of slots of the concurrency
     limits given in `names` are acquired. Requires that all given concurrency
     limits have a slot decay.
@@ -91,11 +103,13 @@ def rate_limit(names: Union[str, List[str]], occupy: int = 1):
     _emit_concurrency_acquisition_events(limits, occupy)
 
 
-def _call_async_function_from_sync(fn, *args, **kwargs):
+def _call_async_function_from_sync(
+    fn: Callable[..., Awaitable[T]], *args: Any, **kwargs: Any
+) -> T:
     loop = get_running_loop()
     call = create_call(fn, *args, **kwargs)
 
     if loop is not None:
         return from_sync.call_soon_in_loop_thread(call).result()
     else:
-        return call()
+        return call()  # type: ignore [return-value]

prefect/context.py

@@ -9,6 +9,7 @@ For more user-accessible information about the current run, see [`prefect.runtim
 import os
 import sys
 import warnings
+import weakref
 from contextlib import ExitStack, contextmanager
 from contextvars import ContextVar, Token
 from pathlib import Path
@@ -17,6 +18,7 @@ from typing import (
     Any,
     Dict,
     Generator,
+    Mapping,
     Optional,
     Set,
     Type,
@@ -32,6 +34,7 @@ from typing_extensions import Self
 import prefect.logging
 import prefect.logging.configuration
 import prefect.settings
+from prefect._internal.compatibility.migration import getattr_migration
 from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_client
 from prefect.client.schemas import FlowRun, TaskRun
 from prefect.events.worker import EventsWorker
@@ -290,8 +293,12 @@ class EngineContext(RunContext):
     # Counter for flow pauses
     observed_flow_pauses: Dict[str, int] = Field(default_factory=dict)
 
-    # Tracking for result from task runs in this flow run
-    task_run_results: Dict[int, State] = Field(default_factory=dict)
+    # Tracking for result from task runs in this flow run for dependency tracking
+    # Holds the ID of the object returned by the task run and task run state
+    # This is a weakref dictionary to avoid undermining garbage collection
+    task_run_results: Mapping[int, State] = Field(
+        default_factory=weakref.WeakValueDictionary
+    )
 
     # Events worker to emit events to Prefect Cloud
     events: Optional[EventsWorker] = None
@@ -608,3 +615,8 @@ def root_settings_context():
 
 
 GLOBAL_SETTINGS_CONTEXT: SettingsContext = root_settings_context()
+
+
+# 2024-07-02: This surfaces an actionable error message for removed objects
+# in Prefect 3.0 upgrade.
+__getattr__ = getattr_migration(__name__)

prefect/deployments/__init__.py

@@ -1,20 +1,33 @@
+from typing import TYPE_CHECKING
 from prefect._internal.compatibility.migration import getattr_migration
-import prefect.deployments.base
-import prefect.deployments.steps
-from prefect.deployments.base import (
-    initialize_project,
-)
 
-from prefect.deployments.runner import (
-    RunnerDeployment,
-    deploy,
-    DockerImage,
-    EntrypointType,
-)
 
+if TYPE_CHECKING:
+    from .flow_runs import run_deployment
+    from .base import initialize_project
+    from .runner import deploy
 
-from prefect.deployments.flow_runs import (
-    run_deployment,
-)
+_public_api: dict[str, tuple[str, str]] = {
+    "initialize_project": (__spec__.parent, ".base"),
+    "run_deployment": (__spec__.parent, ".flow_runs"),
+    "deploy": (__spec__.parent, ".runner"),
+}
 
-__getattr__ = getattr_migration(__name__)
+# Declare API for type-checkers
+__all__ = ["initialize_project", "deploy", "run_deployment"]
+
+
+def __getattr__(attr_name: str) -> object:
+    dynamic_attr = _public_api.get(attr_name)
+    if dynamic_attr is None:
+        return getattr_migration(__name__)(attr_name)
+
+    package, module_name = dynamic_attr
+
+    from importlib import import_module
+
+    if module_name == "__module__":
+        return import_module(f".{attr_name}", package=package)
+    else:
+        module = import_module(module_name, package=package)
+        return getattr(module, attr_name)

prefect/deployments/schedules.py

@@ -1,11 +1,14 @@
-from typing import TYPE_CHECKING, Any, List, Optional
+from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Union
 
 from prefect.client.schemas.actions import DeploymentScheduleCreate
 from prefect.client.schemas.schedules import is_schedule_type
 
 if TYPE_CHECKING:
     from prefect.client.schemas.schedules import SCHEDULE_TYPES
-    from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
+
+FlexibleScheduleList = Sequence[
+    Union[DeploymentScheduleCreate, dict[str, Any], "SCHEDULE_TYPES"]
+]
 
 
 def create_deployment_schedule_create(

prefect/deployments/steps/pull.py

@@ -6,6 +6,7 @@ import os
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
+from prefect._internal.retries import retry_async_fn
 from prefect.logging.loggers import get_logger
 from prefect.runner.storage import BlockStorageAdapter, GitRepository, RemoteStorage
 from prefect.utilities.asyncutils import sync_compatible
@@ -31,6 +32,12 @@ def set_working_directory(directory: str) -> dict:
     return dict(directory=directory)
 
 
+@retry_async_fn(
+    max_attempts=3,
+    base_delay=1,
+    max_delay=10,
+    retry_on_exceptions=(RuntimeError,),
+)
 @sync_compatible
 async def git_clone(
     repository: str,

prefect/events/schemas/automations.py

@@ -187,18 +187,18 @@ class EventTrigger(ResourceTrigger):
         within: Optional[timedelta] = data.get("within")
 
         if isinstance(within, (int, float)):
-            data["within"] = within = timedelta(seconds=within)
+            within = timedelta(seconds=within)
 
         if posture == Posture.Proactive:
             if not within or within == timedelta(0):
-                data["within"] = timedelta(seconds=10.0)
+                within = timedelta(seconds=10.0)
             elif within < timedelta(seconds=10.0):
                 raise ValueError(
                     "`within` for Proactive triggers must be greater than or equal to "
                     "10 seconds"
                 )
 
-        return data
+        return data | {"within": within} if within else data
 
     def describe_for_cli(self, indent: int = 0) -> str:
         """Return a human-readable description of this trigger for the CLI"""

prefect/exceptions.py

@@ -178,7 +178,10 @@ class ParameterTypeError(PrefectException):
 
     @classmethod
     def from_validation_error(cls, exc: ValidationError) -> Self:
-        bad_params = [f'{".".join(err["loc"])}: {err["msg"]}' for err in exc.errors()]
+        bad_params = [
+            f'{".".join(str(item) for item in err["loc"])}: {err["msg"]}'
+            for err in exc.errors()
+        ]
         msg = "Flow run received invalid parameters:\n - " + "\n - ".join(bad_params)
         return cls(msg)
 

prefect/filesystems.py

@@ -95,7 +95,7 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
     def cast_pathlib(cls, value):
         return stringify_path(value)
 
-    def _resolve_path(self, path: str) -> Path:
+    def _resolve_path(self, path: str, validate: bool = False) -> Path:
         # Only resolve the base path at runtime, default to the current directory
         basepath = (
             Path(self.basepath).expanduser().resolve()
@@ -114,11 +114,12 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
             resolved_path = basepath / resolved_path
         else:
             resolved_path = resolved_path.resolve()
+
+        if validate:
             if basepath not in resolved_path.parents and (basepath != resolved_path):
                 raise ValueError(
                     f"Provided path {resolved_path} is outside of the base path {basepath}."
                 )
-
         return resolved_path
 
     @sync_compatible
@@ -184,7 +185,7 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
         Defaults to copying the entire contents of the current working directory to the block's basepath.
         An `ignore_file` path may be provided that can include gitignore style expressions for filepaths to ignore.
         """
-        destination_path = self._resolve_path(to_path)
+        destination_path = self._resolve_path(to_path, validate=True)
 
         if not local_path:
             local_path = Path(".").absolute()