prefect-client 3.1.11__py3-none-any.whl → 3.1.13__py3-none-any.whl

prefect/runner/runner.py

@@ -1,6 +1,8 @@
 """
-Runners are responsible for managing the execution of deployments created and managed by
-either `flow.serve` or the `serve` utility.
+Runners are responsible for managing the execution of all deployments.
+
+When creating a deployment using either `flow.serve` or the `serve` utility,
+they also will poll for scheduled runs.
 
 Example:
     ```python
@@ -30,6 +32,8 @@ Example:
 
 """
 
+from __future__ import annotations
+
 import asyncio
 import datetime
 import logging
@@ -61,13 +65,14 @@ import anyio
 import anyio.abc
 import pendulum
 from cachetools import LRUCache
+from typing_extensions import Self
 
 from prefect._internal.concurrency.api import (
     create_call,
     from_async,
     from_sync,
 )
-from prefect.client.orchestration import get_client
+from prefect.client.orchestration import PrefectClient, get_client
 from prefect.client.schemas.filters import (
     FlowRunFilter,
     FlowRunFilterId,
@@ -91,7 +96,7 @@ from prefect.events.related import tags_as_related_resources
 from prefect.events.schemas.events import RelatedResource
 from prefect.events.utilities import emit_event
 from prefect.exceptions import Abort, ObjectNotFound
-from prefect.flows import Flow, load_flow_from_flow_run
+from prefect.flows import Flow, FlowStateHook, load_flow_from_flow_run
 from prefect.logging.loggers import PrefectLogAdapter, flow_run_logger, get_logger
 from prefect.runner.storage import RunnerStorage
 from prefect.settings import (
@@ -123,10 +128,11 @@ from prefect.utilities.services import (
 from prefect.utilities.slugify import slugify
 
 if TYPE_CHECKING:
+    import concurrent.futures
+
     from prefect.client.schemas.responses import DeploymentResponse
     from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
     from prefect.deployments.runner import RunnerDeployment
-
 __all__ = ["Runner"]
 
 
@@ -193,25 +199,25 @@ class Runner:
 
         if name and ("/" in name or "%" in name):
             raise ValueError("Runner name cannot contain '/' or '%'")
-        self.name = Path(name).stem if name is not None else f"runner-{uuid4()}"
-        self._logger = get_logger("runner")
-
-        self.started = False
-        self.stopping = False
-        self.pause_on_shutdown = pause_on_shutdown
-        self.limit = limit or settings.runner.process_limit
-        self.webserver = webserver
-
-        self.query_seconds = query_seconds or settings.runner.poll_frequency
-        self._prefetch_seconds = prefetch_seconds
-        self.heartbeat_seconds = (
+        self.name: str = Path(name).stem if name is not None else f"runner-{uuid4()}"
+        self._logger: "logging.Logger" = get_logger("runner")
+
+        self.started: bool = False
+        self.stopping: bool = False
+        self.pause_on_shutdown: bool = pause_on_shutdown
+        self.limit: int | None = limit or settings.runner.process_limit
+        self.webserver: bool = webserver
+
+        self.query_seconds: float = query_seconds or settings.runner.poll_frequency
+        self._prefetch_seconds: float = prefetch_seconds
+        self.heartbeat_seconds: float | None = (
             heartbeat_seconds or settings.runner.heartbeat_frequency
         )
         if self.heartbeat_seconds is not None and self.heartbeat_seconds < 30:
             raise ValueError("Heartbeat must be 30 seconds or greater.")
 
-        self._limiter: Optional[anyio.CapacityLimiter] = None
-        self._client = get_client()
+        self._limiter: anyio.CapacityLimiter | None = None
+        self._client: PrefectClient = get_client()
         self._submitting_flow_run_ids: set[UUID] = set()
         self._cancelling_flow_run_ids: set[UUID] = set()
         self._scheduled_task_scopes: set[UUID] = set()
@@ -221,8 +227,8 @@ class Runner:
         self._tmp_dir: Path = (
             Path(tempfile.gettempdir()) / "runner_storage" / str(uuid4())
         )
-        self._storage_objs: List[RunnerStorage] = []
-        self._deployment_storage_map: Dict[UUID, RunnerStorage] = {}
+        self._storage_objs: list[RunnerStorage] = []
+        self._deployment_storage_map: dict[UUID, RunnerStorage] = {}
 
         self._loop: Optional[asyncio.AbstractEventLoop] = None
 
@@ -476,7 +482,7 @@ class Runner:
 
     def execute_in_background(
         self, func: Callable[..., Any], *args: Any, **kwargs: Any
-    ):
+    ) -> "concurrent.futures.Future[Any]":
         """
         Executes a function in the background.
         """
@@ -485,7 +491,7 @@ class Runner:
 
         return asyncio.run_coroutine_threadsafe(func(*args, **kwargs), self._loop)
 
-    async def cancel_all(self):
+    async def cancel_all(self) -> None:
         runs_to_cancel = []
 
         # done to avoid dictionary size changing during iteration
@@ -522,7 +528,7 @@ class Runner:
 
     async def execute_flow_run(
         self, flow_run_id: UUID, entrypoint: Optional[str] = None
-    ):
+    ) -> None:
         """
         Executes a single flow run with the given ID.
 
@@ -774,7 +780,7 @@ class Runner:
         if self.stopping:
             return
         runs_response = await self._get_scheduled_flow_runs()
-        self.last_polled = pendulum.now("UTC")
+        self.last_polled: pendulum.DateTime = pendulum.now("UTC")
         return await self._submit_scheduled_flow_runs(flow_run_response=runs_response)
 
     async def _check_for_cancelled_flow_runs(
@@ -1387,7 +1393,7 @@ class Runner:
 
             await _run_hooks(hooks, flow_run, flow, state)
 
-    async def __aenter__(self):
+    async def __aenter__(self) -> Self:
         self._logger.debug("Starting runner...")
         self._client = get_client()
         self._tmp_dir.mkdir(parents=True)
@@ -1409,7 +1415,7 @@ class Runner:
         self.started = True
         return self
 
-    async def __aexit__(self, *exc_info):
+    async def __aexit__(self, *exc_info: Any) -> None:
         self._logger.debug("Stopping runner...")
         if self.pause_on_shutdown:
             await self._pause_schedules()
@@ -1427,7 +1433,7 @@ class Runner:
         shutil.rmtree(str(self._tmp_dir))
         del self._runs_task_group, self._loops_task_group
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return f"Runner(name={self.name!r})"
 
 
@@ -1437,7 +1443,10 @@ if sys.platform == "win32":
 
 
 async def _run_hooks(
-    hooks: List[Callable[[Flow, "FlowRun", State], None]], flow_run, flow, state
+    hooks: list[FlowStateHook[Any, Any]],
+    flow_run: "FlowRun",
+    flow: "Flow[..., Any]",
+    state: State,
 ):
     logger = flow_run_logger(flow_run, flow)
     for hook in hooks:

prefect/runner/server.py

@@ -26,12 +26,14 @@ from prefect.utilities.asyncutils import run_coro_as_sync
 from prefect.utilities.importtools import load_script_as_module
 
 if TYPE_CHECKING:
+    import logging
+
     from prefect.client.schemas.responses import DeploymentResponse
     from prefect.runner import Runner
 
 from pydantic import BaseModel
 
-logger = get_logger("webserver")
+logger: "logging.Logger" = get_logger("webserver")
 
 RunnableEndpoint = Literal["deployment", "flow", "task"]
 

prefect/runner/storage.py

@@ -33,7 +33,7 @@ class RunnerStorage(Protocol):
     remotely stored flow code.
     """
 
-    def set_base_path(self, path: Path):
+    def set_base_path(self, path: Path) -> None:
         """
         Sets the base path to use when pulling contents from remote storage to
         local storage.
@@ -55,7 +55,7 @@ class RunnerStorage(Protocol):
         """
         ...
 
-    async def pull_code(self):
+    async def pull_code(self) -> None:
         """
         Pulls contents from remote storage to the local filesystem.
         """
@@ -150,7 +150,7 @@ class GitRepository:
     def destination(self) -> Path:
         return self._storage_base_path / self._name
 
-    def set_base_path(self, path: Path):
+    def set_base_path(self, path: Path) -> None:
         self._storage_base_path = path
 
     @property
@@ -221,7 +221,7 @@ class GitRepository:
         except Exception:
             return False
 
-    async def pull_code(self):
+    async def pull_code(self) -> None:
         """
         Pulls the contents of the configured repository to the local filesystem.
         """
@@ -324,7 +324,7 @@ class GitRepository:
                 cwd=self.destination,
             )
 
-    def __eq__(self, __value) -> bool:
+    def __eq__(self, __value: Any) -> bool:
         if isinstance(__value, GitRepository):
             return (
                 self._url == __value._url
@@ -339,7 +339,7 @@ class GitRepository:
             f" branch={self._branch!r})"
         )
 
-    def to_pull_step(self) -> Dict:
+    def to_pull_step(self) -> dict[str, Any]:
         pull_step = {
             "prefect.deployments.steps.git_clone": {
                 "repository": self._url,
@@ -466,7 +466,7 @@ class RemoteStorage:
 
         return fsspec.filesystem(scheme, **settings_with_block_values)
 
-    def set_base_path(self, path: Path):
+    def set_base_path(self, path: Path) -> None:
         self._storage_base_path = path
 
     @property
@@ -492,7 +492,7 @@ class RemoteStorage:
         _, netloc, urlpath, _, _ = urlsplit(self._url)
         return Path(netloc) / Path(urlpath.lstrip("/"))
 
-    async def pull_code(self):
+    async def pull_code(self) -> None:
         """
         Pulls contents from remote storage to the local filesystem.
         """
@@ -522,7 +522,7 @@ class RemoteStorage:
                 f" {self.destination!r}"
             ) from exc
 
-    def to_pull_step(self) -> dict:
+    def to_pull_step(self) -> dict[str, Any]:
         """
         Returns a dictionary representation of the storage object that can be
         used as a deployment pull step.
@@ -551,7 +551,7 @@ class RemoteStorage:
             ] = required_package
         return step
 
-    def __eq__(self, __value) -> bool:
+    def __eq__(self, __value: Any) -> bool:
         """
         Equality check for runner storage objects.
         """
@@ -590,7 +590,7 @@ class BlockStorageAdapter:
             else str(uuid4())
         )
 
-    def set_base_path(self, path: Path):
+    def set_base_path(self, path: Path) -> None:
         self._storage_base_path = path
 
     @property
@@ -601,12 +601,12 @@ class BlockStorageAdapter:
     def destination(self) -> Path:
         return self._storage_base_path / self._name
 
-    async def pull_code(self):
+    async def pull_code(self) -> None:
         if not self.destination.exists():
             self.destination.mkdir(parents=True, exist_ok=True)
         await self._block.get_directory(local_path=str(self.destination))
 
-    def to_pull_step(self) -> dict:
+    def to_pull_step(self) -> dict[str, Any]:
         # Give blocks the change to implement their own pull step
         if hasattr(self._block, "get_pull_step"):
             return self._block.get_pull_step()
@@ -623,7 +623,7 @@ class BlockStorageAdapter:
                 }
             }
 
-    def __eq__(self, __value) -> bool:
+    def __eq__(self, __value: Any) -> bool:
         if isinstance(__value, BlockStorageAdapter):
             return self._block == __value._block
         return False
@@ -658,19 +658,19 @@ class LocalStorage:
     def destination(self) -> Path:
         return self._path
 
-    def set_base_path(self, path: Path):
+    def set_base_path(self, path: Path) -> None:
         self._storage_base_path = path
 
     @property
     def pull_interval(self) -> Optional[int]:
         return self._pull_interval
 
-    async def pull_code(self):
+    async def pull_code(self) -> None:
         # Local storage assumes the code already exists on the local filesystem
         # and does not need to be pulled from a remote location
         pass
 
-    def to_pull_step(self) -> dict:
+    def to_pull_step(self) -> dict[str, Any]:
         """
         Returns a dictionary representation of the storage object that can be
         used as a deployment pull step.
@@ -682,7 +682,7 @@ class LocalStorage:
         }
         return step
 
-    def __eq__(self, __value) -> bool:
+    def __eq__(self, __value: Any) -> bool:
         if isinstance(__value, LocalStorage):
             return self._path == __value._path
         return False

prefect/runner/submit.py

@@ -1,11 +1,13 @@
+from __future__ import annotations
+
 import asyncio
 import inspect
 import uuid
-from typing import Any, Dict, List, Optional, Union, overload
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, overload
 
 import anyio
 import httpx
-from typing_extensions import Literal
+from typing_extensions import Literal, TypeAlias
 
 from prefect.client.orchestration import get_client
 from prefect.client.schemas.filters import FlowRunFilter, TaskRunFilter
@@ -22,12 +24,17 @@ from prefect.states import Pending
 from prefect.tasks import Task
 from prefect.utilities.asyncutils import sync_compatible
 
-logger = get_logger("webserver")
+if TYPE_CHECKING:
+    import logging
+
+logger: "logging.Logger" = get_logger("webserver")
+
+FlowOrTask: TypeAlias = Union[Flow[Any, Any], Task[Any, Any]]
 
 
 async def _submit_flow_to_runner(
-    flow: Flow,
-    parameters: Dict[str, Any],
+    flow: Flow[Any, Any],
+    parameters: dict[str, Any],
     retry_failed_submissions: bool = True,
 ) -> FlowRun:
     """
@@ -91,7 +98,7 @@ async def _submit_flow_to_runner(
 
 @overload
 def submit_to_runner(
-    prefect_callable: Union[Flow, Task],
+    prefect_callable: Union[Flow[Any, Any], Task[Any, Any]],
     parameters: Dict[str, Any],
     retry_failed_submissions: bool = True,
 ) -> FlowRun:
@@ -100,19 +107,19 @@ def submit_to_runner(
 
 @overload
 def submit_to_runner(
-    prefect_callable: Union[Flow, Task],
-    parameters: List[Dict[str, Any]],
+    prefect_callable: Union[Flow[Any, Any], Task[Any, Any]],
+    parameters: list[dict[str, Any]],
     retry_failed_submissions: bool = True,
-) -> List[FlowRun]:
+) -> list[FlowRun]:
     ...
 
 
 @sync_compatible
 async def submit_to_runner(
-    prefect_callable: Union[Flow, Task],
-    parameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
+    prefect_callable: FlowOrTask,
+    parameters: Optional[Union[dict[str, Any], list[dict[str, Any]]]] = None,
     retry_failed_submissions: bool = True,
-) -> Union[FlowRun, List[FlowRun]]:
+) -> Union[FlowRun, list[FlowRun]]:
     """
     Submit a callable in the background via the runner webserver one or more times.
 

prefect/runtime/deployment.py

@@ -25,8 +25,10 @@ Available attributes:
         object or those directly provided via API for this run
 """
 
+from __future__ import annotations
+
 import os
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Callable, List, Optional
 
 from prefect._internal.concurrency.api import create_call, from_sync
 from prefect.client.orchestration import get_client
@@ -34,12 +36,17 @@ from prefect.context import FlowRunContext
 
 from .flow_run import _get_flow_run
 
+if TYPE_CHECKING:
+    from prefect.client.schemas.responses import DeploymentResponse
+
 __all__ = ["id", "flow_run_id", "name", "parameters", "version"]
 
-CACHED_DEPLOYMENT = {}
+CACHED_DEPLOYMENT: dict[str, "DeploymentResponse"] = {}
 
 
-type_cast = {
+type_cast: dict[
+    type[bool] | type[int] | type[float] | type[str] | type[None], Callable[[Any], Any]
+] = {
     bool: lambda x: x.lower() == "true",
     int: int,
     float: float,
@@ -88,7 +95,7 @@ def __dir__() -> List[str]:
     return sorted(__all__)
 
 
-async def _get_deployment(deployment_id):
+async def _get_deployment(deployment_id: str) -> "DeploymentResponse":
     # deployments won't change between calls so let's avoid the lifecycle of a client
     if CACHED_DEPLOYMENT.get(deployment_id):
         return CACHED_DEPLOYMENT[deployment_id]
@@ -115,7 +122,7 @@ def get_id() -> Optional[str]:
         return str(deployment_id)
 
 
-def get_parameters() -> Dict:
+def get_parameters() -> dict[str, Any]:
     run_id = get_flow_run_id()
     if run_id is None:
         return {}
@@ -126,7 +133,7 @@ def get_parameters() -> Dict:
     return flow_run.parameters or {}
 
 
-def get_name() -> Optional[Dict]:
+def get_name() -> Optional[str]:
     dep_id = get_id()
 
     if dep_id is None:
@@ -138,7 +145,7 @@ def get_name() -> Optional[Dict]:
     return deployment.name
 
 
-def get_version() -> Optional[Dict]:
+def get_version() -> Optional[str]:
     dep_id = get_id()
 
     if dep_id is None:
@@ -154,7 +161,7 @@ def get_flow_run_id() -> Optional[str]:
     return os.getenv("PREFECT__FLOW_RUN_ID")
 
 
-FIELDS = {
+FIELDS: dict[str, Callable[[], Any]] = {
     "id": get_id,
     "flow_run_id": get_flow_run_id,
     "parameters": get_parameters,

prefect/runtime/flow_run.py

@@ -20,8 +20,10 @@ Available attributes:
     - `run_count`: the number of times this flow run has been run
 """
 
+from __future__ import annotations
+
 import os
-from typing import Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
 
 import pendulum
 
@@ -30,6 +32,9 @@ from prefect.client.orchestration import get_client
 from prefect.context import FlowRunContext, TaskRunContext
 from prefect.settings import PREFECT_API_URL, PREFECT_UI_URL
 
+if TYPE_CHECKING:
+    from prefect.client.schemas.objects import Flow, FlowRun, TaskRun
+
 __all__ = [
     "id",
     "tags",
@@ -56,7 +61,15 @@ def _pendulum_parse(dt: str) -> pendulum.DateTime:
     return pendulum.parse(dt, tz=None, strict=False).set(tz="UTC")
 
 
-type_cast = {
+type_cast: dict[
+    type[bool]
+    | type[int]
+    | type[float]
+    | type[str]
+    | type[None]
+    | type[pendulum.DateTime],
+    Callable[[Any], Any],
+] = {
     bool: lambda x: x.lower() == "true",
     int: int,
     float: float,
@@ -106,17 +119,17 @@ def __dir__() -> List[str]:
     return sorted(__all__)
 
 
-async def _get_flow_run(flow_run_id):
+async def _get_flow_run(flow_run_id: str) -> "FlowRun":
     async with get_client() as client:
         return await client.read_flow_run(flow_run_id)
 
 
-async def _get_task_run(task_run_id):
+async def _get_task_run(task_run_id: str) -> "TaskRun":
     async with get_client() as client:
         return await client.read_task_run(task_run_id)
 
 
-async def _get_flow_from_run(flow_run_id):
+async def _get_flow_from_run(flow_run_id: str) -> "Flow":
     async with get_client() as client:
         flow_run = await client.read_flow_run(flow_run_id)
         return await client.read_flow(flow_run.flow_id)
@@ -323,7 +336,7 @@ def get_job_variables() -> Optional[Dict[str, Any]]:
     return flow_run_ctx.flow_run.job_variables if flow_run_ctx else None
 
 
-FIELDS = {
+FIELDS: dict[str, Callable[[], Any]] = {
     "id": get_id,
     "tags": get_tags,
     "scheduled_start_time": get_scheduled_start_time,

prefect/runtime/task_run.py

@@ -15,15 +15,19 @@ Available attributes:
     - `task_name`: the name of the task
 """
 
+from __future__ import annotations
+
 import os
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 from prefect.context import TaskRunContext
 
 __all__ = ["id", "tags", "name", "parameters", "run_count", "task_name"]
 
 
-type_cast = {
+type_cast: dict[
+    type[bool] | type[int] | type[float] | type[str] | type[None], Callable[[Any], Any]
+] = {
     bool: lambda x: x.lower() == "true",
     int: int,
     float: float,
@@ -118,7 +122,7 @@ def get_parameters() -> Dict[str, Any]:
         return {}
 
 
-FIELDS = {
+FIELDS: dict[str, Callable[[], Any]] = {
     "id": get_id,
     "tags": get_tags,
     "name": get_name,

prefect/settings/base.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import inspect
 from functools import partial
 from typing import Any, Dict, Tuple, Type
@@ -43,13 +45,17 @@ class PrefectBaseSettings(BaseSettings):
 
         See https://docs.pydantic.dev/latest/concepts/pydantic_settings/#customise-settings-sources
         """
-        env_filter = set()
+        env_filter: set[str] = set()
         for field_name, field in settings_cls.model_fields.items():
             if field.validation_alias is not None and isinstance(
                 field.validation_alias, AliasChoices
             ):
                 for alias in field.validation_alias.choices:
-                    if isinstance(alias, AliasPath) and len(alias.path) > 0:
+                    if (
+                        isinstance(alias, AliasPath)
+                        and len(alias.path) > 0
+                        and isinstance(alias.path[0], str)
+                    ):
                         env_filter.add(alias.path[0])
             env_filter.add(field_name)
         return (
@@ -88,13 +94,12 @@ class PrefectBaseSettings(BaseSettings):
         include_secrets: bool = True,
     ) -> Dict[str, str]:
         """Convert the settings object to a dictionary of environment variables."""
-
         env: Dict[str, Any] = self.model_dump(
             exclude_unset=exclude_unset,
             mode="json",
             context={"include_secrets": include_secrets},
         )
-        env_variables = {}
+        env_variables: dict[str, str] = {}
         for key in self.model_fields.keys():
             if isinstance(child_settings := getattr(self, key), PrefectBaseSettings):
                 child_env = child_settings.to_environment_variables(
@@ -175,7 +180,7 @@ def _add_environment_variables(
     schema: Dict[str, Any], model: Type[PrefectBaseSettings]
 ) -> None:
     for property in schema["properties"]:
-        env_vars = []
+        env_vars: list[str] = []
         schema["properties"][property]["supported_environment_variables"] = env_vars
         field = model.model_fields[property]
         if inspect.isclass(field.annotation) and issubclass(
@@ -191,7 +196,7 @@ def _add_environment_variables(
             env_vars.append(f"{model.model_config.get('env_prefix')}{property.upper()}")
 
 
-def _build_settings_config(
+def build_settings_config(
     path: Tuple[str, ...] = tuple(), frozen: bool = False
 ) -> PrefectSettingsConfigDict:
     env_prefix = f"PREFECT_{'_'.join(path).upper()}_" if path else "PREFECT_"
@@ -207,7 +212,12 @@ def _build_settings_config(
     )
 
 
-def _to_environment_variable_value(value: Any) -> str:
+_build_settings_config = build_settings_config  # noqa # TODO: remove once all usage updated
+
+
+def _to_environment_variable_value(
+    value: list[object] | set[object] | tuple[object] | Any,
+) -> str:
     if isinstance(value, (list, set, tuple)):
         return ",".join(str(v) for v in value)
     return str(value)