prefect-client 2.20.2__py3-none-any.whl → 3.0.0__py3-none-any.whl

prefect/runner/runner.py

@@ -35,7 +35,6 @@ import datetime
 import inspect
 import logging
 import os
-import shlex
 import shutil
 import signal
 import subprocess
@@ -45,7 +44,7 @@ import threading
 from copy import deepcopy
 from functools import partial
 from pathlib import Path
-from typing import Callable, Dict, Iterable, List, Optional, Set, Union
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Set, Union
 from uuid import UUID, uuid4
 
 import anyio
@@ -65,23 +64,15 @@ from prefect.client.schemas.filters import (
     FlowRunFilterStateName,
     FlowRunFilterStateType,
 )
-from prefect.client.schemas.objects import (
-    FlowRun,
-    State,
-    StateType,
-)
-from prefect.client.schemas.schedules import SCHEDULE_TYPES
-from prefect.deployments.deployments import load_flow_from_flow_run
-from prefect.deployments.runner import (
-    EntrypointType,
-    RunnerDeployment,
-)
-from prefect.deployments.schedules import FlexibleScheduleList
+from prefect.client.schemas.objects import Flow as APIFlow
+from prefect.client.schemas.objects import FlowRun, State, StateType
 from prefect.events import DeploymentTriggerTypes, TriggerTypes
+from prefect.events.related import tags_as_related_resources
+from prefect.events.schemas.events import RelatedResource
+from prefect.events.utilities import emit_event
 from prefect.exceptions import Abort, ObjectNotFound
-from prefect.flows import Flow
+from prefect.flows import Flow, load_flow_from_flow_run
 from prefect.logging.loggers import PrefectLogAdapter, flow_run_logger, get_logger
-from prefect.runner.server import start_webserver
 from prefect.runner.storage import RunnerStorage
 from prefect.settings import (
     PREFECT_API_URL,
@@ -91,14 +82,28 @@ from prefect.settings import (
     get_current_settings,
 )
 from prefect.states import Crashed, Pending, exception_to_failed_state
+from prefect.types.entrypoint import EntrypointType
 from prefect.utilities.asyncutils import (
     asyncnullcontext,
     is_async_fn,
     sync_compatible,
 )
 from prefect.utilities.engine import propose_state
-from prefect.utilities.processutils import _register_signal, run_process
-from prefect.utilities.services import critical_service_loop
+from prefect.utilities.processutils import (
+    _register_signal,
+    get_sys_executable,
+    run_process,
+)
+from prefect.utilities.services import (
+    critical_service_loop,
+    start_client_metrics_server,
+)
+from prefect.utilities.slugify import slugify
+
+if TYPE_CHECKING:
+    from prefect.client.schemas.objects import Deployment
+    from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
+    from prefect.deployments.runner import RunnerDeployment
 
 __all__ = ["Runner"]
 
@@ -130,6 +135,7 @@ class Runner:
         Examples:
             Set up a Runner to manage the execute of scheduled flow runs for two flows:
                 ```python
+                import asyncio
                 from prefect import flow, Runner
 
                 @flow
@@ -149,7 +155,7 @@ class Runner:
                     # Run on a cron schedule
                     runner.add_flow(goodbye_flow, schedule={"cron": "0 * * * *"})
 
-                    runner.start()
+                    asyncio.run(runner.start())
                 ```
         """
         if name and ("/" in name or "%" in name):
@@ -166,30 +172,26 @@ class Runner:
         self.query_seconds = query_seconds or PREFECT_RUNNER_POLL_FREQUENCY.value()
         self._prefetch_seconds = prefetch_seconds
 
-        self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
-        self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
-
-        self._limiter: Optional[anyio.CapacityLimiter] = anyio.CapacityLimiter(
-            self.limit
-        )
+        self._limiter: Optional[anyio.CapacityLimiter] = None
         self._client = get_client()
         self._submitting_flow_run_ids = set()
         self._cancelling_flow_run_ids = set()
         self._scheduled_task_scopes = set()
         self._deployment_ids: Set[UUID] = set()
-        self._flow_run_process_map = dict()
+        self._flow_run_process_map: Dict[UUID, Dict] = dict()
 
         self._tmp_dir: Path = (
             Path(tempfile.gettempdir()) / "runner_storage" / str(uuid4())
         )
         self._storage_objs: List[RunnerStorage] = []
         self._deployment_storage_map: Dict[UUID, RunnerStorage] = {}
-        self._loop = asyncio.get_event_loop()
+
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
 
     @sync_compatible
     async def add_deployment(
         self,
-        deployment: RunnerDeployment,
+        deployment: "RunnerDeployment",
     ) -> UUID:
         """
         Registers the deployment with the Prefect API and will monitor for work once
@@ -211,7 +213,7 @@ class Runner:
     async def add_flow(
         self,
         flow: Flow,
-        name: str = None,
+        name: Optional[str] = None,
         interval: Optional[
             Union[
                 Iterable[Union[int, float, datetime.timedelta]],
@@ -223,15 +225,13 @@ class Runner:
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[FlexibleScheduleList] = None,
-        schedule: Optional[SCHEDULE_TYPES] = None,
-        is_schedule_active: Optional[bool] = None,
+        schedules: Optional["FlexibleScheduleList"] = None,
         parameters: Optional[dict] = None,
         triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
         description: Optional[str] = None,
         tags: Optional[List[str]] = None,
         version: Optional[str] = None,
-        enforce_parameter_schema: bool = False,
+        enforce_parameter_schema: bool = True,
         entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
     ) -> UUID:
         """
@@ -248,11 +248,6 @@ class Runner:
                 or a timedelta object. If a number is given, it will be interpreted as seconds.
             cron: A cron schedule of when to execute runs of this flow.
             rrule: An rrule schedule of when to execute runs of this flow.
-            schedule: A schedule object of when to execute runs of this flow. Used for
-                advanced scheduling options like timezone.
-            is_schedule_active: Whether or not to set the schedule for this deployment as active. If
-                not provided when creating a deployment, the schedule will be set as active. If not
-                provided when updating a deployment, the schedule's activation will not be changed.
             triggers: A list of triggers that should kick of a run of this flow.
             parameters: A dictionary of default parameter values to pass to runs of this flow.
             description: A description for the created deployment. Defaults to the flow's
@@ -277,9 +272,7 @@ class Runner:
             cron=cron,
             rrule=rrule,
             schedules=schedules,
-            schedule=schedule,
             paused=paused,
-            is_schedule_active=is_schedule_active,
             triggers=triggers,
             parameters=parameters,
             description=description,
@@ -324,7 +317,6 @@ class Runner:
 
         sys.exit(0)
 
-    @sync_compatible
     async def start(
         self, run_once: bool = False, webserver: Optional[bool] = None
     ) -> None:
@@ -342,6 +334,7 @@ class Runner:
             Initialize a Runner, add two flows, and serve them by starting the Runner:
 
             ```python
+            import asyncio
             from prefect import flow, Runner
 
             @flow
@@ -361,9 +354,11 @@ class Runner:
                 # Run on a cron schedule
                 runner.add_flow(goodbye_flow, schedule={"cron": "0 * * * *"})
 
-                runner.start()
+                asyncio.run(runner.start())
             ```
         """
+        from prefect.runner.server import start_webserver
+
         _register_signal(signal.SIGTERM, self.handle_sigterm)
 
         webserver = webserver if webserver is not None else self.webserver
@@ -381,6 +376,8 @@ class Runner:
             )
             server_thread.start()
 
+        start_client_metrics_server()
+
         async with self as runner:
             async with self._loops_task_group as tg:
                 for storage in self._storage_objs:
@@ -533,7 +530,7 @@ class Runner:
             task_status: anyio task status used to send a message to the caller
                 than the flow run process has started.
         """
-        command = f"{shlex.quote(sys.executable)} -m prefect.engine"
+        command = [get_sys_executable(), "-m", "prefect.engine"]
 
         flow_run_logger = self._get_flow_run_logger(flow_run)
 
@@ -580,7 +577,7 @@ class Runner:
                 setattr(storage, "last_adhoc_pull", datetime.datetime.now())
 
         process = await run_process(
-            shlex.split(command),
+            command=command,
             stream_output=True,
             task_status=task_status,
             env=env,
@@ -693,8 +690,9 @@ class Runner:
         """
         self._logger.info("Pausing all deployments...")
         for deployment_id in self._deployment_ids:
-            self._logger.debug(f"Pausing deployment '{deployment_id}'")
             await self._client.set_deployment_paused_state(deployment_id, True)
+            self._logger.debug(f"Paused deployment '{deployment_id}'")
+
         self._logger.info("All deployments have been paused!")
 
     async def _get_and_submit_flow_runs(self):
@@ -816,8 +814,71 @@ class Runner:
                     "message": state_msg or "Flow run was cancelled successfully."
                 },
             )
+            try:
+                deployment = await self._client.read_deployment(flow_run.deployment_id)
+            except ObjectNotFound:
+                deployment = None
+            try:
+                flow = await self._client.read_flow(flow_run.flow_id)
+            except ObjectNotFound:
+                flow = None
+            self._emit_flow_run_cancelled_event(
+                flow_run=flow_run, flow=flow, deployment=deployment
+            )
             run_logger.info(f"Cancelled flow run '{flow_run.name}'!")
 
+    def _event_resource(self):
+        from prefect import __version__
+
+        return {
+            "prefect.resource.id": f"prefect.runner.{slugify(self.name)}",
+            "prefect.resource.name": self.name,
+            "prefect.version": __version__,
+        }
+
+    def _emit_flow_run_cancelled_event(
+        self,
+        flow_run: "FlowRun",
+        flow: "Optional[APIFlow]",
+        deployment: "Optional[Deployment]",
+    ):
+        related = []
+        tags = []
+        if deployment:
+            related.append(
+                {
+                    "prefect.resource.id": f"prefect.deployment.{deployment.id}",
+                    "prefect.resource.role": "deployment",
+                    "prefect.resource.name": deployment.name,
+                }
+            )
+            tags.extend(deployment.tags)
+        if flow:
+            related.append(
+                {
+                    "prefect.resource.id": f"prefect.flow.{flow.id}",
+                    "prefect.resource.role": "flow",
+                    "prefect.resource.name": flow.name,
+                }
+            )
+        related.append(
+            {
+                "prefect.resource.id": f"prefect.flow-run.{flow_run.id}",
+                "prefect.resource.role": "flow-run",
+                "prefect.resource.name": flow_run.name,
+            }
+        )
+        tags.extend(flow_run.tags)
+
+        related = [RelatedResource.model_validate(r) for r in related]
+        related += tags_as_related_resources(set(tags))
+
+        emit_event(
+            event="prefect.runner.cancelled-flow-run",
+            resource=self._event_resource(),
+            related=related,
+        )
+
     async def _get_scheduled_flow_runs(
         self,
     ) -> List["FlowRun"]:
@@ -954,7 +1015,7 @@ class Runner:
             # If the run is not ready to submit, release the concurrency slot
             self._release_limit_slot(flow_run.id)
 
-        self._submitting_flow_run_ids.remove(flow_run.id)
+        self._submitting_flow_run_ids.discard(flow_run.id)
 
     async def _submit_run_and_capture_errors(
         self,
@@ -1080,7 +1141,7 @@ class Runner:
         state_updates = state_updates or {}
         state_updates.setdefault("name", "Cancelled")
         state_updates.setdefault("type", StateType.CANCELLED)
-        state = flow_run.state.copy(update=state_updates)
+        state = flow_run.state.model_copy(update=state_updates)
 
         await self._client.set_flow_run_state(flow_run.id, state, force=True)
 
@@ -1130,9 +1191,9 @@ class Runner:
         if state.is_cancelling():
             try:
                 flow = await load_flow_from_flow_run(
-                    flow_run, client=self._client, storage_base_path=str(self._tmp_dir)
+                    flow_run, storage_base_path=str(self._tmp_dir)
                 )
-                hooks = flow.on_cancellation or []
+                hooks = flow.on_cancellation_hooks or []
 
                 await _run_hooks(hooks, flow_run, flow, state)
             except ObjectNotFound:
@@ -1151,9 +1212,9 @@ class Runner:
         """
         if state.is_crashed():
             flow = await load_flow_from_flow_run(
-                flow_run, client=self._client, storage_base_path=str(self._tmp_dir)
+                flow_run, storage_base_path=str(self._tmp_dir)
             )
-            hooks = flow.on_crashed or []
+            hooks = flow.on_crashed_hooks or []
 
             await _run_hooks(hooks, flow_run, flow, state)
 
@@ -1161,6 +1222,18 @@ class Runner:
         self._logger.debug("Starting runner...")
         self._client = get_client()
         self._tmp_dir.mkdir(parents=True)
+
+        self._limiter = anyio.CapacityLimiter(self.limit)
+
+        if not hasattr(self, "_loop") or not self._loop:
+            self._loop = asyncio.get_event_loop()
+
+        if not hasattr(self, "_runs_task_group") or not self._runs_task_group:
+            self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
+
+        if not hasattr(self, "_loops_task_group") or not self._loops_task_group:
+            self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
+
         await self._client.__aenter__()
         await self._runs_task_group.__aenter__()
 

prefect/runner/server.py

@@ -3,21 +3,19 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
 
 import pendulum
 import uvicorn
-from prefect._vendor.fastapi import APIRouter, FastAPI, HTTPException, status
-from prefect._vendor.fastapi.responses import JSONResponse
+from fastapi import APIRouter, FastAPI, HTTPException, status
+from fastapi.responses import JSONResponse
 from typing_extensions import Literal
 
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
 from prefect._internal.schemas.validators import validate_values_conform_to_schema
 from prefect.client.orchestration import get_client
 from prefect.exceptions import MissingFlowError, ScriptError
-from prefect.flows import Flow, load_flow_from_entrypoint, load_flows_from_script
+from prefect.flows import Flow, load_flow_from_entrypoint
 from prefect.logging import get_logger
 from prefect.runner.utils import (
     inject_schemas_into_openapi,
 )
 from prefect.settings import (
-    PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS,
     PREFECT_RUNNER_POLL_FREQUENCY,
     PREFECT_RUNNER_SERVER_HOST,
     PREFECT_RUNNER_SERVER_LOG_LEVEL,
@@ -25,15 +23,13 @@ from prefect.settings import (
     PREFECT_RUNNER_SERVER_PORT,
 )
 from prefect.utilities.asyncutils import sync_compatible
+from prefect.utilities.importtools import load_script_as_module
 
 if TYPE_CHECKING:
     from prefect.client.schemas.responses import DeploymentResponse
     from prefect.runner import Runner
 
-if HAS_PYDANTIC_V2:
-    from pydantic.v1 import BaseModel
-else:
-    from pydantic import BaseModel
+from pydantic import BaseModel
 
 logger = get_logger("webserver")
 
@@ -46,7 +42,7 @@ class RunnerGenericFlowRunRequest(BaseModel):
     parent_task_run_id: Optional[uuid.UUID] = None
 
 
-def perform_health_check(runner, delay_threshold: int = None) -> JSONResponse:
+def perform_health_check(runner, delay_threshold: Optional[int] = None) -> JSONResponse:
     if delay_threshold is None:
         delay_threshold = (
             PREFECT_RUNNER_SERVER_MISSED_POLLS_TOLERANCE.value()
@@ -159,9 +155,12 @@ async def get_subflow_schemas(runner: "Runner") -> Dict[str, Dict]:
                 continue
 
             script = deployment.entrypoint.split(":")[0]
-            subflows = load_flows_from_script(script)
+            module = load_script_as_module(script)
+            subflows = [
+                obj for obj in module.__dict__.values() if isinstance(obj, Flow)
+            ]
             for flow in subflows:
-                schemas[flow.name] = flow.parameters.dict()
+                schemas[flow.name] = flow.parameters.model_dump()
 
     return schemas
 
@@ -183,7 +182,7 @@ def _flow_schema_changed(flow: Flow, schemas: Dict[str, Dict]) -> bool:
     flow_name_with_dashes = flow.name.replace("_", "-")
 
     schema = schemas.get(flow.name, None) or schemas.get(flow_name_with_dashes, None)
-    if schema is not None and flow.parameters.dict() != schema:
+    if schema is not None and flow.parameters.model_dump() != schema:
         return True
     return False
 
@@ -236,7 +235,7 @@ def _build_generic_endpoint_for_flows(
 
         return JSONResponse(
             status_code=status.HTTP_201_CREATED,
-            content=flow_run.dict(json_compatible=True),
+            content=flow_run.model_dump(mode="json"),
         )
 
     return _create_flow_run_for_flow_from_fqn
@@ -261,29 +260,28 @@ async def build_server(runner: "Runner") -> FastAPI:
     router.add_api_route("/shutdown", shutdown(runner=runner), methods=["POST"])
     webserver.include_router(router)
 
-    if PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS.value():
-        deployments_router, deployment_schemas = await get_deployment_router(runner)
-        webserver.include_router(deployments_router)
-
-        subflow_schemas = await get_subflow_schemas(runner)
-        webserver.add_api_route(
-            "/flow/run",
-            _build_generic_endpoint_for_flows(runner=runner, schemas=subflow_schemas),
-            methods=["POST"],
-            name="Run flow in background",
-            description="Trigger any flow run as a background task on the runner.",
-            summary="Run flow",
-        )
-
-        def customize_openapi():
-            if webserver.openapi_schema:
-                return webserver.openapi_schema
+    deployments_router, deployment_schemas = await get_deployment_router(runner)
+    webserver.include_router(deployments_router)
+
+    subflow_schemas = await get_subflow_schemas(runner)
+    webserver.add_api_route(
+        "/flow/run",
+        _build_generic_endpoint_for_flows(runner=runner, schemas=subflow_schemas),
+        methods=["POST"],
+        name="Run flow in background",
+        description="Trigger any flow run as a background task on the runner.",
+        summary="Run flow",
+    )
 
-            openapi_schema = inject_schemas_into_openapi(webserver, deployment_schemas)
-            webserver.openapi_schema = openapi_schema
+    def customize_openapi():
+        if webserver.openapi_schema:
             return webserver.openapi_schema
 
-        webserver.openapi = customize_openapi
+        openapi_schema = inject_schemas_into_openapi(webserver, deployment_schemas)
+        webserver.openapi_schema = openapi_schema
+        return webserver.openapi_schema
+
+    webserver.openapi = customize_openapi
 
     return webserver
 

prefect/runner/storage.py

@@ -8,20 +8,15 @@ from uuid import uuid4
 
 import fsspec
 from anyio import run_process
+from pydantic import SecretStr
 
 from prefect._internal.concurrency.api import create_call, from_async
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
 from prefect.blocks.core import Block, BlockNotSavedError
 from prefect.blocks.system import Secret
 from prefect.filesystems import ReadableDeploymentStorage, WritableDeploymentStorage
 from prefect.logging.loggers import get_logger
 from prefect.utilities.collections import visit_collection
 
-if HAS_PYDANTIC_V2:
-    from pydantic.v1 import SecretStr
-else:
-    from pydantic import SecretStr
-
 
 @runtime_checkable
 class RunnerStorage(Protocol):
@@ -159,7 +154,7 @@ class GitRepository:
         url_components = urlparse(self._url)
 
         credentials = (
-            self._credentials.dict()
+            self._credentials.model_dump()
             if isinstance(self._credentials, Block)
             else deepcopy(self._credentials)
         )
@@ -396,7 +391,7 @@ class RemoteStorage:
                 if hasattr(obj, "value"):
                     return obj.value
                 else:
-                    return obj.dict()
+                    return obj.model_dump()
             return obj
 
         settings_with_block_values = visit_collection(
@@ -571,16 +566,12 @@ class BlockStorageAdapter:
 class LocalStorage:
     """
     Sets the working directory in the local filesystem.
-
     Parameters:
         Path: Local file path to set the working directory for the flow
-
     Examples:
         Sets the working directory for the local path to the flow:
-
         ```python
         from prefect.runner.storage import Localstorage
-
         storage = LocalStorage(
             path="/path/to/local/flow_directory",
         )

prefect/runner/submit.py

@@ -14,7 +14,6 @@ from prefect.context import FlowRunContext
 from prefect.flows import Flow
 from prefect.logging import get_logger
 from prefect.settings import (
-    PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS,
     PREFECT_RUNNER_PROCESS_LIMIT,
     PREFECT_RUNNER_SERVER_HOST,
     PREFECT_RUNNER_SERVER_PORT,
@@ -43,7 +42,7 @@ async def _submit_flow_to_runner(
     Returns:
         A `FlowRun` object representing the flow run that was submitted.
     """
-    from prefect.engine import (
+    from prefect.utilities.engine import (
         _dynamic_key_for_task_run,
         collect_task_run_inputs,
         resolve_inputs,
@@ -90,7 +89,7 @@ async def _submit_flow_to_runner(
         )
         response.raise_for_status()
 
-        return FlowRun.parse_obj(response.json())
+        return FlowRun.model_validate(response.json())
 
 
 @overload
@@ -131,13 +130,6 @@ async def submit_to_runner(
             "The `submit_to_runner` utility only supports submitting flows and tasks."
         )
 
-    if not PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS.value():
-        raise ValueError(
-            "The `submit_to_runner` utility requires the `Runner` webserver to be"
-            " running and built with extra endpoints enabled. To enable this, set the"
-            " `PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS` setting to `True`."
-        )
-
     parameters = parameters or {}
     if isinstance(parameters, List):
         return_single = False

prefect/runner/utils.py

@@ -1,8 +1,8 @@
 from copy import deepcopy
 from typing import Any, Dict
 
-from prefect._vendor.fastapi import FastAPI
-from prefect._vendor.fastapi.openapi.utils import get_openapi
+from fastapi import FastAPI
+from fastapi.openapi.utils import get_openapi
 
 from prefect import __version__ as PREFECT_VERSION
 

prefect/runtime/__init__.py

@@ -8,6 +8,7 @@ Example usage:
     print(f"This script is running from deployment {deployment.id} with parameters {deployment.parameters}")
     ```
 """
+
 import prefect.runtime.deployment
 import prefect.runtime.flow_run
 import prefect.runtime.task_run

prefect/runtime/deployment.py

@@ -24,6 +24,7 @@ Available attributes:
         include default values set on the flow function, only the parameter values set on the deployment
         object or those directly provided via API for this run
 """
+
 import os
 from typing import Any, Dict, List, Optional