prefect-client 3.0.0rc10__py3-none-any.whl → 3.0.0rc12__py3-none-any.whl

prefect/runner/runner.py

@@ -65,17 +65,13 @@ from prefect.client.schemas.filters import (
     FlowRunFilterStateName,
     FlowRunFilterStateType,
 )
-from prefect.client.schemas.objects import (
-    FlowRun,
-    State,
-    StateType,
-)
+from prefect.client.schemas.objects import Flow as APIFlow
+from prefect.client.schemas.objects import FlowRun, State, StateType
 from prefect.client.schemas.schedules import SCHEDULE_TYPES
-from prefect.deployments.runner import (
-    EntrypointType,
-    RunnerDeployment,
-)
 from prefect.events import DeploymentTriggerTypes, TriggerTypes
+from prefect.events.related import tags_as_related_resources
+from prefect.events.schemas.events import RelatedResource
+from prefect.events.utilities import emit_event
 from prefect.exceptions import Abort, ObjectNotFound
 from prefect.flows import Flow, load_flow_from_flow_run
 from prefect.logging.loggers import PrefectLogAdapter, flow_run_logger, get_logger
@@ -88,6 +84,7 @@ from prefect.settings import (
     get_current_settings,
 )
 from prefect.states import Crashed, Pending, exception_to_failed_state
+from prefect.types.entrypoint import EntrypointType
 from prefect.utilities.asyncutils import (
     asyncnullcontext,
     is_async_fn,
@@ -96,9 +93,12 @@ from prefect.utilities.asyncutils import (
 from prefect.utilities.engine import propose_state
 from prefect.utilities.processutils import _register_signal, run_process
 from prefect.utilities.services import critical_service_loop
+from prefect.utilities.slugify import slugify
 
 if TYPE_CHECKING:
+    from prefect.client.schemas.objects import Deployment
     from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
+    from prefect.deployments.runner import RunnerDeployment
 
 __all__ = ["Runner"]
 
@@ -130,6 +130,7 @@ class Runner:
         Examples:
             Set up a Runner to manage the execute of scheduled flow runs for two flows:
                 ```python
+                import asyncio
                 from prefect import flow, Runner
 
                 @flow
@@ -149,7 +150,7 @@ class Runner:
                     # Run on a cron schedule
                     runner.add_flow(goodbye_flow, schedule={"cron": "0 * * * *"})
 
-                    runner.start()
+                    asyncio.run(runner.start())
                 ```
         """
         if name and ("/" in name or "%" in name):
@@ -166,30 +167,26 @@ class Runner:
         self.query_seconds = query_seconds or PREFECT_RUNNER_POLL_FREQUENCY.value()
         self._prefetch_seconds = prefetch_seconds
 
-        self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
-        self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
-
-        self._limiter: Optional[anyio.CapacityLimiter] = anyio.CapacityLimiter(
-            self.limit
-        )
+        self._limiter: Optional[anyio.CapacityLimiter] = None
         self._client = get_client()
         self._submitting_flow_run_ids = set()
         self._cancelling_flow_run_ids = set()
         self._scheduled_task_scopes = set()
         self._deployment_ids: Set[UUID] = set()
-        self._flow_run_process_map = dict()
+        self._flow_run_process_map: Dict[UUID, Dict] = dict()
 
         self._tmp_dir: Path = (
             Path(tempfile.gettempdir()) / "runner_storage" / str(uuid4())
         )
         self._storage_objs: List[RunnerStorage] = []
         self._deployment_storage_map: Dict[UUID, RunnerStorage] = {}
-        self._loop = asyncio.get_event_loop()
+
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
 
     @sync_compatible
     async def add_deployment(
         self,
-        deployment: RunnerDeployment,
+        deployment: "RunnerDeployment",
     ) -> UUID:
         """
         Registers the deployment with the Prefect API and will monitor for work once
@@ -324,7 +321,6 @@ class Runner:
 
         sys.exit(0)
 
-    @sync_compatible
     async def start(
         self, run_once: bool = False, webserver: Optional[bool] = None
     ) -> None:
@@ -342,6 +338,7 @@ class Runner:
             Initialize a Runner, add two flows, and serve them by starting the Runner:
 
             ```python
+            import asyncio
             from prefect import flow, Runner
 
             @flow
@@ -361,7 +358,7 @@ class Runner:
                 # Run on a cron schedule
                 runner.add_flow(goodbye_flow, schedule={"cron": "0 * * * *"})
 
-                runner.start()
+                asyncio.run(runner.start())
             ```
         """
         from prefect.runner.server import start_webserver
@@ -695,8 +692,9 @@ class Runner:
         """
         self._logger.info("Pausing all deployments...")
         for deployment_id in self._deployment_ids:
-            self._logger.debug(f"Pausing deployment '{deployment_id}'")
             await self._client.set_deployment_paused_state(deployment_id, True)
+            self._logger.debug(f"Paused deployment '{deployment_id}'")
+
         self._logger.info("All deployments have been paused!")
 
     async def _get_and_submit_flow_runs(self):
@@ -818,8 +816,71 @@ class Runner:
                     "message": state_msg or "Flow run was cancelled successfully."
                 },
             )
+            try:
+                deployment = await self._client.read_deployment(flow_run.deployment_id)
+            except ObjectNotFound:
+                deployment = None
+            try:
+                flow = await self._client.read_flow(flow_run.flow_id)
+            except ObjectNotFound:
+                flow = None
+            self._emit_flow_run_cancelled_event(
+                flow_run=flow_run, flow=flow, deployment=deployment
+            )
             run_logger.info(f"Cancelled flow run '{flow_run.name}'!")
 
+    def _event_resource(self):
+        from prefect import __version__
+
+        return {
+            "prefect.resource.id": f"prefect.runner.{slugify(self.name)}",
+            "prefect.resource.name": self.name,
+            "prefect.version": __version__,
+        }
+
+    def _emit_flow_run_cancelled_event(
+        self,
+        flow_run: "FlowRun",
+        flow: "Optional[APIFlow]",
+        deployment: "Optional[Deployment]",
+    ):
+        related = []
+        tags = []
+        if deployment:
+            related.append(
+                {
+                    "prefect.resource.id": f"prefect.deployment.{deployment.id}",
+                    "prefect.resource.role": "deployment",
+                    "prefect.resource.name": deployment.name,
+                }
+            )
+            tags.extend(deployment.tags)
+        if flow:
+            related.append(
+                {
+                    "prefect.resource.id": f"prefect.flow.{flow.id}",
+                    "prefect.resource.role": "flow",
+                    "prefect.resource.name": flow.name,
+                }
+            )
+        related.append(
+            {
+                "prefect.resource.id": f"prefect.flow-run.{flow_run.id}",
+                "prefect.resource.role": "flow-run",
+                "prefect.resource.name": flow_run.name,
+            }
+        )
+        tags.extend(flow_run.tags)
+
+        related = [RelatedResource.model_validate(r) for r in related]
+        related += tags_as_related_resources(set(tags))
+
+        emit_event(
+            event="prefect.runner.cancelled-flow-run",
+            resource=self._event_resource(),
+            related=related,
+        )
+
     async def _get_scheduled_flow_runs(
         self,
     ) -> List["FlowRun"]:
@@ -956,7 +1017,7 @@ class Runner:
             # If the run is not ready to submit, release the concurrency slot
             self._release_limit_slot(flow_run.id)
 
-        self._submitting_flow_run_ids.remove(flow_run.id)
+        self._submitting_flow_run_ids.discard(flow_run.id)
 
     async def _submit_run_and_capture_errors(
         self,
@@ -1163,6 +1224,18 @@ class Runner:
         self._logger.debug("Starting runner...")
         self._client = get_client()
         self._tmp_dir.mkdir(parents=True)
+
+        self._limiter = anyio.CapacityLimiter(self.limit)
+
+        if not hasattr(self, "_loop") or not self._loop:
+            self._loop = asyncio.get_event_loop()
+
+        if not hasattr(self, "_runs_task_group") or not self._runs_task_group:
+            self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
+
+        if not hasattr(self, "_loops_task_group") or not self._loops_task_group:
+            self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
+
         await self._client.__aenter__()
         await self._runs_task_group.__aenter__()
 

prefect/runner/server.py

@@ -16,7 +16,6 @@ from prefect.runner.utils import (
     inject_schemas_into_openapi,
 )
 from prefect.settings import (
-    PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS,
     PREFECT_RUNNER_POLL_FREQUENCY,
     PREFECT_RUNNER_SERVER_HOST,
     PREFECT_RUNNER_SERVER_LOG_LEVEL,
@@ -202,7 +201,7 @@ def _build_generic_endpoint_for_flows(
 
         try:
             flow = load_flow_from_entrypoint(body.entrypoint)
-        except (MissingFlowError, ScriptError, ModuleNotFoundError):
+        except (FileNotFoundError, MissingFlowError, ScriptError, ModuleNotFoundError):
             return JSONResponse(
                 status_code=status.HTTP_404_NOT_FOUND,
                 content={"message": "Flow not found"},
@@ -261,29 +260,28 @@ async def build_server(runner: "Runner") -> FastAPI:
     router.add_api_route("/shutdown", shutdown(runner=runner), methods=["POST"])
     webserver.include_router(router)
 
-    if PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS.value():
-        deployments_router, deployment_schemas = await get_deployment_router(runner)
-        webserver.include_router(deployments_router)
-
-        subflow_schemas = await get_subflow_schemas(runner)
-        webserver.add_api_route(
-            "/flow/run",
-            _build_generic_endpoint_for_flows(runner=runner, schemas=subflow_schemas),
-            methods=["POST"],
-            name="Run flow in background",
-            description="Trigger any flow run as a background task on the runner.",
-            summary="Run flow",
-        )
-
-        def customize_openapi():
-            if webserver.openapi_schema:
-                return webserver.openapi_schema
+    deployments_router, deployment_schemas = await get_deployment_router(runner)
+    webserver.include_router(deployments_router)
+
+    subflow_schemas = await get_subflow_schemas(runner)
+    webserver.add_api_route(
+        "/flow/run",
+        _build_generic_endpoint_for_flows(runner=runner, schemas=subflow_schemas),
+        methods=["POST"],
+        name="Run flow in background",
+        description="Trigger any flow run as a background task on the runner.",
+        summary="Run flow",
+    )
 
-            openapi_schema = inject_schemas_into_openapi(webserver, deployment_schemas)
-            webserver.openapi_schema = openapi_schema
+    def customize_openapi():
+        if webserver.openapi_schema:
             return webserver.openapi_schema
 
-        webserver.openapi = customize_openapi
+        openapi_schema = inject_schemas_into_openapi(webserver, deployment_schemas)
+        webserver.openapi_schema = openapi_schema
+        return webserver.openapi_schema
+
+    webserver.openapi = customize_openapi
 
     return webserver
 

prefect/runner/submit.py

@@ -14,7 +14,6 @@ from prefect.context import FlowRunContext
 from prefect.flows import Flow
 from prefect.logging import get_logger
 from prefect.settings import (
-    PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS,
     PREFECT_RUNNER_PROCESS_LIMIT,
     PREFECT_RUNNER_SERVER_HOST,
     PREFECT_RUNNER_SERVER_PORT,
@@ -131,13 +130,6 @@ async def submit_to_runner(
             "The `submit_to_runner` utility only supports submitting flows and tasks."
         )
 
-    if not PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS.value():
-        raise ValueError(
-            "The `submit_to_runner` utility requires the `Runner` webserver to be"
-            " running and built with extra endpoints enabled. To enable this, set the"
-            " `PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS` setting to `True`."
-        )
-
     parameters = parameters or {}
     if isinstance(parameters, List):
         return_single = False

prefect/runtime/flow_run.py

@@ -38,6 +38,7 @@ __all__ = [
     "parameters",
     "parent_flow_run_id",
     "parent_deployment_id",
+    "root_flow_run_id",
     "run_count",
     "api_url",
     "ui_url",
@@ -237,11 +238,12 @@ def get_parent_flow_run_id() -> Optional[str]:
         parent_task_run = from_sync.call_soon_in_loop_thread(
             create_call(_get_task_run, parent_task_run_id)
         ).result()
-        return parent_task_run.flow_run_id
+        return str(parent_task_run.flow_run_id) if parent_task_run.flow_run_id else None
+
     return None
 
 
-def get_parent_deployment_id() -> Dict[str, Any]:
+def get_parent_deployment_id() -> Optional[str]:
     parent_flow_run_id = get_parent_flow_run_id()
     if parent_flow_run_id is None:
         return None
@@ -249,7 +251,39 @@ def get_parent_deployment_id() -> Dict[str, Any]:
     parent_flow_run = from_sync.call_soon_in_loop_thread(
         create_call(_get_flow_run, parent_flow_run_id)
     ).result()
-    return parent_flow_run.deployment_id if parent_flow_run else None
+
+    if parent_flow_run:
+        return (
+            str(parent_flow_run.deployment_id)
+            if parent_flow_run.deployment_id
+            else None
+        )
+
+    return None
+
+
+def get_root_flow_run_id() -> str:
+    run_id = get_id()
+    parent_flow_run_id = get_parent_flow_run_id()
+    if parent_flow_run_id is None:
+        return run_id
+
+    def _get_root_flow_run_id(flow_run_id):
+        flow_run = from_sync.call_soon_in_loop_thread(
+            create_call(_get_flow_run, flow_run_id)
+        ).result()
+
+        if flow_run.parent_task_run_id is None:
+            return str(flow_run_id)
+        else:
+            parent_task_run = from_sync.call_soon_in_loop_thread(
+                create_call(_get_task_run, flow_run.parent_task_run_id)
+            ).result()
+            return _get_root_flow_run_id(parent_task_run.flow_run_id)
+
+    root_flow_run_id = _get_root_flow_run_id(parent_flow_run_id)
+
+    return root_flow_run_id
 
 
 def get_flow_run_api_url() -> Optional[str]:
@@ -275,6 +309,7 @@ FIELDS = {
     "parameters": get_parameters,
     "parent_flow_run_id": get_parent_flow_run_id,
     "parent_deployment_id": get_parent_deployment_id,
+    "root_flow_run_id": get_root_flow_run_id,
     "run_count": get_run_count,
     "api_url": get_flow_run_api_url,
     "ui_url": get_flow_run_ui_url,

prefect/settings.py

@@ -481,18 +481,6 @@ PREFECT_HOME = Setting(
 directory may be created automatically when required.
 """
 
-PREFECT_EXTRA_ENTRYPOINTS = Setting(
-    str,
-    default="",
-)
-"""
-Modules for Prefect to import when Prefect is imported.
-
-Values should be separated by commas, e.g. `my_module,my_other_module`.
-Objects within modules may be specified by a ':' partition, e.g. `my_module:my_object`.
-If a callable object is provided, it will be called with no arguments on import.
-"""
-
 PREFECT_DEBUG_MODE = Setting(
     bool,
     default=False,
@@ -1160,6 +1148,11 @@ polled."""
 PREFECT_API_LOG_RETRYABLE_ERRORS = Setting(bool, default=False)
 """If `True`, log retryable errors in the API and it's services."""
 
+PREFECT_API_SERVICES_TASK_RUN_RECORDER_ENABLED = Setting(bool, default=True)
+"""
+Whether or not to start the task run recorder service in the server application.
+"""
+
 
 PREFECT_API_DEFAULT_LIMIT = Setting(
     int,
@@ -1309,17 +1302,13 @@ The maximum number of artifacts to show on a flow run graph on the v2 API
 """
 
 
-PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_CANCELLATION = Setting(bool, default=True)
-"""
-Whether or not to enable experimental enhanced flow run cancellation.
-"""
-
-PREFECT_EXPERIMENTAL_WARN_ENHANCED_CANCELLATION = Setting(bool, default=False)
+PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION = Setting(
+    bool, default=False
+)
 """
-Whether or not to warn when experimental enhanced flow run cancellation is used.
+Whether or not to enable experimental client side task run orchestration.
 """
 
-
 # Prefect Events feature flags
 
 PREFECT_RUNNER_PROCESS_LIMIT = Setting(int, default=5)
@@ -1437,16 +1426,6 @@ a task worker should move a task from PENDING to RUNNING very quickly, so runs s
 PENDING for a while is a sign that the task worker may have crashed.
 """
 
-PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS = Setting(bool, default=False)
-"""
-Whether or not to enable experimental worker webserver endpoints.
-"""
-
-PREFECT_EXPERIMENTAL_DISABLE_SYNC_COMPAT = Setting(bool, default=False)
-"""
-Whether or not to disable the sync_compatible decorator utility.
-"""
-
 PREFECT_EXPERIMENTAL_ENABLE_SCHEDULE_CONCURRENCY = Setting(bool, default=False)
 
 # Defaults -----------------------------------------------------------------------------