prefect-client 3.1.14__py3-none-any.whl → 3.2.0__py3-none-any.whl

prefect/events/utilities.py

@@ -1,8 +1,10 @@
+from __future__ import annotations
+
 from datetime import timedelta
-from typing import Any, Dict, List, Optional, Union
+from typing import Any
 from uuid import UUID
 
-import pendulum
+from prefect.types import DateTime
 
 from .clients import (
     AssertingEventsClient,
@@ -18,16 +20,16 @@ TIGHT_TIMING = timedelta(minutes=5)
 
 def emit_event(
     event: str,
-    resource: Dict[str, str],
-    occurred: Optional[pendulum.DateTime] = None,
-    related: Optional[Union[List[Dict[str, str]], List[RelatedResource]]] = None,
-    payload: Optional[Dict[str, Any]] = None,
-    id: Optional[UUID] = None,
-    follows: Optional[Event] = None,
-    **kwargs: Optional[Dict[str, Any]],
-) -> Optional[Event]:
+    resource: dict[str, str],
+    occurred: DateTime | None = None,
+    related: list[dict[str, str]] | list[RelatedResource] | None = None,
+    payload: dict[str, Any] | None = None,
+    id: UUID | None = None,
+    follows: Event | None = None,
+    **kwargs: dict[str, Any] | None,
+) -> Event | None:
     """
-    Send an event to Prefect Cloud.
+    Send an event to Prefect.
 
     Args:
         event: The name of the event that happened.
@@ -60,14 +62,14 @@ def emit_event(
     if worker_instance.client_type not in operational_clients:
         return None
 
-    event_kwargs: Dict[str, Any] = {
+    event_kwargs: dict[str, Any] = {
         "event": event,
         "resource": resource,
         **kwargs,
     }
 
     if occurred is None:
-        occurred = pendulum.now("UTC")
+        occurred = DateTime.now("UTC")
     event_kwargs["occurred"] = occurred
 
     if related is not None:

prefect/exceptions.py

@@ -161,7 +161,7 @@ class ParameterTypeError(PrefectException):
     @classmethod
     def from_validation_error(cls, exc: ValidationError) -> Self:
         bad_params = [
-            f'{".".join(str(item) for item in err["loc"])}: {err["msg"]}'
+            f"{'.'.join(str(item) for item in err['loc'])}: {err['msg']}"
             for err in exc.errors()
         ]
         msg = "Flow run received invalid parameters:\n - " + "\n - ".join(bad_params)

prefect/flow_engine.py

@@ -2,10 +2,13 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import multiprocessing
+import multiprocessing.context
 import os
 import time
 from contextlib import ExitStack, asynccontextmanager, contextmanager, nullcontext
 from dataclasses import dataclass, field
+from functools import wraps
 from typing import (
     Any,
     AsyncGenerator,
@@ -37,9 +40,12 @@ from prefect.concurrency.v1.context import ConcurrencyContext as ConcurrencyCont
 from prefect.context import (
     AsyncClientContext,
     FlowRunContext,
+    SettingsContext,
     SyncClientContext,
     TagsContext,
+    get_settings_context,
     hydrated_context,
+    serialize_context,
 )
 from prefect.exceptions import (
     Abort,
@@ -62,6 +68,8 @@ from prefect.results import (
     should_persist_result,
 )
 from prefect.settings import PREFECT_DEBUG_MODE
+from prefect.settings.context import get_current_settings
+from prefect.settings.models.root import Settings
 from prefect.states import (
     Failed,
     Pending,
@@ -83,6 +91,7 @@ from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import run_coro_as_sync
 from prefect.utilities.callables import (
     call_with_parameters,
+    cloudpickle_wrapped_call,
     get_call_parameters,
     parameters_to_args_kwargs,
 )
@@ -1509,6 +1518,8 @@ def run_flow(
             ret_val = run_flow_async(**kwargs)
         else:
             ret_val = run_flow_sync(**kwargs)
+    except (Abort, Pause):
+        raise
     except:
         if error_logger:
             error_logger.error(
@@ -1533,3 +1544,111 @@ def _flow_parameters(
     parameters = flow_run.parameters if flow_run else {}
     call_args, call_kwargs = parameters_to_args_kwargs(flow.fn, parameters)
     return get_call_parameters(flow.fn, call_args, call_kwargs)
+
+
+def run_flow_in_subprocess(
+    flow: "Flow[..., Any]",
+    flow_run: "FlowRun | None" = None,
+    parameters: dict[str, Any] | None = None,
+    wait_for: Iterable[PrefectFuture[Any]] | None = None,
+    context: dict[str, Any] | None = None,
+) -> multiprocessing.context.SpawnProcess:
+    """
+    Run a flow in a subprocess.
+
+    Note the result of the flow will only be accessible if the flow is configured to
+    persist its result.
+
+    Args:
+        flow: The flow to run.
+        flow_run: The flow run object containing run metadata.
+        parameters: The parameters to use when invoking the flow.
+        wait_for: The futures to wait for before starting the flow.
+        context: A serialized context to hydrate before running the flow. If not provided,
+            the current context will be used. A serialized context should be provided if
+            this function is called in a separate memory space from the parent run (e.g.
+            in a subprocess or on another machine).
+
+    Returns:
+        A multiprocessing.context.SpawnProcess representing the process that is running the flow.
+    """
+    from prefect.flow_engine import run_flow
+
+    @wraps(run_flow)
+    def run_flow_with_env(
+        *args: Any,
+        env: dict[str, str] | None = None,
+        **kwargs: Any,
+    ):
+        """
+        Wrapper function to update environment variables and settings before running the flow.
+        """
+        engine_logger = logging.getLogger("prefect.engine")
+
+        os.environ.update(env or {})
+        settings_context = get_settings_context()
+        # Create a new settings context with a new settings object to pick up the updated
+        # environment variables
+        with SettingsContext(
+            profile=settings_context.profile,
+            settings=Settings(),
+        ):
+            try:
+                maybe_coro = run_flow(*args, **kwargs)
+                if asyncio.iscoroutine(maybe_coro):
+                    # This is running in a brand new process, so there won't be an existing
+                    # event loop.
+                    asyncio.run(maybe_coro)
+            except Abort:
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' aborted by orchestrator."
+                else:
+                    msg = "Execution aborted by orchestrator."
+                engine_logger.info(msg)
+                exit(0)
+            except Pause:
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' is paused."
+                else:
+                    msg = "Execution is paused."
+                engine_logger.info(msg)
+                exit(0)
+            except Exception:
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' exited with unexpected exception"
+                else:
+                    msg = "Execution exited with unexpected exception"
+                engine_logger.error(msg, exc_info=True)
+                exit(1)
+            except BaseException:
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' interrupted by base exception"
+                else:
+                    msg = "Execution interrupted by base exception"
+                engine_logger.error(msg, exc_info=True)
+                # Let the exit code be determined by the base exception type
+                raise
+
+    ctx = multiprocessing.get_context("spawn")
+
+    context = context or serialize_context()
+
+    process = ctx.Process(
+        target=cloudpickle_wrapped_call(
+            run_flow_with_env,
+            env=get_current_settings().to_environment_variables(exclude_unset=True)
+            | os.environ
+            | {
+                # TODO: make this a thing we can pass into the engine
+                "PREFECT__ENABLE_CANCELLATION_AND_CRASHED_HOOKS": "false",
+            },
+            flow=flow,
+            flow_run=flow_run,
+            parameters=parameters,
+            wait_for=wait_for,
+            context=context,
+        ),
+    )
+    process.start()
+
+    return process

prefect/flow_runs.py

@@ -139,8 +139,7 @@ async def pause_flow_run(
     timeout: int = 3600,
     poll_interval: int = 10,
     key: Optional[str] = None,
-) -> None:
-    ...
+) -> None: ...
 
 
 @overload
@@ -149,8 +148,7 @@ async def pause_flow_run(
     timeout: int = 3600,
     poll_interval: int = 10,
     key: Optional[str] = None,
-) -> T:
-    ...
+) -> T: ...
 
 
 @sync_compatible
@@ -308,8 +306,7 @@ async def suspend_flow_run(
     timeout: Optional[int] = 3600,
     key: Optional[str] = None,
     client: Optional[PrefectClient] = None,
-) -> None:
-    ...
+) -> None: ...
 
 
 @overload
@@ -319,8 +316,7 @@ async def suspend_flow_run(
     timeout: Optional[int] = 3600,
     key: Optional[str] = None,
     client: Optional[PrefectClient] = None,
-) -> T:
-    ...
+) -> T: ...
 
 
 @sync_compatible