prefect-client 3.1.13__py3-none-any.whl → 3.1.15__py3-none-any.whl

prefect/events/filters.py

@@ -1,7 +1,6 @@
-from typing import List, Optional, Tuple, cast
+from typing import Optional
 from uuid import UUID
 
-import pendulum
 from pydantic import Field
 
 from prefect._internal.schemas.bases import PrefectBaseModel
@@ -23,7 +22,7 @@ class AutomationFilterCreated(PrefectBaseModel):
 class AutomationFilterName(PrefectBaseModel):
     """Filter by `Automation.created`."""
 
-    any_: Optional[List[str]] = Field(
+    any_: Optional[list[str]] = Field(
         default=None,
         description="Only include automations with names that match any of these strings",
     )
@@ -41,8 +40,8 @@ class AutomationFilter(PrefectBaseModel):
 class EventDataFilter(PrefectBaseModel, extra="forbid"):  # type: ignore[call-arg]
     """A base class for filtering event data."""
 
-    def get_filters(self) -> List["EventDataFilter"]:
-        filters: List["EventDataFilter"] = [
+    def get_filters(self) -> list["EventDataFilter"]:
+        filters: list["EventDataFilter"] = [
             filter
             for filter in [getattr(self, name) for name in self.model_fields]
             if isinstance(filter, EventDataFilter)
@@ -60,14 +59,11 @@ class EventDataFilter(PrefectBaseModel, extra="forbid"):  # type: ignore[call-ar
 
 class EventOccurredFilter(EventDataFilter):
     since: DateTime = Field(
-        default_factory=lambda: cast(
-            DateTime,
-            pendulum.now("UTC").start_of("day").subtract(days=180),
-        ),
+        default_factory=lambda: DateTime.now("UTC").start_of("day").subtract(days=180),
         description="Only include events after this time (inclusive)",
     )
     until: DateTime = Field(
-        default_factory=lambda: cast(DateTime, pendulum.now("UTC")),
+        default_factory=lambda: DateTime.now("UTC"),
         description="Only include events prior to this time (inclusive)",
     )
 
@@ -76,18 +72,18 @@ class EventOccurredFilter(EventDataFilter):
 
 
 class EventNameFilter(EventDataFilter):
-    prefix: Optional[List[str]] = Field(
+    prefix: Optional[list[str]] = Field(
         default=None, description="Only include events matching one of these prefixes"
     )
-    exclude_prefix: Optional[List[str]] = Field(
+    exclude_prefix: Optional[list[str]] = Field(
         default=None, description="Exclude events matching one of these prefixes"
     )
 
-    name: Optional[List[str]] = Field(
+    name: Optional[list[str]] = Field(
         default=None,
         description="Only include events matching one of these names exactly",
     )
-    exclude_name: Optional[List[str]] = Field(
+    exclude_name: Optional[list[str]] = Field(
         default=None, description="Exclude events matching one of these names exactly"
     )
 
@@ -112,20 +108,20 @@ class EventNameFilter(EventDataFilter):
 
 
 class EventResourceFilter(EventDataFilter):
-    id: Optional[List[str]] = Field(
-        None, description="Only include events for resources with these IDs"
+    id: Optional[list[str]] = Field(
+        default=None, description="Only include events for resources with these IDs"
     )
-    id_prefix: Optional[List[str]] = Field(
-        None,
+    id_prefix: Optional[list[str]] = Field(
+        default=None,
         description=(
             "Only include events for resources with IDs starting with these prefixes."
         ),
     )
     labels: Optional[ResourceSpecification] = Field(
-        None, description="Only include events for resources with these labels"
+        default=None, description="Only include events for resources with these labels"
     )
     distinct: bool = Field(
-        False,
+        default=False,
         description="Only include events for distinct resources",
     )
 
@@ -148,35 +144,39 @@ class EventResourceFilter(EventDataFilter):
 
 
 class EventRelatedFilter(EventDataFilter):
-    id: Optional[List[str]] = Field(
-        None, description="Only include events for related resources with these IDs"
+    id: Optional[list[str]] = Field(
+        default=None,
+        description="Only include events for related resources with these IDs",
     )
-    role: Optional[List[str]] = Field(
-        None, description="Only include events for related resources in these roles"
+    role: Optional[list[str]] = Field(
+        default=None,
+        description="Only include events for related resources in these roles",
     )
-    resources_in_roles: Optional[List[Tuple[str, str]]] = Field(
-        None,
+    resources_in_roles: Optional[list[tuple[str, str]]] = Field(
+        default=None,
         description=(
             "Only include events with specific related resources in specific roles"
         ),
     )
     labels: Optional[ResourceSpecification] = Field(
-        None, description="Only include events for related resources with these labels"
+        default=None,
+        description="Only include events for related resources with these labels",
     )
 
 
 class EventAnyResourceFilter(EventDataFilter):
-    id: Optional[List[str]] = Field(
-        None, description="Only include events for resources with these IDs"
+    id: Optional[list[str]] = Field(
+        default=None, description="Only include events for resources with these IDs"
     )
-    id_prefix: Optional[List[str]] = Field(
-        None,
+    id_prefix: Optional[list[str]] = Field(
+        default=None,
         description=(
             "Only include events for resources with IDs starting with these prefixes"
         ),
     )
     labels: Optional[ResourceSpecification] = Field(
-        None, description="Only include events for related resources with these labels"
+        default=None,
+        description="Only include events for related resources with these labels",
     )
 
     def includes(self, event: Event) -> bool:
@@ -202,8 +202,8 @@ class EventAnyResourceFilter(EventDataFilter):
 
 
 class EventIDFilter(EventDataFilter):
-    id: Optional[List[UUID]] = Field(
-        None, description="Only include events with one of these IDs"
+    id: Optional[list[UUID]] = Field(
+        default=None, description="Only include events with one of these IDs"
     )
 
     def includes(self, event: Event) -> bool:

prefect/flow_engine.py

@@ -2,10 +2,13 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import multiprocessing
+import multiprocessing.context
 import os
 import time
 from contextlib import ExitStack, asynccontextmanager, contextmanager, nullcontext
 from dataclasses import dataclass, field
+from functools import wraps
 from typing import (
     Any,
     AsyncGenerator,
@@ -37,8 +40,12 @@ from prefect.concurrency.v1.context import ConcurrencyContext as ConcurrencyCont
 from prefect.context import (
     AsyncClientContext,
     FlowRunContext,
+    SettingsContext,
     SyncClientContext,
     TagsContext,
+    get_settings_context,
+    hydrated_context,
+    serialize_context,
 )
 from prefect.exceptions import (
     Abort,
@@ -61,6 +68,8 @@ from prefect.results import (
     should_persist_result,
 )
 from prefect.settings import PREFECT_DEBUG_MODE
+from prefect.settings.context import get_current_settings
+from prefect.settings.models.root import Settings
 from prefect.states import (
     Failed,
     Pending,
@@ -82,6 +91,7 @@ from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import run_coro_as_sync
 from prefect.utilities.callables import (
     call_with_parameters,
+    cloudpickle_wrapped_call,
     get_call_parameters,
     parameters_to_args_kwargs,
 )
@@ -137,6 +147,7 @@ class BaseFlowRunEngine(Generic[P, R]):
     flow_run_id: Optional[UUID] = None
     logger: logging.Logger = field(default_factory=lambda: get_logger("engine"))
     wait_for: Optional[Iterable[PrefectFuture[Any]]] = None
+    context: Optional[dict[str, Any]] = None
     # holds the return value from the user code
     _return_value: Union[R, Type[NotSet]] = NotSet
     # holds the exception raised by the user code, if any
@@ -647,65 +658,68 @@ class FlowRunEngine(BaseFlowRunEngine[P, R]):
         """
         Enters a client context and creates a flow run if needed.
         """
-        with SyncClientContext.get_or_create() as client_ctx:
-            self._client = client_ctx.client
-            self._is_started = True
+        with hydrated_context(self.context):
+            with SyncClientContext.get_or_create() as client_ctx:
+                self._client = client_ctx.client
+                self._is_started = True
 
-            if not self.flow_run:
-                self.flow_run = self.create_flow_run(self.client)
-            else:
-                # Update the empirical policy to match the flow if it is not set
-                if self.flow_run.empirical_policy.retry_delay is None:
-                    self.flow_run.empirical_policy.retry_delay = (
-                        self.flow.retry_delay_seconds
-                    )
+                if not self.flow_run:
+                    self.flow_run = self.create_flow_run(self.client)
+                else:
+                    # Update the empirical policy to match the flow if it is not set
+                    if self.flow_run.empirical_policy.retry_delay is None:
+                        self.flow_run.empirical_policy.retry_delay = (
+                            self.flow.retry_delay_seconds
+                        )
 
-                if self.flow_run.empirical_policy.retries is None:
-                    self.flow_run.empirical_policy.retries = self.flow.retries
+                    if self.flow_run.empirical_policy.retries is None:
+                        self.flow_run.empirical_policy.retries = self.flow.retries
 
-                self.client.update_flow_run(
-                    flow_run_id=self.flow_run.id,
-                    flow_version=self.flow.version,
-                    empirical_policy=self.flow_run.empirical_policy,
-                )
+                    self.client.update_flow_run(
+                        flow_run_id=self.flow_run.id,
+                        flow_version=self.flow.version,
+                        empirical_policy=self.flow_run.empirical_policy,
+                    )
 
-            self._telemetry.start_span(
-                run=self.flow_run,
-                client=self.client,
-                parameters=self.parameters,
-            )
+                self._telemetry.start_span(
+                    run=self.flow_run,
+                    client=self.client,
+                    parameters=self.parameters,
+                )
 
-            try:
-                yield self
+                try:
+                    yield self
 
-            except TerminationSignal as exc:
-                self.cancel_all_tasks()
-                self.handle_crash(exc)
-                raise
-            except Exception:
-                # regular exceptions are caught and re-raised to the user
-                raise
-            except (Abort, Pause):
-                raise
-            except GeneratorExit:
-                # Do not capture generator exits as crashes
-                raise
-            except BaseException as exc:
-                # BaseExceptions are caught and handled as crashes
-                self.handle_crash(exc)
-                raise
-            finally:
-                # If debugging, use the more complete `repr` than the usual `str` description
-                display_state = (
-                    repr(self.state) if PREFECT_DEBUG_MODE else str(self.state)
-                )
-                self.logger.log(
-                    level=logging.INFO if self.state.is_completed() else logging.ERROR,
-                    msg=f"Finished in state {display_state}",
-                )
+                except TerminationSignal as exc:
+                    self.cancel_all_tasks()
+                    self.handle_crash(exc)
+                    raise
+                except Exception:
+                    # regular exceptions are caught and re-raised to the user
+                    raise
+                except (Abort, Pause):
+                    raise
+                except GeneratorExit:
+                    # Do not capture generator exits as crashes
+                    raise
+                except BaseException as exc:
+                    # BaseExceptions are caught and handled as crashes
+                    self.handle_crash(exc)
+                    raise
+                finally:
+                    # If debugging, use the more complete `repr` than the usual `str` description
+                    display_state = (
+                        repr(self.state) if PREFECT_DEBUG_MODE else str(self.state)
+                    )
+                    self.logger.log(
+                        level=logging.INFO
+                        if self.state.is_completed()
+                        else logging.ERROR,
+                        msg=f"Finished in state {display_state}",
+                    )
 
-                self._is_started = False
-                self._client = None
+                    self._is_started = False
+                    self._client = None
 
     # --------------------------
     #
@@ -1208,71 +1222,74 @@ class AsyncFlowRunEngine(BaseFlowRunEngine[P, R]):
         """
         Enters a client context and creates a flow run if needed.
         """
-        async with AsyncClientContext.get_or_create() as client_ctx:
-            self._client = client_ctx.client
-            self._is_started = True
+        with hydrated_context(self.context):
+            async with AsyncClientContext.get_or_create() as client_ctx:
+                self._client = client_ctx.client
+                self._is_started = True
+
+                if not self.flow_run:
+                    self.flow_run = await self.create_flow_run(self.client)
+                    flow_run_url = url_for(self.flow_run)
+
+                    if flow_run_url:
+                        self.logger.info(
+                            f"View at {flow_run_url}", extra={"send_to_api": False}
+                        )
+                else:
+                    # Update the empirical policy to match the flow if it is not set
+                    if self.flow_run.empirical_policy.retry_delay is None:
+                        self.flow_run.empirical_policy.retry_delay = (
+                            self.flow.retry_delay_seconds
+                        )
 
-            if not self.flow_run:
-                self.flow_run = await self.create_flow_run(self.client)
-                flow_run_url = url_for(self.flow_run)
+                    if self.flow_run.empirical_policy.retries is None:
+                        self.flow_run.empirical_policy.retries = self.flow.retries
 
-                if flow_run_url:
-                    self.logger.info(
-                        f"View at {flow_run_url}", extra={"send_to_api": False}
-                    )
-            else:
-                # Update the empirical policy to match the flow if it is not set
-                if self.flow_run.empirical_policy.retry_delay is None:
-                    self.flow_run.empirical_policy.retry_delay = (
-                        self.flow.retry_delay_seconds
+                    await self.client.update_flow_run(
+                        flow_run_id=self.flow_run.id,
+                        flow_version=self.flow.version,
+                        empirical_policy=self.flow_run.empirical_policy,
                     )
 
-                if self.flow_run.empirical_policy.retries is None:
-                    self.flow_run.empirical_policy.retries = self.flow.retries
-
-                await self.client.update_flow_run(
-                    flow_run_id=self.flow_run.id,
-                    flow_version=self.flow.version,
-                    empirical_policy=self.flow_run.empirical_policy,
+                await self._telemetry.async_start_span(
+                    run=self.flow_run,
+                    client=self.client,
+                    parameters=self.parameters,
                 )
 
-            await self._telemetry.async_start_span(
-                run=self.flow_run,
-                client=self.client,
-                parameters=self.parameters,
-            )
-
-            try:
-                yield self
+                try:
+                    yield self
 
-            except TerminationSignal as exc:
-                self.cancel_all_tasks()
-                await self.handle_crash(exc)
-                raise
-            except Exception:
-                # regular exceptions are caught and re-raised to the user
-                raise
-            except (Abort, Pause):
-                raise
-            except GeneratorExit:
-                # Do not capture generator exits as crashes
-                raise
-            except BaseException as exc:
-                # BaseExceptions are caught and handled as crashes
-                await self.handle_crash(exc)
-                raise
-            finally:
-                # If debugging, use the more complete `repr` than the usual `str` description
-                display_state = (
-                    repr(self.state) if PREFECT_DEBUG_MODE else str(self.state)
-                )
-                self.logger.log(
-                    level=logging.INFO if self.state.is_completed() else logging.ERROR,
-                    msg=f"Finished in state {display_state}",
-                )
+                except TerminationSignal as exc:
+                    self.cancel_all_tasks()
+                    await self.handle_crash(exc)
+                    raise
+                except Exception:
+                    # regular exceptions are caught and re-raised to the user
+                    raise
+                except (Abort, Pause):
+                    raise
+                except GeneratorExit:
+                    # Do not capture generator exits as crashes
+                    raise
+                except BaseException as exc:
+                    # BaseExceptions are caught and handled as crashes
+                    await self.handle_crash(exc)
+                    raise
+                finally:
+                    # If debugging, use the more complete `repr` than the usual `str` description
+                    display_state = (
+                        repr(self.state) if PREFECT_DEBUG_MODE else str(self.state)
+                    )
+                    self.logger.log(
+                        level=logging.INFO
+                        if self.state.is_completed()
+                        else logging.ERROR,
+                        msg=f"Finished in state {display_state}",
+                    )
 
-                self._is_started = False
-                self._client = None
+                    self._is_started = False
+                    self._client = None
 
     # --------------------------
     #
@@ -1330,12 +1347,14 @@ def run_flow_sync(
     parameters: Optional[Dict[str, Any]] = None,
     wait_for: Optional[Iterable[PrefectFuture[Any]]] = None,
     return_type: Literal["state", "result"] = "result",
+    context: Optional[dict[str, Any]] = None,
 ) -> Union[R, State, None]:
     engine = FlowRunEngine[P, R](
         flow=flow,
         parameters=parameters,
         flow_run=flow_run,
         wait_for=wait_for,
+        context=context,
     )
 
     with engine.start():
@@ -1352,9 +1371,14 @@ async def run_flow_async(
     parameters: Optional[Dict[str, Any]] = None,
     wait_for: Optional[Iterable[PrefectFuture[Any]]] = None,
     return_type: Literal["state", "result"] = "result",
+    context: Optional[dict[str, Any]] = None,
 ) -> Union[R, State, None]:
     engine = AsyncFlowRunEngine[P, R](
-        flow=flow, parameters=parameters, flow_run=flow_run, wait_for=wait_for
+        flow=flow,
+        parameters=parameters,
+        flow_run=flow_run,
+        wait_for=wait_for,
+        context=context,
     )
 
     async with engine.start():
@@ -1371,12 +1395,17 @@ def run_generator_flow_sync(
     parameters: Optional[Dict[str, Any]] = None,
     wait_for: Optional[Iterable[PrefectFuture[Any]]] = None,
     return_type: Literal["state", "result"] = "result",
+    context: Optional[dict[str, Any]] = None,
 ) -> Generator[R, None, None]:
     if return_type != "result":
         raise ValueError("The return_type for a generator flow must be 'result'")
 
     engine = FlowRunEngine[P, R](
-        flow=flow, parameters=parameters, flow_run=flow_run, wait_for=wait_for
+        flow=flow,
+        parameters=parameters,
+        flow_run=flow_run,
+        wait_for=wait_for,
+        context=context,
     )
 
     with engine.start():
@@ -1407,12 +1436,17 @@ async def run_generator_flow_async(
     parameters: Optional[Dict[str, Any]] = None,
     wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
     return_type: Literal["state", "result"] = "result",
+    context: Optional[dict[str, Any]] = None,
 ) -> AsyncGenerator[R, None]:
     if return_type != "result":
         raise ValueError("The return_type for a generator flow must be 'result'")
 
     engine = AsyncFlowRunEngine[P, R](
-        flow=flow, parameters=parameters, flow_run=flow_run, wait_for=wait_for
+        flow=flow,
+        parameters=parameters,
+        flow_run=flow_run,
+        wait_for=wait_for,
+        context=context,
     )
 
     async with engine.start():
@@ -1446,8 +1480,23 @@ def run_flow(
     wait_for: Optional[Iterable[PrefectFuture[R]]] = None,
     return_type: Literal["state", "result"] = "result",
     error_logger: Optional[logging.Logger] = None,
-) -> Union[R, State, None]:
-    ret_val: Union[R, State, None] = None
+    context: Optional[dict[str, Any]] = None,
+) -> (
+    R
+    | State
+    | None
+    | Coroutine[Any, Any, R | State | None]
+    | Generator[R, None, None]
+    | AsyncGenerator[R, None]
+):
+    ret_val: Union[
+        R,
+        State,
+        None,
+        Coroutine[Any, Any, R | State | None],
+        Generator[R, None, None],
+        AsyncGenerator[R, None],
+    ] = None
 
     try:
         kwargs: dict[str, Any] = dict(
@@ -1458,6 +1507,7 @@ def run_flow(
             ),
             wait_for=wait_for,
             return_type=return_type,
+            context=context,
         )
 
         if flow.isasync and flow.isgenerator:
@@ -1492,3 +1542,113 @@ def _flow_parameters(
     parameters = flow_run.parameters if flow_run else {}
     call_args, call_kwargs = parameters_to_args_kwargs(flow.fn, parameters)
     return get_call_parameters(flow.fn, call_args, call_kwargs)
+
+
+def run_flow_in_subprocess(
+    flow: "Flow[..., Any]",
+    flow_run: "FlowRun | None" = None,
+    parameters: dict[str, Any] | None = None,
+    wait_for: Iterable[PrefectFuture[Any]] | None = None,
+    context: dict[str, Any] | None = None,
+) -> multiprocessing.context.SpawnProcess:
+    """
+    Run a flow in a subprocess.
+
+    Note the result of the flow will only be accessible if the flow is configured to
+    persist its result.
+
+    Args:
+        flow: The flow to run.
+        flow_run: The flow run object containing run metadata.
+        parameters: The parameters to use when invoking the flow.
+        wait_for: The futures to wait for before starting the flow.
+        context: A serialized context to hydrate before running the flow. If not provided,
+            the current context will be used. A serialized context should be provided if
+            this function is called in a separate memory space from the parent run (e.g.
+            in a subprocess or on another machine).
+
+    Returns:
+        A multiprocessing.context.SpawnProcess representing the process that is running the flow.
+    """
+    from prefect.flow_engine import run_flow
+
+    @wraps(run_flow)
+    def run_flow_with_env(
+        *args: Any,
+        env: dict[str, str] | None = None,
+        **kwargs: Any,
+    ):
+        """
+        Wrapper function to update environment variables and settings before running the flow.
+        """
+        engine_logger = logging.getLogger("prefect.engine")
+
+        os.environ.update(env or {})
+        settings_context = get_settings_context()
+        # Create a new settings context with a new settings object to pick up the updated
+        # environment variables
+        with SettingsContext(
+            profile=settings_context.profile,
+            settings=Settings(),
+        ):
+            try:
+                maybe_coro = run_flow(*args, **kwargs)
+                if asyncio.iscoroutine(maybe_coro):
+                    # This is running in a brand new process, so there won't be an existing
+                    # event loop.
+                    asyncio.run(maybe_coro)
+            except Abort as abort_signal:
+                abort_signal: Abort
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' aborted by orchestrator: {abort_signal}"
+                else:
+                    msg = f"Execution aborted by orchestrator: {abort_signal}"
+                engine_logger.info(msg)
+                exit(0)
+            except Pause as pause_signal:
+                pause_signal: Pause
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' is paused: {pause_signal}"
+                else:
+                    msg = f"Execution is paused: {pause_signal}"
+                engine_logger.info(msg)
+                exit(0)
+            except Exception:
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' exited with unexpected exception"
+                else:
+                    msg = "Execution exited with unexpected exception"
+                engine_logger.error(msg, exc_info=True)
+                exit(1)
+            except BaseException:
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' interrupted by base exception"
+                else:
+                    msg = "Execution interrupted by base exception"
+                engine_logger.error(msg, exc_info=True)
+                # Let the exit code be determined by the base exception type
+                raise
+
+    ctx = multiprocessing.get_context("spawn")
+
+    context = context or serialize_context()
+
+    process = ctx.Process(
+        target=cloudpickle_wrapped_call(
+            run_flow_with_env,
+            env=get_current_settings().to_environment_variables(exclude_unset=True)
+            | os.environ
+            | {
+                # TODO: make this a thing we can pass into the engine
+                "PREFECT__ENABLE_CANCELLATION_AND_CRASHED_HOOKS": "false",
+            },
+            flow=flow,
+            flow_run=flow_run,
+            parameters=parameters,
+            wait_for=wait_for,
+            context=context,
+        ),
+    )
+    process.start()
+
+    return process