prefect-client 3.3.5.dev3__py3-none-any.whl → 3.3.6.dev1__py3-none-any.whl

prefect/_build_info.py

@@ -1,5 +1,5 @@
 # Generated by versioningit
-__version__ = "3.3.5.dev3"
-__build_date__ = "2025-04-16 08:08:24.630955+00:00"
-__git_commit__ = "d01b30e8ec3cffd839c23ab42edc62983c3cb1a4"
+__version__ = "3.3.6.dev1"
+__build_date__ = "2025-04-19 08:07:12.903915+00:00"
+__git_commit__ = "c9c018d468694ea61ce3a482948bf4d03bb67227"
 __dirty__ = False

prefect/_experimental/bundles.py

@@ -8,6 +8,7 @@ import multiprocessing.context
 import os
 import subprocess
 import sys
+from pathlib import Path
 from typing import Any, TypedDict
 
 import cloudpickle
@@ -99,7 +100,9 @@ def extract_flow_from_bundle(bundle: SerializedBundle) -> Flow[Any, Any]:
 
 
 def _extract_and_run_flow(
-    bundle: SerializedBundle, env: dict[str, Any] | None = None
+    bundle: SerializedBundle,
+    cwd: Path | str | None = None,
+    env: dict[str, Any] | None = None,
 ) -> None:
     """
     Extracts a flow from a bundle and runs it.
@@ -108,6 +111,7 @@ def _extract_and_run_flow(
 
     Args:
         bundle: The bundle to extract and run.
+        cwd: The working directory to use when running the flow.
         env: The environment to use when running the flow.
     """
 
@@ -120,6 +124,9 @@ def _extract_and_run_flow(
     context = _deserialize_bundle_object(bundle["context"])
     flow_run = FlowRun.model_validate(bundle["flow_run"])
 
+    if cwd:
+        os.chdir(cwd)
+
     with SettingsContext(
         profile=settings_context.profile,
         settings=Settings(),
@@ -138,6 +145,8 @@ def _extract_and_run_flow(
 
 def execute_bundle_in_subprocess(
     bundle: SerializedBundle,
+    env: dict[str, Any] | None = None,
+    cwd: Path | str | None = None,
 ) -> multiprocessing.context.SpawnProcess:
     """
     Executes a bundle in a subprocess.
@@ -150,6 +159,7 @@ def execute_bundle_in_subprocess(
     """
 
     ctx = multiprocessing.get_context("spawn")
+    env = env or {}
 
     # Install dependencies if necessary
     if dependencies := bundle.get("dependencies"):
@@ -164,7 +174,9 @@ def execute_bundle_in_subprocess(
         kwargs={
             "bundle": bundle,
             "env": get_current_settings().to_environment_variables(exclude_unset=True)
-            | os.environ,
+            | os.environ
+            | env,
+            "cwd": cwd,
         },
     )
 

prefect/cache_policies.py

@@ -33,9 +33,9 @@ def _register_stable_transforms() -> None:
     so that cache keys that utilize them are deterministic across invocations.
     """
     try:
-        import pandas as pd
+        import pandas as pd  # pyright: ignore
 
-        STABLE_TRANSFORMS[pd.DataFrame] = lambda df: [
+        STABLE_TRANSFORMS[pd.DataFrame] = lambda df: [  # pyright: ignore
             df[col] for col in sorted(df.columns)
         ]
     except (ImportError, ModuleNotFoundError):
@@ -183,7 +183,7 @@ class CompoundCachePolicy(CachePolicy):
     Any keys that return `None` will be ignored.
     """
 
-    policies: list[CachePolicy] = field(default_factory=list)
+    policies: list[CachePolicy] = field(default_factory=lambda: [])
 
     def __post_init__(self) -> None:
         # flatten any CompoundCachePolicies
@@ -349,7 +349,7 @@ class Inputs(CachePolicy):
     Policy that computes a cache key based on a hash of the runtime inputs provided to the task..
     """
 
-    exclude: list[str] = field(default_factory=list)
+    exclude: list[str] = field(default_factory=lambda: [])
 
     def compute_key(
         self,

prefect/concurrency/context.py

@@ -13,7 +13,9 @@ class ConcurrencyContext(ContextModel):
     # Track the slots that have been acquired but were not able to be released
     # due to cancellation or some other error. These slots are released when
     # the context manager exits.
-    cleanup_slots: list[tuple[list[str], int, float]] = Field(default_factory=list)
+    cleanup_slots: list[tuple[list[str], int, float]] = Field(
+        default_factory=lambda: []
+    )
 
     def __exit__(self, *exc_info: Any) -> None:
         if self.cleanup_slots:

prefect/context.py

@@ -18,8 +18,6 @@ from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, TypeVar, Un
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
 from typing_extensions import Self
 
-import prefect.logging
-import prefect.logging.configuration
 import prefect.settings
 import prefect.types._datetime
 from prefect._internal.compatibility.migration import getattr_migration
@@ -128,7 +126,7 @@ class ContextModel(BaseModel):
         def __init__(self, **kwargs: Any) -> None: ...
 
     # The context variable for storing data must be defined by the child class
-    __var__: ClassVar[ContextVar[Self]]
+    __var__: ClassVar[ContextVar[Any]]
     _token: Optional[Token[Self]] = PrivateAttr(None)
     model_config: ClassVar[ConfigDict] = ConfigDict(
         arbitrary_types_allowed=True,

prefect/flows.py

@@ -40,6 +40,7 @@ from typing import (
 from uuid import UUID
 
 import pydantic
+from exceptiongroup import BaseExceptionGroup, ExceptionGroup
 from pydantic.v1 import BaseModel as V1BaseModel
 from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
 from pydantic.v1.errors import ConfigError  # TODO
@@ -105,6 +106,9 @@ from ._internal.pydantic.v2_validated_func import (
     V2ValidatedFunction as ValidatedFunction,
 )
 
+if TYPE_CHECKING:
+    from prefect.workers.base import BaseWorker
+
 T = TypeVar("T")  # Generic type var for capturing the inner return type of async funcs
 R = TypeVar("R")  # The return type of the user's function
 P = ParamSpec("P")  # The parameters of the flow
@@ -1984,6 +1988,141 @@ class FlowDecorator:
 flow: FlowDecorator = FlowDecorator()
 
 
+class InfrastructureBoundFlow(Flow[P, R]):
+    """
+    EXPERIMENTAL: This class is experimental and may be removed or changed in future
+        releases.
+
+    A flow that is bound to running on a specific infrastructure.
+
+    Attributes:
+        work_pool: The name of the work pool to run the flow on. The base job
+            configuration of the work pool will determine the configuration of the
+            infrastructure the flow will run on.
+        job_variables: Infrastructure configuration that will override the base job
+            configuration of the work pool.
+        worker_cls: The class of the worker to use to spin up infrastructure and submit
+            the flow to it.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        work_pool: str,
+        job_variables: dict[str, Any],
+        worker_cls: type["BaseWorker[Any, Any, Any]"],
+        **kwargs: Any,
+    ):
+        super().__init__(*args, **kwargs)
+        self.work_pool = work_pool
+        self.job_variables = job_variables
+        self.worker_cls = worker_cls
+
+    @overload
+    def __call__(self: "Flow[P, NoReturn]", *args: P.args, **kwargs: P.kwargs) -> None:
+        # `NoReturn` matches if a type can't be inferred for the function which stops a
+        # sync function from matching the `Coroutine` overload
+        ...
+
+    @overload
+    def __call__(
+        self: "Flow[P, Coroutine[Any, Any, T]]",
+        *args: P.args,
+        **kwargs: P.kwargs,
+    ) -> Coroutine[Any, Any, T]: ...
+
+    @overload
+    def __call__(
+        self: "Flow[P, T]",
+        *args: P.args,
+        **kwargs: P.kwargs,
+    ) -> T: ...
+
+    @overload
+    def __call__(
+        self: "Flow[P, Coroutine[Any, Any, T]]",
+        *args: P.args,
+        return_state: Literal[True],
+        **kwargs: P.kwargs,
+    ) -> Awaitable[State[T]]: ...
+
+    @overload
+    def __call__(
+        self: "Flow[P, T]",
+        *args: P.args,
+        return_state: Literal[True],
+        **kwargs: P.kwargs,
+    ) -> State[T]: ...
+
+    def __call__(
+        self,
+        *args: "P.args",
+        return_state: bool = False,
+        wait_for: Optional[Iterable[PrefectFuture[Any]]] = None,
+        **kwargs: "P.kwargs",
+    ):
+        async def modified_call(
+            *args: P.args,
+            return_state: bool = False,
+            # TODO: Handle wait_for once we have an asynchronous way to wait for futures
+            # We should wait locally for futures to resolve before spinning up
+            # infrastructure.
+            wait_for: Optional[Iterable[PrefectFuture[Any]]] = None,
+            **kwargs: P.kwargs,
+        ) -> R | State[R]:
+            try:
+                async with self.worker_cls(work_pool_name=self.work_pool) as worker:
+                    parameters = get_call_parameters(self, args, kwargs)
+                    future = await worker.submit(
+                        flow=self,
+                        parameters=parameters,
+                        job_variables=self.job_variables,
+                    )
+                    if return_state:
+                        await future.wait_async()
+                        return future.state
+                    return await future.aresult()
+            except (ExceptionGroup, BaseExceptionGroup) as exc:
+                # For less verbose tracebacks
+                exceptions = exc.exceptions
+                if len(exceptions) == 1:
+                    raise exceptions[0] from None
+                else:
+                    raise
+
+        if inspect.iscoroutinefunction(self.fn):
+            return modified_call(
+                *args, return_state=return_state, wait_for=wait_for, **kwargs
+            )
+        else:
+            return run_coro_as_sync(
+                modified_call(
+                    *args,
+                    return_state=return_state,
+                    wait_for=wait_for,
+                    **kwargs,
+                )
+            )
+
+
+def bind_flow_to_infrastructure(
+    flow: Flow[P, R],
+    work_pool: str,
+    worker_cls: type["BaseWorker[Any, Any, Any]"],
+    job_variables: dict[str, Any] | None = None,
+) -> InfrastructureBoundFlow[P, R]:
+    new = InfrastructureBoundFlow[P, R](
+        flow.fn,
+        work_pool=work_pool,
+        job_variables=job_variables or {},
+        worker_cls=worker_cls,
+    )
+    # Copy all attributes from the original flow
+    for attr, value in flow.__dict__.items():
+        setattr(new, attr, value)
+    return new
+
+
 def _raise_on_name_with_banned_characters(name: Optional[str]) -> Optional[str]:
     """
     Raise an InvalidNameError if the given name contains any invalid

prefect/runner/runner.py

@@ -636,7 +636,12 @@ class Runner:
 
                     return process
 
-    async def execute_bundle(self, bundle: SerializedBundle) -> None:
+    async def execute_bundle(
+        self,
+        bundle: SerializedBundle,
+        cwd: Path | str | None = None,
+        env: dict[str, str | None] | None = None,
+    ) -> None:
         """
         Executes a bundle in a subprocess.
         """
@@ -651,7 +656,7 @@ class Runner:
             if not self._acquire_limit_slot(flow_run.id):
                 return
 
-            process = execute_bundle_in_subprocess(bundle)
+            process = execute_bundle_in_subprocess(bundle, cwd=cwd, env=env)
 
             if process.pid is None:
                 # This shouldn't happen because `execute_bundle_in_subprocess` starts the process
@@ -776,7 +781,7 @@ class Runner:
         if command is None:
             runner_command = [get_sys_executable(), "-m", "prefect.engine"]
         else:
-            runner_command = shlex.split(command)
+            runner_command = shlex.split(command, posix=(os.name != "nt"))
 
         flow_run_logger = self._get_flow_run_logger(flow_run)
 

prefect/utilities/_ast.py

@@ -0,0 +1,117 @@
+import ast
+import math
+from typing import TYPE_CHECKING, Literal
+
+import anyio
+from typing_extensions import TypeAlias
+
+from prefect.logging.loggers import get_logger
+from prefect.settings import get_current_settings
+from prefect.utilities.asyncutils import LazySemaphore
+from prefect.utilities.filesystem import get_open_file_limit
+
+# Only allow half of the open file limit to be open at once to allow for other
+# actors to open files.
+OPEN_FILE_SEMAPHORE = LazySemaphore(lambda: math.floor(get_open_file_limit() * 0.5))
+
+# this potentially could be a TypedDict, but you
+# need some way to convince the type checker that
+# Literal["flow_name", "task_name"] are being provided
+DecoratedFnMetadata: TypeAlias = dict[str, str]
+
+
+async def find_prefect_decorated_functions_in_file(
+    path: anyio.Path, decorator_module: str, decorator_name: Literal["flow", "task"]
+) -> list[DecoratedFnMetadata]:
+    logger = get_logger()
+    decorator_name_key = f"{decorator_name}_name"
+    decorated_functions: list[DecoratedFnMetadata] = []
+
+    async with OPEN_FILE_SEMAPHORE:
+        try:
+            async with await anyio.open_file(path) as f:
+                try:
+                    tree = ast.parse(await f.read())
+                except SyntaxError:
+                    if get_current_settings().debug_mode:
+                        logger.debug(
+                            f"Could not parse {path} as a Python file. Skipping."
+                        )
+                    return decorated_functions
+        except Exception as exc:
+            if get_current_settings().debug_mode:
+                logger.debug(f"Could not open {path}: {exc}. Skipping.")
+            return decorated_functions
+
+    for node in ast.walk(tree):
+        if isinstance(
+            node,
+            (
+                ast.FunctionDef,
+                ast.AsyncFunctionDef,
+            ),
+        ):
+            for decorator in node.decorator_list:
+                # handles @decorator_name
+                is_name_match = (
+                    isinstance(decorator, ast.Name) and decorator.id == decorator_name
+                )
+                # handles @decorator_name()
+                is_func_name_match = (
+                    isinstance(decorator, ast.Call)
+                    and isinstance(decorator.func, ast.Name)
+                    and decorator.func.id == decorator_name
+                )
+                # handles @decorator_module.decorator_name
+                is_module_attribute_match = (
+                    isinstance(decorator, ast.Attribute)
+                    and isinstance(decorator.value, ast.Name)
+                    and decorator.value.id == decorator_module
+                    and decorator.attr == decorator_name
+                )
+                # handles @decorator_module.decorator_name()
+                is_module_attribute_func_match = (
+                    isinstance(decorator, ast.Call)
+                    and isinstance(decorator.func, ast.Attribute)
+                    and decorator.func.attr == decorator_name
+                    and isinstance(decorator.func.value, ast.Name)
+                    and decorator.func.value.id == decorator_module
+                )
+                if is_name_match or is_module_attribute_match:
+                    decorated_functions.append(
+                        {
+                            decorator_name_key: node.name,
+                            "function_name": node.name,
+                            "filepath": str(path),
+                        }
+                    )
+                if is_func_name_match or is_module_attribute_func_match:
+                    name_kwarg_node = None
+                    if TYPE_CHECKING:
+                        assert isinstance(decorator, ast.Call)
+                    for kw in decorator.keywords:
+                        if kw.arg == "name":
+                            name_kwarg_node = kw
+                            break
+                    if name_kwarg_node is not None and isinstance(
+                        name_kwarg_node.value, ast.Constant
+                    ):
+                        decorated_fn_name = name_kwarg_node.value.value
+                    else:
+                        decorated_fn_name = node.name
+                    decorated_functions.append(
+                        {
+                            decorator_name_key: decorated_fn_name,
+                            "function_name": node.name,
+                            "filepath": str(path),
+                        }
+                    )
+    return decorated_functions
+
+
+async def find_flow_functions_in_file(path: anyio.Path) -> list[DecoratedFnMetadata]:
+    return await find_prefect_decorated_functions_in_file(path, "prefect", "flow")
+
+
+async def find_task_functions_in_file(path: anyio.Path) -> list[DecoratedFnMetadata]:
+    return await find_prefect_decorated_functions_in_file(path, "prefect", "task")

prefect/workers/base.py

@@ -3,7 +3,11 @@ from __future__ import annotations
 import abc
 import asyncio
 import datetime
+import json
+import subprocess
+import tempfile
 import threading
+import uuid
 import warnings
 from contextlib import AsyncExitStack
 from functools import partial
@@ -21,6 +25,7 @@ from zoneinfo import ZoneInfo
 import anyio
 import anyio.abc
 import httpx
+from exceptiongroup import BaseExceptionGroup, ExceptionGroup
 from importlib_metadata import (
     distributions,  # type: ignore[reportUnknownVariableType] incomplete typing
 )
@@ -35,6 +40,7 @@ from prefect._internal.schemas.validators import return_v_or_none
 from prefect.client.base import ServerType
 from prefect.client.orchestration import PrefectClient, get_client
 from prefect.client.schemas.actions import WorkPoolCreate, WorkPoolUpdate
+from prefect.client.schemas.objects import Flow as APIFlow
 from prefect.client.schemas.objects import (
     Integration,
     StateType,
@@ -48,6 +54,7 @@ from prefect.exceptions import (
     Abort,
     ObjectNotFound,
 )
+from prefect.futures import PrefectFlowRunFuture
 from prefect.logging.loggers import (
     PrefectLogAdapter,
     flow_run_logger,
@@ -81,11 +88,12 @@ from prefect.utilities.templating import (
 from prefect.utilities.urls import url_for
 
 if TYPE_CHECKING:
-    from prefect.client.schemas.objects import Flow, FlowRun
+    from prefect.client.schemas.objects import FlowRun
     from prefect.client.schemas.responses import (
         DeploymentResponse,
         WorkerFlowRunResponse,
     )
+    from prefect.flows import Flow
 
 
 class BaseJobConfiguration(BaseModel):
@@ -217,7 +225,7 @@ class BaseJobConfiguration(BaseModel):
         self,
         flow_run: "FlowRun",
         deployment: "DeploymentResponse | None" = None,
-        flow: "Flow | None" = None,
+        flow: "APIFlow | None" = None,
         work_pool: "WorkPool | None" = None,
         worker_name: str | None = None,
     ) -> None:
@@ -314,7 +322,7 @@ class BaseJobConfiguration(BaseModel):
         return labels
 
     @staticmethod
-    def _base_flow_labels(flow: "Flow | None") -> dict[str, str]:
+    def _base_flow_labels(flow: "APIFlow | None") -> dict[str, str]:
         if flow is None:
             return {}
 
@@ -418,6 +426,7 @@ class BaseWorkerResult(BaseModel, abc.ABC):
 C = TypeVar("C", bound=BaseJobConfiguration)
 V = TypeVar("V", bound=BaseVariables)
 R = TypeVar("R", bound=BaseWorkerResult)
+FR = TypeVar("FR")  # used to capture the return type of a flow
 
 
 @register_base_type
@@ -685,6 +694,146 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
             "Workers must implement a method for running submitted flow runs"
         )
 
+    async def submit(
+        self,
+        flow: "Flow[..., FR]",
+        parameters: dict[str, Any] | None = None,
+        job_variables: dict[str, Any] | None = None,
+    ) -> "PrefectFlowRunFuture[FR]":
+        """
+        EXPERIMENTAL: The interface for this method is subject to change.
+
+        Submits a flow to run via the worker.
+
+        Args:
+            flow: The flow to submit
+            parameters: The parameters to pass to the flow
+
+        Returns:
+            A flow run object
+        """
+        warnings.warn(
+            "Ad-hoc flow submission via workers is experimental. The interface "
+            "and behavior of this feature are subject to change.",
+            category=FutureWarning,
+        )
+        if self._runs_task_group is None:
+            raise RuntimeError("Worker not properly initialized")
+
+        flow_run = await self._runs_task_group.start(
+            partial(
+                self._submit_adhoc_run,
+                flow=flow,
+                parameters=parameters,
+                job_variables=job_variables,
+            ),
+        )
+        return PrefectFlowRunFuture(flow_run_id=flow_run.id)
+
+    async def _submit_adhoc_run(
+        self,
+        flow: "Flow[..., FR]",
+        parameters: dict[str, Any] | None = None,
+        job_variables: dict[str, Any] | None = None,
+        task_status: anyio.abc.TaskStatus["FlowRun"] | None = None,
+    ):
+        """
+        Submits a flow run to the Kubernetes worker.
+        """
+        from prefect._experimental.bundles import (
+            convert_step_to_command,
+            create_bundle_for_flow_run,
+        )
+
+        if (
+            self.work_pool.storage_configuration.bundle_upload_step is None
+            or self.work_pool.storage_configuration.bundle_execution_step is None
+        ):
+            raise RuntimeError(
+                f"Storage is not configured for work pool {self.work_pool.name!r}. "
+                "Please configure storage for the work pool by running `prefect "
+                "work-pool storage configure`."
+            )
+
+        bundle_key = str(uuid.uuid4())
+        upload_command = convert_step_to_command(
+            self.work_pool.storage_configuration.bundle_upload_step,
+            bundle_key,
+            quiet=True,
+        )
+        execute_command = convert_step_to_command(
+            self.work_pool.storage_configuration.bundle_execution_step, bundle_key
+        )
+
+        job_variables = (job_variables or {}) | {"command": " ".join(execute_command)}
+        flow_run = await self.client.create_flow_run(
+            flow,
+            parameters=parameters,
+            state=Pending(),
+            job_variables=job_variables,
+            work_pool_name=self.work_pool.name,
+        )
+        if task_status is not None:
+            # Emit the flow run object to .submit to allow it to return a future as soon as possible
+            task_status.started(flow_run)
+        # Avoid an API call to get the flow
+        api_flow = APIFlow(id=flow_run.flow_id, name=flow.name, labels={})
+        logger = self.get_flow_run_logger(flow_run)
+
+        configuration = await self.job_configuration.from_template_and_values(
+            base_job_template=self.work_pool.base_job_template,
+            values=job_variables,
+            client=self._client,
+        )
+        configuration.prepare_for_flow_run(
+            flow_run=flow_run,
+            flow=api_flow,
+            work_pool=self.work_pool,
+            worker_name=self.name,
+        )
+
+        bundle = create_bundle_for_flow_run(flow=flow, flow_run=flow_run)
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            await (
+                anyio.Path(temp_dir)
+                .joinpath(bundle_key)
+                .write_bytes(json.dumps(bundle).encode("utf-8"))
+            )
+
+            try:
+                full_command = upload_command + [bundle_key]
+                logger.debug(
+                    "Uploading execution bundle with command: %s", full_command
+                )
+                await anyio.run_process(
+                    full_command,
+                    cwd=temp_dir,
+                )
+            except subprocess.CalledProcessError as e:
+                raise RuntimeError(e.stderr.decode("utf-8")) from e
+
+        logger.debug("Successfully uploaded execution bundle")
+
+        try:
+            result = await self.run(flow_run, configuration)
+
+            if result.status_code != 0:
+                await self._propose_crashed_state(
+                    flow_run,
+                    (
+                        "Flow run infrastructure exited with non-zero status code"
+                        f" {result.status_code}."
+                    ),
+                )
+        except Exception as exc:
+            # This flow run was being submitted and did not start successfully
+            logger.exception(
+                f"Failed to submit flow run '{flow_run.id}' to infrastructure."
+            )
+            message = f"Flow run could not be submitted to infrastructure:\n{exc!r}"
+            await self._propose_crashed_state(flow_run, message, client=self.client)
+
     @classmethod
     def __dispatch_key__(cls) -> str | None:
         if cls.__name__ == "BaseWorker":
@@ -1230,11 +1379,13 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
                 exc_info=True,
             )
 
-    async def _propose_crashed_state(self, flow_run: "FlowRun", message: str) -> None:
+    async def _propose_crashed_state(
+        self, flow_run: "FlowRun", message: str, client: PrefectClient | None = None
+    ) -> None:
         run_logger = self.get_flow_run_logger(flow_run)
         try:
             state = await propose_state(
-                self.client,
+                client or self.client,
                 Crashed(message=message),
                 flow_run_id=flow_run.id,
             )
@@ -1345,8 +1496,16 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
         return self
 
     async def __aexit__(self, *exc_info: Any) -> None:
-        self._logger.debug("Exiting worker context...")
-        await self.teardown(*exc_info)
+        try:
+            self._logger.debug("Exiting worker context...")
+            await self.teardown(*exc_info)
+        except (ExceptionGroup, BaseExceptionGroup) as exc:
+            # For less verbose tracebacks
+            exceptions = exc.exceptions
+            if len(exceptions) == 1:
+                raise exceptions[0] from None
+            else:
+                raise
 
     def __repr__(self) -> str:
         return f"Worker(pool={self._work_pool_name!r}, name={self.name!r})"

prefect/workers/process.py

@@ -22,15 +22,17 @@ import tempfile
 import threading
 from functools import partial
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Optional
+from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar
 
 import anyio
 import anyio.abc
 from pydantic import Field, field_validator
 
 from prefect._internal.schemas.validators import validate_working_dir
+from prefect.client.schemas.objects import Flow as APIFlow
 from prefect.runner.runner import Runner
 from prefect.settings import PREFECT_WORKER_QUERY_SECONDS
+from prefect.states import Pending
 from prefect.utilities.processutils import get_sys_executable
 from prefect.utilities.services import critical_service_loop
 from prefect.workers.base import (
@@ -41,8 +43,11 @@ from prefect.workers.base import (
 )
 
 if TYPE_CHECKING:
-    from prefect.client.schemas.objects import Flow, FlowRun, WorkPool
+    from prefect.client.schemas.objects import FlowRun, WorkPool
     from prefect.client.schemas.responses import DeploymentResponse
+    from prefect.flows import Flow
+
+FR = TypeVar("FR")  # used to capture the return type of a flow
 
 
 class ProcessJobConfiguration(BaseJobConfiguration):
@@ -60,7 +65,7 @@ class ProcessJobConfiguration(BaseJobConfiguration):
         self,
         flow_run: "FlowRun",
         deployment: "DeploymentResponse | None" = None,
-        flow: "Flow | None" = None,
+        flow: "APIFlow | None" = None,
         work_pool: "WorkPool | None" = None,
         worker_name: str | None = None,
     ) -> None:
@@ -231,6 +236,58 @@ class ProcessWorker(
             status_code=process.returncode, identifier=str(process.pid)
         )
 
+    async def _submit_adhoc_run(
+        self,
+        flow: "Flow[..., FR]",
+        parameters: dict[str, Any] | None = None,
+        job_variables: dict[str, Any] | None = None,
+        task_status: anyio.abc.TaskStatus["FlowRun"] | None = None,
+    ):
+        from prefect._experimental.bundles import (
+            create_bundle_for_flow_run,
+        )
+
+        flow_run = await self.client.create_flow_run(
+            flow,
+            parameters=parameters,
+            state=Pending(),
+            job_variables=job_variables,
+            work_pool_name=self.work_pool.name,
+        )
+        if task_status is not None:
+            # Emit the flow run object to .submit to allow it to return a future as soon as possible
+            task_status.started(flow_run)
+
+        api_flow = APIFlow(id=flow_run.flow_id, name=flow.name, labels={})
+        logger = self.get_flow_run_logger(flow_run)
+
+        configuration = await self.job_configuration.from_template_and_values(
+            base_job_template=self.work_pool.base_job_template,
+            values=job_variables or {},
+            client=self._client,
+        )
+        configuration.prepare_for_flow_run(
+            flow_run=flow_run,
+            flow=api_flow,
+            work_pool=self.work_pool,
+            worker_name=self.name,
+        )
+
+        bundle = create_bundle_for_flow_run(flow=flow, flow_run=flow_run)
+
+        logger.debug("Executing flow run bundle in subprocess...")
+        try:
+            await self._runner.execute_bundle(
+                bundle=bundle,
+                cwd=configuration.working_dir,
+                env=configuration.env,
+            )
+        except Exception:
+            logger.exception("Error executing flow run bundle in subprocess")
+            await self._propose_crashed_state(flow_run, "Flow run execution failed")
+        finally:
+            logger.debug("Flow run bundle execution complete")
+
     async def __aenter__(self) -> ProcessWorker:
         await super().__aenter__()
         self._runner = await self._exit_stack.enter_async_context(

{prefect_client-3.3.5.dev3.dist-info → prefect_client-3.3.6.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect-client
-Version: 3.3.5.dev3
+Version: 3.3.6.dev1
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io
@@ -47,7 +47,7 @@ Requires-Dist: prometheus-client>=0.20.0
 Requires-Dist: pydantic!=2.10.0,<3.0.0,>=2.9
 Requires-Dist: pydantic-core<3.0.0,>=2.12.0
 Requires-Dist: pydantic-extra-types<3.0.0,>=2.8.2
-Requires-Dist: pydantic-settings>2.2.1
+Requires-Dist: pydantic-settings!=2.9.0,<3.0.0,>2.2.1
 Requires-Dist: python-dateutil<3.0.0,>=2.8.2
 Requires-Dist: python-slugify<9.0,>=5.0
 Requires-Dist: python-socks[asyncio]<3.0,>=2.5.3

{prefect_client-3.3.5.dev3.dist-info → prefect_client-3.3.6.dev1.dist-info}/RECORD

@@ -1,20 +1,20 @@
 prefect/.prefectignore,sha256=awSprvKT0vI8a64mEOLrMxhxqcO-b0ERQeYpA2rNKVQ,390
 prefect/__init__.py,sha256=iCdcC5ZmeewikCdnPEP6YBAjPNV5dvfxpYCTpw30Hkw,3685
 prefect/__main__.py,sha256=WFjw3kaYJY6pOTA7WDOgqjsz8zUEUZHCcj3P5wyVa-g,66
-prefect/_build_info.py,sha256=8rZCXd-Nz2-o4Nm_4yyvy6osOkdGCLiuhickzD7KvFM,185
+prefect/_build_info.py,sha256=jisNjC8uPjptK7U97v_vxNXQXL36_gwcQrQgC-M9VF0,185
 prefect/_result_records.py,sha256=S6QmsODkehGVSzbMm6ig022PYbI6gNKz671p_8kBYx4,7789
 prefect/_waiters.py,sha256=Ia2ITaXdHzevtyWIgJoOg95lrEXQqNEOquHvw3T33UQ,9026
 prefect/agent.py,sha256=dPvG1jDGD5HSH7aM2utwtk6RaJ9qg13XjkA0lAIgQmY,287
 prefect/artifacts.py,sha256=dMBUOAWnUamzjb5HSqwB5-GR2Qb-Gxee26XG5NDCUuw,22720
 prefect/automations.py,sha256=ZzPxn2tINdlXTQo805V4rIlbXuNWxd7cdb3gTJxZIeY,12567
-prefect/cache_policies.py,sha256=Kwdei4JjitNfx42OepKpDNxwPtEwRgUUAn_soxsnNzI,12699
-prefect/context.py,sha256=LYEOlz7ZkuSDj7TmrE4mByy3N3TquFkIE2hEy0WHW1Y,23798
+prefect/cache_policies.py,sha256=jH1aDW6vItTcsEytuTCrNYyjbq87IQPwdOgF0yxiUts,12749
+prefect/context.py,sha256=IXS_ddSkQVFKFCQhjWk7Fwgfstfu6ITCmNeZ1beablY,23737
 prefect/engine.py,sha256=uB5JN4l045i5JTlRQNT1x7MwlSiGQ5Bop2Q6jHHOgxY,3699
 prefect/exceptions.py,sha256=wZLQQMRB_DyiYkeEdIC5OKwbba5A94Dlnics-lrWI7A,11581
 prefect/filesystems.py,sha256=v5YqGB4uXf9Ew2VuB9VCSkawvYMMVvEtZf7w1VmAmr8,18036
 prefect/flow_engine.py,sha256=hZpTYEtwTPMtwVoTCrfD93igN7rlKeG_0kyCvdU4aYE,58876
 prefect/flow_runs.py,sha256=dbHcXsOq1UsNM7vyJV9gboCTylmdUwQ_-W4NQt4R4ds,17267
-prefect/flows.py,sha256=kBLT6M903ZFD4TUvmvRN-zOdNcjSlIO9d2kHhPYq5Oo,109547
+prefect/flows.py,sha256=8gWWoZB8S8j8Iwz0TTc5F-f_8sTFucGm53aaue5vUi4,114116
 prefect/futures.py,sha256=ZD5rdgUHA4sfxwHaPToumOUKlyn4d989JHR7eI97-Hs,23271
 prefect/main.py,sha256=8V-qLB4GjEVCkGRgGXeaIk-JIXY8Z9FozcNluj4Sm9E,2589
 prefect/plugins.py,sha256=FPRLR2mWVBMuOnlzeiTD9krlHONZH2rtYLD753JQDNQ,2516
@@ -31,7 +31,7 @@ prefect/tasks.py,sha256=EpMw5O1B9pAFVraC0KzytMOKi8iy7ZYnKWRs7WtvogU,74742
 prefect/transactions.py,sha256=uIoPNudzJzH6NrMJhrgr5lyh6JxOJQqT1GvrXt69yNw,26068
 prefect/variables.py,sha256=dCK3vX7TbkqXZhnNT_v7rcGh3ISRqoR6pJVLpoll3Js,8342
 prefect/_experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-prefect/_experimental/bundles.py,sha256=EIZc6hsjQS8V2CH1RbsxOiFeR11l1IgvhiXTSO3uDyM,6386
+prefect/_experimental/bundles.py,sha256=E5nRaLVTbYCrACXZcRJCd4ssOcQU-Z26ewCb_7tPeTM,6687
 prefect/_experimental/lineage.py,sha256=8LssReoq7eLtQScUCu-7FCtrWoRZstXKRdpO0PxgbKg,9958
 prefect/_experimental/sla/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 prefect/_experimental/sla/client.py,sha256=XTkYHFZiBy_O7RgUyGEdl9MxaHP-6fEAKBk3ksNQobU,3611
@@ -122,7 +122,7 @@ prefect/concurrency/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 prefect/concurrency/_asyncio.py,sha256=uHjC3vQAiznRz_ueZE1RQ4x28zTcPJPoO2MMi0J41vU,2575
 prefect/concurrency/_events.py,sha256=KWHDldCWE3b5AH9eZ7kfmajvp36lRFCjCXIEx77jtKk,1825
 prefect/concurrency/asyncio.py,sha256=SUnRfqwBdBGwQll7SvywugVQnVbEzePqPFcUfIcTNMs,4505
-prefect/concurrency/context.py,sha256=8ZXs3G7NOF5Q2NqydK-K3zfjmYNnmfer-25hH6r6MgA,1009
+prefect/concurrency/context.py,sha256=kJWE2zGuoel9qiGOqHW5qnSyzV1INlsicTmeEEexoFo,1029
 prefect/concurrency/services.py,sha256=U_1Y8Mm-Fd4Nvn0gxiWc_UdacdqT-vKjzex-oJpUt50,2288
 prefect/concurrency/sync.py,sha256=MMRJvxK-Yzyt0WEEu95C2RaMwfLdYgYH6vejCqfSUmw,4687
 prefect/concurrency/v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -182,7 +182,7 @@ prefect/logging/highlighters.py,sha256=BCf_LNhFInIfGPqwuu8YVrGa4wVxNc4YXo2pYgftp
 prefect/logging/loggers.py,sha256=rwFJv0i3dhdKr25XX-xUkQy4Vv4dy18bTy366jrC0OQ,12741
 prefect/logging/logging.yml,sha256=tT7gTyC4NmngFSqFkCdHaw7R0GPNPDDsTCGZQByiJAQ,3169
 prefect/runner/__init__.py,sha256=pQBd9wVrUVUDUFJlgiweKSnbahoBZwqnd2O2jkhrULY,158
-prefect/runner/runner.py,sha256=D2sTbcUvFW5eiQLsQgzUO8hSTyWMvwQ7-nTg35twuIY,64962
+prefect/runner/runner.py,sha256=jv87XyaJ89uK0VzKpMzL3HfXgKZky8JlRs-gW04no5Y,65117
 prefect/runner/server.py,sha256=YRYFNoYddA9XfiTIYtudxrnD1vCX-PaOLhvyGUOb9AQ,11966
 prefect/runner/storage.py,sha256=L7aSjie5L6qbXYCDqYDX3ouQ_NsNMlmfjPeaWOC-ncs,28043
 prefect/runner/submit.py,sha256=qOEj-NChQ6RYFV35hHEVMTklrNmKwaGs2mR78ku9H0o,9474
@@ -278,6 +278,7 @@ prefect/types/__init__.py,sha256=yBjKxiQmSC7jXoo0UNmM3KZil1NBFS-BWGPfwSEaoJo,462
 prefect/types/_datetime.py,sha256=Cy6z7MxPDV_-jH2vxqC3PNA2G74IdUDIB07Jaakdj5w,7294
 prefect/types/entrypoint.py,sha256=2FF03-wLPgtnqR_bKJDB2BsXXINPdu8ptY9ZYEZnXg8,328
 prefect/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+prefect/utilities/_ast.py,sha256=sgEPUWElih-3cp4PoAy1IOyPtu8E27lL0Dldf3ijnYY,4905
 prefect/utilities/_deprecated.py,sha256=b3pqRSoFANdVJAc8TJkygBcP-VjZtLJUxVIWC7kwspI,1303
 prefect/utilities/_engine.py,sha256=9GW4X1lyAbmPwCuXXIubVJ7Z0DMT3dykkEUtp9tm5hI,3356
 prefect/utilities/_git.py,sha256=bPYWQdr9xvH0BqxR1ll1RkaSb3x0vhwylhYD5EilkKU,863
@@ -310,13 +311,13 @@ prefect/utilities/schema_tools/__init__.py,sha256=At3rMHd2g_Em2P3_dFQlFgqR_EpBwr
 prefect/utilities/schema_tools/hydration.py,sha256=NkRhWkNfxxFmVGhNDfmxdK_xeKaEhs3a42q83Sg9cT4,9436
 prefect/utilities/schema_tools/validation.py,sha256=Wix26IVR-ZJ32-6MX2pHhrwm3reB-Q4iB6_phn85OKE,10743
 prefect/workers/__init__.py,sha256=EaM1F0RZ-XIJaGeTKLsXDnfOPHzVWk5bk0_c4BVS44M,64
-prefect/workers/base.py,sha256=2GtGIJ4eAEVUaZwi6q1OMEbU_JhkEQNUmEznzj8HKbE,53364
+prefect/workers/base.py,sha256=l7ghNeSJh6DXiDdsnLmQor2-b82YaZJIz5TpFSn2dFY,59171
 prefect/workers/block.py,sha256=dPvG1jDGD5HSH7aM2utwtk6RaJ9qg13XjkA0lAIgQmY,287
 prefect/workers/cloud.py,sha256=dPvG1jDGD5HSH7aM2utwtk6RaJ9qg13XjkA0lAIgQmY,287
-prefect/workers/process.py,sha256=uxOwcqA2Ps-V-W6WeSdKCQMINrCxBEVx1K1Un8pb7vs,8973
+prefect/workers/process.py,sha256=Yi5D0U5AQ51wHT86GdwtImXSefe0gJf3LGq4r4z9zwM,11090
 prefect/workers/server.py,sha256=2pmVeJZiVbEK02SO6BEZaBIvHMsn6G8LzjW8BXyiTtk,1952
 prefect/workers/utilities.py,sha256=VfPfAlGtTuDj0-Kb8WlMgAuOfgXCdrGAnKMapPSBrwc,2483
-prefect_client-3.3.5.dev3.dist-info/METADATA,sha256=LcejbNc0zC2HoeBq6bkhgj5Pyne1GPZwmi9SLwEwCBc,7456
-prefect_client-3.3.5.dev3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-prefect_client-3.3.5.dev3.dist-info/licenses/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
-prefect_client-3.3.5.dev3.dist-info/RECORD,,
+prefect_client-3.3.6.dev1.dist-info/METADATA,sha256=PMgTb0Zkd5VixWs2eGC-cb6L7wdl9TRKu5ISxI5KoIo,7471
+prefect_client-3.3.6.dev1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+prefect_client-3.3.6.dev1.dist-info/licenses/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
+prefect_client-3.3.6.dev1.dist-info/RECORD,,