prefect-client 2.14.20__py3-none-any.whl → 2.15.0__py3-none-any.whl

prefect/engine.py

@@ -192,6 +192,8 @@ from prefect.states import (
 from prefect.task_runners import (
     CONCURRENCY_MESSAGES,
     BaseTaskRunner,
+    ConcurrentTaskRunner,
+    SequentialTaskRunner,
     TaskConcurrencyType,
 )
 from prefect.tasks import Task
@@ -850,7 +852,11 @@ async def orchestrate_flow_run(
                     not parent_flow_run_context
                     or (
                         parent_flow_run_context
-                        and parent_flow_run_context.flow.isasync == flow.isasync
+                        and
+                        # Unless the parent is async and the child is sync, run the
+                        # child flow in the parent thread; running a sync child in
+                        # an async parent could be bad for async performance.
+                        not (parent_flow_run_context.flow.isasync and not flow.isasync)
                     )
                 ):
                     from_async.call_soon_in_waiting_thread(
@@ -1341,12 +1347,13 @@ async def resume_flow_run(flow_run_id, run_input: Optional[Dict] = None):
         run_input: a dictionary of inputs to provide to the flow run.
     """
     client = get_client()
-    flow_run = await client.read_flow_run(flow_run_id)
+    async with client:
+        flow_run = await client.read_flow_run(flow_run_id)
 
-    if not flow_run.state.is_paused():
-        raise NotPausedError("Cannot resume a run that isn't paused!")
+        if not flow_run.state.is_paused():
+            raise NotPausedError("Cannot resume a run that isn't paused!")
 
-    response = await client.resume_flow_run(flow_run_id, run_input=run_input)
+        response = await client.resume_flow_run(flow_run_id, run_input=run_input)
 
     if response.status == SetStateStatus.REJECT:
         if response.state.type == StateType.FAILED:
@@ -1369,7 +1376,17 @@ def enter_task_run_engine(
 
     if not flow_run_context:
         if PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING.value():
-            return _create_autonomous_task_run(task=task, parameters=parameters)
+            create_autonomous_task_run = create_call(
+                _create_autonomous_task_run, task=task, parameters=parameters
+            )
+            if task.isasync:
+                return from_async.wait_for_call_in_loop_thread(
+                    create_autonomous_task_run
+                )
+            else:
+                return from_sync.wait_for_call_in_loop_thread(
+                    create_autonomous_task_run
+                )
 
         raise RuntimeError(
             "Tasks cannot be run outside of a flow"
@@ -1394,6 +1411,7 @@ def enter_task_run_engine(
         wait_for=wait_for,
         return_type=return_type,
         task_runner=task_runner,
+        user_thread=threading.current_thread(),
     )
 
     if task.isasync and flow_run_context.flow.isasync:
@@ -1410,6 +1428,7 @@ async def begin_task_map(
     wait_for: Optional[Iterable[PrefectFuture]],
     return_type: EngineReturnType,
     task_runner: Optional[BaseTaskRunner],
+    user_thread: threading.Thread,
 ) -> List[Union[PrefectFuture, Awaitable[PrefectFuture]]]:
     """Async entrypoint for task mapping"""
     # We need to resolve some futures to map over their data, collect the upstream
@@ -1487,6 +1506,7 @@ async def begin_task_map(
                 return_type=return_type,
                 task_runner=task_runner,
                 extra_task_inputs=task_inputs,
+                user_thread=user_thread,
             )
         )
 
@@ -1551,6 +1571,7 @@ async def get_task_call_return_value(
     wait_for: Optional[Iterable[PrefectFuture]],
     return_type: EngineReturnType,
     task_runner: Optional[BaseTaskRunner],
+    user_thread: threading.Thread,
     extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
 ):
     extra_task_inputs = extra_task_inputs or {}
@@ -1562,6 +1583,7 @@ async def get_task_call_return_value(
         wait_for=wait_for,
         task_runner=task_runner,
         extra_task_inputs=extra_task_inputs,
+        user_thread=user_thread,
     )
     if return_type == "future":
         return future
@@ -1580,6 +1602,7 @@ async def create_task_run_future(
     wait_for: Optional[Iterable[PrefectFuture]],
     task_runner: Optional[BaseTaskRunner],
     extra_task_inputs: Dict[str, Set[TaskRunInput]],
+    user_thread: threading.Thread,
 ) -> PrefectFuture:
     # Default to the flow run's task runner
     task_runner = task_runner or flow_run_context.task_runner
@@ -1617,6 +1640,7 @@ async def create_task_run_future(
             wait_for=wait_for,
             task_runner=task_runner,
             extra_task_inputs=extra_task_inputs,
+            user_thread=user_thread,
         )
     )
 
@@ -1640,6 +1664,7 @@ async def create_task_run_then_submit(
     wait_for: Optional[Iterable[PrefectFuture]],
     task_runner: BaseTaskRunner,
     extra_task_inputs: Dict[str, Set[TaskRunInput]],
+    user_thread: threading.Thread,
 ) -> None:
     task_run = (
         await create_task_run(
@@ -1666,6 +1691,7 @@ async def create_task_run_then_submit(
         task_run=task_run,
         wait_for=wait_for,
         task_runner=task_runner,
+        user_thread=user_thread,
     )
 
     future._submitted.set()
@@ -1713,6 +1739,7 @@ async def submit_task_run(
     task_run: TaskRun,
     wait_for: Optional[Iterable[PrefectFuture]],
     task_runner: BaseTaskRunner,
+    user_thread: threading.Thread,
 ) -> PrefectFuture:
     logger = get_run_logger(flow_run_context)
 
@@ -1722,6 +1749,10 @@ async def submit_task_run(
     ):
         logger.info(f"Executing {task_run.name!r} immediately...")
 
+    if not isinstance(task_runner, (ConcurrentTaskRunner, SequentialTaskRunner)):
+        # Only pass the user thread to "local" task runners
+        user_thread = None
+
     future = await task_runner.submit(
         key=future.key,
         call=partial(
@@ -1735,10 +1766,15 @@ async def submit_task_run(
             ),
             log_prints=should_log_prints(task),
             settings=prefect.context.SettingsContext.get().copy(),
+            user_thread=user_thread,
+            concurrency_type=task_runner.concurrency_type,
         ),
     )
 
-    if task_runner.concurrency_type != TaskConcurrencyType.SEQUENTIAL:
+    if (
+        task_runner.concurrency_type != TaskConcurrencyType.SEQUENTIAL
+        and not flow_run_context.autonomous_task_run
+    ):
         logger.info(f"Submitted task run {task_run.name!r} for execution.")
 
     return future
@@ -1752,6 +1788,8 @@ async def begin_task_run(
     result_factory: ResultFactory,
     log_prints: bool,
     settings: prefect.context.SettingsContext,
+    user_thread: Optional[threading.Thread],
+    concurrency_type: TaskConcurrencyType,
 ):
     """
     Entrypoint for task run execution.
@@ -1822,6 +1860,8 @@ async def begin_task_run(
                 log_prints=log_prints,
                 interruptible=interruptible,
                 client=client,
+                user_thread=user_thread,
+                concurrency_type=concurrency_type,
             )
 
             if not maybe_flow_run_context:
@@ -1872,6 +1912,8 @@ async def orchestrate_task_run(
     log_prints: bool,
     interruptible: bool,
     client: PrefectClient,
+    concurrency_type: TaskConcurrencyType,
+    user_thread: Optional[threading.Thread],
 ) -> State:
     """
     Execute a task run
@@ -1903,6 +1945,7 @@ async def orchestrate_task_run(
         flow_run = flow_run_context.flow_run
     else:
         flow_run = await client.read_flow_run(task_run.flow_run_id)
+
     logger = task_run_logger(task_run, task=task, flow_run=flow_run)
 
     partial_task_run_context = PartialModel(
@@ -2101,9 +2144,41 @@ async def orchestrate_task_run(
                         "Beginning execution...", extra={"state_message": True}
                     )
 
-                call = from_async.call_soon_in_new_thread(
-                    create_call(task.fn, *args, **kwargs), timeout=task.timeout_seconds
-                )
+                call = create_call(task.fn, *args, **kwargs)
+
+                if (
+                    flow_run_context
+                    and user_thread
+                    and (
+                        # Async and sync tasks can be executed on synchronous flows
+                        # if the task runner is sequential; if the task is sync and a
+                        # concurrent task runner is used, we must execute it in a worker
+                        # thread instead.
+                        (
+                            concurrency_type == TaskConcurrencyType.SEQUENTIAL
+                            and (
+                                flow_run_context.flow
+                                and not flow_run_context.flow.isasync
+                            )
+                        )
+                        # Async tasks can always be executed on asynchronous flow; if the
+                        # flow is async we do not want to block the event loop with
+                        # synchronous tasks
+                        or (
+                            flow_run_context.flow
+                            and flow_run_context.flow.isasync
+                            and task.isasync
+                        )
+                    )
+                ):
+                    from_async.call_soon_in_waiting_thread(
+                        call, thread=user_thread, timeout=task.timeout_seconds
+                    )
+                else:
+                    from_async.call_soon_in_new_thread(
+                        call, timeout=task.timeout_seconds
+                    )
+
                 result = await call.aresult()
 
             except (CancelledError, asyncio.CancelledError) as exc:
@@ -2200,7 +2275,6 @@ async def orchestrate_task_run(
         level=logging.INFO if state.is_completed() else logging.ERROR,
         msg=f"Finished in state {display_state}",
     )
-    logger.warning(f"Task run {task_run.name!r} finished in state {display_state}")
     return state
 
 
@@ -2433,8 +2507,7 @@ async def resolve_inputs(
             #       incorrectly evaluate to false — to resolve this, we must track all
             #       annotations wrapping the current expression but this is not yet
             #       implemented.
-            isinstance(context.get("annotation"), allow_failure)
-            and state.is_failed()
+            isinstance(context.get("annotation"), allow_failure) and state.is_failed()
         ):
             raise UpstreamTaskError(
                 f"Upstream task run '{state.state_details.task_run_id}' did not reach a"
@@ -2914,15 +2987,14 @@ def _emit_task_run_state_change_event(
     )
 
 
-@sync_compatible
 async def _create_autonomous_task_run(
     task: Task, parameters: Dict[str, Any]
 ) -> TaskRun:
     async with get_client() as client:
-        scheduled = Scheduled()
+        state = Scheduled()
         if parameters:
             parameters_id = uuid4()
-            scheduled.state_details.task_parameters_id = parameters_id
+            state.state_details.task_parameters_id = parameters_id
 
             # TODO: We want to use result storage for parameters, but we'll need
             # a better way to use it than this.
@@ -2934,7 +3006,7 @@ async def _create_autonomous_task_run(
             task=task,
             flow_run_id=None,
             dynamic_key=f"{task.task_key}-{str(uuid4())[:NUM_CHARS_DYNAMIC_KEY]}",
-            state=scheduled,
+            state=state,
         )
 
         engine_logger.debug(f"Submitted run of task {task.name!r} for execution")

prefect/events/clients.py

@@ -21,7 +21,7 @@ try:
     from cachetools import TTLCache
 except ImportError:
     pass
-from starlette.status import WS_1008_POLICY_VIOLATION
+from prefect._vendor.starlette.status import WS_1008_POLICY_VIOLATION
 from websockets.client import WebSocketClientProtocol, connect
 from websockets.exceptions import (
     ConnectionClosed,

prefect/events/utilities.py

@@ -8,7 +8,7 @@ from prefect._internal.schemas.fields import DateTimeTZ
 
 from .clients import AssertingEventsClient, PrefectCloudEventsClient
 from .schemas import Event, RelatedResource
-from .worker import EventsWorker
+from .worker import EventsWorker, emit_events_to_cloud
 
 TIGHT_TIMING = timedelta(minutes=5)
 
@@ -42,6 +42,9 @@ def emit_event(
         The event that was emitted if worker is using a client that emit
         events, otherwise None.
     """
+    if not emit_events_to_cloud():
+        return None
+
     operational_clients = [AssertingEventsClient, PrefectCloudEventsClient]
     worker_instance = EventsWorker.instance()
 

prefect/events/worker.py

@@ -14,6 +14,15 @@ from .related import related_resources_from_run_context
 from .schemas import Event
 
 
+def emit_events_to_cloud() -> bool:
+    api = PREFECT_API_URL.value()
+    return (
+        experiment_enabled("events_client")
+        and api
+        and api.startswith(PREFECT_CLOUD_API_URL.value())
+    )
+
+
 class EventsWorker(QueueService[Event]):
     def __init__(
         self, client_type: Type[EventsClient], client_options: Tuple[Tuple[str, Any]]
@@ -52,12 +61,7 @@ class EventsWorker(QueueService[Event]):
 
         # Select a client type for this worker based on settings
         if client_type is None:
-            api = PREFECT_API_URL.value()
-            if (
-                experiment_enabled("events_client")
-                and api
-                and api.startswith(PREFECT_CLOUD_API_URL.value())
-            ):
+            if emit_events_to_cloud():
                 client_type = PrefectCloudEventsClient
                 client_kwargs = {
                     "api_url": PREFECT_API_URL.value(),

prefect/filesystems.py

@@ -709,13 +709,13 @@ class Azure(WritableFileSystem, WritableDeploymentStorage):
     def filesystem(self) -> RemoteFileSystem:
         settings = {}
         if self.azure_storage_connection_string:
-            settings["connection_string"] = (
-                self.azure_storage_connection_string.get_secret_value()
-            )
+            settings[
+                "connection_string"
+            ] = self.azure_storage_connection_string.get_secret_value()
         if self.azure_storage_account_name:
-            settings["account_name"] = (
-                self.azure_storage_account_name.get_secret_value()
-            )
+            settings[
+                "account_name"
+            ] = self.azure_storage_account_name.get_secret_value()
         if self.azure_storage_account_key:
             settings["account_key"] = self.azure_storage_account_key.get_secret_value()
         if self.azure_storage_tenant_id:
@@ -723,9 +723,9 @@ class Azure(WritableFileSystem, WritableDeploymentStorage):
         if self.azure_storage_client_id:
             settings["client_id"] = self.azure_storage_client_id.get_secret_value()
         if self.azure_storage_client_secret:
-            settings["client_secret"] = (
-                self.azure_storage_client_secret.get_secret_value()
-            )
+            settings[
+                "client_secret"
+            ] = self.azure_storage_client_secret.get_secret_value()
         settings["anon"] = self.azure_storage_anon
         self._remote_file_system = RemoteFileSystem(
             basepath=self.basepath, settings=settings

prefect/flow_runs.py

@@ -7,6 +7,7 @@ from prefect.client.orchestration import PrefectClient
 from prefect.client.schemas import FlowRun
 from prefect.client.utilities import inject_client
 from prefect.exceptions import FlowRunWaitTimeout
+from prefect.logging import get_logger
 
 
 @inject_client
@@ -15,6 +16,7 @@ async def wait_for_flow_run(
     timeout: Optional[int] = 10800,
     poll_interval: int = 5,
     client: Optional[PrefectClient] = None,
+    log_states: bool = False,
 ) -> FlowRun:
     """
     Waits for the prefect flow run to finish and returns the FlowRun
@@ -71,14 +73,16 @@ async def wait_for_flow_run(
             ```
     """
     assert client is not None, "Client injection failed"
+    logger = get_logger()
     with anyio.move_on_after(timeout):
         while True:
             flow_run = await client.read_flow_run(flow_run_id)
             flow_state = flow_run.state
+            if log_states:
+                logger.info(f"Flow run is in state {flow_run.state.name!r}")
             if flow_state and flow_state.is_final():
                 return flow_run
             await anyio.sleep(poll_interval)
-
     raise FlowRunWaitTimeout(
         f"Flow run with ID {flow_run_id} exceeded watch timeout of {timeout} seconds"
     )

prefect/futures.py

@@ -360,7 +360,7 @@ async def resolve_futures_to_data(
 
 
 async def resolve_futures_to_states(
-    expr: Union[PrefectFuture[R, Any], Any]
+    expr: Union[PrefectFuture[R, Any], Any],
 ) -> Union[State[R], Any]:
     """
     Given a Python built-in collection, recursively find `PrefectFutures` and build a

prefect/infrastructure/container.py

@@ -398,9 +398,9 @@ class DockerContainer(Infrastructure):
             return await super().generate_work_pool_base_job_template()
         for key, value in self.dict(exclude_unset=True, exclude_defaults=True).items():
             if key == "command":
-                base_job_template["variables"]["properties"]["command"]["default"] = (
-                    shlex.join(value)
-                )
+                base_job_template["variables"]["properties"]["command"][
+                    "default"
+                ] = shlex.join(value)
             elif key == "image_registry":
                 self.logger.warning(
                     "Image registry blocks are not supported by Docker"

prefect/infrastructure/kubernetes.py

@@ -376,9 +376,9 @@ class KubernetesJob(Infrastructure):
         ), "Failed to retrieve default base job template."
         for key, value in self.dict(exclude_unset=True, exclude_defaults=True).items():
             if key == "command":
-                base_job_template["variables"]["properties"]["command"]["default"] = (
-                    shlex.join(value)
-                )
+                base_job_template["variables"]["properties"]["command"][
+                    "default"
+                ] = shlex.join(value)
             elif key in [
                 "type",
                 "block_type_slug",
@@ -892,9 +892,7 @@ class KubernetesJob(Infrastructure):
                     prefix,
                     max_length=253,
                     regex_pattern=r"[^a-zA-Z0-9-\.]+",
-                ).strip(
-                    "_-."
-                )  # Must start or end with alphanumeric characters
+                ).strip("_-.")  # Must start or end with alphanumeric characters
                 or prefix
             )
 

prefect/infrastructure/process.py

@@ -264,9 +264,9 @@ class Process(Infrastructure):
         ), "Failed to generate default base job template for Process worker."
         for key, value in self.dict(exclude_unset=True, exclude_defaults=True).items():
             if key == "command":
-                base_job_template["variables"]["properties"]["command"]["default"] = (
-                    shlex.join(value)
-                )
+                base_job_template["variables"]["properties"]["command"][
+                    "default"
+                ] = shlex.join(value)
             elif key in [
                 "type",
                 "block_type_slug",

prefect/input/run_input.py

@@ -379,7 +379,7 @@ class AutomaticRunInput(RunInput, Generic[T]):
 
 
 def run_input_subclass_from_type(
-    _type: Union[Type[R], Type[T], pydantic.BaseModel]
+    _type: Union[Type[R], Type[T], pydantic.BaseModel],
 ) -> Union[Type[AutomaticRunInput[T]], Type[R]]:
     """
     Create a new `RunInput` subclass from the given type.

prefect/logging/formatters.py

@@ -80,7 +80,7 @@ class PrefectFormatter(logging.Formatter):
         *,
         defaults=None,
         task_run_fmt: str = None,
-        flow_run_fmt: str = None
+        flow_run_fmt: str = None,
     ) -> None:
         """
         Implementation of the standard Python formatter with support for multiple

prefect/results.py

@@ -185,10 +185,9 @@ class ResultFactory(pydantic.BaseModel):
                 persist_result=(
                     flow.persist_result
                     if flow.persist_result is not None
-                    else
                     # !! Child flows persist their result by default if the it or the
                     #    parent flow uses a feature that requires it
-                    (
+                    else (
                         flow_features_require_result_persistence(flow)
                         or flow_features_require_child_result_persistence(ctx.flow)
                         or get_default_persist_setting()
@@ -209,10 +208,9 @@ class ResultFactory(pydantic.BaseModel):
                 persist_result=(
                     flow.persist_result
                     if flow.persist_result is not None
-                    else
                     # !! Flows persist their result by default if uses a feature that
                     #    requires it
-                    (
+                    else (
                         flow_features_require_result_persistence(flow)
                         or get_default_persist_setting()
                     )
@@ -246,10 +244,9 @@ class ResultFactory(pydantic.BaseModel):
         persist_result = (
             task.persist_result
             if task.persist_result is not None
-            else
             # !! Tasks persist their result by default if their parent flow uses a
             #    feature that requires it or the task uses a feature that requires it
-            (
+            else (
                 (
                     flow_features_require_child_result_persistence(ctx.flow)
                     if ctx

prefect/runner/server.py

@@ -87,7 +87,7 @@ async def _build_endpoint_for_deployment(
     deployment: "DeploymentResponse", runner: "Runner"
 ) -> Callable:
     async def _create_flow_run_for_deployment(
-        body: Optional[Dict[Any, Any]] = None
+        body: Optional[Dict[Any, Any]] = None,
     ) -> JSONResponse:
         body = body or {}
         if deployment.enforce_parameter_schema and deployment.parameter_openapi_schema:
@@ -139,9 +139,9 @@ async def get_deployment_router(
             )
 
             # Used for updating the route schemas later on
-            schemas[f"{deployment.name}-{deployment_id}"] = (
-                deployment.parameter_openapi_schema
-            )
+            schemas[
+                f"{deployment.name}-{deployment_id}"
+            ] = deployment.parameter_openapi_schema
             schemas[deployment_id] = deployment.name
     return router, schemas
 

prefect/settings.py

@@ -39,6 +39,7 @@ settings to be dynamically modified on retrieval. This allows us to make setting
 dependent on the value of other settings or perform other dynamic effects.
 
 """
+
 import logging
 import os
 import string
@@ -420,9 +421,7 @@ def warn_on_misconfigured_api_url(values):
             )
 
         if warnings_list:
-            example = (
-                'e.g. PREFECT_API_URL="https://api.prefect.cloud/api/accounts/[ACCOUNT-ID]/workspaces/[WORKSPACE-ID]"'
-            )
+            example = 'e.g. PREFECT_API_URL="https://api.prefect.cloud/api/accounts/[ACCOUNT-ID]/workspaces/[WORKSPACE-ID]"'
             warnings_list.append(example)
 
             warnings.warn("\n".join(warnings_list), stacklevel=2)
@@ -1416,6 +1415,22 @@ PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS = Setting(
 Whether or not to delete failed task submissions from the database.
 """
 
+PREFECT_TASK_SCHEDULING_MAX_SCHEDULED_QUEUE_SIZE = Setting(
+    int,
+    default=1000,
+)
+"""
+The maximum number of scheduled tasks to queue for submission.
+"""
+
+PREFECT_TASK_SCHEDULING_MAX_RETRY_QUEUE_SIZE = Setting(
+    int,
+    default=100,
+)
+"""
+The maximum number of retries to queue for submission.
+"""
+
 PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS = Setting(bool, default=False)
 """
 Whether or not to enable experimental worker webserver endpoints.
@@ -1447,6 +1462,11 @@ PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING = Setting(bool, default=False)
 Whether or not to enable experimental task scheduling.
 """
 
+PREFECT_EXPERIMENTAL_ENABLE_WORK_QUEUE_STATUS = Setting(bool, default=True)
+"""
+Whether or not to enable experimental work queue status in-place of work queue health.
+"""
+
 # Defaults -----------------------------------------------------------------------------
 
 PREFECT_DEFAULT_RESULT_STORAGE_BLOCK = Setting(

prefect/software/pip.py

@@ -68,7 +68,7 @@ def _is_editable_install(dist: "importlib_metadata.Distribution") -> bool:
 
 
 def _remove_distributions_required_by_others(
-    dists: Dict[str, "importlib_metadata.Distribution"]
+    dists: Dict[str, "importlib_metadata.Distribution"],
 ) -> Dict[str, "importlib_metadata.Distribution"]:
     # Collect all child requirements
     child_requirement_names = set()