prefect-client 2.18.0__py3-none-any.whl → 2.18.2__py3-none-any.whl

prefect/settings.py

@@ -1214,6 +1214,36 @@ PREFECT_API_SERVICES_CANCELLATION_CLEANUP_LOOP_SECONDS = Setting(
 this often. Defaults to `20`.
 """
 
+PREFECT_API_SERVICES_FOREMAN_ENABLED = Setting(bool, default=True)
+"""Whether or not to start the Foreman service in the server application."""
+
+PREFECT_API_SERVICES_FOREMAN_LOOP_SECONDS = Setting(float, default=15)
+"""The number of seconds to wait between each iteration of the Foreman loop which checks
+for offline workers and updates work pool status."""
+
+
+PREFECT_API_SERVICES_FOREMAN_INACTIVITY_HEARTBEAT_MULTIPLE = Setting(int, default=3)
+"The number of heartbeats that must be missed before a worker is marked as offline."
+
+PREFECT_API_SERVICES_FOREMAN_FALLBACK_HEARTBEAT_INTERVAL_SECONDS = Setting(
+    int, default=30
+)
+"""The number of seconds to use for online/offline evaluation if a worker's heartbeat
+interval is not set."""
+
+PREFECT_API_SERVICES_FOREMAN_DEPLOYMENT_LAST_POLLED_TIMEOUT_SECONDS = Setting(
+    int, default=60
+)
+"""The number of seconds before a deployment is marked as not ready if it has not been
+polled."""
+
+PREFECT_API_SERVICES_FOREMAN_WORK_QUEUE_LAST_POLLED_TIMEOUT_SECONDS = Setting(
+    int, default=60
+)
+"""The number of seconds before a work queue is marked as not ready if it has not been
+polled."""
+
+
 PREFECT_API_DEFAULT_LIMIT = Setting(
     int,
     default=200,
@@ -1579,6 +1609,11 @@ PREFECT_EXPERIMENTAL_ENABLE_WORK_QUEUE_STATUS = Setting(bool, default=True)
 Whether or not to enable experimental work queue status in-place of work queue health.
 """
 
+PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE = Setting(bool, default=False)
+"""
+Whether or not to enable experimental new engine.
+"""
+
 
 # Defaults -----------------------------------------------------------------------------
 
@@ -1693,11 +1728,23 @@ PREFECT_API_SERVICES_EVENT_PERSISTER_FLUSH_INTERVAL = Setting(float, default=5,
 The maximum number of seconds between flushes of the event persister.
 """
 
+PREFECT_EVENTS_RETENTION_PERIOD = Setting(timedelta, default=timedelta(days=7))
+"""
+The amount of time to retain events in the database.
+"""
+
 PREFECT_API_EVENTS_STREAM_OUT_ENABLED = Setting(bool, default=True)
 """
 Whether or not to allow streaming events out of via websockets.
 """
 
+PREFECT_API_EVENTS_RELATED_RESOURCE_CACHE_TTL = Setting(
+    timedelta, default=timedelta(minutes=5)
+)
+"""
+How long to cache related resource data for emitting server-side vents
+"""
+
 # Deprecated settings ------------------------------------------------------------------
 
 

prefect/states.py

@@ -486,8 +486,10 @@ class StateGroup:
 
 
 def Scheduled(
-    cls: Type[State] = State, scheduled_time: datetime.datetime = None, **kwargs
-) -> State:
+    cls: Type[State[R]] = State,
+    scheduled_time: Optional[datetime.datetime] = None,
+    **kwargs: Any,
+) -> State[R]:
     """Convenience function for creating `Scheduled` states.
 
     Returns:
@@ -503,7 +505,7 @@ def Scheduled(
     return cls(type=StateType.SCHEDULED, state_details=state_details, **kwargs)
 
 
-def Completed(cls: Type[State] = State, **kwargs) -> State:
+def Completed(cls: Type[State[R]] = State, **kwargs: Any) -> State[R]:
     """Convenience function for creating `Completed` states.
 
     Returns:
@@ -512,7 +514,7 @@ def Completed(cls: Type[State] = State, **kwargs) -> State:
     return cls(type=StateType.COMPLETED, **kwargs)
 
 
-def Running(cls: Type[State] = State, **kwargs) -> State:
+def Running(cls: Type[State[R]] = State, **kwargs: Any) -> State[R]:
     """Convenience function for creating `Running` states.
 
     Returns:
@@ -521,7 +523,7 @@ def Running(cls: Type[State] = State, **kwargs) -> State:
     return cls(type=StateType.RUNNING, **kwargs)
 
 
-def Failed(cls: Type[State] = State, **kwargs) -> State:
+def Failed(cls: Type[State[R]] = State, **kwargs: Any) -> State[R]:
     """Convenience function for creating `Failed` states.
 
     Returns:
@@ -530,7 +532,7 @@ def Failed(cls: Type[State] = State, **kwargs) -> State:
     return cls(type=StateType.FAILED, **kwargs)
 
 
-def Crashed(cls: Type[State] = State, **kwargs) -> State:
+def Crashed(cls: Type[State[R]] = State, **kwargs: Any) -> State[R]:
     """Convenience function for creating `Crashed` states.
 
     Returns:
@@ -539,7 +541,7 @@ def Crashed(cls: Type[State] = State, **kwargs) -> State:
     return cls(type=StateType.CRASHED, **kwargs)
 
 
-def Cancelling(cls: Type[State] = State, **kwargs) -> State:
+def Cancelling(cls: Type[State[R]] = State, **kwargs: Any) -> State[R]:
     """Convenience function for creating `Cancelling` states.
 
     Returns:
@@ -548,7 +550,7 @@ def Cancelling(cls: Type[State] = State, **kwargs) -> State:
     return cls(type=StateType.CANCELLING, **kwargs)
 
 
-def Cancelled(cls: Type[State] = State, **kwargs) -> State:
+def Cancelled(cls: Type[State[R]] = State, **kwargs: Any) -> State[R]:
     """Convenience function for creating `Cancelled` states.
 
     Returns:
@@ -557,7 +559,7 @@ def Cancelled(cls: Type[State] = State, **kwargs) -> State:
     return cls(type=StateType.CANCELLED, **kwargs)
 
 
-def Pending(cls: Type[State] = State, **kwargs) -> State:
+def Pending(cls: Type[State[R]] = State, **kwargs: Any) -> State[R]:
     """Convenience function for creating `Pending` states.
 
     Returns:
@@ -567,13 +569,13 @@ def Pending(cls: Type[State] = State, **kwargs) -> State:
 
 
 def Paused(
-    cls: Type[State] = State,
+    cls: Type[State[R]] = State,
     timeout_seconds: Optional[int] = None,
     pause_expiration_time: Optional[datetime.datetime] = None,
     reschedule: bool = False,
     pause_key: Optional[str] = None,
-    **kwargs,
-) -> State:
+    **kwargs: Any,
+) -> State[R]:
     """Convenience function for creating `Paused` states.
 
     Returns:
@@ -603,11 +605,11 @@ def Paused(
 
 
 def Suspended(
-    cls: Type[State] = State,
+    cls: Type[State[R]] = State,
     timeout_seconds: Optional[int] = None,
     pause_expiration_time: Optional[datetime.datetime] = None,
     pause_key: Optional[str] = None,
-    **kwargs,
+    **kwargs: Any,
 ):
     """Convenience function for creating `Suspended` states.
 
@@ -626,8 +628,10 @@ def Suspended(
 
 
 def AwaitingRetry(
-    cls: Type[State] = State, scheduled_time: datetime.datetime = None, **kwargs
-) -> State:
+    cls: Type[State[R]] = State,
+    scheduled_time: Optional[datetime.datetime] = None,
+    **kwargs: Any,
+) -> State[R]:
     """Convenience function for creating `AwaitingRetry` states.
 
     Returns:
@@ -638,7 +642,7 @@ def AwaitingRetry(
     )
 
 
-def Retrying(cls: Type[State] = State, **kwargs) -> State:
+def Retrying(cls: Type[State[R]] = State, **kwargs: Any) -> State[R]:
     """Convenience function for creating `Retrying` states.
 
     Returns:
@@ -648,8 +652,10 @@ def Retrying(cls: Type[State] = State, **kwargs) -> State:
 
 
 def Late(
-    cls: Type[State] = State, scheduled_time: datetime.datetime = None, **kwargs
-) -> State:
+    cls: Type[State[R]] = State,
+    scheduled_time: Optional[datetime.datetime] = None,
+    **kwargs: Any,
+) -> State[R]:
     """Convenience function for creating `Late` states.
 
     Returns:

prefect/tasks.py

@@ -22,25 +22,30 @@ from typing import (
     List,
     NoReturn,
     Optional,
+    Set,
     TypeVar,
     Union,
     cast,
     overload,
 )
+from uuid import uuid4
 
 from typing_extensions import Literal, ParamSpec
 
 from prefect._internal.concurrency.api import create_call, from_async, from_sync
 from prefect.client.schemas import TaskRun
-from prefect.context import FlowRunContext, PrefectObjectRegistry
+from prefect.client.schemas.objects import TaskRunInput
+from prefect.context import FlowRunContext, PrefectObjectRegistry, TagsContext
 from prefect.futures import PrefectFuture
+from prefect.logging.loggers import get_logger, get_run_logger
 from prefect.results import ResultSerializer, ResultStorage
 from prefect.settings import (
+    PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE,
     PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING,
     PREFECT_TASK_DEFAULT_RETRIES,
     PREFECT_TASK_DEFAULT_RETRY_DELAY_SECONDS,
 )
-from prefect.states import State
+from prefect.states import Pending, State
 from prefect.task_runners import BaseTaskRunner
 from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import Async, Sync
@@ -64,6 +69,8 @@ T = TypeVar("T")  # Generic type var for capturing the inner return type of asyn
 R = TypeVar("R")  # The return type of the user's function
 P = ParamSpec("P")  # The parameters of the task
 
+logger = get_logger("tasks")
+
 
 def task_input_hash(
     context: "TaskRunContext", arguments: Dict[str, Any]
@@ -189,14 +196,14 @@ class Task(Generic[P, R]):
     def __init__(
         self,
         fn: Callable[P, R],
-        name: str = None,
-        description: str = None,
-        tags: Iterable[str] = None,
-        version: str = None,
-        cache_key_fn: Callable[
-            ["TaskRunContext", Dict[str, Any]], Optional[str]
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        tags: Optional[Iterable[str]] = None,
+        version: Optional[str] = None,
+        cache_key_fn: Optional[
+            Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
         ] = None,
-        cache_expiration: datetime.timedelta = None,
+        cache_expiration: Optional[datetime.timedelta] = None,
         task_run_name: Optional[Union[Callable[[], str], str]] = None,
         retries: Optional[int] = None,
         retry_delay_seconds: Optional[
@@ -213,7 +220,7 @@ class Task(Generic[P, R]):
         result_serializer: Optional[ResultSerializer] = None,
         result_storage_key: Optional[str] = None,
         cache_result_in_memory: bool = True,
-        timeout_seconds: Union[int, float] = None,
+        timeout_seconds: Union[int, float, None] = None,
         log_prints: Optional[bool] = False,
         refresh_cache: Optional[bool] = None,
         on_completion: Optional[List[Callable[["Task", TaskRun, State], None]]] = None,
@@ -325,6 +332,7 @@ class Task(Generic[P, R]):
         self.result_serializer = result_serializer
         self.result_storage_key = result_storage_key
         self.cache_result_in_memory = cache_result_in_memory
+
         self.timeout_seconds = float(timeout_seconds) if timeout_seconds else None
         # Warn if this task's `name` conflicts with another task while having a
         # different function. This is to detect the case where two or more tasks
@@ -529,6 +537,53 @@ class Task(Generic[P, R]):
             viz_return_value=viz_return_value or self.viz_return_value,
         )
 
+    async def create_run(
+        self,
+        flow_run_context: FlowRunContext,
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]],
+        extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
+    ) -> TaskRun:
+        # TODO: Investigate if we can replace create_task_run on the task run engine
+        # with this method. Would require updating to work without the flow run context.
+        from prefect.utilities.engine import (
+            _dynamic_key_for_task_run,
+            collect_task_run_inputs,
+        )
+
+        dynamic_key = _dynamic_key_for_task_run(flow_run_context, self)
+        task_inputs = {
+            k: await collect_task_run_inputs(v) for k, v in parameters.items()
+        }
+        if wait_for:
+            task_inputs["wait_for"] = await collect_task_run_inputs(wait_for)
+
+        # Join extra task inputs
+        extra_task_inputs = extra_task_inputs or {}
+        for k, extras in extra_task_inputs.items():
+            task_inputs[k] = task_inputs[k].union(extras)
+
+        flow_run_logger = get_run_logger(flow_run_context)
+
+        task_run = await flow_run_context.client.create_task_run(
+            task=self,
+            name=f"{self.name} - {dynamic_key}",
+            flow_run_id=flow_run_context.flow_run.id,
+            dynamic_key=dynamic_key,
+            state=Pending(),
+            extra_tags=TagsContext.get().current_tags,
+            task_inputs=task_inputs,
+        )
+
+        if flow_run_context.flow_run:
+            flow_run_logger.info(
+                f"Created task run {task_run.name!r} for task {self.name!r}"
+            )
+        else:
+            logger.info(f"Created task run {task_run.name!r} for task {self.name!r}")
+
+        return task_run
+
     @overload
     def __call__(
         self: "Task[P, NoReturn]",
@@ -582,6 +637,22 @@ class Task(Generic[P, R]):
                 self.isasync, self.name, parameters, self.viz_return_value
             )
 
+        # new engine currently only compatible with async tasks
+        if PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE.value():
+            from prefect.new_task_engine import run_task, run_task_sync
+
+            run_kwargs = dict(
+                task=self,
+                parameters=parameters,
+                wait_for=wait_for,
+                return_type=return_type,
+            )
+            if self.isasync:
+                # this returns an awaitable coroutine
+                return run_task(**run_kwargs)
+            else:
+                return run_task_sync(**run_kwargs)
+
         if (
             PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING.value()
             and not FlowRunContext.get()
@@ -810,6 +881,7 @@ class Task(Generic[P, R]):
         # Convert the call args/kwargs to a parameter dict
         parameters = get_call_parameters(self.fn, args, kwargs)
         return_type = "state" if return_state else "future"
+        flow_run_context = FlowRunContext.get()
 
         task_viz_tracker = get_task_viz_tracker()
         if task_viz_tracker:
@@ -817,10 +889,7 @@ class Task(Generic[P, R]):
                 "`task.submit()` is not currently supported by `flow.visualize()`"
             )
 
-        if (
-            PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING.value()
-            and not FlowRunContext.get()
-        ):
+        if PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING and not flow_run_context:
             create_autonomous_task_run_call = create_call(
                 create_autonomous_task_run, task=self, parameters=parameters
             )
@@ -832,16 +901,74 @@ class Task(Generic[P, R]):
                 return from_sync.wait_for_call_in_loop_thread(
                     create_autonomous_task_run_call
                 )
+        if PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE and flow_run_context:
+            if self.isasync:
+                return self._submit_async(
+                    parameters=parameters,
+                    flow_run_context=flow_run_context,
+                    wait_for=wait_for,
+                    return_state=return_state,
+                )
+            else:
+                raise NotImplementedError(
+                    "Submitting sync tasks with the new engine has not be implemented yet."
+                )
 
-        return enter_task_run_engine(
-            self,
+        else:
+            return enter_task_run_engine(
+                self,
+                parameters=parameters,
+                wait_for=wait_for,
+                return_type=return_type,
+                task_runner=None,  # Use the flow's task runner
+                mapped=False,
+            )
+
+    async def _submit_async(
+        self,
+        parameters: Dict[str, Any],
+        flow_run_context: FlowRunContext,
+        wait_for: Optional[Iterable[PrefectFuture]],
+        return_state: bool,
+    ):
+        from prefect.new_task_engine import run_task
+
+        task_runner = flow_run_context.task_runner
+
+        task_run = await self.create_run(
+            flow_run_context=flow_run_context,
             parameters=parameters,
             wait_for=wait_for,
-            return_type=return_type,
-            task_runner=None,  # Use the flow's task runner
-            mapped=False,
         )
 
+        future = PrefectFuture(
+            name=task_run.name,
+            key=uuid4(),
+            task_runner=task_runner,
+            asynchronous=(self.isasync and flow_run_context.flow.isasync),
+        )
+        future.task_run = task_run
+        flow_run_context.task_run_futures.append(future)
+        await task_runner.submit(
+            key=future.key,
+            call=partial(
+                run_task,
+                task=self,
+                task_run=task_run,
+                parameters=parameters,
+                wait_for=wait_for,
+                return_type="state",
+            ),
+        )
+        # TODO: I don't like this. Can we move responsibility for creating the future
+        # and setting this anyio.Event to the task runner?
+        future._submitted.set()
+
+        if return_state:
+            return await future.wait()
+        else:
+            return future
+
     @overload
     def map(
         self: "Task[P, NoReturn]",

prefect/utilities/asyncutils.py

@@ -1,12 +1,15 @@
 """
 Utilities for interoperability with async functions and workers from various contexts.
 """
+
 import asyncio
 import ctypes
 import inspect
 import threading
 import warnings
+from concurrent.futures import ThreadPoolExecutor
 from contextlib import asynccontextmanager
+from contextvars import copy_context
 from functools import partial, wraps
 from threading import Thread
 from typing import (
@@ -20,6 +23,7 @@ from typing import (
     Type,
     TypeVar,
     Union,
+    cast,
 )
 from uuid import UUID, uuid4
 
@@ -78,6 +82,43 @@ def is_async_gen_fn(func):
     return inspect.isasyncgenfunction(func)
 
 
+def run_sync(coroutine: Coroutine[Any, Any, T]) -> T:
+    """
+    Runs a coroutine from a synchronous context. A thread will be spawned
+    to run the event loop if necessary, which allows coroutines to run in
+    environments like Jupyter notebooks where the event loop runs on the main
+    thread.
+
+    Args:
+        coroutine: The coroutine to run.
+
+    Returns:
+        The return value of the coroutine.
+
+    Example:
+        Basic usage:
+        ```python
+        async def my_async_function(x: int) -> int:
+            return x + 1
+
+        run_sync(my_async_function(1))
+        ```
+    """
+    # ensure context variables are properly copied to the async frame
+    context = copy_context()
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = None
+
+    if loop and loop.is_running():
+        with ThreadPoolExecutor() as executor:
+            future = executor.submit(context.run, asyncio.run, coroutine)
+            return cast(T, future.result())
+    else:
+        return context.run(asyncio.run, coroutine)
+
+
 async def run_sync_in_worker_thread(
     __fn: Callable[..., T], *args: Any, **kwargs: Any
 ) -> T:

prefect/utilities/engine.py

@@ -11,6 +11,7 @@ from typing import (
     Iterable,
     Optional,
     Set,
+    TypeVar,
     Union,
 )
 from uuid import UUID, uuid4
@@ -66,6 +67,7 @@ from prefect.utilities.text import truncated_to
 API_HEALTHCHECKS = {}
 UNTRACKABLE_TYPES = {bool, type(None), type(...), type(NotImplemented)}
 engine_logger = get_logger("engine")
+T = TypeVar("T")
 
 
 async def collect_task_run_inputs(expr: Any, max_depth: int = -1) -> Set[TaskRunInput]:
@@ -308,11 +310,11 @@ async def resolve_inputs(
 
 async def propose_state(
     client: PrefectClient,
-    state: State,
+    state: State[object],
     force: bool = False,
-    task_run_id: UUID = None,
-    flow_run_id: UUID = None,
-) -> State:
+    task_run_id: Optional[UUID] = None,
+    flow_run_id: Optional[UUID] = None,
+) -> State[object]:
     """
     Propose a new state for a flow run or task run, invoking Prefect orchestration logic.
 

prefect/utilities/schema_tools/validation.py

@@ -232,7 +232,7 @@ def preprocess_schema(schema):
         process_properties(schema["properties"], required_fields)
 
     if "definitions" in schema:  # Also process definitions for reused models
-        for definition in schema["definitions"].values():
+        for definition in (schema["definitions"] or {}).values():
             if "properties" in definition:
                 required_fields = definition.get("required", [])
                 process_properties(definition["properties"], required_fields)

prefect/workers/process.py

@@ -13,6 +13,7 @@ to poll for flow runs.
 For more information about work pools and workers,
 checkout out the [Prefect docs](/concepts/work-pools/).
 """
+
 import asyncio
 import contextlib
 import os
@@ -142,7 +143,7 @@ class ProcessWorker(BaseWorker):
         "Execute flow runs as subprocesses on a worker. Works well for local execution"
         " when first getting started."
     )
-    _display_name = "Local Subprocess"
+    _display_name = "Process"
     _documentation_url = (
         "https://docs.prefect.io/latest/api-ref/prefect/workers/process/"
     )

{prefect_client-2.18.0.dist-info → prefect_client-2.18.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 2.18.0
+Version: 2.18.2
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.