prefect-client 3.0.0rc7__py3-none-any.whl → 3.0.0rc9__py3-none-any.whl

prefect/settings.py

@@ -92,21 +92,6 @@ T = TypeVar("T")
 
 DEFAULT_PROFILES_PATH = Path(__file__).parent.joinpath("profiles.toml")
 
-# When we remove the experimental settings we also want to add them to the set of REMOVED_EXPERIMENTAL_FLAGS.
-# The reason for this is removing the settings entirely causes the CLI to crash for anyone who has them in one or more of their profiles.
-# Adding them to REMOVED_EXPERIMENTAL_FLAGS will make it so that the user is warned about it and they have time to take action.
-REMOVED_EXPERIMENTAL_FLAGS = {
-    "PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_SCHEDULING_UI",
-    "PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_DEPLOYMENT_PARAMETERS",
-    "PREFECT_EXPERIMENTAL_ENABLE_EVENTS_CLIENT",
-    "PREFECT_EXPERIMENTAL_ENABLE_EVENTS",
-    "PREFECT_EXPERIMENTAL_EVENTS",
-    "PREFECT_EXPERIMENTAL_WARN_EVENTS_CLIENT",
-    "PREFECT_EXPERIMENTAL_ENABLE_FLOW_RUN_INFRA_OVERRIDES",
-    "PREFECT_EXPERIMENTAL_WARN_FLOW_RUN_INFRA_OVERRIDES",
-    "PREFECT_EXPERIMENTAL_ENABLE_WORK_POOLS",
-}
-
 
 class Setting(Generic[T]):
     """
@@ -423,7 +408,7 @@ def default_result_storage_block_name(
     settings: Optional["Settings"] = None, value: Optional[str] = None
 ):
     """
-    `value_callback` for `PREFECT_DEFAULT_RESULT_STORAGE_BLOCK_NAME` that sets the default
+    `value_callback` for `PREFECT_DEFAULT_RESULT_STORAGE_BLOCK` that sets the default
     value to the hostname of the machine.
     """
     if value is None:
@@ -959,41 +944,6 @@ interpreted and lead to incomplete output, e.g.
 `DROP TABLE [dbo].[SomeTable];"` outputs `DROP TABLE .[SomeTable];`.
 """
 
-PREFECT_TASK_INTROSPECTION_WARN_THRESHOLD = Setting(
-    float,
-    default=10.0,
-)
-"""
-Threshold time in seconds for logging a warning if task parameter introspection
-exceeds this duration. Parameter introspection can be a significant performance hit
-when the parameter is a large collection object, e.g. a large dictionary or DataFrame,
-and each element needs to be inspected. See `prefect.utilities.annotations.quote`
-for more details.
-Defaults to `10.0`.
-Set to `0` to disable logging the warning.
-"""
-
-PREFECT_AGENT_QUERY_INTERVAL = Setting(
-    float,
-    default=15,
-)
-"""
-The agent loop interval, in seconds. Agents will check for new runs this often.
-Defaults to `15`.
-"""
-
-PREFECT_AGENT_PREFETCH_SECONDS = Setting(
-    int,
-    default=15,
-)
-"""
-Agents will look for scheduled runs this many seconds in
-the future and attempt to run them. This accounts for any additional
-infrastructure spin-up time or latency in preparing a flow run. Note
-flow runs will not start before their scheduled time, even if they are
-prefetched. Defaults to `15`.
-"""
-
 PREFECT_ASYNC_FETCH_STATE_RESULT = Setting(bool, default=False)
 """
 Determines whether `State.result()` fetches results automatically or not.
@@ -1373,16 +1323,6 @@ PREFECT_API_MAX_FLOW_RUN_GRAPH_ARTIFACTS = Setting(int, default=10000)
 The maximum number of artifacts to show on a flow run graph on the v2 API
 """
 
-PREFECT_EXPERIMENTAL_ENABLE_ARTIFACTS_ON_FLOW_RUN_GRAPH = Setting(bool, default=True)
-"""
-Whether or not to enable artifacts on the flow run graph.
-"""
-
-PREFECT_EXPERIMENTAL_ENABLE_STATES_ON_FLOW_RUN_GRAPH = Setting(bool, default=True)
-"""
-Whether or not to enable flow run states on the flow run graph.
-"""
-
 PREFECT_EXPERIMENTAL_ENABLE_WORKERS = Setting(bool, default=True)
 """
 Whether or not to enable experimental Prefect workers.
@@ -1393,11 +1333,6 @@ PREFECT_EXPERIMENTAL_WARN_WORKERS = Setting(bool, default=False)
 Whether or not to warn when experimental Prefect workers are used.
 """
 
-PREFECT_EXPERIMENTAL_WARN_VISUALIZE = Setting(bool, default=False)
-"""
-Whether or not to warn when experimental Prefect visualize is used.
-"""
-
 PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_CANCELLATION = Setting(bool, default=True)
 """
 Whether or not to enable experimental enhanced flow run cancellation.
@@ -1408,26 +1343,6 @@ PREFECT_EXPERIMENTAL_WARN_ENHANCED_CANCELLATION = Setting(bool, default=False)
 Whether or not to warn when experimental enhanced flow run cancellation is used.
 """
 
-PREFECT_EXPERIMENTAL_ENABLE_DEPLOYMENT_STATUS = Setting(bool, default=True)
-"""
-Whether or not to enable deployment status in the UI
-"""
-
-PREFECT_EXPERIMENTAL_WARN_DEPLOYMENT_STATUS = Setting(bool, default=False)
-"""
-Whether or not to warn when deployment status is used.
-"""
-
-PREFECT_EXPERIMENTAL_FLOW_RUN_INPUT = Setting(bool, default=False)
-"""
-Whether or not to enable flow run input.
-"""
-
-PREFECT_EXPERIMENTAL_WARN_FLOW_RUN_INPUT = Setting(bool, default=True)
-"""
-Whether or not to enable flow run input.
-"""
-
 
 # Prefect Events feature flags
 
@@ -1554,31 +1469,6 @@ PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS = Setting(bool, default=False
 Whether or not to enable experimental worker webserver endpoints.
 """
 
-PREFECT_EXPERIMENTAL_ENABLE_ARTIFACTS = Setting(bool, default=True)
-"""
-Whether or not to enable experimental Prefect artifacts.
-"""
-
-PREFECT_EXPERIMENTAL_WARN_ARTIFACTS = Setting(bool, default=False)
-"""
-Whether or not to warn when experimental Prefect artifacts are used.
-"""
-
-PREFECT_EXPERIMENTAL_ENABLE_WORKSPACE_DASHBOARD = Setting(bool, default=True)
-"""
-Whether or not to enable the experimental workspace dashboard.
-"""
-
-PREFECT_EXPERIMENTAL_WARN_WORKSPACE_DASHBOARD = Setting(bool, default=False)
-"""
-Whether or not to warn when the experimental workspace dashboard is enabled.
-"""
-
-PREFECT_EXPERIMENTAL_ENABLE_WORK_QUEUE_STATUS = Setting(bool, default=True)
-"""
-Whether or not to enable experimental work queue status in-place of work queue health.
-"""
-
 PREFECT_EXPERIMENTAL_DISABLE_SYNC_COMPAT = Setting(bool, default=False)
 """
 Whether or not to disable the sync_compatible decorator utility.
@@ -1663,11 +1553,6 @@ PREFECT_EVENTS_MAXIMUM_SIZE_BYTES = Setting(int, default=1_500_000)
 The maximum size of an Event when serialized to JSON
 """
 
-PREFECT_API_SERVICES_EVENT_LOGGER_ENABLED = Setting(bool, default=True)
-"""
-Whether or not to start the event debug logger service in the server application.
-"""
-
 PREFECT_API_SERVICES_TRIGGERS_ENABLED = Setting(bool, default=True)
 """
 Whether or not to start the triggers service in the server application.
@@ -1715,6 +1600,18 @@ PREFECT_API_EVENTS_RELATED_RESOURCE_CACHE_TTL = Setting(
 How long to cache related resource data for emitting server-side vents
 """
 
+PREFECT_EVENTS_MAXIMUM_WEBSOCKET_BACKFILL = Setting(
+    timedelta, default=timedelta(minutes=15)
+)
+"""
+The maximum range to look back for backfilling events for a websocket subscriber
+"""
+
+PREFECT_EVENTS_WEBSOCKET_BACKFILL_PAGE_SIZE = Setting(int, default=250, gt=0)
+"""
+The page size for the queries to backfill events for websocket subscribers
+"""
+
 
 # Deprecated settings ------------------------------------------------------------------
 
@@ -2215,26 +2112,6 @@ class ProfilesCollection:
         )
 
 
-def _handle_removed_flags(
-    profile_name: str, settings: Dict[str, Any]
-) -> Dict[str, Any]:
-    to_remove = [name for name in settings if name in REMOVED_EXPERIMENTAL_FLAGS]
-
-    for name in to_remove:
-        warnings.warn(
-            (
-                f"Experimental flag {name!r} has been removed, please "
-                f"update your {profile_name!r} profile."
-            ),
-            UserWarning,
-            stacklevel=3,
-        )
-
-        settings.pop(name)
-
-    return settings
-
-
 def _read_profiles_from(path: Path) -> ProfilesCollection:
     """
     Read profiles from a path into a new `ProfilesCollection`.
@@ -2253,7 +2130,6 @@ def _read_profiles_from(path: Path) -> ProfilesCollection:
 
     profiles = []
     for name, settings in raw_profiles.items():
-        settings = _handle_removed_flags(name, settings)
         profiles.append(Profile(name=name, settings=settings, source=path))
 
     return ProfilesCollection(profiles, active=active_profile)

prefect/states.py

@@ -1,3 +1,4 @@
+import asyncio
 import datetime
 import sys
 import traceback
@@ -22,15 +23,21 @@ from prefect.exceptions import (
     TerminationSignal,
     UnfinishedRun,
 )
+from prefect.logging.loggers import get_logger
 from prefect.results import BaseResult, R, ResultFactory
 from prefect.settings import PREFECT_ASYNC_FETCH_STATE_RESULT
 from prefect.utilities.annotations import BaseAnnotation
 from prefect.utilities.asyncutils import in_async_main_thread, sync_compatible
 from prefect.utilities.collections import ensure_iterable
 
+logger = get_logger("states")
+
 
 def get_state_result(
-    state: State[R], raise_on_failure: bool = True, fetch: Optional[bool] = None
+    state: State[R],
+    raise_on_failure: bool = True,
+    fetch: Optional[bool] = None,
+    retry_result_failure: bool = True,
 ) -> R:
     """
     Get the result from a state.
@@ -58,11 +65,50 @@ def get_state_result(
 
         return state.data
     else:
-        return _get_state_result(state, raise_on_failure=raise_on_failure)
+        return _get_state_result(
+            state,
+            raise_on_failure=raise_on_failure,
+            retry_result_failure=retry_result_failure,
+        )
+
+
+RESULT_READ_MAXIMUM_ATTEMPTS = 10
+RESULT_READ_RETRY_DELAY = 0.25
+
+
+async def _get_state_result_data_with_retries(
+    state: State[R], retry_result_failure: bool = True
+) -> R:
+    # Results may be written asynchronously, possibly after their corresponding
+    # state has been written and events have been emitted, so we should give some
+    # grace here about missing results.  The exception below could come in the form
+    # of a missing file, a short read, or other types of errors depending on the
+    # result storage backend.
+    if retry_result_failure is False:
+        max_attempts = 1
+    else:
+        max_attempts = RESULT_READ_MAXIMUM_ATTEMPTS
+
+    for i in range(1, max_attempts + 1):
+        try:
+            return await state.data.get()
+        except Exception as e:
+            if i == max_attempts:
+                raise
+            logger.debug(
+                "Exception %r while reading result, retry %s/%s in %ss...",
+                e,
+                i,
+                max_attempts,
+                RESULT_READ_RETRY_DELAY,
+            )
+            await asyncio.sleep(RESULT_READ_RETRY_DELAY)
 
 
 @sync_compatible
-async def _get_state_result(state: State[R], raise_on_failure: bool) -> R:
+async def _get_state_result(
+    state: State[R], raise_on_failure: bool, retry_result_failure: bool = True
+) -> R:
     """
     Internal implementation for `get_state_result` without async backwards compatibility
     """
@@ -81,7 +127,10 @@ async def _get_state_result(state: State[R], raise_on_failure: bool) -> R:
         raise await get_state_exception(state)
 
     if isinstance(state.data, BaseResult):
-        result = await state.data.get()
+        result = await _get_state_result_data_with_retries(
+            state, retry_result_failure=retry_result_failure
+        )
+
     elif state.data is None:
         if state.is_failed() or state.is_crashed() or state.is_cancelled():
             return await get_state_exception(state)
@@ -352,7 +401,7 @@ async def get_state_exception(state: State) -> BaseException:
         raise ValueError(f"Expected failed or crashed state got {state!r}.")
 
     if isinstance(state.data, BaseResult):
-        result = await state.data.get()
+        result = await _get_state_result_data_with_retries(state)
     elif state.data is None:
         result = None
     else:

prefect/task_engine.py

@@ -249,6 +249,16 @@ class TaskRunEngine(Generic[P, R]):
         new_state = Running()
         state = self.set_state(new_state)
 
+        # TODO: this is temporary until the API stops rejecting state transitions
+        # and the client / transaction store becomes the source of truth
+        # this is a bandaid caused by the API storing a Completed state with a bad
+        # result reference that no longer exists
+        if state.is_completed():
+            try:
+                state.result(retry_result_failure=False, _sync=True)
+            except Exception:
+                state = self.set_state(new_state, force=True)
+
         BACKOFF_MAX = 10
         backoff_count = 0
 
@@ -592,6 +602,7 @@ class TaskRunEngine(Generic[P, R]):
             key=self.compute_transaction_key(),
             store=ResultFactoryStore(result_factory=result_factory),
             overwrite=overwrite,
+            logger=self.logger,
         ) as txn:
             yield txn
 

prefect/task_runners.py

@@ -4,7 +4,18 @@ import sys
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from contextvars import copy_context
-from typing import TYPE_CHECKING, Any, Dict, Generic, Iterable, Optional, Set
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Coroutine,
+    Dict,
+    Generic,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    overload,
+)
 
 from typing_extensions import ParamSpec, Self, TypeVar
 
@@ -14,6 +25,7 @@ from prefect.futures import (
     PrefectConcurrentFuture,
     PrefectDistributedFuture,
     PrefectFuture,
+    PrefectFutureList,
 )
 from prefect.logging.loggers import get_logger, get_run_logger
 from prefect.utilities.annotations import allow_failure, quote, unmapped
@@ -30,6 +42,7 @@ if TYPE_CHECKING:
 P = ParamSpec("P")
 T = TypeVar("T")
 F = TypeVar("F", bound=PrefectFuture)
+R = TypeVar("R")
 
 
 class TaskRunner(abc.ABC, Generic[F]):
@@ -85,7 +98,7 @@ class TaskRunner(abc.ABC, Generic[F]):
         task: "Task",
         parameters: Dict[str, Any],
         wait_for: Optional[Iterable[PrefectFuture]] = None,
-    ) -> Iterable[F]:
+    ) -> PrefectFutureList[F]:
         """
         Submit multiple tasks to the task run engine.
 
@@ -157,7 +170,7 @@ class TaskRunner(abc.ABC, Generic[F]):
 
         map_length = list(lengths)[0]
 
-        futures = []
+        futures: List[PrefectFuture] = []
         for i in range(map_length):
             call_parameters = {
                 key: value[i] for key, value in iterable_parameters.items()
@@ -187,7 +200,7 @@ class TaskRunner(abc.ABC, Generic[F]):
                 )
             )
 
-        return futures
+        return PrefectFutureList(futures)
 
     def __enter__(self):
         if self._started:
@@ -211,13 +224,33 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
     def duplicate(self) -> "ThreadPoolTaskRunner":
         return type(self)(max_workers=self._max_workers)
 
+    @overload
+    def submit(
+        self,
+        task: "Task[P, Coroutine[Any, Any, R]]",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+        dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
+    ) -> PrefectConcurrentFuture[R]:
+        ...
+
+    @overload
+    def submit(
+        self,
+        task: "Task[Any, R]",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+        dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
+    ) -> PrefectConcurrentFuture[R]:
+        ...
+
     def submit(
         self,
         task: "Task",
         parameters: Dict[str, Any],
         wait_for: Optional[Iterable[PrefectFuture]] = None,
         dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
-    ) -> PrefectConcurrentFuture:
+    ):
         """
         Submit a task to the task run engine running in a separate thread.
 
@@ -278,6 +311,32 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
         )
         return prefect_future
 
+    @overload
+    def map(
+        self,
+        task: "Task[P, Coroutine[Any, Any, R]]",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+    ) -> PrefectFutureList[PrefectConcurrentFuture[R]]:
+        ...
+
+    @overload
+    def map(
+        self,
+        task: "Task[Any, R]",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+    ) -> PrefectFutureList[PrefectConcurrentFuture[R]]:
+        ...
+
+    def map(
+        self,
+        task: "Task",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+    ):
+        return super().map(task, parameters, wait_for)
+
     def __enter__(self):
         super().__enter__()
         self._executor = ThreadPoolExecutor(max_workers=self._max_workers)
@@ -306,13 +365,33 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
     def duplicate(self) -> "PrefectTaskRunner":
         return type(self)()
 
+    @overload
+    def submit(
+        self,
+        task: "Task[P, Coroutine[Any, Any, R]]",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+        dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
+    ) -> PrefectDistributedFuture[R]:
+        ...
+
+    @overload
+    def submit(
+        self,
+        task: "Task[Any, R]",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+        dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
+    ) -> PrefectDistributedFuture[R]:
+        ...
+
     def submit(
         self,
         task: "Task",
         parameters: Dict[str, Any],
         wait_for: Optional[Iterable[PrefectFuture]] = None,
         dependencies: Optional[Dict[str, Set[TaskRunInput]]] = None,
-    ) -> PrefectDistributedFuture:
+    ):
         """
         Submit a task to the task run engine running in a separate thread.
 
@@ -342,3 +421,29 @@ class PrefectTaskRunner(TaskRunner[PrefectDistributedFuture]):
         return task.apply_async(
             kwargs=parameters, wait_for=wait_for, dependencies=dependencies
         )
+
+    @overload
+    def map(
+        self,
+        task: "Task[P, Coroutine[Any, Any, R]]",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+    ) -> PrefectFutureList[PrefectDistributedFuture[R]]:
+        ...
+
+    @overload
+    def map(
+        self,
+        task: "Task[Any, R]",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+    ) -> PrefectFutureList[PrefectDistributedFuture[R]]:
+        ...
+
+    def map(
+        self,
+        task: "Task",
+        parameters: Dict[str, Any],
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
+    ):
+        return super().map(task, parameters, wait_for)

prefect/task_worker.py

@@ -7,7 +7,7 @@ import sys
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import AsyncExitStack
 from contextvars import copy_context
-from typing import List, Optional
+from typing import Optional
 from uuid import UUID
 
 import anyio
@@ -20,6 +20,7 @@ from websockets.exceptions import InvalidStatusCode
 
 from prefect import Task
 from prefect._internal.concurrency.api import create_call, from_sync
+from prefect.cache_policies import DEFAULT, NONE
 from prefect.client.orchestration import get_client
 from prefect.client.schemas.objects import TaskRun
 from prefect.client.subscriptions import Subscription
@@ -32,6 +33,7 @@ from prefect.settings import (
 )
 from prefect.states import Pending
 from prefect.task_engine import run_task_async, run_task_sync
+from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import asyncnullcontext, sync_compatible
 from prefect.utilities.engine import emit_task_run_state_change_event, propose_state
 from prefect.utilities.processutils import _register_signal
@@ -76,7 +78,16 @@ class TaskWorker:
         *tasks: Task,
         limit: Optional[int] = 10,
     ):
-        self.tasks: List[Task] = list(tasks)
+        self.tasks = []
+        for t in tasks:
+            if isinstance(t, Task):
+                if t.cache_policy in [None, NONE, NotSet]:
+                    self.tasks.append(
+                        t.with_options(persist_result=True, cache_policy=DEFAULT)
+                    )
+                else:
+                    self.tasks.append(t.with_options(persist_result=True))
+
         self.task_keys = set(t.task_key for t in tasks if isinstance(t, Task))
 
         self._started_at: Optional[pendulum.DateTime] = None