prefect-client 3.0.0rc6__py3-none-any.whl → 3.0.0rc8__py3-none-any.whl

prefect/results.py

@@ -142,38 +142,6 @@ def get_default_persist_setting() -> bool:
     return PREFECT_RESULTS_PERSIST_BY_DEFAULT.value()
 
 
-def flow_features_require_result_persistence(flow: "Flow") -> bool:
-    """
-    Returns `True` if the given flow uses features that require its result to be
-    persisted.
-    """
-    if not flow.cache_result_in_memory:
-        return True
-    return False
-
-
-def flow_features_require_child_result_persistence(flow: "Flow") -> bool:
-    """
-    Returns `True` if the given flow uses features that require child flow and task
-    runs to persist their results.
-    """
-    if flow and flow.retries:
-        return True
-    return False
-
-
-def task_features_require_result_persistence(task: "Task") -> bool:
-    """
-    Returns `True` if the given task uses features that require its result to be
-    persisted.
-    """
-    if task.cache_key_fn:
-        return True
-    if not task.cache_result_in_memory:
-        return True
-    return False
-
-
 def _format_user_supplied_storage_key(key: str) -> str:
     # Note here we are pinning to task runs since flow runs do not support storage keys
     # yet; we'll need to split logic in the future or have two separate functions
@@ -235,17 +203,7 @@ class ResultFactory(BaseModel):
                 result_storage=flow.result_storage or ctx.result_factory.storage_block,
                 result_serializer=flow.result_serializer
                 or ctx.result_factory.serializer,
-                persist_result=(
-                    flow.persist_result
-                    if flow.persist_result is not None
-                    # !! Child flows persist their result by default if the it or the
-                    #    parent flow uses a feature that requires it
-                    else (
-                        flow_features_require_result_persistence(flow)
-                        or flow_features_require_child_result_persistence(ctx.flow)
-                        or get_default_persist_setting()
-                    )
-                ),
+                persist_result=flow.persist_result,
                 cache_result_in_memory=flow.cache_result_in_memory,
                 storage_key_fn=DEFAULT_STORAGE_KEY_FN,
                 client=client,
@@ -258,16 +216,7 @@ class ResultFactory(BaseModel):
                 client=client,
                 result_storage=flow.result_storage,
                 result_serializer=flow.result_serializer,
-                persist_result=(
-                    flow.persist_result
-                    if flow.persist_result is not None
-                    # !! Flows persist their result by default if uses a feature that
-                    #    requires it
-                    else (
-                        flow_features_require_result_persistence(flow)
-                        or get_default_persist_setting()
-                    )
-                ),
+                persist_result=flow.persist_result,
                 cache_result_in_memory=flow.cache_result_in_memory,
                 storage_key_fn=DEFAULT_STORAGE_KEY_FN,
             )
@@ -318,21 +267,7 @@ class ResultFactory(BaseModel):
             if ctx and ctx.result_factory
             else get_default_result_serializer()
         )
-        persist_result = (
-            task.persist_result
-            if task.persist_result is not None
-            # !! Tasks persist their result by default if their parent flow uses a
-            #    feature that requires it or the task uses a feature that requires it
-            else (
-                (
-                    flow_features_require_child_result_persistence(ctx.flow)
-                    if ctx
-                    else False
-                )
-                or task_features_require_result_persistence(task)
-                or get_default_persist_setting()
-            )
-        )
+        persist_result = task.persist_result
 
         cache_result_in_memory = task.cache_result_in_memory
 
@@ -355,11 +290,14 @@ class ResultFactory(BaseModel):
         cls: Type[Self],
         result_storage: ResultStorage,
         result_serializer: ResultSerializer,
-        persist_result: bool,
+        persist_result: Optional[bool],
         cache_result_in_memory: bool,
         storage_key_fn: Callable[[], str],
         client: "PrefectClient",
     ) -> Self:
+        if persist_result is None:
+            persist_result = get_default_persist_setting()
+
         storage_block_id, storage_block = await cls.resolve_storage_block(
             result_storage, client=client, persist_result=persist_result
         )
@@ -614,7 +552,6 @@ class PersistedResult(BaseResult):
         """
         Retrieve the data and deserialize it into the original object.
         """
-
         if self.has_cached_object():
             return self._cache
 
@@ -680,7 +617,13 @@ class PersistedResult(BaseResult):
             # this could error if the serializer requires kwargs
             serializer = Serializer(type=self.serializer_type)
 
-        data = serializer.dumps(obj)
+        try:
+            data = serializer.dumps(obj)
+        except Exception as exc:
+            raise ValueError(
+                f"Failed to serialize object of type {type(obj).__name__!r} with "
+                f"serializer {serializer.type!r}."
+            ) from exc
         blob = PersistedResultBlob(
             serializer=serializer, data=data, expiration=self.expiration
         )

prefect/runner/storage.py

@@ -559,8 +559,70 @@ class BlockStorageAdapter:
         return False
 
 
-def create_storage_from_url(
-    url: str, pull_interval: Optional[int] = 60
+class LocalStorage:
+    """
+    Sets the working directory in the local filesystem.
+    Parameters:
+        Path: Local file path to set the working directory for the flow
+    Examples:
+        Sets the working directory for the local path to the flow:
+        ```python
+        from prefect.runner.storage import Localstorage
+        storage = LocalStorage(
+            path="/path/to/local/flow_directory",
+        )
+        ```
+    """
+
+    def __init__(
+        self,
+        path: str,
+        pull_interval: Optional[int] = None,
+    ):
+        self._path = Path(path).resolve()
+        self._logger = get_logger("runner.storage.local-storage")
+        self._storage_base_path = Path.cwd()
+        self._pull_interval = pull_interval
+
+    @property
+    def destination(self) -> Path:
+        return self._path
+
+    def set_base_path(self, path: Path):
+        self._storage_base_path = path
+
+    @property
+    def pull_interval(self) -> Optional[int]:
+        return self._pull_interval
+
+    async def pull_code(self):
+        # Local storage assumes the code already exists on the local filesystem
+        # and does not need to be pulled from a remote location
+        pass
+
+    def to_pull_step(self) -> dict:
+        """
+        Returns a dictionary representation of the storage object that can be
+        used as a deployment pull step.
+        """
+        step = {
+            "prefect.deployments.steps.set_working_directory": {
+                "directory": str(self.destination)
+            }
+        }
+        return step
+
+    def __eq__(self, __value) -> bool:
+        if isinstance(__value, LocalStorage):
+            return self._path == __value._path
+        return False
+
+    def __repr__(self) -> str:
+        return f"LocalStorage(path={self._path!r})"
+
+
+def create_storage_from_source(
+    source: str, pull_interval: Optional[int] = 60
 ) -> RunnerStorage:
     """
     Creates a storage object from a URL.
@@ -574,11 +636,18 @@ def create_storage_from_url(
     Returns:
         RunnerStorage: A runner storage compatible object
     """
-    parsed_url = urlparse(url)
-    if parsed_url.scheme == "git" or parsed_url.path.endswith(".git"):
-        return GitRepository(url=url, pull_interval=pull_interval)
+    logger = get_logger("runner.storage")
+    parsed_source = urlparse(source)
+    if parsed_source.scheme == "git" or parsed_source.path.endswith(".git"):
+        return GitRepository(url=source, pull_interval=pull_interval)
+    elif parsed_source.scheme in ("file", "local"):
+        source_path = source.split("://", 1)[-1]
+        return LocalStorage(path=source_path, pull_interval=pull_interval)
+    elif parsed_source.scheme in fsspec.available_protocols():
+        return RemoteStorage(url=source, pull_interval=pull_interval)
     else:
-        return RemoteStorage(url=url, pull_interval=pull_interval)
+        logger.debug("No valid fsspec protocol found for URL, assuming local storage.")
+        return LocalStorage(path=source, pull_interval=pull_interval)
 
 
 def _format_token_from_credentials(netloc: str, credentials: dict) -> str:

prefect/settings.py

@@ -92,21 +92,6 @@ T = TypeVar("T")
 
 DEFAULT_PROFILES_PATH = Path(__file__).parent.joinpath("profiles.toml")
 
-# When we remove the experimental settings we also want to add them to the set of REMOVED_EXPERIMENTAL_FLAGS.
-# The reason for this is removing the settings entirely causes the CLI to crash for anyone who has them in one or more of their profiles.
-# Adding them to REMOVED_EXPERIMENTAL_FLAGS will make it so that the user is warned about it and they have time to take action.
-REMOVED_EXPERIMENTAL_FLAGS = {
-    "PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_SCHEDULING_UI",
-    "PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_DEPLOYMENT_PARAMETERS",
-    "PREFECT_EXPERIMENTAL_ENABLE_EVENTS_CLIENT",
-    "PREFECT_EXPERIMENTAL_ENABLE_EVENTS",
-    "PREFECT_EXPERIMENTAL_EVENTS",
-    "PREFECT_EXPERIMENTAL_WARN_EVENTS_CLIENT",
-    "PREFECT_EXPERIMENTAL_ENABLE_FLOW_RUN_INFRA_OVERRIDES",
-    "PREFECT_EXPERIMENTAL_WARN_FLOW_RUN_INFRA_OVERRIDES",
-    "PREFECT_EXPERIMENTAL_ENABLE_WORK_POOLS",
-}
-
 
 class Setting(Generic[T]):
     """
@@ -423,7 +408,7 @@ def default_result_storage_block_name(
     settings: Optional["Settings"] = None, value: Optional[str] = None
 ):
     """
-    `value_callback` for `PREFECT_DEFAULT_RESULT_STORAGE_BLOCK_NAME` that sets the default
+    `value_callback` for `PREFECT_DEFAULT_RESULT_STORAGE_BLOCK` that sets the default
     value to the hostname of the machine.
     """
     if value is None:
@@ -749,7 +734,7 @@ PREFECT_RESULTS_DEFAULT_SERIALIZER = Setting(
 
 PREFECT_RESULTS_PERSIST_BY_DEFAULT = Setting(
     bool,
-    default=False,
+    default=True,
 )
 """
 The default setting for persisting results when not otherwise specified. If enabled,
@@ -959,41 +944,6 @@ interpreted and lead to incomplete output, e.g.
 `DROP TABLE [dbo].[SomeTable];"` outputs `DROP TABLE .[SomeTable];`.
 """
 
-PREFECT_TASK_INTROSPECTION_WARN_THRESHOLD = Setting(
-    float,
-    default=10.0,
-)
-"""
-Threshold time in seconds for logging a warning if task parameter introspection
-exceeds this duration. Parameter introspection can be a significant performance hit
-when the parameter is a large collection object, e.g. a large dictionary or DataFrame,
-and each element needs to be inspected. See `prefect.utilities.annotations.quote`
-for more details.
-Defaults to `10.0`.
-Set to `0` to disable logging the warning.
-"""
-
-PREFECT_AGENT_QUERY_INTERVAL = Setting(
-    float,
-    default=15,
-)
-"""
-The agent loop interval, in seconds. Agents will check for new runs this often.
-Defaults to `15`.
-"""
-
-PREFECT_AGENT_PREFETCH_SECONDS = Setting(
-    int,
-    default=15,
-)
-"""
-Agents will look for scheduled runs this many seconds in
-the future and attempt to run them. This accounts for any additional
-infrastructure spin-up time or latency in preparing a flow run. Note
-flow runs will not start before their scheduled time, even if they are
-prefetched. Defaults to `15`.
-"""
-
 PREFECT_ASYNC_FETCH_STATE_RESULT = Setting(bool, default=False)
 """
 Determines whether `State.result()` fetches results automatically or not.
@@ -1373,16 +1323,6 @@ PREFECT_API_MAX_FLOW_RUN_GRAPH_ARTIFACTS = Setting(int, default=10000)
 The maximum number of artifacts to show on a flow run graph on the v2 API
 """
 
-PREFECT_EXPERIMENTAL_ENABLE_ARTIFACTS_ON_FLOW_RUN_GRAPH = Setting(bool, default=True)
-"""
-Whether or not to enable artifacts on the flow run graph.
-"""
-
-PREFECT_EXPERIMENTAL_ENABLE_STATES_ON_FLOW_RUN_GRAPH = Setting(bool, default=True)
-"""
-Whether or not to enable flow run states on the flow run graph.
-"""
-
 PREFECT_EXPERIMENTAL_ENABLE_WORKERS = Setting(bool, default=True)
 """
 Whether or not to enable experimental Prefect workers.
@@ -1393,11 +1333,6 @@ PREFECT_EXPERIMENTAL_WARN_WORKERS = Setting(bool, default=False)
 Whether or not to warn when experimental Prefect workers are used.
 """
 
-PREFECT_EXPERIMENTAL_WARN_VISUALIZE = Setting(bool, default=False)
-"""
-Whether or not to warn when experimental Prefect visualize is used.
-"""
-
 PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_CANCELLATION = Setting(bool, default=True)
 """
 Whether or not to enable experimental enhanced flow run cancellation.
@@ -1408,26 +1343,6 @@ PREFECT_EXPERIMENTAL_WARN_ENHANCED_CANCELLATION = Setting(bool, default=False)
 Whether or not to warn when experimental enhanced flow run cancellation is used.
 """
 
-PREFECT_EXPERIMENTAL_ENABLE_DEPLOYMENT_STATUS = Setting(bool, default=True)
-"""
-Whether or not to enable deployment status in the UI
-"""
-
-PREFECT_EXPERIMENTAL_WARN_DEPLOYMENT_STATUS = Setting(bool, default=False)
-"""
-Whether or not to warn when deployment status is used.
-"""
-
-PREFECT_EXPERIMENTAL_FLOW_RUN_INPUT = Setting(bool, default=False)
-"""
-Whether or not to enable flow run input.
-"""
-
-PREFECT_EXPERIMENTAL_WARN_FLOW_RUN_INPUT = Setting(bool, default=True)
-"""
-Whether or not to enable flow run input.
-"""
-
 
 # Prefect Events feature flags
 
@@ -1554,31 +1469,6 @@ PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS = Setting(bool, default=False
 Whether or not to enable experimental worker webserver endpoints.
 """
 
-PREFECT_EXPERIMENTAL_ENABLE_ARTIFACTS = Setting(bool, default=True)
-"""
-Whether or not to enable experimental Prefect artifacts.
-"""
-
-PREFECT_EXPERIMENTAL_WARN_ARTIFACTS = Setting(bool, default=False)
-"""
-Whether or not to warn when experimental Prefect artifacts are used.
-"""
-
-PREFECT_EXPERIMENTAL_ENABLE_WORKSPACE_DASHBOARD = Setting(bool, default=True)
-"""
-Whether or not to enable the experimental workspace dashboard.
-"""
-
-PREFECT_EXPERIMENTAL_WARN_WORKSPACE_DASHBOARD = Setting(bool, default=False)
-"""
-Whether or not to warn when the experimental workspace dashboard is enabled.
-"""
-
-PREFECT_EXPERIMENTAL_ENABLE_WORK_QUEUE_STATUS = Setting(bool, default=True)
-"""
-Whether or not to enable experimental work queue status in-place of work queue health.
-"""
-
 PREFECT_EXPERIMENTAL_DISABLE_SYNC_COMPAT = Setting(bool, default=False)
 """
 Whether or not to disable the sync_compatible decorator utility.
@@ -1663,11 +1553,6 @@ PREFECT_EVENTS_MAXIMUM_SIZE_BYTES = Setting(int, default=1_500_000)
 The maximum size of an Event when serialized to JSON
 """
 
-PREFECT_API_SERVICES_EVENT_LOGGER_ENABLED = Setting(bool, default=True)
-"""
-Whether or not to start the event debug logger service in the server application.
-"""
-
 PREFECT_API_SERVICES_TRIGGERS_ENABLED = Setting(bool, default=True)
 """
 Whether or not to start the triggers service in the server application.
@@ -1715,6 +1600,18 @@ PREFECT_API_EVENTS_RELATED_RESOURCE_CACHE_TTL = Setting(
 How long to cache related resource data for emitting server-side vents
 """
 
+PREFECT_EVENTS_MAXIMUM_WEBSOCKET_BACKFILL = Setting(
+    timedelta, default=timedelta(minutes=15)
+)
+"""
+The maximum range to look back for backfilling events for a websocket subscriber
+"""
+
+PREFECT_EVENTS_WEBSOCKET_BACKFILL_PAGE_SIZE = Setting(int, default=250, gt=0)
+"""
+The page size for the queries to backfill events for websocket subscribers
+"""
+
 
 # Deprecated settings ------------------------------------------------------------------
 
@@ -2215,26 +2112,6 @@ class ProfilesCollection:
         )
 
 
-def _handle_removed_flags(
-    profile_name: str, settings: Dict[str, Any]
-) -> Dict[str, Any]:
-    to_remove = [name for name in settings if name in REMOVED_EXPERIMENTAL_FLAGS]
-
-    for name in to_remove:
-        warnings.warn(
-            (
-                f"Experimental flag {name!r} has been removed, please "
-                f"update your {profile_name!r} profile."
-            ),
-            UserWarning,
-            stacklevel=3,
-        )
-
-        settings.pop(name)
-
-    return settings
-
-
 def _read_profiles_from(path: Path) -> ProfilesCollection:
     """
     Read profiles from a path into a new `ProfilesCollection`.
@@ -2253,7 +2130,6 @@ def _read_profiles_from(path: Path) -> ProfilesCollection:
 
     profiles = []
     for name, settings in raw_profiles.items():
-        settings = _handle_removed_flags(name, settings)
         profiles.append(Profile(name=name, settings=settings, source=path))
 
     return ProfilesCollection(profiles, active=active_profile)

prefect/states.py

@@ -1,3 +1,4 @@
+import asyncio
 import datetime
 import sys
 import traceback
@@ -22,15 +23,21 @@ from prefect.exceptions import (
     TerminationSignal,
     UnfinishedRun,
 )
+from prefect.logging.loggers import get_logger
 from prefect.results import BaseResult, R, ResultFactory
 from prefect.settings import PREFECT_ASYNC_FETCH_STATE_RESULT
 from prefect.utilities.annotations import BaseAnnotation
 from prefect.utilities.asyncutils import in_async_main_thread, sync_compatible
 from prefect.utilities.collections import ensure_iterable
 
+logger = get_logger("states")
+
 
 def get_state_result(
-    state: State[R], raise_on_failure: bool = True, fetch: Optional[bool] = None
+    state: State[R],
+    raise_on_failure: bool = True,
+    fetch: Optional[bool] = None,
+    retry_result_failure: bool = True,
 ) -> R:
     """
     Get the result from a state.
@@ -58,11 +65,50 @@ def get_state_result(
 
         return state.data
     else:
-        return _get_state_result(state, raise_on_failure=raise_on_failure)
+        return _get_state_result(
+            state,
+            raise_on_failure=raise_on_failure,
+            retry_result_failure=retry_result_failure,
+        )
+
+
+RESULT_READ_MAXIMUM_ATTEMPTS = 10
+RESULT_READ_RETRY_DELAY = 0.25
+
+
+async def _get_state_result_data_with_retries(
+    state: State[R], retry_result_failure: bool = True
+) -> R:
+    # Results may be written asynchronously, possibly after their corresponding
+    # state has been written and events have been emitted, so we should give some
+    # grace here about missing results.  The exception below could come in the form
+    # of a missing file, a short read, or other types of errors depending on the
+    # result storage backend.
+    if retry_result_failure is False:
+        max_attempts = 1
+    else:
+        max_attempts = RESULT_READ_MAXIMUM_ATTEMPTS
+
+    for i in range(1, max_attempts + 1):
+        try:
+            return await state.data.get()
+        except Exception as e:
+            if i == max_attempts:
+                raise
+            logger.debug(
+                "Exception %r while reading result, retry %s/%s in %ss...",
+                e,
+                i,
+                max_attempts,
+                RESULT_READ_RETRY_DELAY,
+            )
+            await asyncio.sleep(RESULT_READ_RETRY_DELAY)
 
 
 @sync_compatible
-async def _get_state_result(state: State[R], raise_on_failure: bool) -> R:
+async def _get_state_result(
+    state: State[R], raise_on_failure: bool, retry_result_failure: bool = True
+) -> R:
     """
     Internal implementation for `get_state_result` without async backwards compatibility
     """
@@ -81,7 +127,10 @@ async def _get_state_result(state: State[R], raise_on_failure: bool) -> R:
         raise await get_state_exception(state)
 
     if isinstance(state.data, BaseResult):
-        result = await state.data.get()
+        result = await _get_state_result_data_with_retries(
+            state, retry_result_failure=retry_result_failure
+        )
+
     elif state.data is None:
         if state.is_failed() or state.is_crashed() or state.is_cancelled():
             return await get_state_exception(state)
@@ -352,7 +401,7 @@ async def get_state_exception(state: State) -> BaseException:
         raise ValueError(f"Expected failed or crashed state got {state!r}.")
 
     if isinstance(state.data, BaseResult):
-        result = await state.data.get()
+        result = await _get_state_result_data_with_retries(state)
     elif state.data is None:
         result = None
     else:

prefect/task_engine.py

@@ -249,6 +249,16 @@ class TaskRunEngine(Generic[P, R]):
         new_state = Running()
         state = self.set_state(new_state)
 
+        # TODO: this is temporary until the API stops rejecting state transitions
+        # and the client / transaction store becomes the source of truth
+        # this is a bandaid caused by the API storing a Completed state with a bad
+        # result reference that no longer exists
+        if state.is_completed():
+            try:
+                state.result(retry_result_failure=False, _sync=True)
+            except Exception:
+                state = self.set_state(new_state, force=True)
+
         BACKOFF_MAX = 10
         backoff_count = 0