prefect-client 3.0.0rc1__py3-none-any.whl → 3.0.0rc3__py3-none-any.whl

prefect/records/cache_policies.py

@@ -0,0 +1,179 @@
+import inspect
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, Optional
+
+from prefect.context import TaskRunContext
+from prefect.utilities.hashing import hash_objects
+
+
+@dataclass
+class CachePolicy:
+    @classmethod
+    def from_cache_key_fn(
+        cls, cache_key_fn: Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
+    ) -> "CacheKeyFnPolicy":
+        """
+        Given a function generates a key policy.
+        """
+        return CacheKeyFnPolicy(cache_key_fn=cache_key_fn)
+
+    def compute_key(
+        self,
+        task_ctx: TaskRunContext,
+        inputs: Dict[str, Any],
+        flow_parameters: Dict[str, Any],
+        **kwargs,
+    ) -> Optional[str]:
+        raise NotImplementedError
+
+    def __sub__(self, other: str) -> "CompoundCachePolicy":
+        if not isinstance(other, str):
+            raise TypeError("Can only subtract strings from key policies.")
+        if isinstance(self, Inputs):
+            exclude = self.exclude or []
+            return Inputs(exclude=exclude + [other])
+        elif isinstance(self, CompoundCachePolicy):
+            new = Inputs(exclude=[other])
+            policies = self.policies or []
+            return CompoundCachePolicy(policies=policies + [new])
+        else:
+            new = Inputs(exclude=[other])
+            return CompoundCachePolicy(policies=[self, new])
+
+    def __add__(self, other: "CachePolicy") -> "CompoundCachePolicy":
+        # adding _None is a no-op
+        if isinstance(other, _None):
+            return self
+        elif isinstance(self, _None):
+            return other
+
+        if isinstance(self, CompoundCachePolicy):
+            policies = self.policies or []
+            return CompoundCachePolicy(policies=policies + [other])
+        elif isinstance(other, CompoundCachePolicy):
+            policies = other.policies or []
+            return CompoundCachePolicy(policies=policies + [self])
+        else:
+            return CompoundCachePolicy(policies=[self, other])
+
+
+@dataclass
+class CacheKeyFnPolicy(CachePolicy):
+    # making it optional for tests
+    cache_key_fn: Optional[
+        Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
+    ] = None
+
+    def compute_key(
+        self,
+        task_ctx: TaskRunContext,
+        inputs: Dict[str, Any],
+        flow_parameters: Dict[str, Any],
+        **kwargs,
+    ) -> Optional[str]:
+        if self.cache_key_fn:
+            return self.cache_key_fn(task_ctx, inputs)
+
+
+@dataclass
+class CompoundCachePolicy(CachePolicy):
+    policies: Optional[list] = None
+
+    def compute_key(
+        self,
+        task_ctx: TaskRunContext,
+        inputs: Dict[str, Any],
+        flow_parameters: Dict[str, Any],
+        **kwargs,
+    ) -> Optional[str]:
+        keys = []
+        for policy in self.policies or []:
+            keys.append(
+                policy.compute_key(
+                    task_ctx=task_ctx,
+                    inputs=inputs,
+                    flow_parameters=flow_parameters,
+                    **kwargs,
+                )
+            )
+        return hash_objects(*keys)
+
+
+@dataclass
+class Default(CachePolicy):
+    "Execution run ID only"
+
+    def compute_key(
+        self,
+        task_ctx: TaskRunContext,
+        inputs: Dict[str, Any],
+        flow_parameters: Dict[str, Any],
+        **kwargs,
+    ) -> Optional[str]:
+        return str(task_ctx.task_run.id)
+
+
+@dataclass
+class _None(CachePolicy):
+    "ignore key policies altogether, always run - prevents persistence"
+
+    def compute_key(
+        self,
+        task_ctx: TaskRunContext,
+        inputs: Dict[str, Any],
+        flow_parameters: Dict[str, Any],
+        **kwargs,
+    ) -> Optional[str]:
+        return None
+
+
+@dataclass
+class TaskDef(CachePolicy):
+    def compute_key(
+        self,
+        task_ctx: TaskRunContext,
+        inputs: Dict[str, Any],
+        flow_parameters: Dict[str, Any],
+        **kwargs,
+    ) -> Optional[str]:
+        lines = inspect.getsource(task_ctx.task)
+        return hash_objects(lines)
+
+
+@dataclass
+class FlowParameters(CachePolicy):
+    pass
+
+
+@dataclass
+class Inputs(CachePolicy):
+    """
+    Exposes flag for whether to include flow parameters as well.
+
+    And exclude/include config.
+    """
+
+    exclude: Optional[list] = None
+
+    def compute_key(
+        self,
+        task_ctx: TaskRunContext,
+        inputs: Dict[str, Any],
+        flow_parameters: Dict[str, Any],
+        **kwargs,
+    ) -> Optional[str]:
+        hashed_inputs = {}
+        inputs = inputs or {}
+        exclude = self.exclude or []
+
+        for key, val in inputs.items():
+            if key not in exclude:
+                hashed_inputs[key] = val
+
+        return hash_objects(hashed_inputs)
+
+
+DEFAULT = Default()
+INPUTS = Inputs()
+NONE = _None()
+TASKDEF = TaskDef()

prefect/records/result_store.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
 from typing import Any
 
-from prefect.exceptions import ObjectNotFound
+import pendulum
+
 from prefect.results import BaseResult, PersistedResult, ResultFactory
 from prefect.utilities.asyncutils import run_coro_as_sync
 
@@ -17,9 +18,15 @@ class ResultFactoryStore(RecordStore):
         try:
             result = self.read(key)
             result.get(_sync=True)
+            if result.expiration:
+                # if the result has an expiration,
+                # check if it is still in the future
+                exists = result.expiration > pendulum.now("utc")
+            else:
+                exists = True
             self.cache = result
-            return True
-        except (ObjectNotFound, ValueError):
+            return exists
+        except Exception:
             return False
 
     def read(self, key: str) -> BaseResult:

prefect/results.py

@@ -18,6 +18,7 @@ from uuid import UUID
 
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, ValidationError
 from pydantic_core import PydanticUndefinedType
+from pydantic_extra_types.pendulum_dt import DateTime
 from typing_extensions import ParamSpec, Self
 
 import prefect
@@ -37,6 +38,7 @@ from prefect.settings import (
     PREFECT_RESULTS_DEFAULT_SERIALIZER,
     PREFECT_RESULTS_PERSIST_BY_DEFAULT,
     PREFECT_TASK_SCHEDULING_DEFAULT_STORAGE_BLOCK,
+    default_result_storage_block_name,
 )
 from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import sync_compatible
@@ -66,11 +68,18 @@ async def get_default_result_storage() -> ResultStorage:
     """
     Generate a default file system for result storage.
     """
-    return (
-        await Block.load(PREFECT_DEFAULT_RESULT_STORAGE_BLOCK.value())
-        if PREFECT_DEFAULT_RESULT_STORAGE_BLOCK.value() is not None
-        else LocalFileSystem(basepath=PREFECT_LOCAL_STORAGE_PATH.value())
-    )
+    try:
+        return await Block.load(PREFECT_DEFAULT_RESULT_STORAGE_BLOCK.value())
+    except ValueError as e:
+        if "Unable to find" not in str(e):
+            raise e
+        elif (
+            PREFECT_DEFAULT_RESULT_STORAGE_BLOCK.value()
+            == default_result_storage_block_name()
+        ):
+            return LocalFileSystem(basepath=PREFECT_LOCAL_STORAGE_PATH.value())
+        else:
+            raise
 
 
 _default_task_scheduling_storages: Dict[Tuple[str, str], WritableFileSystem] = {}
@@ -271,13 +280,6 @@ class ResultFactory(BaseModel):
         """
         Create a new result factory for a task.
         """
-        from prefect.context import FlowRunContext
-
-        ctx = FlowRunContext.get()
-
-        if ctx and ctx.autonomous_task_run:
-            return await cls.from_autonomous_task(task, client=client)
-
         return await cls._from_task(task, get_default_result_storage, client=client)
 
     @classmethod
@@ -426,16 +428,16 @@ class ResultFactory(BaseModel):
             )
 
     @sync_compatible
-    async def create_result(self, obj: R, key: str = None) -> Union[R, "BaseResult[R]"]:
+    async def create_result(
+        self, obj: R, key: Optional[str] = None, expiration: Optional[DateTime] = None
+    ) -> Union[R, "BaseResult[R]"]:
         """
         Create a result type for the given object.
 
         If persistence is disabled, the object is wrapped in an `UnpersistedResult` and
         returned.
 
-        If persistence is enabled:
-        - Bool and null types are converted into `LiteralResult`.
-        - Other types are serialized, persisted to storage, and a reference is returned.
+        If persistence is enabled the object is serialized, persisted to storage, and a reference is returned.
         """
         # Null objects are "cached" in memory at no cost
         should_cache_object = self.cache_result_in_memory or obj is None
@@ -443,9 +445,6 @@ class ResultFactory(BaseModel):
         if not self.persist_result:
             return await UnpersistedResult.create(obj, cache_object=should_cache_object)
 
-        if type(obj) in LITERAL_TYPES:
-            return await LiteralResult.create(obj)
-
         if key:
 
             def key_fn():
@@ -462,6 +461,7 @@ class ResultFactory(BaseModel):
             storage_key_fn=storage_key_fn,
             serializer=self.serializer,
             cache_object=should_cache_object,
+            expiration=expiration,
         )
 
     @sync_compatible
@@ -569,41 +569,6 @@ class UnpersistedResult(BaseResult):
         return result
 
 
-class LiteralResult(BaseResult):
-    """
-    Result type for literal values like `None`, `True`, `False`.
-
-    These values are stored inline and JSON serialized when sent to the Prefect API.
-    They are not persisted to external result storage.
-    """
-
-    type: str = "literal"
-    value: Any = None
-
-    def has_cached_object(self) -> bool:
-        # This result type always has the object cached in memory
-        return True
-
-    @sync_compatible
-    async def get(self) -> R:
-        return self.value
-
-    @classmethod
-    @sync_compatible
-    async def create(
-        cls: "Type[LiteralResult]",
-        obj: R,
-    ) -> "LiteralResult[R]":
-        if type(obj) not in LITERAL_TYPES:
-            raise TypeError(
-                f"Unsupported type {type(obj).__name__!r} for result literal. Expected"
-                f" one of: {', '.join(type_.__name__ for type_ in LITERAL_TYPES)}"
-            )
-
-        description = f"Result with value `{obj}` persisted to Prefect."
-        return cls(value=obj, artifact_type="result", artifact_description=description)
-
-
 class PersistedResult(BaseResult):
     """
     Result type which stores a reference to a persisted result.
@@ -619,6 +584,7 @@ class PersistedResult(BaseResult):
     serializer_type: str
     storage_block_id: uuid.UUID
     storage_key: str
+    expiration: Optional[DateTime] = None
 
     _should_cache_object: bool = PrivateAttr(default=True)
 
@@ -634,6 +600,7 @@ class PersistedResult(BaseResult):
 
         blob = await self._read_blob(client=client)
         obj = blob.serializer.loads(blob.data)
+        self.expiration = blob.expiration
 
         if self._should_cache_object:
             self._cache_object(obj)
@@ -673,6 +640,7 @@ class PersistedResult(BaseResult):
         storage_key_fn: Callable[[], str],
         serializer: Serializer,
         cache_object: bool = True,
+        expiration: Optional[DateTime] = None,
     ) -> "PersistedResult[R]":
         """
         Create a new result reference from a user's object.
@@ -684,7 +652,9 @@ class PersistedResult(BaseResult):
             storage_block_id is not None
         ), "Unexpected storage block ID. Was it persisted?"
         data = serializer.dumps(obj)
-        blob = PersistedResultBlob(serializer=serializer, data=data)
+        blob = PersistedResultBlob(
+            serializer=serializer, data=data, expiration=expiration
+        )
 
         key = storage_key_fn()
         if not isinstance(key, str):
@@ -709,6 +679,7 @@ class PersistedResult(BaseResult):
             storage_key=key,
             artifact_type="result",
             artifact_description=description,
+            expiration=expiration,
         )
 
         if cache_object:
@@ -730,6 +701,7 @@ class PersistedResultBlob(BaseModel):
     serializer: Serializer
     data: bytes
     prefect_version: str = Field(default=prefect.__version__)
+    expiration: Optional[DateTime] = None
 
     def to_bytes(self) -> bytes:
         return self.model_dump_json(serialize_as_any=True).encode()

prefect/runner/runner.py

@@ -209,7 +209,7 @@ class Runner:
     async def add_flow(
         self,
         flow: Flow,
-        name: str = None,
+        name: Optional[str] = None,
         interval: Optional[
             Union[
                 Iterable[Union[int, float, datetime.timedelta]],

prefect/runner/server.py

@@ -42,7 +42,7 @@ class RunnerGenericFlowRunRequest(BaseModel):
     parent_task_run_id: Optional[uuid.UUID] = None
 
 
-def perform_health_check(runner, delay_threshold: int = None) -> JSONResponse:
+def perform_health_check(runner, delay_threshold: Optional[int] = None) -> JSONResponse:
     if delay_threshold is None:
         delay_threshold = (
             PREFECT_RUNNER_SERVER_MISSED_POLLS_TOLERANCE.value()

prefect/runtime/__init__.py

@@ -8,6 +8,7 @@ Example usage:
     print(f"This script is running from deployment {deployment.id} with parameters {deployment.parameters}")
     ```
 """
+
 import prefect.runtime.deployment
 import prefect.runtime.flow_run
 import prefect.runtime.task_run

prefect/runtime/deployment.py

@@ -24,6 +24,7 @@ Available attributes:
         include default values set on the flow function, only the parameter values set on the deployment
         object or those directly provided via API for this run
 """
+
 import os
 from typing import Any, Dict, List, Optional
 

prefect/runtime/flow_run.py

@@ -18,6 +18,7 @@ Available attributes:
     - `parent_deployment_id`: the ID of the deployment that triggered this run, if any
     - `run_count`: the number of times this flow run has been run
 """
+
 import os
 from typing import Any, Dict, List, Optional
 

prefect/runtime/task_run.py

@@ -14,6 +14,7 @@ Available attributes:
     - `run_count`: the number of times this task run has been run
     - `task_name`: the name of the task
 """
+
 import os
 from typing import Any, Dict, List, Optional
 

prefect/settings.py

@@ -42,6 +42,7 @@ dependent on the value of other settings or perform other dynamic effects.
 
 import logging
 import os
+import socket
 import string
 import warnings
 from contextlib import contextmanager
@@ -84,6 +85,7 @@ from prefect._internal.schemas.validators import validate_settings
 from prefect.exceptions import MissingProfileError
 from prefect.utilities.names import OBFUSCATED_PREFIX, obfuscate
 from prefect.utilities.pydantic import add_cloudpickle_reduction
+from prefect.utilities.slugify import slugify
 
 T = TypeVar("T")
 
@@ -417,6 +419,18 @@ def warn_on_misconfigured_api_url(values):
     return values
 
 
+def default_result_storage_block_name(
+    settings: Optional["Settings"] = None, value: Optional[str] = None
+):
+    """
+    `value_callback` for `PREFECT_DEFAULT_RESULT_STORAGE_BLOCK_NAME` that sets the default
+    value to the hostname of the machine.
+    """
+    if value is None:
+        return f"local-file-system/{slugify(socket.gethostname())}-storage"
+    return value
+
+
 def default_database_connection_url(settings, value):
     templater = template_with_settings(PREFECT_HOME, PREFECT_API_DATABASE_PASSWORD)
 
@@ -1208,6 +1222,9 @@ PREFECT_API_SERVICES_FOREMAN_WORK_QUEUE_LAST_POLLED_TIMEOUT_SECONDS = Setting(
 """The number of seconds before a work queue is marked as not ready if it has not been
 polled."""
 
+PREFECT_API_LOG_RETRYABLE_ERRORS = Setting(bool, default=False)
+"""If `True`, log retryable errors in the API and it's services."""
+
 
 PREFECT_API_DEFAULT_LIMIT = Setting(
     int,
@@ -1527,9 +1544,9 @@ PREFECT_TASK_SCHEDULING_PENDING_TASK_TIMEOUT = Setting(
     default=timedelta(seconds=30),
 )
 """
-How long before a PENDING task are made available to another task server.  In practice,
-a task server should move a task from PENDING to RUNNING very quickly, so runs stuck in
-PENDING for a while is a sign that the task server may have crashed.
+How long before a PENDING task are made available to another task worker.  In practice,
+a task worker should move a task from PENDING to RUNNING very quickly, so runs stuck in
+PENDING for a while is a sign that the task worker may have crashed.
 """
 
 PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS = Setting(bool, default=False)
@@ -1572,8 +1589,7 @@ PREFECT_EXPERIMENTAL_ENABLE_SCHEDULE_CONCURRENCY = Setting(bool, default=False)
 # Defaults -----------------------------------------------------------------------------
 
 PREFECT_DEFAULT_RESULT_STORAGE_BLOCK = Setting(
-    Optional[str],
-    default=None,
+    Optional[str], default=None, value_callback=default_result_storage_block_name
 )
 """The `block-type/block-document` slug of a block to use as the default result storage."""
 

prefect/states.py

@@ -204,7 +204,12 @@ async def exception_to_failed_state(
     return state
 
 
-async def return_value_to_state(retval: R, result_factory: ResultFactory) -> State[R]:
+async def return_value_to_state(
+    retval: R,
+    result_factory: ResultFactory,
+    key: Optional[str] = None,
+    expiration: Optional[datetime.datetime] = None,
+) -> State[R]:
     """
     Given a return value from a user's function, create a `State` the run should
     be placed in.
@@ -236,7 +241,9 @@ async def return_value_to_state(retval: R, result_factory: ResultFactory) -> Sta
         # Unless the user has already constructed a result explicitly, use the factory
         # to update the data to the correct type
         if not isinstance(state.data, BaseResult):
-            state.data = await result_factory.create_result(state.data)
+            state.data = await result_factory.create_result(
+                state.data, key=key, expiration=expiration
+            )
 
         return state
 
@@ -276,7 +283,9 @@ async def return_value_to_state(retval: R, result_factory: ResultFactory) -> Sta
         return State(
             type=new_state_type,
             message=message,
-            data=await result_factory.create_result(retval),
+            data=await result_factory.create_result(
+                retval, key=key, expiration=expiration
+            ),
         )
 
     # Generators aren't portable, implicitly convert them to a list.
@@ -289,7 +298,11 @@ async def return_value_to_state(retval: R, result_factory: ResultFactory) -> Sta
     if isinstance(data, BaseResult):
         return Completed(data=data)
     else:
-        return Completed(data=await result_factory.create_result(data))
+        return Completed(
+            data=await result_factory.create_result(
+                data, key=key, expiration=expiration
+            )
+        )
 
 
 @sync_compatible