prefect-client 3.0.0rc5__py3-none-any.whl → 3.0.0rc7__py3-none-any.whl

prefect/{records/cache_policies.py → cache_policies.py}

@@ -8,6 +8,10 @@ from prefect.utilities.hashing import hash_objects
 
 @dataclass
 class CachePolicy:
+    """
+    Base class for all cache policies.
+    """
+
     @classmethod
     def from_cache_key_fn(
         cls, cache_key_fn: Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
@@ -59,6 +63,11 @@ class CachePolicy:
 
 @dataclass
 class CacheKeyFnPolicy(CachePolicy):
+    """
+    This policy accepts a custom function with signature f(task_run_context, task_parameters, flow_parameters) -> str
+    and uses it to compute a task run cache key.
+    """
+
     # making it optional for tests
     cache_key_fn: Optional[
         Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
@@ -77,6 +86,13 @@ class CacheKeyFnPolicy(CachePolicy):
 
 @dataclass
 class CompoundCachePolicy(CachePolicy):
+    """
+    This policy is constructed from two or more other cache policies and works by computing the keys
+    for each policy individually, and then hashing a sorted tuple of all computed keys.
+
+    Any keys that return `None` will be ignored.
+    """
+
     policies: Optional[list] = None
 
     def compute_key(
@@ -88,20 +104,25 @@ class CompoundCachePolicy(CachePolicy):
     ) -> Optional[str]:
         keys = []
         for policy in self.policies or []:
-            keys.append(
-                policy.compute_key(
-                    task_ctx=task_ctx,
-                    inputs=inputs,
-                    flow_parameters=flow_parameters,
-                    **kwargs,
-                )
+            policy_key = policy.compute_key(
+                task_ctx=task_ctx,
+                inputs=inputs,
+                flow_parameters=flow_parameters,
+                **kwargs,
             )
+            if policy_key is not None:
+                keys.append(policy_key)
+        if not keys:
+            return None
         return hash_objects(*keys)
 
 
 @dataclass
-class Default(CachePolicy):
-    "Execution run ID only"
+class _None(CachePolicy):
+    """
+    Policy that always returns `None` for the computed cache key.
+    This policy prevents persistence.
+    """
 
     def compute_key(
         self,
@@ -110,12 +131,14 @@ class Default(CachePolicy):
         flow_parameters: Dict[str, Any],
         **kwargs,
     ) -> Optional[str]:
-        return str(task_ctx.task_run.id)
+        return None
 
 
 @dataclass
-class _None(CachePolicy):
-    "ignore key policies altogether, always run - prevents persistence"
+class TaskSource(CachePolicy):
+    """
+    Policy for computing a cache key based on the source code of the task.
+    """
 
     def compute_key(
         self,
@@ -124,11 +147,22 @@ class _None(CachePolicy):
         flow_parameters: Dict[str, Any],
         **kwargs,
     ) -> Optional[str]:
-        return None
+        if not task_ctx:
+            return None
+        try:
+            lines = inspect.getsource(task_ctx.task)
+        except TypeError:
+            lines = inspect.getsource(task_ctx.task.fn.__class__)
+
+        return hash_objects(lines)
 
 
 @dataclass
-class TaskDef(CachePolicy):
+class FlowParameters(CachePolicy):
+    """
+    Policy that computes the cache key based on a hash of the flow parameters.
+    """
+
     def compute_key(
         self,
         task_ctx: TaskRunContext,
@@ -136,21 +170,37 @@ class TaskDef(CachePolicy):
         flow_parameters: Dict[str, Any],
         **kwargs,
     ) -> Optional[str]:
-        lines = inspect.getsource(task_ctx.task)
-        return hash_objects(lines)
+        if not flow_parameters:
+            return None
+        return hash_objects(flow_parameters)
 
 
 @dataclass
-class FlowParameters(CachePolicy):
-    pass
+class RunId(CachePolicy):
+    """
+    Returns either the prevailing flow run ID, or if not found, the prevailing task
+    run ID.
+    """
+
+    def compute_key(
+        self,
+        task_ctx: TaskRunContext,
+        inputs: Dict[str, Any],
+        flow_parameters: Dict[str, Any],
+        **kwargs,
+    ) -> Optional[str]:
+        if not task_ctx:
+            return None
+        run_id = task_ctx.task_run.flow_run_id
+        if run_id is None:
+            run_id = task_ctx.task_run.id
+        return str(run_id)
 
 
 @dataclass
 class Inputs(CachePolicy):
     """
-    Exposes flag for whether to include flow parameters as well.
-
-    And exclude/include config.
+    Policy that computes a cache key based on a hash of the runtime inputs provided to the task..
     """
 
     exclude: Optional[list] = None
@@ -166,6 +216,9 @@ class Inputs(CachePolicy):
         inputs = inputs or {}
         exclude = self.exclude or []
 
+        if not inputs:
+            return None
+
         for key, val in inputs.items():
             if key not in exclude:
                 hashed_inputs[key] = val
@@ -173,7 +226,9 @@ class Inputs(CachePolicy):
         return hash_objects(hashed_inputs)
 
 
-DEFAULT = Default()
 INPUTS = Inputs()
 NONE = _None()
-TASKDEF = TaskDef()
+TASK_SOURCE = TaskSource()
+FLOW_PARAMETERS = FlowParameters()
+RUN_ID = RunId()
+DEFAULT = INPUTS + TASK_SOURCE + RUN_ID

prefect/task_engine.py

@@ -174,11 +174,18 @@ class TaskRunEngine(Generic[P, R]):
     def compute_transaction_key(self) -> str:
         key = None
         if self.task.cache_policy:
+            flow_run_context = FlowRunContext.get()
             task_run_context = TaskRunContext.get()
+
+            if flow_run_context:
+                parameters = flow_run_context.parameters
+            else:
+                parameters = None
+
             key = self.task.cache_policy.compute_key(
                 task_ctx=task_run_context,
                 inputs=self.parameters,
-                flow_parameters=None,
+                flow_parameters=parameters,
             )
         elif self.task.result_storage_key is not None:
             key = _format_user_supplied_storage_key(self.task.result_storage_key)

prefect/task_runners.py

@@ -1,5 +1,6 @@
 import abc
 import asyncio
+import sys
 import uuid
 from concurrent.futures import ThreadPoolExecutor
 from contextvars import copy_context
@@ -205,7 +206,7 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
     def __init__(self, max_workers: Optional[int] = None):
         super().__init__()
         self._executor: Optional[ThreadPoolExecutor] = None
-        self._max_workers = max_workers
+        self._max_workers = sys.maxsize if max_workers is None else max_workers
 
     def duplicate(self) -> "ThreadPoolTaskRunner":
         return type(self)(max_workers=self._max_workers)

prefect/tasks.py

@@ -32,6 +32,7 @@ from uuid import UUID, uuid4
 
 from typing_extensions import Literal, ParamSpec
 
+from prefect.cache_policies import DEFAULT, NONE, CachePolicy
 from prefect.client.orchestration import get_client
 from prefect.client.schemas import TaskRun
 from prefect.client.schemas.objects import TaskRunInput, TaskRunResult
@@ -43,7 +44,6 @@ from prefect.context import (
 )
 from prefect.futures import PrefectDistributedFuture, PrefectFuture
 from prefect.logging.loggers import get_logger
-from prefect.records.cache_policies import DEFAULT, NONE, CachePolicy
 from prefect.results import ResultFactory, ResultSerializer, ResultStorage
 from prefect.settings import (
     PREFECT_TASK_DEFAULT_RETRIES,
@@ -51,7 +51,10 @@ from prefect.settings import (
 )
 from prefect.states import Pending, Scheduled, State
 from prefect.utilities.annotations import NotSet
-from prefect.utilities.asyncutils import run_coro_as_sync
+from prefect.utilities.asyncutils import (
+    run_coro_as_sync,
+    sync_compatible,
+)
 from prefect.utilities.callables import (
     expand_mapping_parameters,
     get_call_parameters,
@@ -1284,7 +1287,8 @@ class Task(Generic[P, R]):
         """
         return self.apply_async(args=args, kwargs=kwargs)
 
-    def serve(self) -> "Task":
+    @sync_compatible
+    async def serve(self) -> NoReturn:
         """Serve the task using the provided task runner. This method is used to
         establish a websocket connection with the Prefect server and listen for
         submitted task runs to execute.
@@ -1303,7 +1307,7 @@ class Task(Generic[P, R]):
         """
         from prefect.task_worker import serve
 
-        serve(self)
+        await serve(self)
 
 
 @overload

{prefect_client-3.0.0rc5.dist-info → prefect_client-3.0.0rc7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 3.0.0rc5
+Version: 3.0.0rc7
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.

{prefect_client-3.0.0rc5.dist-info → prefect_client-3.0.0rc7.dist-info}/RECORD

@@ -3,6 +3,7 @@ prefect/__init__.py,sha256=YmPor6iCXeKOr8jfH08qD9CBiRsEY6pXs4mRSQRo6ro,2873
 prefect/_version.py,sha256=I9JsXwt7BjAAbMEZgtmE3a6dJ2jqV-wqWto9D6msb3k,24597
 prefect/artifacts.py,sha256=G-jCyce3XGtTyQpCk_s3L7e-TWFyJY8Dcnk_i4_CsY4,12647
 prefect/automations.py,sha256=NlQ62GPJzy-gnWQqX7c6CQJKw7p60WLGDAFcy82vtg4,5613
+prefect/cache_policies.py,sha256=uEKNGO-PJ3N35B2tjhRDtQULN6ok72D9raIoJaUyXk0,6365
 prefect/context.py,sha256=OEmbC61D3l0E50HIaMlVNNJShhYC6I1-4TQhpP321tw,19480
 prefect/engine.py,sha256=asH7iMb1IfEOOIVIxM3ZalfvCe9PUp7f9ceKyT6isa8,2019
 prefect/exceptions.py,sha256=kRiEX6qpT9errs0SuYJDYG7ioMNddTvqK7gT8RVFajk,11076
@@ -19,11 +20,11 @@ prefect/results.py,sha256=qG-5mAlZTRV7GneuQVS7GpHHt3QXcZbwLwn03fHAi4M,27326
 prefect/serializers.py,sha256=8ON--RmaLX3Td3Rpd1lshGcqWyjlCFkmO3sblxsdT_c,8699
 prefect/settings.py,sha256=JkCgcFO7Zw0Kv_azK9XLabx08ypVgL76E6r8nD-HZLM,74598
 prefect/states.py,sha256=JdN01UMYFelFybPoAEKbiPJNuPaj6pksLJ3o0_oNz5Q,20690
-prefect/task_engine.py,sha256=dzMID0AhbRlJnrm7U-PjJ8DxvW2J0j2jStOvDRuZDJs,32317
-prefect/task_runners.py,sha256=TQHyQATPkZE6BNVJ_JQBNmiL1kdgZRjY_Fjs3-N1UiE,11869
+prefect/task_engine.py,sha256=wGN-jFfiLROth8pA8be9Y9_ghi5VS3T2Zpu4ZxTtHSk,32519
+prefect/task_runners.py,sha256=fjOWg2-ds_bE-x5O5Eshr25IS7EMxcEvo7NSmRRtdm8,11920
 prefect/task_runs.py,sha256=eDWYH5H1K4SyduhKmn3GzO6vM3fZSwOZxAb8KhkMGsk,7798
 prefect/task_worker.py,sha256=iawQZn4hNcrXR-CHtM4jzhlnotqeNHiRuHc-eumJ9Oc,16788
-prefect/tasks.py,sha256=-j9ZCxvfvgHkyUsaRuj9EhEWmfO4NLN76Pg2_yGCaL0,60243
+prefect/tasks.py,sha256=rA7FAqcb58hwhrju7ocxL-e982vXEKx86Zl8G-LYcQY,60300
 prefect/transactions.py,sha256=XhEXdhid1457m5V7VTz1U8JCek6U6jSFD7EffGhCLag,9149
 prefect/variables.py,sha256=-t5LVY0N-K4f0fa6YwruVVQqwnU3fGWBMYXXE32XPkA,4821
 prefect/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -129,7 +130,6 @@ prefect/logging/highlighters.py,sha256=BpSXOy0n3lFVvlKWa7jC-HetAiClFi9jnQtEq5-rg
 prefect/logging/loggers.py,sha256=qWM-5IxN3U5MlK7srfALOC9sCpGqt20Vu9WSxpU2zIs,11527
 prefect/logging/logging.yml,sha256=UkEewf0c3_dURI2uCU4RrxkhI5Devoa1s93fl7hilcg,3160
 prefect/records/__init__.py,sha256=7q-lwyevfVgb5S7K9frzawmiJmpZ5ET0m5yXIHBYcVA,31
-prefect/records/cache_policies.py,sha256=2R6tQioujG2qr5rQgg7kPK-SLMM1lUHplEKcOfJbrh0,4761
 prefect/records/result_store.py,sha256=6Yh9zqqXMWjn0qWSfcjQBSfXCM7jVg9pve5TVsOodDc,1734
 prefect/records/store.py,sha256=eQM1p2vZDshXZYg6SkJwL-DP3kUehL_Zgs8xa2-0DZs,224
 prefect/runner/__init__.py,sha256=7U-vAOXFkzMfRz1q8Uv6Otsvc0OrPYLLP44srwkJ_8s,89
@@ -179,8 +179,8 @@ prefect/workers/base.py,sha256=62E0Q41pPr3eQdSBSUBfiR4WYx01OfuqUp5INRqHGgo,46942
 prefect/workers/process.py,sha256=vylkSSswaSCew-V65YW0HcxIxyKI-uqWkbSKpkkLamQ,9372
 prefect/workers/server.py,sha256=EfPiMxI7TVgkqpHkdPwSaYG-ydi99sG7jwXhkAcACbI,1519
 prefect/workers/utilities.py,sha256=VfPfAlGtTuDj0-Kb8WlMgAuOfgXCdrGAnKMapPSBrwc,2483
-prefect_client-3.0.0rc5.dist-info/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
-prefect_client-3.0.0rc5.dist-info/METADATA,sha256=fGHLD4XaLt3FzjMoZxnEJcK-rByrMfxN-Bpu27Y66Ao,7392
-prefect_client-3.0.0rc5.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-prefect_client-3.0.0rc5.dist-info/top_level.txt,sha256=MJZYJgFdbRc2woQCeB4vM6T33tr01TmkEhRcns6H_H4,8
-prefect_client-3.0.0rc5.dist-info/RECORD,,
+prefect_client-3.0.0rc7.dist-info/LICENSE,sha256=MCxsn8osAkzfxKC4CC_dLcUkU8DZLkyihZ8mGs3Ah3Q,11357
+prefect_client-3.0.0rc7.dist-info/METADATA,sha256=XGHFsFT4rRszItqxIOYgPy9KGgecgj7YZSVN0tZB-OE,7392
+prefect_client-3.0.0rc7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+prefect_client-3.0.0rc7.dist-info/top_level.txt,sha256=MJZYJgFdbRc2woQCeB4vM6T33tr01TmkEhRcns6H_H4,8
+prefect_client-3.0.0rc7.dist-info/RECORD,,