wt-task 0.1.3__tar.gz → 0.2.0__tar.gz

wt_task-0.2.0/CHANGELOG.md

@@ -0,0 +1,25 @@
+# Changelog
+
+## v0.2.0 — 2026-11-03
+
+- Add a content-addressed step cache: `.cached(CacheSpec)` keys a step's result on code identity, fixed inputs and upstream output digests. Identical calls share entries across workflows, and an unchanged output stops recomputation downstream ([#275](https://github.com/wildlife-dynamics/wt/pull/275))
+- Add `CacheSettings` for runtime cache configuration (`WT_CACHE__*` env vars, `.env`, `pyproject.toml`): store, epoch, scope, io TTL, verify, strict ([#275](https://github.com/wildlife-dynamics/wt/pull/275))
+- Each run writes a per-step cache report (`cache-report.json`), including how far a failed run got ([#275](https://github.com/wildlife-dynamics/wt/pull/275))
+- New dependencies: `packaging` and `pydantic-settings`. Object-store backends need the new `cache` extra (`obstore`)
+- Raise the `wt-contracts` floor to `>=0.3.0`
+
+## v0.1.3 — 2026-05-13
+
+- Standardize ruff lint config to enforce type annotations and Google-style docstrings; convert module/class/function docstrings accordingly ([#155](https://github.com/wildlife-dynamics/wt/pull/155))
+
+## v0.1.2 — 2026-03-13
+
+- `configure_tracer()` and `attach_context()` now warn instead of raising `ImportError` when tracing deps are missing ([#60](https://github.com/wildlife-dynamics/wt/pull/60))
+
+## v0.1.1 — 2026-03-05
+
+- Fix incorrect license metadata
+
+## v0.1.0 — 2026-03-05
+
+- Initial release

{wt_task-0.1.3/src/wt_task.egg-info → wt_task-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wt-task
-Version: 0.1.3
+Version: 0.2.0
 Summary: Task decorator and execution features for wt ecosystem
 Author: Ecoscope Workflows Team
 License: BSD-3-Clause
@@ -14,9 +14,13 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: wt-contracts<1.0.0,>=0.1.0
+Requires-Dist: wt-contracts<1.0.0,>=0.3.0
 Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: packaging>=23.0
+Requires-Dist: pydantic-settings<3.0.0,>=2.3.0
 Requires-Dist: typing-extensions>=4.0.0; python_version < "3.11"
+Provides-Extra: cache
+Requires-Dist: obstore>=0.6.0; extra == "cache"
 Provides-Extra: gcp
 Requires-Dist: opentelemetry-api>=1.20.0; extra == "gcp"
 Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "gcp"

{wt_task-0.1.3 → wt_task-0.2.0}/pyproject.toml

@@ -23,12 +23,19 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "wt-contracts>=0.1.0,<1.0.0",
+    "wt-contracts>=0.3.0,<1.0.0",
     "pydantic>=2.0.0,<3.0.0",
+    # Parses the requirement metadata the cache key's dependency fingerprint walks.
+    "packaging>=23.0",
+    # Backs CacheSettings: env, .env and pyproject.toml sources for the cache knobs.
+    "pydantic-settings>=2.3.0,<3.0.0",
     "typing-extensions>=4.0.0; python_version < '3.11'",
 ]
 
 [project.optional-dependencies]
+cache = [
+    "obstore>=0.6.0",
+]
 gcp = [
     "opentelemetry-api>=1.20.0",
     "opentelemetry-sdk>=1.20.0",
@@ -89,6 +96,13 @@ module = [
 ]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+module = [
+    "obstore",
+    "obstore.*",
+]
+ignore_missing_imports = true
+
 [[tool.mypy.overrides]]
 module = [
     "wt_task.tracing._config",
@@ -101,6 +115,8 @@ line-length = 100
 target-version = "py310"
 
 [tool.ruff.lint]
+# Ruff's legacy defaults; extend-select below is the source of truth.
+select = ["E4", "E7", "E9", "F"]
 extend-select = [
     # already-explicit families (kept; F dropped — it's a default)
     "E",       # pycodestyle errors (extends defaults E4/E7/E9 with E1/E2/E5)
@@ -159,7 +175,7 @@ convention = "google"
 
 [tool.pixi.package]
 name = "wt-task"
-version = "0.1.3"
+version = "0.2.0"
 
 [tool.pixi.package.build]
 backend = { name = "pixi-build-python", version = "*", channels = ["conda-forge", "https://prefix.dev/pixi-build-backends"] }
@@ -176,4 +192,6 @@ pip = "*"
 [tool.pixi.package.run-dependencies]
 python = ">=3.10,<3.16"
 pydantic = ">=2.0.0,<3.0.0"
-wt-contracts = ">=0.1.0,<1.0.0"
+packaging = ">=23.0"
+pydantic-settings = ">=2.3.0,<3.0.0"
+wt-contracts = ">=0.3.0,<1.0.0"

{wt_task-0.1.3 → wt_task-0.2.0}/src/wt_task/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.3'
-__version_tuple__ = version_tuple = (0, 1, 3)
+__version__ = version = '0.2.0'
+__version_tuple__ = version_tuple = (0, 2, 0)
 
-__commit_id__ = commit_id = 'g5f0f25f'
+__commit_id__ = commit_id = 'gf28c604'

{wt_task-0.1.3 → wt_task-0.2.0}/src/wt_task/base.py

@@ -7,10 +7,11 @@ for SyncTask and AsyncTask implementations.
 from __future__ import annotations
 
 import functools
+import inspect
 import sys
 import warnings
-from dataclasses import dataclass, replace
-from typing import TYPE_CHECKING, Generic, Literal, ParamSpec, TypeVar, overload
+from dataclasses import dataclass, field, replace
+from typing import TYPE_CHECKING, Any, Generic, Literal, ParamSpec, TypeVar, overload
 
 if sys.version_info >= (3, 11):
     from typing import Self
@@ -21,10 +22,12 @@ if TYPE_CHECKING:
     from collections.abc import Callable
 
     from .async_task import AsyncTask
+    from .cache import CacheSpec, TaskEncoding
     from .sync_task import SyncTask
 
 from pydantic import validate_call
 
+from .cache import cached_call, normalized_params, resolve_task, step_params
 from .exceptions import handle_errors as handle_errors_fn
 from .executors import AsyncExecutor, SyncExecutor
 from .executors.python import PythonExecutor
@@ -35,6 +38,10 @@ P = ParamSpec("P")
 R = TypeVar("R")
 K = TypeVar("K")
 V = TypeVar("V")
+# Scoped to `_through_cache`, not a class parameter: it passes through call's R
+# and map's Sequence[R] alike. Module-level only for <3.12, which lacks
+# per-function type parameters.
+T = TypeVar("T")
 
 
 @dataclass(frozen=True)
@@ -56,12 +63,22 @@ class _Task(Generic[P, R, K, V]):
         tags: List of tags for categorization
         description: Optional description of the task
         task_instance_id: Optional unique identifier for this task instance
+        cache_spec: Optional cache identity for this step, set by the compiler when
+            the workflow is compiled with caching enabled
+        encoding: What `.cached()` worked out from the wrapped function
     """
 
     func: Callable[P, R]
     tags: list[str]
     description: str | None
     task_instance_id: str | None
+    # kw_only so that a default here does not force one onto `AsyncTask.executor`,
+    # which has no sensible default.
+    cache_spec: CacheSpec | None = field(default=None, kw_only=True)
+    # Set by `validate`: the function then receives validated arguments, so the
+    # cache can key on them instead of on their raw spelling.
+    validated: bool = field(default=False, kw_only=True)
+    encoding: TaskEncoding | None = field(default=None, kw_only=True, repr=False, compare=False)
 
     def set_task_instance_id(self, task_instance_id: str, /) -> Self:
         """Return a new Task with the task_instance_id set.
@@ -148,6 +165,7 @@ class _Task(Generic[P, R, K, V]):
                 validate_return=True,
                 config={"arbitrary_types_allowed": True},
             ),
+            validated=True,
         )
 
     def with_tracing(self) -> Self:
@@ -242,6 +260,65 @@ class _Task(Generic[P, R, K, V]):
             func=skipif_fn(self.func, conditions=conditions, unpack_depth=unpack_depth),
         )
 
+    def cached(self, spec: CacheSpec) -> Self:
+        """Return a new Task that resolves its result through the step cache.
+
+        Unlike the other builders this sets a field rather than wrapping `func`:
+        `map` and `mapvalues` call `func` once per item, so a wrapper would key
+        per item, and every item of one mapped step shares a single upstream
+        identity. Caching at the method level keeps one entry per step.
+
+        The rest of the step's identity — where the task lives, and how its
+        result encodes — is worked out here from the wrapped function, which
+        is why this runs before `.partial()` erases it.
+
+        Args:
+            spec: Compile-time cache identity for this task instance
+
+        Returns:
+            New task carrying the cache spec
+
+        Examples:
+            >>> from wt_task import task
+            >>> from wt_task.cache import CacheSpec
+            >>> @task
+            ... def add(a: int, b: int) -> int:
+            ...     return a + b
+            >>> cached = add.cached(CacheSpec(signature="abc123"))
+            >>> cached.encoding.function
+            'add'
+            >>> cached.call(1, 2)  # inert with WT_CACHE__STORE unset
+            3
+        """
+        return replace(self, cache_spec=spec, encoding=resolve_task(inspect.unwrap(self.func)))
+
+    def _through_cache(
+        self,
+        run: Callable[[], T],
+        span: Any = None,  # noqa: ANN401  # an OTel span, or None when tracing is unavailable
+        method: str = "call",
+    ) -> T:
+        """Resolve `run()` through the cache when one is configured.
+
+        Shared by every execution method so that `call`, `map` and `mapvalues`
+        agree on what one cache entry covers: the whole step. The method is
+        passed in rather than stored, because it is the call that picks it.
+        """
+        if self.cache_spec is None or self.encoding is None:
+            return run()
+        task_instance_id = self.task_instance_id or ""
+        params = step_params(task_instance_id)
+        if self.validated:
+            # Only then is the validated value what the function receives; an
+            # unvalidated task may behave differently for "2" and 2.
+            params = normalized_params(self.func, params, self.cache_spec.upstream_inputs)
+        result, hit = cached_call(
+            self.cache_spec, run, self.encoding, task_instance_id, method, params
+        )
+        if span is not None:
+            span.set_attribute("cache.hit", hit)
+        return result
+
     @overload
     def set_executor(
         self,
@@ -302,9 +379,13 @@ class _Task(Generic[P, R, K, V]):
                     tags=self.tags,
                     description=self.description,
                     task_instance_id=self.task_instance_id,
+                    cache_spec=self.cache_spec,
+                    encoding=self.encoding,
                     executor=PythonExecutor(),
                 )
             case AsyncExecutor():
+                # An async method hands back a future, which has no stored
+                # form, so the cache spec is not carried over.
                 return AsyncTask(
                     self.func,
                     tags=self.tags,
@@ -318,6 +399,8 @@ class _Task(Generic[P, R, K, V]):
                     tags=self.tags,
                     description=self.description,
                     task_instance_id=self.task_instance_id,
+                    cache_spec=self.cache_spec,
+                    encoding=self.encoding,
                     executor=name_or_executor,
                 )
             case _:

wt_task-0.2.0/src/wt_task/cache/__init__.py

@@ -0,0 +1,64 @@
+"""Content-addressed step caching for compiled DAGs.
+
+Opt-in at compile time. A step is keyed by its code, its arguments, and the
+content digests of the steps feeding it, and a hit returns the stored value
+instead of executing. What a result *is* comes from the task's return type —
+`wt_contracts.declared_structure` resolves it at run time, and
+`resolve_structure` turns it into the `Tree` that encodes the whole step.
+Everything degrades to running normally: with `WT_CACHE__STORE` unset, or for any
+return type that declares no codec, the cache is inert.
+
+Runtime knobs live in `CacheSettings`, read once per `run_scope`.
+"""
+
+from ._codec import BYTES, JSON, PATH, SKIP, URL, Codec, Undigestible, enum_codec
+from ._entry import CacheEntry
+from ._key import (
+    CacheRecord,
+    normalized_params,
+    record,
+    run_report,
+    run_scope,
+    step_identity,
+    step_params,
+)
+from ._report import REPORT_NAME, cache_report, put_cache_report
+from ._runtime import CacheCorruptionError, CacheMismatchWarning, cached_call
+from ._settings import CacheSettings, load_settings
+from ._spec import CacheSpec, TaskEncoding, resolve_task
+from ._store import open_store
+from ._tree import StructureMismatch, Tree, resolve_structure
+
+__all__ = [
+    "BYTES",
+    "JSON",
+    "PATH",
+    "REPORT_NAME",
+    "SKIP",
+    "URL",
+    "CacheCorruptionError",
+    "CacheEntry",
+    "CacheMismatchWarning",
+    "CacheRecord",
+    "CacheSettings",
+    "CacheSpec",
+    "Codec",
+    "StructureMismatch",
+    "TaskEncoding",
+    "Tree",
+    "Undigestible",
+    "cache_report",
+    "cached_call",
+    "enum_codec",
+    "load_settings",
+    "normalized_params",
+    "open_store",
+    "put_cache_report",
+    "record",
+    "resolve_structure",
+    "resolve_task",
+    "run_report",
+    "run_scope",
+    "step_identity",
+    "step_params",
+]

wt_task-0.2.0/src/wt_task/cache/_codec.py

@@ -0,0 +1,175 @@
+"""Leaf codecs: how one declared value becomes bytes, and back.
+
+Which codec applies is not decided here — the task's return type declares it
+(`wt_contracts.codecs`) and `_tree` resolves it. This module only holds the
+builtin leaves and the shape a task library's own codecs must have.
+
+There is deliberately no pickle fallback: pickle couples every stored entry to
+the exact version of the code that wrote it and turns the cache into a
+deserialization surface. Refusing is always safe — the step runs normally.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import pathlib
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from pydantic_core import Url
+
+from wt_task.skip import SKIP_SENTINEL, SkipSentinel
+
+if TYPE_CHECKING:
+    import enum
+    from collections.abc import Callable
+
+
+class Undigestible(Exception):  # noqa: N818  # named for what a codec does, not for a failure
+    """Raised by a codec that cannot handle the value it was handed.
+
+    The step then runs normally and records itself uncacheable, so a codec can
+    decline a value (an unhashable cell, a subclass it does not cover) without
+    the caller having to probe it first.
+    """
+
+
+@dataclass(frozen=True)
+class Codec:
+    """Encodes one family of runtime values to bytes, and digests them.
+
+    `digest` is separate from `encode` on purpose: it must be stable across
+    encodings, because an unstable digest silently invalidates every downstream
+    cache entry on each run. Parquet in particular is not byte-stable, so a
+    frame codec digests a canonical form rather than the bytes it writes.
+
+    Attributes:
+        codec_id: Stable identifier recorded in the payload header. Changing it
+            makes old entries miss rather than decode wrongly.
+        media_type: Media type of the encoded payload.
+        types: Runtime types this codec accepts. Used only to pick an arm of a
+            declared union; at a non-union position the declaration already
+            settled it.
+        encode: Value to bytes.
+        decode: Bytes back to a value.
+        digest: Stable content digest. May raise `Undigestible`.
+        accepts_subclasses: Match with `isinstance` rather than
+            `type(value) is types[0]`. Cleared by class-parameterized codecs,
+            which must not accept a subclass their encoder would silently narrow.
+    """
+
+    codec_id: str
+    media_type: str
+    types: tuple[type, ...]
+    encode: Callable[[Any], bytes]
+    decode: Callable[[bytes], Any]
+    digest: Callable[[Any], str]
+    accepts_subclasses: bool = True
+
+
+def json_digest(value: Any) -> str:  # noqa: ANN401  # digests arbitrary JSON-able values
+    """Return a stable sha256 over a JSON-able value, stringifying what it must.
+
+    The lenient `default=str` is for cache *keys*, whose inputs are compiler
+    literals and params rather than task results. Payload digests use the
+    strict `JSON` codec instead.
+
+    Examples:
+        >>> json_digest({"b": 1, "a": 2}) == json_digest({"a": 2, "b": 1})
+        True
+    """
+    return hashlib.sha256(
+        json.dumps(value, sort_keys=True, default=str).encode("utf-8")
+    ).hexdigest()
+
+
+def _strict_json(value: Any) -> bytes:  # noqa: ANN401  # encodes arbitrary JSON-able values
+    """Serialize `value` as JSON, declining anything JSON cannot hold losslessly."""
+    try:
+        return json.dumps(value, sort_keys=True, allow_nan=False).encode("utf-8")
+    except (TypeError, ValueError) as e:
+        raise Undigestible(f"not JSON-native: {e}") from e
+
+
+JSON = Codec(
+    codec_id="json/v1",
+    media_type="application/json",
+    types=(str, int, float, bool, list, dict, type(None)),
+    encode=_strict_json,
+    decode=json.loads,
+    digest=lambda v: hashlib.sha256(_strict_json(v)).hexdigest(),
+)
+"""JSON-native values. Reached only through an explicit `CodecRef`, as the
+declaration that a loosely-typed return really does hold plain JSON."""
+
+BYTES = Codec(
+    codec_id="bytes/v1",
+    media_type="application/octet-stream",
+    types=(bytes,),
+    encode=lambda v: v,
+    decode=lambda d: d,
+    digest=lambda v: hashlib.sha256(v).hexdigest(),
+)
+
+PATH = Codec(
+    codec_id="path/v1",
+    media_type="text/plain",
+    types=(pathlib.PurePath,),
+    encode=lambda v: str(v).encode("utf-8"),
+    decode=lambda d: pathlib.Path(d.decode("utf-8")),
+    digest=lambda v: hashlib.sha256(str(v).encode("utf-8")).hexdigest(),
+)
+"""Paths travel as the path, not as the bytes they point at. A task returning
+one is claiming the location is the result; `no-cache` is the way to say
+otherwise."""
+
+URL = Codec(
+    codec_id="url/v1",
+    media_type="text/plain",
+    types=(Url,),
+    encode=lambda v: str(v).encode("utf-8"),
+    decode=lambda d: Url(d.decode("utf-8")),
+    digest=lambda v: hashlib.sha256(str(v).encode("utf-8")).hexdigest(),
+)
+
+_SKIP_MARKER = b'{"wt_task":"skip"}'
+
+SKIP = Codec(
+    codec_id="skip/v1",
+    media_type="application/json",
+    types=(SkipSentinel,),
+    encode=lambda _v: _SKIP_MARKER,
+    decode=lambda _d: SKIP_SENTINEL,
+    digest=lambda _v: hashlib.sha256(_SKIP_MARKER).hexdigest(),
+)
+"""A skipped step is cached like any other result. Without this the sentinel
+would have no codec and every step below a skip would lose its cache."""
+
+
+def enum_codec(cls: type[enum.Enum]) -> Codec:
+    """Build the codec for one enum class: the member's value, round-tripped.
+
+    Args:
+        cls: The enum class, as named by the declared structure.
+
+    Returns:
+        A codec accepting exactly `cls`.
+
+    Examples:
+        >>> import enum
+        >>> class Unit(enum.Enum):
+        ...     km = "km"
+        >>> codec = enum_codec(Unit)
+        >>> codec.decode(codec.encode(Unit.km))
+        <Unit.km: 'km'>
+    """
+    return Codec(
+        codec_id=f"enum/v1/{cls.__module__}.{cls.__qualname__}",
+        media_type="application/json",
+        types=(cls,),
+        encode=lambda v: _strict_json(v.value),
+        decode=lambda d: cls(json.loads(d)),
+        digest=lambda v: hashlib.sha256(_strict_json(v.value)).hexdigest(),
+        accepts_subclasses=False,
+    )

wt_task-0.2.0/src/wt_task/cache/_entry.py

@@ -0,0 +1,34 @@
+"""The metadata record stored alongside a cached payload."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from pydantic import BaseModel, Field
+
+
+def utcnow() -> datetime:
+    """Now, in UTC. One place, so the freshness check can be driven in tests."""
+    return datetime.now(timezone.utc)
+
+
+class CacheEntry(BaseModel):
+    """Points a step's key at its payload, and says how to read it back.
+
+    Keys and payloads are stored separately so that two steps computing the same
+    value share one payload, and so that an entry still describes what a step
+    produced after its payload has been evicted.
+
+    Attributes:
+        content_id: `<codec>/<hex>`: the step's root codec and the value's
+            content digest, which together locate the payload under `content/`.
+        codec_id: Codec that wrote the payload and must read it back.
+        media_type: Media type of the payload.
+        created_at: When the entry was written. Read by the freshness
+            check that `WT_CACHE__IO_TTL` puts on io-tagged steps.
+    """
+
+    content_id: str
+    codec_id: str
+    media_type: str
+    created_at: datetime = Field(default_factory=utcnow)