wt-task 0.1.2__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. wt_task-0.2.0/CHANGELOG.md +25 -0
  2. {wt_task-0.1.2/src/wt_task.egg-info → wt_task-0.2.0}/PKG-INFO +6 -2
  3. {wt_task-0.1.2 → wt_task-0.2.0}/pyproject.toml +67 -13
  4. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/__init__.py +9 -13
  5. wt_task-0.2.0/src/wt_task/_version.py +24 -0
  6. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/async_task.py +7 -2
  7. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/base.py +90 -6
  8. wt_task-0.2.0/src/wt_task/cache/__init__.py +64 -0
  9. wt_task-0.2.0/src/wt_task/cache/_codec.py +175 -0
  10. wt_task-0.2.0/src/wt_task/cache/_entry.py +34 -0
  11. wt_task-0.2.0/src/wt_task/cache/_key.py +628 -0
  12. wt_task-0.2.0/src/wt_task/cache/_report.py +123 -0
  13. wt_task-0.2.0/src/wt_task/cache/_runtime.py +294 -0
  14. wt_task-0.2.0/src/wt_task/cache/_settings.py +213 -0
  15. wt_task-0.2.0/src/wt_task/cache/_spec.py +194 -0
  16. wt_task-0.2.0/src/wt_task/cache/_store.py +135 -0
  17. wt_task-0.2.0/src/wt_task/cache/_tree.py +519 -0
  18. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/exceptions.py +1 -1
  19. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/executors/base.py +4 -4
  20. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/skip.py +4 -4
  21. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/sync_task.py +25 -18
  22. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/testing.py +14 -11
  23. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/tracing/_config.py +2 -5
  24. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/tracing/_decorator.py +1 -1
  25. {wt_task-0.1.2 → wt_task-0.2.0/src/wt_task.egg-info}/PKG-INFO +6 -2
  26. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task.egg-info/SOURCES.txt +19 -0
  27. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task.egg-info/requires.txt +6 -1
  28. wt_task-0.2.0/src/wt_task.egg-info/scm_file_list.json +50 -0
  29. wt_task-0.2.0/src/wt_task.egg-info/scm_version.json +8 -0
  30. wt_task-0.2.0/tests/test_cache.py +1381 -0
  31. wt_task-0.2.0/tests/test_codec.py +174 -0
  32. wt_task-0.2.0/tests/test_depfingerprint.py +375 -0
  33. wt_task-0.2.0/tests/test_io_ttl.py +338 -0
  34. wt_task-0.2.0/tests/test_report.py +170 -0
  35. wt_task-0.2.0/tests/test_settings.py +246 -0
  36. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_testing.py +10 -5
  37. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_tracing.py +4 -2
  38. wt_task-0.2.0/tests/test_tree.py +285 -0
  39. {wt_task-0.1.2 → wt_task-0.2.0}/uv.lock +316 -23
  40. wt_task-0.1.2/CHANGELOG.md +0 -13
  41. wt_task-0.1.2/src/wt_task/_version.py +0 -34
  42. {wt_task-0.1.2 → wt_task-0.2.0}/README.md +0 -0
  43. {wt_task-0.1.2 → wt_task-0.2.0}/setup.cfg +0 -0
  44. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/decorator.py +0 -0
  45. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/executors/__init__.py +1 -1
  46. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/executors/python.py +0 -0
  47. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/py.typed +0 -0
  48. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/tracing/__init__.py +1 -1
  49. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task.egg-info/dependency_links.txt +0 -0
  50. {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task.egg-info/top_level.txt +0 -0
  51. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_decorator.py +0 -0
  52. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_exceptions.py +0 -0
  53. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_executors.py +0 -0
  54. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_map.py +0 -0
  55. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_mapvalues.py +0 -0
  56. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_partial.py +0 -0
  57. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_skip.py +0 -0
  58. {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_validation.py +0 -0
@@ -0,0 +1,25 @@
1
+ # Changelog
2
+
3
+ ## v0.2.0 — 2026-11-03
4
+
5
+ - Add a content-addressed step cache: `.cached(CacheSpec)` keys a step's result on code identity, fixed inputs and upstream output digests. Identical calls share entries across workflows, and an unchanged output stops recomputation downstream ([#275](https://github.com/wildlife-dynamics/wt/pull/275))
6
+ - Add `CacheSettings` for runtime cache configuration (`WT_CACHE__*` env vars, `.env`, `pyproject.toml`): store, epoch, scope, io TTL, verify, strict ([#275](https://github.com/wildlife-dynamics/wt/pull/275))
7
+ - Each run writes a per-step cache report (`cache-report.json`), including how far a failed run got ([#275](https://github.com/wildlife-dynamics/wt/pull/275))
8
+ - New dependencies: `packaging` and `pydantic-settings`. Object-store backends need the new `cache` extra (`obstore`)
9
+ - Raise the `wt-contracts` floor to `>=0.3.0`
10
+
11
+ ## v0.1.3 — 2026-05-13
12
+
13
+ - Standardize ruff lint config to enforce type annotations and Google-style docstrings; convert module/class/function docstrings accordingly ([#155](https://github.com/wildlife-dynamics/wt/pull/155))
14
+
15
+ ## v0.1.2 — 2026-03-13
16
+
17
+ - `configure_tracer()` and `attach_context()` now warn instead of raising `ImportError` when tracing deps are missing ([#60](https://github.com/wildlife-dynamics/wt/pull/60))
18
+
19
+ ## v0.1.1 — 2026-03-05
20
+
21
+ - Fix incorrect license metadata
22
+
23
+ ## v0.1.0 — 2026-03-05
24
+
25
+ - Initial release
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: wt-task
3
- Version: 0.1.2
3
+ Version: 0.2.0
4
4
  Summary: Task decorator and execution features for wt ecosystem
5
5
  Author: Ecoscope Workflows Team
6
6
  License: BSD-3-Clause
@@ -14,9 +14,13 @@ Classifier: Programming Language :: Python :: 3.11
14
14
  Classifier: Programming Language :: Python :: 3.12
15
15
  Requires-Python: >=3.10
16
16
  Description-Content-Type: text/markdown
17
- Requires-Dist: wt-contracts<1.0.0,>=0.1.0
17
+ Requires-Dist: wt-contracts<1.0.0,>=0.3.0
18
18
  Requires-Dist: pydantic<3.0.0,>=2.0.0
19
+ Requires-Dist: packaging>=23.0
20
+ Requires-Dist: pydantic-settings<3.0.0,>=2.3.0
19
21
  Requires-Dist: typing-extensions>=4.0.0; python_version < "3.11"
22
+ Provides-Extra: cache
23
+ Requires-Dist: obstore>=0.6.0; extra == "cache"
20
24
  Provides-Extra: gcp
21
25
  Requires-Dist: opentelemetry-api>=1.20.0; extra == "gcp"
22
26
  Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "gcp"
@@ -23,12 +23,19 @@ classifiers = [
23
23
  "Programming Language :: Python :: 3.12",
24
24
  ]
25
25
  dependencies = [
26
- "wt-contracts>=0.1.0,<1.0.0",
26
+ "wt-contracts>=0.3.0,<1.0.0",
27
27
  "pydantic>=2.0.0,<3.0.0",
28
+ # Parses the requirement metadata the cache key's dependency fingerprint walks.
29
+ "packaging>=23.0",
30
+ # Backs CacheSettings: env, .env and pyproject.toml sources for the cache knobs.
31
+ "pydantic-settings>=2.3.0,<3.0.0",
28
32
  "typing-extensions>=4.0.0; python_version < '3.11'",
29
33
  ]
30
34
 
31
35
  [project.optional-dependencies]
36
+ cache = [
37
+ "obstore>=0.6.0",
38
+ ]
32
39
  gcp = [
33
40
  "opentelemetry-api>=1.20.0",
34
41
  "opentelemetry-sdk>=1.20.0",
@@ -89,6 +96,13 @@ module = [
89
96
  ]
90
97
  ignore_missing_imports = true
91
98
 
99
+ [[tool.mypy.overrides]]
100
+ module = [
101
+ "obstore",
102
+ "obstore.*",
103
+ ]
104
+ ignore_missing_imports = true
105
+
92
106
  [[tool.mypy.overrides]]
93
107
  module = [
94
108
  "wt_task.tracing._config",
@@ -101,21 +115,59 @@ line-length = 100
101
115
  target-version = "py310"
102
116
 
103
117
  [tool.ruff.lint]
104
- select = [
105
- "E", # pycodestyle errors
106
- "W", # pycodestyle warnings
107
- "F", # pyflakes
108
- "I", # isort
109
- "B", # flake8-bugbear
110
- "C4", # flake8-comprehensions
111
- "UP", # pyupgrade
118
+ # Ruff's legacy defaults; extend-select below is the source of truth.
119
+ select = ["E4", "E7", "E9", "F"]
120
+ extend-select = [
121
+ # already-explicit families (kept; F dropped — it's a default)
122
+ "E", # pycodestyle errors (extends defaults E4/E7/E9 with E1/E2/E5)
123
+ "W", # pycodestyle warnings
124
+ "I", # isort
125
+ "B", # flake8-bugbear
126
+ "C4", # flake8-comprehensions
127
+ "UP", # pyupgrade
128
+ "N", # pep8-naming
129
+ "RUF", # ruff-specific
130
+ "PLC0415", # import-outside-top-level (mandatory per CLAUDE.md)
131
+
132
+ # CLAUDE.md mandates, now enforced
133
+ "ANN", # flake8-annotations (require complete type annotations)
134
+ "D", # pydocstyle (require docstrings on public API)
135
+
136
+ # high-value / low-noise modernizers
137
+ "SIM", # flake8-simplify
138
+ "RET", # flake8-return
139
+ "PIE", # flake8-pie
140
+ "PTH", # flake8-use-pathlib
141
+ "ICN", # flake8-import-conventions
142
+ "TID", # flake8-tidy-imports
143
+ "TC", # flake8-type-checking
144
+ "ISC", # flake8-implicit-str-concat
145
+ "YTT", # flake8-2020
146
+ "PERF", # Perflint
147
+ "FURB", # refurb
148
+ "LOG", # flake8-logging
149
+ "G", # flake8-logging-format
150
+ "PT", # flake8-pytest-style
151
+ "ASYNC", # flake8-async
152
+
153
+ # security / strictness (require per-file-ignores)
154
+ "S", # flake8-bandit
155
+ "ARG", # flake8-unused-arguments
156
+ "BLE", # flake8-blind-except
157
+ "PLE", # Pylint errors
158
+ "PLW", # Pylint warnings
112
159
  ]
113
160
  ignore = [
114
- "E501", # line too long (handled by formatter)
161
+ "E501", # line length — handled by formatter
162
+ "ISC001", # conflicts with formatter (ruff docs recommend disabling)
115
163
  ]
116
164
 
165
+ [tool.ruff.lint.pydocstyle]
166
+ convention = "google"
167
+
117
168
  [tool.ruff.lint.per-file-ignores]
118
- "__init__.py" = ["F401"] # unused imports in __init__.py are often intentional
169
+ "__init__.py" = ["F401", "D104"] # re-exports + missing module docstring
170
+ "tests/**/*.py" = ["D", "S101", "ANN", "ARG", "PT011"] # tests don't need docstrings/annotations; assert OK
119
171
 
120
172
  # ============================================================================
121
173
  # Pixi Build Configuration (conda package building)
@@ -123,7 +175,7 @@ ignore = [
123
175
 
124
176
  [tool.pixi.package]
125
177
  name = "wt-task"
126
- version = "0.1.2"
178
+ version = "0.2.0"
127
179
 
128
180
  [tool.pixi.package.build]
129
181
  backend = { name = "pixi-build-python", version = "*", channels = ["conda-forge", "https://prefix.dev/pixi-build-backends"] }
@@ -140,4 +192,6 @@ pip = "*"
140
192
  [tool.pixi.package.run-dependencies]
141
193
  python = ">=3.10,<3.16"
142
194
  pydantic = ">=2.0.0,<3.0.0"
143
- wt-contracts = ">=0.1.0,<1.0.0"
195
+ packaging = ">=23.0"
196
+ pydantic-settings = ">=2.3.0,<3.0.0"
197
+ wt-contracts = ">=0.3.0,<1.0.0"
@@ -24,25 +24,21 @@ from .skip import (
24
24
  from .sync_task import SyncTask
25
25
 
26
26
  __all__ = [
27
- # Main decorator and task types
28
- "task",
29
- "Task",
30
- "SyncTask",
31
- "AsyncTask",
32
- # Exceptions
33
- "TaskInstanceError",
34
- "handle_errors",
35
- # Executors
36
- "SyncExecutor",
27
+ "SKIP_SENTINEL",
37
28
  "AsyncExecutor",
38
- "PythonExecutor",
29
+ "AsyncTask",
39
30
  "Future",
40
31
  "FutureSequence",
41
- # Skip utilities
32
+ "PythonExecutor",
42
33
  "SkipSentinel",
43
- "SKIP_SENTINEL",
44
34
  "SkippedDependencyFallback",
35
+ "SyncExecutor",
36
+ "SyncTask",
37
+ "Task",
38
+ "TaskInstanceError",
39
+ "handle_errors",
45
40
  "skipif",
41
+ "task",
46
42
  "unpack_listlike",
47
43
  ]
48
44
 
@@ -0,0 +1,24 @@
1
+ # file generated by vcs-versioning
2
+ # don't change, don't track in version control
3
+ from __future__ import annotations
4
+
5
+ __all__ = [
6
+ "__version__",
7
+ "__version_tuple__",
8
+ "version",
9
+ "version_tuple",
10
+ "__commit_id__",
11
+ "commit_id",
12
+ ]
13
+
14
+ version: str
15
+ __version__: str
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
20
+
21
+ __version__ = version = '0.2.0'
22
+ __version_tuple__ = version_tuple = (0, 2, 0)
23
+
24
+ __commit_id__ = commit_id = 'gf28c604'
@@ -6,7 +6,6 @@ returns Future objects.
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- from collections.abc import Sequence
10
9
  from dataclasses import dataclass
11
10
 
12
11
  try:
@@ -16,8 +15,9 @@ try:
16
15
  except ImportError:
17
16
  TRACING_AVAILABLE = False
18
17
 
18
+ from typing import TYPE_CHECKING
19
+
19
20
  from .base import K, P, R, V, _Task
20
- from .executors import AsyncExecutor, Future, FutureSequence
21
21
  from .sync_task import (
22
22
  _create_kwargs_iterable,
23
23
  _create_mapvalues_kwargs_iterable,
@@ -25,6 +25,11 @@ from .sync_task import (
25
25
  _wrap_for_mapvalues,
26
26
  )
27
27
 
28
+ if TYPE_CHECKING:
29
+ from collections.abc import Sequence
30
+
31
+ from .executors import AsyncExecutor, Future, FutureSequence
32
+
28
33
 
29
34
  @dataclass(frozen=True)
30
35
  class AsyncTask(_Task[P, R, K, V]):
@@ -7,11 +7,11 @@ for SyncTask and AsyncTask implementations.
7
7
  from __future__ import annotations
8
8
 
9
9
  import functools
10
+ import inspect
10
11
  import sys
11
12
  import warnings
12
- from collections.abc import Callable
13
- from dataclasses import dataclass, replace
14
- from typing import TYPE_CHECKING, Generic, Literal, ParamSpec, TypeVar, overload
13
+ from dataclasses import dataclass, field, replace
14
+ from typing import TYPE_CHECKING, Any, Generic, Literal, ParamSpec, TypeVar, overload
15
15
 
16
16
  if sys.version_info >= (3, 11):
17
17
  from typing import Self
@@ -19,11 +19,15 @@ else:
19
19
  from typing_extensions import Self
20
20
 
21
21
  if TYPE_CHECKING:
22
+ from collections.abc import Callable
23
+
22
24
  from .async_task import AsyncTask
25
+ from .cache import CacheSpec, TaskEncoding
23
26
  from .sync_task import SyncTask
24
27
 
25
28
  from pydantic import validate_call
26
29
 
30
+ from .cache import cached_call, normalized_params, resolve_task, step_params
27
31
  from .exceptions import handle_errors as handle_errors_fn
28
32
  from .executors import AsyncExecutor, SyncExecutor
29
33
  from .executors.python import PythonExecutor
@@ -34,6 +38,10 @@ P = ParamSpec("P")
34
38
  R = TypeVar("R")
35
39
  K = TypeVar("K")
36
40
  V = TypeVar("V")
41
+ # Scoped to `_through_cache`, not a class parameter: it passes through call's R
42
+ # and map's Sequence[R] alike. Module-level only for <3.12, which lacks
43
+ # per-function type parameters.
44
+ T = TypeVar("T")
37
45
 
38
46
 
39
47
  @dataclass(frozen=True)
@@ -55,12 +63,22 @@ class _Task(Generic[P, R, K, V]):
55
63
  tags: List of tags for categorization
56
64
  description: Optional description of the task
57
65
  task_instance_id: Optional unique identifier for this task instance
66
+ cache_spec: Optional cache identity for this step, set by the compiler when
67
+ the workflow is compiled with caching enabled
68
+ encoding: What `.cached()` worked out from the wrapped function
58
69
  """
59
70
 
60
71
  func: Callable[P, R]
61
72
  tags: list[str]
62
73
  description: str | None
63
74
  task_instance_id: str | None
75
+ # kw_only so that a default here does not force one onto `AsyncTask.executor`,
76
+ # which has no sensible default.
77
+ cache_spec: CacheSpec | None = field(default=None, kw_only=True)
78
+ # Set by `validate`: the function then receives validated arguments, so the
79
+ # cache can key on them instead of on their raw spelling.
80
+ validated: bool = field(default=False, kw_only=True)
81
+ encoding: TaskEncoding | None = field(default=None, kw_only=True, repr=False, compare=False)
64
82
 
65
83
  def set_task_instance_id(self, task_instance_id: str, /) -> Self:
66
84
  """Return a new Task with the task_instance_id set.
@@ -147,6 +165,7 @@ class _Task(Generic[P, R, K, V]):
147
165
  validate_return=True,
148
166
  config={"arbitrary_types_allowed": True},
149
167
  ),
168
+ validated=True,
150
169
  )
151
170
 
152
171
  def with_tracing(self) -> Self:
@@ -241,6 +260,65 @@ class _Task(Generic[P, R, K, V]):
241
260
  func=skipif_fn(self.func, conditions=conditions, unpack_depth=unpack_depth),
242
261
  )
243
262
 
263
+ def cached(self, spec: CacheSpec) -> Self:
264
+ """Return a new Task that resolves its result through the step cache.
265
+
266
+ Unlike the other builders this sets a field rather than wrapping `func`:
267
+ `map` and `mapvalues` call `func` once per item, so a wrapper would key
268
+ per item, and every item of one mapped step shares a single upstream
269
+ identity. Caching at the method level keeps one entry per step.
270
+
271
+ The rest of the step's identity — where the task lives, and how its
272
+ result encodes — is worked out here from the wrapped function, which
273
+ is why this runs before `.partial()` erases it.
274
+
275
+ Args:
276
+ spec: Compile-time cache identity for this task instance
277
+
278
+ Returns:
279
+ New task carrying the cache spec
280
+
281
+ Examples:
282
+ >>> from wt_task import task
283
+ >>> from wt_task.cache import CacheSpec
284
+ >>> @task
285
+ ... def add(a: int, b: int) -> int:
286
+ ... return a + b
287
+ >>> cached = add.cached(CacheSpec(signature="abc123"))
288
+ >>> cached.encoding.function
289
+ 'add'
290
+ >>> cached.call(1, 2) # inert with WT_CACHE__STORE unset
291
+ 3
292
+ """
293
+ return replace(self, cache_spec=spec, encoding=resolve_task(inspect.unwrap(self.func)))
294
+
295
+ def _through_cache(
296
+ self,
297
+ run: Callable[[], T],
298
+ span: Any = None, # noqa: ANN401 # an OTel span, or None when tracing is unavailable
299
+ method: str = "call",
300
+ ) -> T:
301
+ """Resolve `run()` through the cache when one is configured.
302
+
303
+ Shared by every execution method so that `call`, `map` and `mapvalues`
304
+ agree on what one cache entry covers: the whole step. The method is
305
+ passed in rather than stored, because it is the call that picks it.
306
+ """
307
+ if self.cache_spec is None or self.encoding is None:
308
+ return run()
309
+ task_instance_id = self.task_instance_id or ""
310
+ params = step_params(task_instance_id)
311
+ if self.validated:
312
+ # Only then is the validated value what the function receives; an
313
+ # unvalidated task may behave differently for "2" and 2.
314
+ params = normalized_params(self.func, params, self.cache_spec.upstream_inputs)
315
+ result, hit = cached_call(
316
+ self.cache_spec, run, self.encoding, task_instance_id, method, params
317
+ )
318
+ if span is not None:
319
+ span.set_attribute("cache.hit", hit)
320
+ return result
321
+
244
322
  @overload
245
323
  def set_executor(
246
324
  self,
@@ -290,9 +368,9 @@ class _Task(Generic[P, R, K, V]):
290
368
  >>> type(custom.executor).__name__
291
369
  'PythonExecutor'
292
370
  """
293
- # Import here to avoid circular imports
294
- from .async_task import AsyncTask
295
- from .sync_task import SyncTask
371
+ # circular import: AsyncTask/SyncTask subclass _Task defined here
372
+ from .async_task import AsyncTask # noqa: PLC0415
373
+ from .sync_task import SyncTask # noqa: PLC0415
296
374
 
297
375
  match name_or_executor:
298
376
  case "python":
@@ -301,9 +379,13 @@ class _Task(Generic[P, R, K, V]):
301
379
  tags=self.tags,
302
380
  description=self.description,
303
381
  task_instance_id=self.task_instance_id,
382
+ cache_spec=self.cache_spec,
383
+ encoding=self.encoding,
304
384
  executor=PythonExecutor(),
305
385
  )
306
386
  case AsyncExecutor():
387
+ # An async method hands back a future, which has no stored
388
+ # form, so the cache spec is not carried over.
307
389
  return AsyncTask(
308
390
  self.func,
309
391
  tags=self.tags,
@@ -317,6 +399,8 @@ class _Task(Generic[P, R, K, V]):
317
399
  tags=self.tags,
318
400
  description=self.description,
319
401
  task_instance_id=self.task_instance_id,
402
+ cache_spec=self.cache_spec,
403
+ encoding=self.encoding,
320
404
  executor=name_or_executor,
321
405
  )
322
406
  case _:
@@ -0,0 +1,64 @@
1
+ """Content-addressed step caching for compiled DAGs.
2
+
3
+ Opt-in at compile time. A step is keyed by its code, its arguments, and the
4
+ content digests of the steps feeding it, and a hit returns the stored value
5
+ instead of executing. What a result *is* comes from the task's return type —
6
+ `wt_contracts.declared_structure` resolves it at run time, and
7
+ `resolve_structure` turns it into the `Tree` that encodes the whole step.
8
+ Everything degrades to running normally: with `WT_CACHE__STORE` unset, or for any
9
+ return type that declares no codec, the cache is inert.
10
+
11
+ Runtime knobs live in `CacheSettings`, read once per `run_scope`.
12
+ """
13
+
14
+ from ._codec import BYTES, JSON, PATH, SKIP, URL, Codec, Undigestible, enum_codec
15
+ from ._entry import CacheEntry
16
+ from ._key import (
17
+ CacheRecord,
18
+ normalized_params,
19
+ record,
20
+ run_report,
21
+ run_scope,
22
+ step_identity,
23
+ step_params,
24
+ )
25
+ from ._report import REPORT_NAME, cache_report, put_cache_report
26
+ from ._runtime import CacheCorruptionError, CacheMismatchWarning, cached_call
27
+ from ._settings import CacheSettings, load_settings
28
+ from ._spec import CacheSpec, TaskEncoding, resolve_task
29
+ from ._store import open_store
30
+ from ._tree import StructureMismatch, Tree, resolve_structure
31
+
32
+ __all__ = [
33
+ "BYTES",
34
+ "JSON",
35
+ "PATH",
36
+ "REPORT_NAME",
37
+ "SKIP",
38
+ "URL",
39
+ "CacheCorruptionError",
40
+ "CacheEntry",
41
+ "CacheMismatchWarning",
42
+ "CacheRecord",
43
+ "CacheSettings",
44
+ "CacheSpec",
45
+ "Codec",
46
+ "StructureMismatch",
47
+ "TaskEncoding",
48
+ "Tree",
49
+ "Undigestible",
50
+ "cache_report",
51
+ "cached_call",
52
+ "enum_codec",
53
+ "load_settings",
54
+ "normalized_params",
55
+ "open_store",
56
+ "put_cache_report",
57
+ "record",
58
+ "resolve_structure",
59
+ "resolve_task",
60
+ "run_report",
61
+ "run_scope",
62
+ "step_identity",
63
+ "step_params",
64
+ ]
@@ -0,0 +1,175 @@
1
+ """Leaf codecs: how one declared value becomes bytes, and back.
2
+
3
+ Which codec applies is not decided here — the task's return type declares it
4
+ (`wt_contracts.codecs`) and `_tree` resolves it. This module only holds the
5
+ builtin leaves and the shape a task library's own codecs must have.
6
+
7
+ There is deliberately no pickle fallback: pickle couples every stored entry to
8
+ the exact version of the code that wrote it and turns the cache into a
9
+ deserialization surface. Refusing is always safe — the step runs normally.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import hashlib
15
+ import json
16
+ import pathlib
17
+ from dataclasses import dataclass
18
+ from typing import TYPE_CHECKING, Any
19
+
20
+ from pydantic_core import Url
21
+
22
+ from wt_task.skip import SKIP_SENTINEL, SkipSentinel
23
+
24
+ if TYPE_CHECKING:
25
+ import enum
26
+ from collections.abc import Callable
27
+
28
+
29
+ class Undigestible(Exception): # noqa: N818 # named for what a codec does, not for a failure
30
+ """Raised by a codec that cannot handle the value it was handed.
31
+
32
+ The step then runs normally and records itself uncacheable, so a codec can
33
+ decline a value (an unhashable cell, a subclass it does not cover) without
34
+ the caller having to probe it first.
35
+ """
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class Codec:
40
+ """Encodes one family of runtime values to bytes, and digests them.
41
+
42
+ `digest` is separate from `encode` on purpose: it must be stable across
43
+ encodings, because an unstable digest silently invalidates every downstream
44
+ cache entry on each run. Parquet in particular is not byte-stable, so a
45
+ frame codec digests a canonical form rather than the bytes it writes.
46
+
47
+ Attributes:
48
+ codec_id: Stable identifier recorded in the payload header. Changing it
49
+ makes old entries miss rather than decode wrongly.
50
+ media_type: Media type of the encoded payload.
51
+ types: Runtime types this codec accepts. Used only to pick an arm of a
52
+ declared union; at a non-union position the declaration already
53
+ settled it.
54
+ encode: Value to bytes.
55
+ decode: Bytes back to a value.
56
+ digest: Stable content digest. May raise `Undigestible`.
57
+ accepts_subclasses: Match with `isinstance` rather than
58
+ `type(value) is types[0]`. Cleared by class-parameterized codecs,
59
+ which must not accept a subclass their encoder would silently narrow.
60
+ """
61
+
62
+ codec_id: str
63
+ media_type: str
64
+ types: tuple[type, ...]
65
+ encode: Callable[[Any], bytes]
66
+ decode: Callable[[bytes], Any]
67
+ digest: Callable[[Any], str]
68
+ accepts_subclasses: bool = True
69
+
70
+
71
+ def json_digest(value: Any) -> str: # noqa: ANN401 # digests arbitrary JSON-able values
72
+ """Return a stable sha256 over a JSON-able value, stringifying what it must.
73
+
74
+ The lenient `default=str` is for cache *keys*, whose inputs are compiler
75
+ literals and params rather than task results. Payload digests use the
76
+ strict `JSON` codec instead.
77
+
78
+ Examples:
79
+ >>> json_digest({"b": 1, "a": 2}) == json_digest({"a": 2, "b": 1})
80
+ True
81
+ """
82
+ return hashlib.sha256(
83
+ json.dumps(value, sort_keys=True, default=str).encode("utf-8")
84
+ ).hexdigest()
85
+
86
+
87
+ def _strict_json(value: Any) -> bytes: # noqa: ANN401 # encodes arbitrary JSON-able values
88
+ """Serialize `value` as JSON, declining anything JSON cannot hold losslessly."""
89
+ try:
90
+ return json.dumps(value, sort_keys=True, allow_nan=False).encode("utf-8")
91
+ except (TypeError, ValueError) as e:
92
+ raise Undigestible(f"not JSON-native: {e}") from e
93
+
94
+
95
+ JSON = Codec(
96
+ codec_id="json/v1",
97
+ media_type="application/json",
98
+ types=(str, int, float, bool, list, dict, type(None)),
99
+ encode=_strict_json,
100
+ decode=json.loads,
101
+ digest=lambda v: hashlib.sha256(_strict_json(v)).hexdigest(),
102
+ )
103
+ """JSON-native values. Reached only through an explicit `CodecRef`, as the
104
+ declaration that a loosely-typed return really does hold plain JSON."""
105
+
106
+ BYTES = Codec(
107
+ codec_id="bytes/v1",
108
+ media_type="application/octet-stream",
109
+ types=(bytes,),
110
+ encode=lambda v: v,
111
+ decode=lambda d: d,
112
+ digest=lambda v: hashlib.sha256(v).hexdigest(),
113
+ )
114
+
115
+ PATH = Codec(
116
+ codec_id="path/v1",
117
+ media_type="text/plain",
118
+ types=(pathlib.PurePath,),
119
+ encode=lambda v: str(v).encode("utf-8"),
120
+ decode=lambda d: pathlib.Path(d.decode("utf-8")),
121
+ digest=lambda v: hashlib.sha256(str(v).encode("utf-8")).hexdigest(),
122
+ )
123
+ """Paths travel as the path, not as the bytes they point at. A task returning
124
+ one is claiming the location is the result; `no-cache` is the way to say
125
+ otherwise."""
126
+
127
+ URL = Codec(
128
+ codec_id="url/v1",
129
+ media_type="text/plain",
130
+ types=(Url,),
131
+ encode=lambda v: str(v).encode("utf-8"),
132
+ decode=lambda d: Url(d.decode("utf-8")),
133
+ digest=lambda v: hashlib.sha256(str(v).encode("utf-8")).hexdigest(),
134
+ )
135
+
136
+ _SKIP_MARKER = b'{"wt_task":"skip"}'
137
+
138
+ SKIP = Codec(
139
+ codec_id="skip/v1",
140
+ media_type="application/json",
141
+ types=(SkipSentinel,),
142
+ encode=lambda _v: _SKIP_MARKER,
143
+ decode=lambda _d: SKIP_SENTINEL,
144
+ digest=lambda _v: hashlib.sha256(_SKIP_MARKER).hexdigest(),
145
+ )
146
+ """A skipped step is cached like any other result. Without this the sentinel
147
+ would have no codec and every step below a skip would lose its cache."""
148
+
149
+
150
+ def enum_codec(cls: type[enum.Enum]) -> Codec:
151
+ """Build the codec for one enum class: the member's value, round-tripped.
152
+
153
+ Args:
154
+ cls: The enum class, as named by the declared structure.
155
+
156
+ Returns:
157
+ A codec accepting exactly `cls`.
158
+
159
+ Examples:
160
+ >>> import enum
161
+ >>> class Unit(enum.Enum):
162
+ ... km = "km"
163
+ >>> codec = enum_codec(Unit)
164
+ >>> codec.decode(codec.encode(Unit.km))
165
+ <Unit.km: 'km'>
166
+ """
167
+ return Codec(
168
+ codec_id=f"enum/v1/{cls.__module__}.{cls.__qualname__}",
169
+ media_type="application/json",
170
+ types=(cls,),
171
+ encode=lambda v: _strict_json(v.value),
172
+ decode=lambda d: cls(json.loads(d)),
173
+ digest=lambda v: hashlib.sha256(_strict_json(v.value)).hexdigest(),
174
+ accepts_subclasses=False,
175
+ )