wt-task 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wt_task-0.2.0/CHANGELOG.md +25 -0
- {wt_task-0.1.2/src/wt_task.egg-info → wt_task-0.2.0}/PKG-INFO +6 -2
- {wt_task-0.1.2 → wt_task-0.2.0}/pyproject.toml +67 -13
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/__init__.py +9 -13
- wt_task-0.2.0/src/wt_task/_version.py +24 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/async_task.py +7 -2
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/base.py +90 -6
- wt_task-0.2.0/src/wt_task/cache/__init__.py +64 -0
- wt_task-0.2.0/src/wt_task/cache/_codec.py +175 -0
- wt_task-0.2.0/src/wt_task/cache/_entry.py +34 -0
- wt_task-0.2.0/src/wt_task/cache/_key.py +628 -0
- wt_task-0.2.0/src/wt_task/cache/_report.py +123 -0
- wt_task-0.2.0/src/wt_task/cache/_runtime.py +294 -0
- wt_task-0.2.0/src/wt_task/cache/_settings.py +213 -0
- wt_task-0.2.0/src/wt_task/cache/_spec.py +194 -0
- wt_task-0.2.0/src/wt_task/cache/_store.py +135 -0
- wt_task-0.2.0/src/wt_task/cache/_tree.py +519 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/exceptions.py +1 -1
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/executors/base.py +4 -4
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/skip.py +4 -4
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/sync_task.py +25 -18
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/testing.py +14 -11
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/tracing/_config.py +2 -5
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/tracing/_decorator.py +1 -1
- {wt_task-0.1.2 → wt_task-0.2.0/src/wt_task.egg-info}/PKG-INFO +6 -2
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task.egg-info/SOURCES.txt +19 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task.egg-info/requires.txt +6 -1
- wt_task-0.2.0/src/wt_task.egg-info/scm_file_list.json +50 -0
- wt_task-0.2.0/src/wt_task.egg-info/scm_version.json +8 -0
- wt_task-0.2.0/tests/test_cache.py +1381 -0
- wt_task-0.2.0/tests/test_codec.py +174 -0
- wt_task-0.2.0/tests/test_depfingerprint.py +375 -0
- wt_task-0.2.0/tests/test_io_ttl.py +338 -0
- wt_task-0.2.0/tests/test_report.py +170 -0
- wt_task-0.2.0/tests/test_settings.py +246 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_testing.py +10 -5
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_tracing.py +4 -2
- wt_task-0.2.0/tests/test_tree.py +285 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/uv.lock +316 -23
- wt_task-0.1.2/CHANGELOG.md +0 -13
- wt_task-0.1.2/src/wt_task/_version.py +0 -34
- {wt_task-0.1.2 → wt_task-0.2.0}/README.md +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/setup.cfg +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/decorator.py +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/executors/__init__.py +1 -1
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/executors/python.py +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/py.typed +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task/tracing/__init__.py +1 -1
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task.egg-info/dependency_links.txt +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/src/wt_task.egg-info/top_level.txt +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_decorator.py +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_exceptions.py +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_executors.py +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_map.py +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_mapvalues.py +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_partial.py +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_skip.py +0 -0
- {wt_task-0.1.2 → wt_task-0.2.0}/tests/test_validation.py +0 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## v0.2.0 — 2026-11-03
|
|
4
|
+
|
|
5
|
+
- Add a content-addressed step cache: `.cached(CacheSpec)` keys a step's result on code identity, fixed inputs and upstream output digests. Identical calls share entries across workflows, and an unchanged output stops recomputation downstream ([#275](https://github.com/wildlife-dynamics/wt/pull/275))
|
|
6
|
+
- Add `CacheSettings` for runtime cache configuration (`WT_CACHE__*` env vars, `.env`, `pyproject.toml`): store, epoch, scope, io TTL, verify, strict ([#275](https://github.com/wildlife-dynamics/wt/pull/275))
|
|
7
|
+
- Each run writes a per-step cache report (`cache-report.json`), including how far a failed run got ([#275](https://github.com/wildlife-dynamics/wt/pull/275))
|
|
8
|
+
- New dependencies: `packaging` and `pydantic-settings`. Object-store backends need the new `cache` extra (`obstore`)
|
|
9
|
+
- Raise the `wt-contracts` floor to `>=0.3.0`
|
|
10
|
+
|
|
11
|
+
## v0.1.3 — 2026-05-13
|
|
12
|
+
|
|
13
|
+
- Standardize ruff lint config to enforce type annotations and Google-style docstrings; convert module/class/function docstrings accordingly ([#155](https://github.com/wildlife-dynamics/wt/pull/155))
|
|
14
|
+
|
|
15
|
+
## v0.1.2 — 2026-03-13
|
|
16
|
+
|
|
17
|
+
- `configure_tracer()` and `attach_context()` now warn instead of raising `ImportError` when tracing deps are missing ([#60](https://github.com/wildlife-dynamics/wt/pull/60))
|
|
18
|
+
|
|
19
|
+
## v0.1.1 — 2026-03-05
|
|
20
|
+
|
|
21
|
+
- Fix incorrect license metadata
|
|
22
|
+
|
|
23
|
+
## v0.1.0 — 2026-03-05
|
|
24
|
+
|
|
25
|
+
- Initial release
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wt-task
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Task decorator and execution features for wt ecosystem
|
|
5
5
|
Author: Ecoscope Workflows Team
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -14,9 +14,13 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
15
|
Requires-Python: >=3.10
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
|
-
Requires-Dist: wt-contracts<1.0.0,>=0.
|
|
17
|
+
Requires-Dist: wt-contracts<1.0.0,>=0.3.0
|
|
18
18
|
Requires-Dist: pydantic<3.0.0,>=2.0.0
|
|
19
|
+
Requires-Dist: packaging>=23.0
|
|
20
|
+
Requires-Dist: pydantic-settings<3.0.0,>=2.3.0
|
|
19
21
|
Requires-Dist: typing-extensions>=4.0.0; python_version < "3.11"
|
|
22
|
+
Provides-Extra: cache
|
|
23
|
+
Requires-Dist: obstore>=0.6.0; extra == "cache"
|
|
20
24
|
Provides-Extra: gcp
|
|
21
25
|
Requires-Dist: opentelemetry-api>=1.20.0; extra == "gcp"
|
|
22
26
|
Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "gcp"
|
|
@@ -23,12 +23,19 @@ classifiers = [
|
|
|
23
23
|
"Programming Language :: Python :: 3.12",
|
|
24
24
|
]
|
|
25
25
|
dependencies = [
|
|
26
|
-
"wt-contracts>=0.
|
|
26
|
+
"wt-contracts>=0.3.0,<1.0.0",
|
|
27
27
|
"pydantic>=2.0.0,<3.0.0",
|
|
28
|
+
# Parses the requirement metadata the cache key's dependency fingerprint walks.
|
|
29
|
+
"packaging>=23.0",
|
|
30
|
+
# Backs CacheSettings: env, .env and pyproject.toml sources for the cache knobs.
|
|
31
|
+
"pydantic-settings>=2.3.0,<3.0.0",
|
|
28
32
|
"typing-extensions>=4.0.0; python_version < '3.11'",
|
|
29
33
|
]
|
|
30
34
|
|
|
31
35
|
[project.optional-dependencies]
|
|
36
|
+
cache = [
|
|
37
|
+
"obstore>=0.6.0",
|
|
38
|
+
]
|
|
32
39
|
gcp = [
|
|
33
40
|
"opentelemetry-api>=1.20.0",
|
|
34
41
|
"opentelemetry-sdk>=1.20.0",
|
|
@@ -89,6 +96,13 @@ module = [
|
|
|
89
96
|
]
|
|
90
97
|
ignore_missing_imports = true
|
|
91
98
|
|
|
99
|
+
[[tool.mypy.overrides]]
|
|
100
|
+
module = [
|
|
101
|
+
"obstore",
|
|
102
|
+
"obstore.*",
|
|
103
|
+
]
|
|
104
|
+
ignore_missing_imports = true
|
|
105
|
+
|
|
92
106
|
[[tool.mypy.overrides]]
|
|
93
107
|
module = [
|
|
94
108
|
"wt_task.tracing._config",
|
|
@@ -101,21 +115,59 @@ line-length = 100
|
|
|
101
115
|
target-version = "py310"
|
|
102
116
|
|
|
103
117
|
[tool.ruff.lint]
|
|
104
|
-
select
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
"
|
|
109
|
-
"
|
|
110
|
-
"
|
|
111
|
-
"
|
|
118
|
+
# Ruff's legacy defaults; extend-select below is the source of truth.
|
|
119
|
+
select = ["E4", "E7", "E9", "F"]
|
|
120
|
+
extend-select = [
|
|
121
|
+
# already-explicit families (kept; F dropped — it's a default)
|
|
122
|
+
"E", # pycodestyle errors (extends defaults E4/E7/E9 with E1/E2/E5)
|
|
123
|
+
"W", # pycodestyle warnings
|
|
124
|
+
"I", # isort
|
|
125
|
+
"B", # flake8-bugbear
|
|
126
|
+
"C4", # flake8-comprehensions
|
|
127
|
+
"UP", # pyupgrade
|
|
128
|
+
"N", # pep8-naming
|
|
129
|
+
"RUF", # ruff-specific
|
|
130
|
+
"PLC0415", # import-outside-top-level (mandatory per CLAUDE.md)
|
|
131
|
+
|
|
132
|
+
# CLAUDE.md mandates, now enforced
|
|
133
|
+
"ANN", # flake8-annotations (require complete type annotations)
|
|
134
|
+
"D", # pydocstyle (require docstrings on public API)
|
|
135
|
+
|
|
136
|
+
# high-value / low-noise modernizers
|
|
137
|
+
"SIM", # flake8-simplify
|
|
138
|
+
"RET", # flake8-return
|
|
139
|
+
"PIE", # flake8-pie
|
|
140
|
+
"PTH", # flake8-use-pathlib
|
|
141
|
+
"ICN", # flake8-import-conventions
|
|
142
|
+
"TID", # flake8-tidy-imports
|
|
143
|
+
"TC", # flake8-type-checking
|
|
144
|
+
"ISC", # flake8-implicit-str-concat
|
|
145
|
+
"YTT", # flake8-2020
|
|
146
|
+
"PERF", # Perflint
|
|
147
|
+
"FURB", # refurb
|
|
148
|
+
"LOG", # flake8-logging
|
|
149
|
+
"G", # flake8-logging-format
|
|
150
|
+
"PT", # flake8-pytest-style
|
|
151
|
+
"ASYNC", # flake8-async
|
|
152
|
+
|
|
153
|
+
# security / strictness (require per-file-ignores)
|
|
154
|
+
"S", # flake8-bandit
|
|
155
|
+
"ARG", # flake8-unused-arguments
|
|
156
|
+
"BLE", # flake8-blind-except
|
|
157
|
+
"PLE", # Pylint errors
|
|
158
|
+
"PLW", # Pylint warnings
|
|
112
159
|
]
|
|
113
160
|
ignore = [
|
|
114
|
-
"E501",
|
|
161
|
+
"E501", # line length — handled by formatter
|
|
162
|
+
"ISC001", # conflicts with formatter (ruff docs recommend disabling)
|
|
115
163
|
]
|
|
116
164
|
|
|
165
|
+
[tool.ruff.lint.pydocstyle]
|
|
166
|
+
convention = "google"
|
|
167
|
+
|
|
117
168
|
[tool.ruff.lint.per-file-ignores]
|
|
118
|
-
"__init__.py" = ["F401"]
|
|
169
|
+
"__init__.py" = ["F401", "D104"] # re-exports + missing module docstring
|
|
170
|
+
"tests/**/*.py" = ["D", "S101", "ANN", "ARG", "PT011"] # tests don't need docstrings/annotations; assert OK
|
|
119
171
|
|
|
120
172
|
# ============================================================================
|
|
121
173
|
# Pixi Build Configuration (conda package building)
|
|
@@ -123,7 +175,7 @@ ignore = [
|
|
|
123
175
|
|
|
124
176
|
[tool.pixi.package]
|
|
125
177
|
name = "wt-task"
|
|
126
|
-
version = "0.
|
|
178
|
+
version = "0.2.0"
|
|
127
179
|
|
|
128
180
|
[tool.pixi.package.build]
|
|
129
181
|
backend = { name = "pixi-build-python", version = "*", channels = ["conda-forge", "https://prefix.dev/pixi-build-backends"] }
|
|
@@ -140,4 +192,6 @@ pip = "*"
|
|
|
140
192
|
[tool.pixi.package.run-dependencies]
|
|
141
193
|
python = ">=3.10,<3.16"
|
|
142
194
|
pydantic = ">=2.0.0,<3.0.0"
|
|
143
|
-
|
|
195
|
+
packaging = ">=23.0"
|
|
196
|
+
pydantic-settings = ">=2.3.0,<3.0.0"
|
|
197
|
+
wt-contracts = ">=0.3.0,<1.0.0"
|
|
@@ -24,25 +24,21 @@ from .skip import (
|
|
|
24
24
|
from .sync_task import SyncTask
|
|
25
25
|
|
|
26
26
|
__all__ = [
|
|
27
|
-
|
|
28
|
-
"task",
|
|
29
|
-
"Task",
|
|
30
|
-
"SyncTask",
|
|
31
|
-
"AsyncTask",
|
|
32
|
-
# Exceptions
|
|
33
|
-
"TaskInstanceError",
|
|
34
|
-
"handle_errors",
|
|
35
|
-
# Executors
|
|
36
|
-
"SyncExecutor",
|
|
27
|
+
"SKIP_SENTINEL",
|
|
37
28
|
"AsyncExecutor",
|
|
38
|
-
"
|
|
29
|
+
"AsyncTask",
|
|
39
30
|
"Future",
|
|
40
31
|
"FutureSequence",
|
|
41
|
-
|
|
32
|
+
"PythonExecutor",
|
|
42
33
|
"SkipSentinel",
|
|
43
|
-
"SKIP_SENTINEL",
|
|
44
34
|
"SkippedDependencyFallback",
|
|
35
|
+
"SyncExecutor",
|
|
36
|
+
"SyncTask",
|
|
37
|
+
"Task",
|
|
38
|
+
"TaskInstanceError",
|
|
39
|
+
"handle_errors",
|
|
45
40
|
"skipif",
|
|
41
|
+
"task",
|
|
46
42
|
"unpack_listlike",
|
|
47
43
|
]
|
|
48
44
|
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.2.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 2, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'gf28c604'
|
|
@@ -6,7 +6,6 @@ returns Future objects.
|
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
|
-
from collections.abc import Sequence
|
|
10
9
|
from dataclasses import dataclass
|
|
11
10
|
|
|
12
11
|
try:
|
|
@@ -16,8 +15,9 @@ try:
|
|
|
16
15
|
except ImportError:
|
|
17
16
|
TRACING_AVAILABLE = False
|
|
18
17
|
|
|
18
|
+
from typing import TYPE_CHECKING
|
|
19
|
+
|
|
19
20
|
from .base import K, P, R, V, _Task
|
|
20
|
-
from .executors import AsyncExecutor, Future, FutureSequence
|
|
21
21
|
from .sync_task import (
|
|
22
22
|
_create_kwargs_iterable,
|
|
23
23
|
_create_mapvalues_kwargs_iterable,
|
|
@@ -25,6 +25,11 @@ from .sync_task import (
|
|
|
25
25
|
_wrap_for_mapvalues,
|
|
26
26
|
)
|
|
27
27
|
|
|
28
|
+
if TYPE_CHECKING:
|
|
29
|
+
from collections.abc import Sequence
|
|
30
|
+
|
|
31
|
+
from .executors import AsyncExecutor, Future, FutureSequence
|
|
32
|
+
|
|
28
33
|
|
|
29
34
|
@dataclass(frozen=True)
|
|
30
35
|
class AsyncTask(_Task[P, R, K, V]):
|
|
@@ -7,11 +7,11 @@ for SyncTask and AsyncTask implementations.
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import functools
|
|
10
|
+
import inspect
|
|
10
11
|
import sys
|
|
11
12
|
import warnings
|
|
12
|
-
from
|
|
13
|
-
from
|
|
14
|
-
from typing import TYPE_CHECKING, Generic, Literal, ParamSpec, TypeVar, overload
|
|
13
|
+
from dataclasses import dataclass, field, replace
|
|
14
|
+
from typing import TYPE_CHECKING, Any, Generic, Literal, ParamSpec, TypeVar, overload
|
|
15
15
|
|
|
16
16
|
if sys.version_info >= (3, 11):
|
|
17
17
|
from typing import Self
|
|
@@ -19,11 +19,15 @@ else:
|
|
|
19
19
|
from typing_extensions import Self
|
|
20
20
|
|
|
21
21
|
if TYPE_CHECKING:
|
|
22
|
+
from collections.abc import Callable
|
|
23
|
+
|
|
22
24
|
from .async_task import AsyncTask
|
|
25
|
+
from .cache import CacheSpec, TaskEncoding
|
|
23
26
|
from .sync_task import SyncTask
|
|
24
27
|
|
|
25
28
|
from pydantic import validate_call
|
|
26
29
|
|
|
30
|
+
from .cache import cached_call, normalized_params, resolve_task, step_params
|
|
27
31
|
from .exceptions import handle_errors as handle_errors_fn
|
|
28
32
|
from .executors import AsyncExecutor, SyncExecutor
|
|
29
33
|
from .executors.python import PythonExecutor
|
|
@@ -34,6 +38,10 @@ P = ParamSpec("P")
|
|
|
34
38
|
R = TypeVar("R")
|
|
35
39
|
K = TypeVar("K")
|
|
36
40
|
V = TypeVar("V")
|
|
41
|
+
# Scoped to `_through_cache`, not a class parameter: it passes through call's R
|
|
42
|
+
# and map's Sequence[R] alike. Module-level only for <3.12, which lacks
|
|
43
|
+
# per-function type parameters.
|
|
44
|
+
T = TypeVar("T")
|
|
37
45
|
|
|
38
46
|
|
|
39
47
|
@dataclass(frozen=True)
|
|
@@ -55,12 +63,22 @@ class _Task(Generic[P, R, K, V]):
|
|
|
55
63
|
tags: List of tags for categorization
|
|
56
64
|
description: Optional description of the task
|
|
57
65
|
task_instance_id: Optional unique identifier for this task instance
|
|
66
|
+
cache_spec: Optional cache identity for this step, set by the compiler when
|
|
67
|
+
the workflow is compiled with caching enabled
|
|
68
|
+
encoding: What `.cached()` worked out from the wrapped function
|
|
58
69
|
"""
|
|
59
70
|
|
|
60
71
|
func: Callable[P, R]
|
|
61
72
|
tags: list[str]
|
|
62
73
|
description: str | None
|
|
63
74
|
task_instance_id: str | None
|
|
75
|
+
# kw_only so that a default here does not force one onto `AsyncTask.executor`,
|
|
76
|
+
# which has no sensible default.
|
|
77
|
+
cache_spec: CacheSpec | None = field(default=None, kw_only=True)
|
|
78
|
+
# Set by `validate`: the function then receives validated arguments, so the
|
|
79
|
+
# cache can key on them instead of on their raw spelling.
|
|
80
|
+
validated: bool = field(default=False, kw_only=True)
|
|
81
|
+
encoding: TaskEncoding | None = field(default=None, kw_only=True, repr=False, compare=False)
|
|
64
82
|
|
|
65
83
|
def set_task_instance_id(self, task_instance_id: str, /) -> Self:
|
|
66
84
|
"""Return a new Task with the task_instance_id set.
|
|
@@ -147,6 +165,7 @@ class _Task(Generic[P, R, K, V]):
|
|
|
147
165
|
validate_return=True,
|
|
148
166
|
config={"arbitrary_types_allowed": True},
|
|
149
167
|
),
|
|
168
|
+
validated=True,
|
|
150
169
|
)
|
|
151
170
|
|
|
152
171
|
def with_tracing(self) -> Self:
|
|
@@ -241,6 +260,65 @@ class _Task(Generic[P, R, K, V]):
|
|
|
241
260
|
func=skipif_fn(self.func, conditions=conditions, unpack_depth=unpack_depth),
|
|
242
261
|
)
|
|
243
262
|
|
|
263
|
+
def cached(self, spec: CacheSpec) -> Self:
|
|
264
|
+
"""Return a new Task that resolves its result through the step cache.
|
|
265
|
+
|
|
266
|
+
Unlike the other builders this sets a field rather than wrapping `func`:
|
|
267
|
+
`map` and `mapvalues` call `func` once per item, so a wrapper would key
|
|
268
|
+
per item, and every item of one mapped step shares a single upstream
|
|
269
|
+
identity. Caching at the method level keeps one entry per step.
|
|
270
|
+
|
|
271
|
+
The rest of the step's identity — where the task lives, and how its
|
|
272
|
+
result encodes — is worked out here from the wrapped function, which
|
|
273
|
+
is why this runs before `.partial()` erases it.
|
|
274
|
+
|
|
275
|
+
Args:
|
|
276
|
+
spec: Compile-time cache identity for this task instance
|
|
277
|
+
|
|
278
|
+
Returns:
|
|
279
|
+
New task carrying the cache spec
|
|
280
|
+
|
|
281
|
+
Examples:
|
|
282
|
+
>>> from wt_task import task
|
|
283
|
+
>>> from wt_task.cache import CacheSpec
|
|
284
|
+
>>> @task
|
|
285
|
+
... def add(a: int, b: int) -> int:
|
|
286
|
+
... return a + b
|
|
287
|
+
>>> cached = add.cached(CacheSpec(signature="abc123"))
|
|
288
|
+
>>> cached.encoding.function
|
|
289
|
+
'add'
|
|
290
|
+
>>> cached.call(1, 2) # inert with WT_CACHE__STORE unset
|
|
291
|
+
3
|
|
292
|
+
"""
|
|
293
|
+
return replace(self, cache_spec=spec, encoding=resolve_task(inspect.unwrap(self.func)))
|
|
294
|
+
|
|
295
|
+
def _through_cache(
|
|
296
|
+
self,
|
|
297
|
+
run: Callable[[], T],
|
|
298
|
+
span: Any = None, # noqa: ANN401 # an OTel span, or None when tracing is unavailable
|
|
299
|
+
method: str = "call",
|
|
300
|
+
) -> T:
|
|
301
|
+
"""Resolve `run()` through the cache when one is configured.
|
|
302
|
+
|
|
303
|
+
Shared by every execution method so that `call`, `map` and `mapvalues`
|
|
304
|
+
agree on what one cache entry covers: the whole step. The method is
|
|
305
|
+
passed in rather than stored, because it is the call that picks it.
|
|
306
|
+
"""
|
|
307
|
+
if self.cache_spec is None or self.encoding is None:
|
|
308
|
+
return run()
|
|
309
|
+
task_instance_id = self.task_instance_id or ""
|
|
310
|
+
params = step_params(task_instance_id)
|
|
311
|
+
if self.validated:
|
|
312
|
+
# Only then is the validated value what the function receives; an
|
|
313
|
+
# unvalidated task may behave differently for "2" and 2.
|
|
314
|
+
params = normalized_params(self.func, params, self.cache_spec.upstream_inputs)
|
|
315
|
+
result, hit = cached_call(
|
|
316
|
+
self.cache_spec, run, self.encoding, task_instance_id, method, params
|
|
317
|
+
)
|
|
318
|
+
if span is not None:
|
|
319
|
+
span.set_attribute("cache.hit", hit)
|
|
320
|
+
return result
|
|
321
|
+
|
|
244
322
|
@overload
|
|
245
323
|
def set_executor(
|
|
246
324
|
self,
|
|
@@ -290,9 +368,9 @@ class _Task(Generic[P, R, K, V]):
|
|
|
290
368
|
>>> type(custom.executor).__name__
|
|
291
369
|
'PythonExecutor'
|
|
292
370
|
"""
|
|
293
|
-
#
|
|
294
|
-
from .async_task import AsyncTask
|
|
295
|
-
from .sync_task import SyncTask
|
|
371
|
+
# circular import: AsyncTask/SyncTask subclass _Task defined here
|
|
372
|
+
from .async_task import AsyncTask # noqa: PLC0415
|
|
373
|
+
from .sync_task import SyncTask # noqa: PLC0415
|
|
296
374
|
|
|
297
375
|
match name_or_executor:
|
|
298
376
|
case "python":
|
|
@@ -301,9 +379,13 @@ class _Task(Generic[P, R, K, V]):
|
|
|
301
379
|
tags=self.tags,
|
|
302
380
|
description=self.description,
|
|
303
381
|
task_instance_id=self.task_instance_id,
|
|
382
|
+
cache_spec=self.cache_spec,
|
|
383
|
+
encoding=self.encoding,
|
|
304
384
|
executor=PythonExecutor(),
|
|
305
385
|
)
|
|
306
386
|
case AsyncExecutor():
|
|
387
|
+
# An async method hands back a future, which has no stored
|
|
388
|
+
# form, so the cache spec is not carried over.
|
|
307
389
|
return AsyncTask(
|
|
308
390
|
self.func,
|
|
309
391
|
tags=self.tags,
|
|
@@ -317,6 +399,8 @@ class _Task(Generic[P, R, K, V]):
|
|
|
317
399
|
tags=self.tags,
|
|
318
400
|
description=self.description,
|
|
319
401
|
task_instance_id=self.task_instance_id,
|
|
402
|
+
cache_spec=self.cache_spec,
|
|
403
|
+
encoding=self.encoding,
|
|
320
404
|
executor=name_or_executor,
|
|
321
405
|
)
|
|
322
406
|
case _:
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Content-addressed step caching for compiled DAGs.
|
|
2
|
+
|
|
3
|
+
Opt-in at compile time. A step is keyed by its code, its arguments, and the
|
|
4
|
+
content digests of the steps feeding it, and a hit returns the stored value
|
|
5
|
+
instead of executing. What a result *is* comes from the task's return type —
|
|
6
|
+
`wt_contracts.declared_structure` resolves it at run time, and
|
|
7
|
+
`resolve_structure` turns it into the `Tree` that encodes the whole step.
|
|
8
|
+
Everything degrades to running normally: with `WT_CACHE__STORE` unset, or for any
|
|
9
|
+
return type that declares no codec, the cache is inert.
|
|
10
|
+
|
|
11
|
+
Runtime knobs live in `CacheSettings`, read once per `run_scope`.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from ._codec import BYTES, JSON, PATH, SKIP, URL, Codec, Undigestible, enum_codec
|
|
15
|
+
from ._entry import CacheEntry
|
|
16
|
+
from ._key import (
|
|
17
|
+
CacheRecord,
|
|
18
|
+
normalized_params,
|
|
19
|
+
record,
|
|
20
|
+
run_report,
|
|
21
|
+
run_scope,
|
|
22
|
+
step_identity,
|
|
23
|
+
step_params,
|
|
24
|
+
)
|
|
25
|
+
from ._report import REPORT_NAME, cache_report, put_cache_report
|
|
26
|
+
from ._runtime import CacheCorruptionError, CacheMismatchWarning, cached_call
|
|
27
|
+
from ._settings import CacheSettings, load_settings
|
|
28
|
+
from ._spec import CacheSpec, TaskEncoding, resolve_task
|
|
29
|
+
from ._store import open_store
|
|
30
|
+
from ._tree import StructureMismatch, Tree, resolve_structure
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"BYTES",
|
|
34
|
+
"JSON",
|
|
35
|
+
"PATH",
|
|
36
|
+
"REPORT_NAME",
|
|
37
|
+
"SKIP",
|
|
38
|
+
"URL",
|
|
39
|
+
"CacheCorruptionError",
|
|
40
|
+
"CacheEntry",
|
|
41
|
+
"CacheMismatchWarning",
|
|
42
|
+
"CacheRecord",
|
|
43
|
+
"CacheSettings",
|
|
44
|
+
"CacheSpec",
|
|
45
|
+
"Codec",
|
|
46
|
+
"StructureMismatch",
|
|
47
|
+
"TaskEncoding",
|
|
48
|
+
"Tree",
|
|
49
|
+
"Undigestible",
|
|
50
|
+
"cache_report",
|
|
51
|
+
"cached_call",
|
|
52
|
+
"enum_codec",
|
|
53
|
+
"load_settings",
|
|
54
|
+
"normalized_params",
|
|
55
|
+
"open_store",
|
|
56
|
+
"put_cache_report",
|
|
57
|
+
"record",
|
|
58
|
+
"resolve_structure",
|
|
59
|
+
"resolve_task",
|
|
60
|
+
"run_report",
|
|
61
|
+
"run_scope",
|
|
62
|
+
"step_identity",
|
|
63
|
+
"step_params",
|
|
64
|
+
]
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Leaf codecs: how one declared value becomes bytes, and back.
|
|
2
|
+
|
|
3
|
+
Which codec applies is not decided here — the task's return type declares it
|
|
4
|
+
(`wt_contracts.codecs`) and `_tree` resolves it. This module only holds the
|
|
5
|
+
builtin leaves and the shape a task library's own codecs must have.
|
|
6
|
+
|
|
7
|
+
There is deliberately no pickle fallback: pickle couples every stored entry to
|
|
8
|
+
the exact version of the code that wrote it and turns the cache into a
|
|
9
|
+
deserialization surface. Refusing is always safe — the step runs normally.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import pathlib
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from typing import TYPE_CHECKING, Any
|
|
19
|
+
|
|
20
|
+
from pydantic_core import Url
|
|
21
|
+
|
|
22
|
+
from wt_task.skip import SKIP_SENTINEL, SkipSentinel
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
import enum
|
|
26
|
+
from collections.abc import Callable
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Undigestible(Exception): # noqa: N818 # named for what a codec does, not for a failure
|
|
30
|
+
"""Raised by a codec that cannot handle the value it was handed.
|
|
31
|
+
|
|
32
|
+
The step then runs normally and records itself uncacheable, so a codec can
|
|
33
|
+
decline a value (an unhashable cell, a subclass it does not cover) without
|
|
34
|
+
the caller having to probe it first.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class Codec:
|
|
40
|
+
"""Encodes one family of runtime values to bytes, and digests them.
|
|
41
|
+
|
|
42
|
+
`digest` is separate from `encode` on purpose: it must be stable across
|
|
43
|
+
encodings, because an unstable digest silently invalidates every downstream
|
|
44
|
+
cache entry on each run. Parquet in particular is not byte-stable, so a
|
|
45
|
+
frame codec digests a canonical form rather than the bytes it writes.
|
|
46
|
+
|
|
47
|
+
Attributes:
|
|
48
|
+
codec_id: Stable identifier recorded in the payload header. Changing it
|
|
49
|
+
makes old entries miss rather than decode wrongly.
|
|
50
|
+
media_type: Media type of the encoded payload.
|
|
51
|
+
types: Runtime types this codec accepts. Used only to pick an arm of a
|
|
52
|
+
declared union; at a non-union position the declaration already
|
|
53
|
+
settled it.
|
|
54
|
+
encode: Value to bytes.
|
|
55
|
+
decode: Bytes back to a value.
|
|
56
|
+
digest: Stable content digest. May raise `Undigestible`.
|
|
57
|
+
accepts_subclasses: Match with `isinstance` rather than
|
|
58
|
+
`type(value) is types[0]`. Cleared by class-parameterized codecs,
|
|
59
|
+
which must not accept a subclass their encoder would silently narrow.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
codec_id: str
|
|
63
|
+
media_type: str
|
|
64
|
+
types: tuple[type, ...]
|
|
65
|
+
encode: Callable[[Any], bytes]
|
|
66
|
+
decode: Callable[[bytes], Any]
|
|
67
|
+
digest: Callable[[Any], str]
|
|
68
|
+
accepts_subclasses: bool = True
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def json_digest(value: Any) -> str: # noqa: ANN401 # digests arbitrary JSON-able values
|
|
72
|
+
"""Return a stable sha256 over a JSON-able value, stringifying what it must.
|
|
73
|
+
|
|
74
|
+
The lenient `default=str` is for cache *keys*, whose inputs are compiler
|
|
75
|
+
literals and params rather than task results. Payload digests use the
|
|
76
|
+
strict `JSON` codec instead.
|
|
77
|
+
|
|
78
|
+
Examples:
|
|
79
|
+
>>> json_digest({"b": 1, "a": 2}) == json_digest({"a": 2, "b": 1})
|
|
80
|
+
True
|
|
81
|
+
"""
|
|
82
|
+
return hashlib.sha256(
|
|
83
|
+
json.dumps(value, sort_keys=True, default=str).encode("utf-8")
|
|
84
|
+
).hexdigest()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _strict_json(value: Any) -> bytes: # noqa: ANN401 # encodes arbitrary JSON-able values
|
|
88
|
+
"""Serialize `value` as JSON, declining anything JSON cannot hold losslessly."""
|
|
89
|
+
try:
|
|
90
|
+
return json.dumps(value, sort_keys=True, allow_nan=False).encode("utf-8")
|
|
91
|
+
except (TypeError, ValueError) as e:
|
|
92
|
+
raise Undigestible(f"not JSON-native: {e}") from e
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
JSON = Codec(
|
|
96
|
+
codec_id="json/v1",
|
|
97
|
+
media_type="application/json",
|
|
98
|
+
types=(str, int, float, bool, list, dict, type(None)),
|
|
99
|
+
encode=_strict_json,
|
|
100
|
+
decode=json.loads,
|
|
101
|
+
digest=lambda v: hashlib.sha256(_strict_json(v)).hexdigest(),
|
|
102
|
+
)
|
|
103
|
+
"""JSON-native values. Reached only through an explicit `CodecRef`, as the
|
|
104
|
+
declaration that a loosely-typed return really does hold plain JSON."""
|
|
105
|
+
|
|
106
|
+
BYTES = Codec(
|
|
107
|
+
codec_id="bytes/v1",
|
|
108
|
+
media_type="application/octet-stream",
|
|
109
|
+
types=(bytes,),
|
|
110
|
+
encode=lambda v: v,
|
|
111
|
+
decode=lambda d: d,
|
|
112
|
+
digest=lambda v: hashlib.sha256(v).hexdigest(),
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
PATH = Codec(
|
|
116
|
+
codec_id="path/v1",
|
|
117
|
+
media_type="text/plain",
|
|
118
|
+
types=(pathlib.PurePath,),
|
|
119
|
+
encode=lambda v: str(v).encode("utf-8"),
|
|
120
|
+
decode=lambda d: pathlib.Path(d.decode("utf-8")),
|
|
121
|
+
digest=lambda v: hashlib.sha256(str(v).encode("utf-8")).hexdigest(),
|
|
122
|
+
)
|
|
123
|
+
"""Paths travel as the path, not as the bytes they point at. A task returning
|
|
124
|
+
one is claiming the location is the result; `no-cache` is the way to say
|
|
125
|
+
otherwise."""
|
|
126
|
+
|
|
127
|
+
URL = Codec(
|
|
128
|
+
codec_id="url/v1",
|
|
129
|
+
media_type="text/plain",
|
|
130
|
+
types=(Url,),
|
|
131
|
+
encode=lambda v: str(v).encode("utf-8"),
|
|
132
|
+
decode=lambda d: Url(d.decode("utf-8")),
|
|
133
|
+
digest=lambda v: hashlib.sha256(str(v).encode("utf-8")).hexdigest(),
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
_SKIP_MARKER = b'{"wt_task":"skip"}'
|
|
137
|
+
|
|
138
|
+
SKIP = Codec(
|
|
139
|
+
codec_id="skip/v1",
|
|
140
|
+
media_type="application/json",
|
|
141
|
+
types=(SkipSentinel,),
|
|
142
|
+
encode=lambda _v: _SKIP_MARKER,
|
|
143
|
+
decode=lambda _d: SKIP_SENTINEL,
|
|
144
|
+
digest=lambda _v: hashlib.sha256(_SKIP_MARKER).hexdigest(),
|
|
145
|
+
)
|
|
146
|
+
"""A skipped step is cached like any other result. Without this the sentinel
|
|
147
|
+
would have no codec and every step below a skip would lose its cache."""
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def enum_codec(cls: type[enum.Enum]) -> Codec:
|
|
151
|
+
"""Build the codec for one enum class: the member's value, round-tripped.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
cls: The enum class, as named by the declared structure.
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
A codec accepting exactly `cls`.
|
|
158
|
+
|
|
159
|
+
Examples:
|
|
160
|
+
>>> import enum
|
|
161
|
+
>>> class Unit(enum.Enum):
|
|
162
|
+
... km = "km"
|
|
163
|
+
>>> codec = enum_codec(Unit)
|
|
164
|
+
>>> codec.decode(codec.encode(Unit.km))
|
|
165
|
+
<Unit.km: 'km'>
|
|
166
|
+
"""
|
|
167
|
+
return Codec(
|
|
168
|
+
codec_id=f"enum/v1/{cls.__module__}.{cls.__qualname__}",
|
|
169
|
+
media_type="application/json",
|
|
170
|
+
types=(cls,),
|
|
171
|
+
encode=lambda v: _strict_json(v.value),
|
|
172
|
+
decode=lambda d: cls(json.loads(d)),
|
|
173
|
+
digest=lambda v: hashlib.sha256(_strict_json(v.value)).hexdigest(),
|
|
174
|
+
accepts_subclasses=False,
|
|
175
|
+
)
|