pymc-cache 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pymc_cache/__init__.py +13 -0
- pymc_cache/_version.py +1 -0
- pymc_cache/api.py +127 -0
- pymc_cache/backends.py +262 -0
- pymc_cache/cli.py +50 -0
- pymc_cache/identity.py +246 -0
- pymc_cache/io.py +269 -0
- pymc_cache/naming.py +28 -0
- pymc_cache/policy.py +65 -0
- pymc_cache/py.typed +0 -0
- pymc_cache/session.py +146 -0
- pymc_cache-0.1.0.dist-info/METADATA +70 -0
- pymc_cache-0.1.0.dist-info/RECORD +16 -0
- pymc_cache-0.1.0.dist-info/WHEEL +4 -0
- pymc_cache-0.1.0.dist-info/entry_points.txt +2 -0
- pymc_cache-0.1.0.dist-info/licenses/LICENSE +21 -0
pymc_cache/identity.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""Versioned, fail-closed structural identity for PyMC sampling results.
|
|
2
|
+
|
|
3
|
+
A false miss wastes CPU. A false hit returns a posterior from a different model,
|
|
4
|
+
which is much worse; unsupported semantic state therefore disables automatic caching.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import importlib.metadata
|
|
10
|
+
import json
|
|
11
|
+
import math
|
|
12
|
+
import sys
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
SCHEMA_VERSION = 3
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class UncacheableError(ValueError):
|
|
21
|
+
"""Raised when the identity of a model or sampling option cannot be established."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _qualified(obj) -> str:
|
|
25
|
+
cls = obj if isinstance(obj, type) else type(obj)
|
|
26
|
+
return cls.__module__ + "." + cls.__qualname__
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _array(value: np.ndarray):
|
|
30
|
+
if value is None:
|
|
31
|
+
return ["none-constant"]
|
|
32
|
+
arr = np.asarray(value)
|
|
33
|
+
if arr.dtype.hasobject:
|
|
34
|
+
raise UncacheableError("object-dtype array has no safe content fingerprint")
|
|
35
|
+
canonical = np.ascontiguousarray(arr)
|
|
36
|
+
return ["array", str(arr.dtype), list(arr.shape), hashlib.sha256(canonical.tobytes()).hexdigest()]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _token(value, *, sampler_key=None):
|
|
40
|
+
"""Typed, JSON-serializable value without repr or memory-address fallbacks."""
|
|
41
|
+
if value is None:
|
|
42
|
+
return ["none"]
|
|
43
|
+
if isinstance(value, np.random.Generator):
|
|
44
|
+
return ["generator", _qualified(value.bit_generator), _token(value.bit_generator.state)]
|
|
45
|
+
if isinstance(value, np.random.RandomState):
|
|
46
|
+
return ["randomstate", _token(value.get_state())]
|
|
47
|
+
if isinstance(value, np.ndarray):
|
|
48
|
+
return _array(value)
|
|
49
|
+
if isinstance(value, np.generic):
|
|
50
|
+
return ["numpy-scalar", str(value.dtype), _token(value.item())]
|
|
51
|
+
if isinstance(value, bool):
|
|
52
|
+
return ["bool", value]
|
|
53
|
+
if isinstance(value, int):
|
|
54
|
+
return ["int", str(value)]
|
|
55
|
+
if isinstance(value, float):
|
|
56
|
+
if math.isnan(value):
|
|
57
|
+
return ["float", "NaN"]
|
|
58
|
+
if math.isinf(value):
|
|
59
|
+
return ["float", "+Inf" if value > 0 else "-Inf"]
|
|
60
|
+
return ["float", value.hex()]
|
|
61
|
+
if isinstance(value, str):
|
|
62
|
+
return ["str", value]
|
|
63
|
+
if isinstance(value, bytes):
|
|
64
|
+
return ["bytes", value.hex()]
|
|
65
|
+
if isinstance(value, tuple):
|
|
66
|
+
return ["tuple", [_token(v, sampler_key=sampler_key) for v in value]]
|
|
67
|
+
if isinstance(value, list):
|
|
68
|
+
return ["list", [_token(v, sampler_key=sampler_key) for v in value]]
|
|
69
|
+
if isinstance(value, (set, frozenset)):
|
|
70
|
+
items = [_token(v, sampler_key=sampler_key) for v in value]
|
|
71
|
+
return ["set", sorted(items, key=lambda v: _json(v))]
|
|
72
|
+
if isinstance(value, Mapping):
|
|
73
|
+
items = [(_token(k, sampler_key=sampler_key), _token(v, sampler_key=sampler_key))
|
|
74
|
+
for k, v in value.items()]
|
|
75
|
+
return ["mapping", sorted(items, key=lambda pair: _json(pair[0]))]
|
|
76
|
+
if isinstance(value, type) and value.__module__.startswith(("pymc", "pytensor")):
|
|
77
|
+
return ["builtin-class", _qualified(value)]
|
|
78
|
+
if sampler_key is not None and callable(value):
|
|
79
|
+
return ["explicit-sampler-key", sampler_key, _qualified(value)]
|
|
80
|
+
raise UncacheableError(f"unsupported identity value: {_qualified(value)}")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _json(value) -> str:
|
|
84
|
+
return json.dumps(value, separators=(",", ":"), sort_keys=True, ensure_ascii=True)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _variable_type(var):
|
|
88
|
+
t = var.type
|
|
89
|
+
return ["variable", _qualified(t), getattr(t, "dtype", None),
|
|
90
|
+
_token(tuple(getattr(t, "shape", ()) or ())),
|
|
91
|
+
_token(tuple(getattr(t, "broadcastable", ()) or ()))]
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _op_property(value, model_key):
|
|
95
|
+
"""Preserve structured builtin Op properties, including nested scalar types."""
|
|
96
|
+
if isinstance(value, (tuple, list)):
|
|
97
|
+
return [type(value).__name__, [_op_property(v, model_key) for v in value]]
|
|
98
|
+
if type(value).__module__.startswith(("pytensor.", "pymc.")):
|
|
99
|
+
return _op_token(value, model_key)
|
|
100
|
+
return _token(value)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _op_token(op, model_key):
|
|
104
|
+
name = _qualified(op)
|
|
105
|
+
if not type(op).__module__.startswith(("pytensor.", "pymc.")) and model_key is None:
|
|
106
|
+
raise UncacheableError(f"custom Op {name} requires model_key")
|
|
107
|
+
props = getattr(op, "__props__", ())
|
|
108
|
+
try:
|
|
109
|
+
recorded = [(p, _op_property(getattr(op, p), model_key)) for p in props]
|
|
110
|
+
except (AttributeError, UncacheableError) as exc:
|
|
111
|
+
if model_key is None:
|
|
112
|
+
raise UncacheableError(f"Op {name} has unhashable properties") from exc
|
|
113
|
+
recorded = [["custom-op", model_key]]
|
|
114
|
+
scalar_op = getattr(op, "scalar_op", None)
|
|
115
|
+
if scalar_op is not None:
|
|
116
|
+
recorded.append(["scalar_op", _op_token(scalar_op, model_key)])
|
|
117
|
+
inner = getattr(op, "inner_outputs", None)
|
|
118
|
+
if inner is not None:
|
|
119
|
+
recorded.append(["inner_graph", _graph(list(inner), model_key)])
|
|
120
|
+
return [name, recorded]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _graph(roots, model_key):
|
|
124
|
+
try:
|
|
125
|
+
from pytensor.compile.sharedvalue import SharedVariable
|
|
126
|
+
from pytensor.graph.basic import Constant
|
|
127
|
+
from pytensor.graph.traversal import graph_inputs, io_toposort
|
|
128
|
+
from pytensor.tensor.random.type import RandomType
|
|
129
|
+
except ImportError:
|
|
130
|
+
from pytensor.compile.sharedvalue import SharedVariable
|
|
131
|
+
from pytensor.graph.basic import Constant, graph_inputs, io_toposort
|
|
132
|
+
from pytensor.tensor.random.type import RandomType
|
|
133
|
+
|
|
134
|
+
roots = list(roots)
|
|
135
|
+
inputs = list(graph_inputs(roots))
|
|
136
|
+
nodes = io_toposort(inputs, roots)
|
|
137
|
+
refs = {}
|
|
138
|
+
entries = []
|
|
139
|
+
|
|
140
|
+
def ref(var):
|
|
141
|
+
if var in refs:
|
|
142
|
+
return refs[var]
|
|
143
|
+
if getattr(var, "owner", None) is not None:
|
|
144
|
+
raise UncacheableError("non-topological graph node")
|
|
145
|
+
index = len(entries)
|
|
146
|
+
refs[var] = index
|
|
147
|
+
record = ["input", _variable_type(var), getattr(var, "name", None)]
|
|
148
|
+
if isinstance(var, Constant):
|
|
149
|
+
record.append(["constant", _array(var.data)])
|
|
150
|
+
elif isinstance(var, SharedVariable):
|
|
151
|
+
if isinstance(var.type, RandomType):
|
|
152
|
+
record.append(["model-construction-rng"])
|
|
153
|
+
else:
|
|
154
|
+
try:
|
|
155
|
+
record.append(["shared-data", _array(var.get_value(borrow=True))])
|
|
156
|
+
except Exception as exc:
|
|
157
|
+
raise UncacheableError("cannot fingerprint shared model data") from exc
|
|
158
|
+
entries.append(record)
|
|
159
|
+
return index
|
|
160
|
+
|
|
161
|
+
for node in nodes:
|
|
162
|
+
parents = [ref(v) for v in node.inputs]
|
|
163
|
+
op = _op_token(node.op, model_key)
|
|
164
|
+
for position, var in enumerate(node.outputs):
|
|
165
|
+
refs[var] = len(entries)
|
|
166
|
+
entries.append(["node", op, parents, position, _variable_type(var),
|
|
167
|
+
getattr(var, "name", None)])
|
|
168
|
+
return [entries, [ref(var) for var in roots]]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def data_digest(model) -> str:
|
|
172
|
+
"""Hash all reachable shared inputs and observed values by full contents."""
|
|
173
|
+
try:
|
|
174
|
+
roots = list(model.observed_RVs) + list(model.deterministics) + list(model.free_RVs)
|
|
175
|
+
token = _graph(roots, model_key="data-only")
|
|
176
|
+
# Graph still participates: a changed graph also changes this conservative digest.
|
|
177
|
+
return hashlib.sha256(_json(token).encode()).hexdigest()
|
|
178
|
+
except UncacheableError:
|
|
179
|
+
raise
|
|
180
|
+
except Exception as exc:
|
|
181
|
+
raise UncacheableError("cannot fingerprint observed or shared model data") from exc
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def model_digest(model, *, model_key: str | None = None) -> str:
|
|
185
|
+
"""Content hash of model graph, data, outputs and coordinates.
|
|
186
|
+
|
|
187
|
+
Custom Python operations require a user-maintained ``model_key``. Their
|
|
188
|
+
implementation cannot be inferred reliably from an Op's class name.
|
|
189
|
+
"""
|
|
190
|
+
if model_key is not None and (not isinstance(model_key, str) or not model_key):
|
|
191
|
+
raise ValueError("model_key must be a nonempty string")
|
|
192
|
+
try:
|
|
193
|
+
roots = [model.logp()]
|
|
194
|
+
roots.extend(model.free_RVs)
|
|
195
|
+
roots.extend(model.observed_RVs)
|
|
196
|
+
roots.extend(model.deterministics)
|
|
197
|
+
roots.extend(model.potentials)
|
|
198
|
+
dims = getattr(model, "named_vars_to_dims", {}) or {}
|
|
199
|
+
metadata = [
|
|
200
|
+
[(name, _variable_type(var), _token(tuple(dims.get(name, ()))))
|
|
201
|
+
for name, var in sorted(model.named_vars.items())],
|
|
202
|
+
_token(model.coords or {}),
|
|
203
|
+
[rv.name for rv in model.free_RVs],
|
|
204
|
+
[rv.name for rv in model.deterministics],
|
|
205
|
+
model_key,
|
|
206
|
+
]
|
|
207
|
+
content = ["pymc-cache-model-v1", metadata, _graph(roots, model_key)]
|
|
208
|
+
return hashlib.sha256(_json(content).encode()).hexdigest()
|
|
209
|
+
except UncacheableError:
|
|
210
|
+
raise
|
|
211
|
+
except Exception as exc:
|
|
212
|
+
raise UncacheableError(f"cannot safely fingerprint model: {exc}") from exc
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def software_versions() -> dict[str, str]:
|
|
216
|
+
"""Exact library versions that affect posterior generation or file format."""
|
|
217
|
+
names = ("pymc", "pytensor", "numpy", "scipy", "arviz", "arviz-base", "arviz-stats",
|
|
218
|
+
"arviz-plots", "nutpie", "numpyro", "blackjax")
|
|
219
|
+
found = {"python": f"{sys.version_info.major}.{sys.version_info.minor}"}
|
|
220
|
+
for name in names:
|
|
221
|
+
try:
|
|
222
|
+
found[name] = importlib.metadata.version(name)
|
|
223
|
+
except importlib.metadata.PackageNotFoundError:
|
|
224
|
+
continue
|
|
225
|
+
return found
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def cache_key(model_identity: str | bytes, sample_kwargs: Mapping, *, operation="sample",
|
|
229
|
+
versions: Mapping | None = None, namespace="", sampler_key: str | None = None) -> str:
|
|
230
|
+
"""Full SHA-256 identity of a model, sampler settings and environment."""
|
|
231
|
+
if not isinstance(model_identity, (str, bytes)):
|
|
232
|
+
raise TypeError("model_identity must be str or bytes")
|
|
233
|
+
if not isinstance(operation, str) or not operation:
|
|
234
|
+
raise ValueError("operation must be a nonempty string")
|
|
235
|
+
if not isinstance(namespace, str):
|
|
236
|
+
raise TypeError("namespace must be a string")
|
|
237
|
+
if sampler_key is not None and (not isinstance(sampler_key, str) or not sampler_key):
|
|
238
|
+
raise ValueError("sampler_key must be a nonempty string")
|
|
239
|
+
if not isinstance(sample_kwargs, Mapping):
|
|
240
|
+
raise TypeError("sample_kwargs must be a mapping")
|
|
241
|
+
# Integration versions are additional identity, never a way to mask installed versions.
|
|
242
|
+
effective_versions = [software_versions(), dict(versions or {})]
|
|
243
|
+
content = ["pymc-cache-key", SCHEMA_VERSION, _token(model_identity), operation,
|
|
244
|
+
namespace, _token(effective_versions),
|
|
245
|
+
_token(sample_kwargs, sampler_key=sampler_key), sampler_key]
|
|
246
|
+
return hashlib.sha256(_json(content).encode()).hexdigest()
|
pymc_cache/io.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"""Lossless-enough, NetCDF-safe storage for PyMC 5 and 6 inference objects."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import base64
|
|
5
|
+
import binascii
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
_CODEC_ATTR = "_pymc_cache_attrs_codec_v1"
|
|
12
|
+
_TEXT_CODEC_ATTR = "_pymc_cache_text_codec_v1"
|
|
13
|
+
_TEXT_GROUPS = {"sample_stats", "warmup_sample_stats"}
|
|
14
|
+
_TEXT_VARS = {"divergence_message", "warning"}
|
|
15
|
+
_TEXT_MARKER = "json-tagged-v2"
|
|
16
|
+
_REAL_FLOAT_TYPES = frozenset(np.dtype(code).type for code in np.typecodes["Float"])
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def group_names(idata) -> list[str]:
|
|
20
|
+
"""List group names for both ArviZ InferenceData and xarray DataTree."""
|
|
21
|
+
raw = idata.groups() if callable(idata.groups) else idata.groups
|
|
22
|
+
return [name.strip("/") for name in raw if name.strip("/")]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_group(idata, name):
|
|
26
|
+
"""Return a group without relying on ArviZ's changed groups interface."""
|
|
27
|
+
try:
|
|
28
|
+
group = getattr(idata, name)
|
|
29
|
+
if group is not None:
|
|
30
|
+
return group
|
|
31
|
+
except (AttributeError, KeyError):
|
|
32
|
+
pass
|
|
33
|
+
try:
|
|
34
|
+
return idata[name]
|
|
35
|
+
except (KeyError, TypeError):
|
|
36
|
+
return None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def group_attrs(idata, name=None):
|
|
40
|
+
"""Attributes of the root or a named group."""
|
|
41
|
+
return getattr(idata if name is None else get_group(idata, name), "attrs", None)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def add_group(idata, name: str, group) -> None:
|
|
45
|
+
"""Attach an xarray Dataset group across ArviZ's two APIs."""
|
|
46
|
+
import xarray as xr
|
|
47
|
+
|
|
48
|
+
ds = group if isinstance(group, xr.Dataset) else group.to_dataset()
|
|
49
|
+
if callable(getattr(idata, "add_groups", None)):
|
|
50
|
+
idata.add_groups({name: ds})
|
|
51
|
+
else:
|
|
52
|
+
idata[name] = ds
|
|
53
|
+
if name not in group_names(idata):
|
|
54
|
+
raise RuntimeError(f"could not attach group {name!r}")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _json_encode(value):
|
|
58
|
+
if isinstance(value, np.generic):
|
|
59
|
+
return value.item()
|
|
60
|
+
if isinstance(value, np.ndarray):
|
|
61
|
+
return {"__ndarray__": value.tolist(), "dtype": str(value.dtype)}
|
|
62
|
+
if isinstance(value, tuple):
|
|
63
|
+
return {"__tuple__": [_json_encode(v) for v in value]}
|
|
64
|
+
if isinstance(value, list):
|
|
65
|
+
return [_json_encode(v) for v in value]
|
|
66
|
+
if isinstance(value, dict):
|
|
67
|
+
if not all(isinstance(k, str) for k in value):
|
|
68
|
+
raise TypeError("attribute dict keys must be strings")
|
|
69
|
+
return {k: _json_encode(v) for k, v in value.items()}
|
|
70
|
+
if value is None or isinstance(value, (str, int, float, bool)):
|
|
71
|
+
return value
|
|
72
|
+
raise TypeError(f"unsupported NetCDF attribute type {type(value).__name__}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _json_decode(value):
|
|
76
|
+
if isinstance(value, dict):
|
|
77
|
+
if set(value) == {"__ndarray__", "dtype"}:
|
|
78
|
+
return np.asarray(value["__ndarray__"], dtype=value["dtype"])
|
|
79
|
+
if set(value) == {"__tuple__"}:
|
|
80
|
+
return tuple(_json_decode(v) for v in value["__tuple__"])
|
|
81
|
+
return {k: _json_decode(v) for k, v in value.items()}
|
|
82
|
+
if isinstance(value, list):
|
|
83
|
+
return [_json_decode(v) for v in value]
|
|
84
|
+
return value
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _sanitize_attrs(attrs):
|
|
88
|
+
if attrs is None:
|
|
89
|
+
return
|
|
90
|
+
if _CODEC_ATTR in attrs:
|
|
91
|
+
raise ValueError(f"reserved cache attribute {_CODEC_ATTR} already exists")
|
|
92
|
+
encoded = []
|
|
93
|
+
for key, value in list(attrs.items()):
|
|
94
|
+
if isinstance(value, (dict, tuple)) or value is None:
|
|
95
|
+
attrs[key] = json.dumps(_json_encode(value), ensure_ascii=True)
|
|
96
|
+
encoded.append(key)
|
|
97
|
+
elif isinstance(value, list):
|
|
98
|
+
# NetCDF rejects mixed/empty lists; encode them consistently.
|
|
99
|
+
attrs[key] = json.dumps(_json_encode(value), ensure_ascii=True)
|
|
100
|
+
encoded.append(key)
|
|
101
|
+
elif isinstance(value, np.ndarray) and value.dtype.hasobject:
|
|
102
|
+
raise TypeError(f"object-dtype attribute {key!r} cannot be stored safely")
|
|
103
|
+
elif not isinstance(value, (str, bytes, int, float, bool, np.ndarray, np.number, np.bool_)):
|
|
104
|
+
raise TypeError(f"unsupported NetCDF attribute {key!r}: {type(value).__name__}")
|
|
105
|
+
if encoded:
|
|
106
|
+
attrs[_CODEC_ATTR] = json.dumps(encoded)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _restore_attrs(attrs):
|
|
110
|
+
if attrs is None or _CODEC_ATTR not in attrs:
|
|
111
|
+
return
|
|
112
|
+
encoded = json.loads(attrs.pop(_CODEC_ATTR))
|
|
113
|
+
for key in encoded:
|
|
114
|
+
attrs[key] = _json_decode(json.loads(attrs[key]))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _containers(idata):
|
|
118
|
+
yield idata
|
|
119
|
+
for name in group_names(idata):
|
|
120
|
+
group = get_group(idata, name)
|
|
121
|
+
yield group
|
|
122
|
+
for collection in (getattr(group, "data_vars", None), getattr(group, "coords", None)):
|
|
123
|
+
if collection is not None:
|
|
124
|
+
for key in collection:
|
|
125
|
+
yield group[key]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _encode_text(data, group, var):
|
|
129
|
+
records = []
|
|
130
|
+
for index in np.ndindex(data.shape):
|
|
131
|
+
value = data.values[index]
|
|
132
|
+
if value is None:
|
|
133
|
+
record = [0]
|
|
134
|
+
elif type(value) in (str, np.str_):
|
|
135
|
+
record = [1, str(value)]
|
|
136
|
+
elif type(value) in (bytes, np.bytes_):
|
|
137
|
+
record = [2, base64.b64encode(bytes(value)).decode("ascii")]
|
|
138
|
+
elif (type(value) is float or type(value) in _REAL_FLOAT_TYPES) and np.isnan(value):
|
|
139
|
+
record = [3]
|
|
140
|
+
else:
|
|
141
|
+
raise TypeError(f"unsupported diagnostic text {group}.{var} at index {index}: "
|
|
142
|
+
f"{type(value).__name__}")
|
|
143
|
+
records.append(json.dumps(record, ensure_ascii=True, allow_nan=False))
|
|
144
|
+
width = max((len(record) for record in records), default=1)
|
|
145
|
+
encoded = np.asarray(records, dtype=f"U{width}").reshape(data.shape)
|
|
146
|
+
result = data.copy(data=encoded)
|
|
147
|
+
result.attrs[_TEXT_CODEC_ATTR] = _TEXT_MARKER
|
|
148
|
+
for key in ("dtype", "_FillValue", "missing_value"):
|
|
149
|
+
result.encoding.pop(key, None)
|
|
150
|
+
return result
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _reject_json_constant(value):
|
|
154
|
+
raise ValueError("JSON constants are not diagnostic text records")
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _decode_text(data, group, var):
|
|
158
|
+
marker = data.attrs[_TEXT_CODEC_ATTR]
|
|
159
|
+
if (marker != _TEXT_MARKER and
|
|
160
|
+
not (marker == "json-tagged" and var == "divergence_message")):
|
|
161
|
+
raise ValueError(f"unknown diagnostic text codec for {group}.{var}")
|
|
162
|
+
values = np.empty(data.shape, dtype=object)
|
|
163
|
+
for index in np.ndindex(data.shape):
|
|
164
|
+
try:
|
|
165
|
+
raw = data.values[index]
|
|
166
|
+
if not isinstance(raw, (str, np.str_)) or not raw.isascii():
|
|
167
|
+
raise ValueError("text codec record must be an ASCII string")
|
|
168
|
+
record = json.loads(raw, parse_constant=_reject_json_constant)
|
|
169
|
+
if not isinstance(record, list) or not record or type(record[0]) is not int:
|
|
170
|
+
raise ValueError("invalid text codec record")
|
|
171
|
+
if record == [0]:
|
|
172
|
+
value = None
|
|
173
|
+
elif len(record) == 2 and record[0] == 1 and type(record[1]) is str:
|
|
174
|
+
value = record[1]
|
|
175
|
+
elif len(record) == 2 and record[0] == 2 and type(record[1]) is str:
|
|
176
|
+
value = base64.b64decode(record[1], validate=True)
|
|
177
|
+
if base64.b64encode(value).decode("ascii") != record[1]:
|
|
178
|
+
raise ValueError("non-canonical base64")
|
|
179
|
+
elif record == [3] and marker == _TEXT_MARKER:
|
|
180
|
+
value = float("nan")
|
|
181
|
+
else:
|
|
182
|
+
raise ValueError("invalid text codec record")
|
|
183
|
+
values[index] = value
|
|
184
|
+
except (ValueError, TypeError, binascii.Error) as exc:
|
|
185
|
+
raise ValueError(f"malformed diagnostic text codec record {group}.{var} "
|
|
186
|
+
f"at index {index}") from exc
|
|
187
|
+
result = data.copy(data=values)
|
|
188
|
+
# xarray normalizes mixed object None/NaN to NaN while constructing a
|
|
189
|
+
# DataArray. Restore the decoded entries after construction, before the
|
|
190
|
+
# caller attaches the array to its group.
|
|
191
|
+
result.values[...] = values
|
|
192
|
+
result.attrs.pop(_TEXT_CODEC_ATTR)
|
|
193
|
+
for key in ("dtype", "_FillValue", "missing_value"):
|
|
194
|
+
result.encoding.pop(key, None)
|
|
195
|
+
return result
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def sanitize_inference_data(idata, copy: bool = True):
|
|
199
|
+
"""Return an independent copy whose attrs and data can be written to NetCDF.
|
|
200
|
+
|
|
201
|
+
Diagnostic text uses a tagged codec in sample_stats and warmup_sample_stats;
|
|
202
|
+
explicitly numeric object arrays in SMC sample_stats are coerced. Other
|
|
203
|
+
object arrays are rejected.
|
|
204
|
+
"""
|
|
205
|
+
result = __import__("copy").deepcopy(idata) if copy else idata
|
|
206
|
+
for container in _containers(result):
|
|
207
|
+
_sanitize_attrs(getattr(container, "attrs", None))
|
|
208
|
+
for name in group_names(result):
|
|
209
|
+
group = get_group(result, name)
|
|
210
|
+
for var in list(getattr(group, "data_vars", ())):
|
|
211
|
+
data = group[var]
|
|
212
|
+
if _TEXT_CODEC_ATTR in data.attrs:
|
|
213
|
+
raise ValueError(f"reserved cache attribute {_TEXT_CODEC_ATTR} already exists")
|
|
214
|
+
if name in _TEXT_GROUPS and var in _TEXT_VARS and data.dtype == object:
|
|
215
|
+
group[var] = _encode_text(data, name, var)
|
|
216
|
+
continue
|
|
217
|
+
if data.dtype == object:
|
|
218
|
+
if name == "sample_stats" and var in {
|
|
219
|
+
"beta", "accept_rate", "log_marginal_likelihood"
|
|
220
|
+
}:
|
|
221
|
+
group[var] = data.astype("float64")
|
|
222
|
+
else:
|
|
223
|
+
raise TypeError(f"object-dtype variable {name}.{var} cannot be cached")
|
|
224
|
+
return result
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def load_all(idata) -> None:
|
|
228
|
+
"""Strictly load every group: never return lazy data backed by a closed file."""
|
|
229
|
+
for name in group_names(idata):
|
|
230
|
+
group = get_group(idata, name)
|
|
231
|
+
loader = getattr(group, "load", None)
|
|
232
|
+
if loader is not None:
|
|
233
|
+
loader()
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def load_inference_data(path: str | Path):
|
|
237
|
+
"""Load and close a NetCDF cache entry, restoring typed attribute values."""
|
|
238
|
+
import arviz as az
|
|
239
|
+
|
|
240
|
+
idata = az.from_netcdf(str(path), engine="h5netcdf")
|
|
241
|
+
try:
|
|
242
|
+
load_all(idata)
|
|
243
|
+
finally:
|
|
244
|
+
close = getattr(idata, "close", None)
|
|
245
|
+
if callable(close):
|
|
246
|
+
close()
|
|
247
|
+
for container in _containers(idata):
|
|
248
|
+
_restore_attrs(getattr(container, "attrs", None))
|
|
249
|
+
for name in group_names(idata):
|
|
250
|
+
group = get_group(idata, name)
|
|
251
|
+
for var in list(group.data_vars):
|
|
252
|
+
data = group[var]
|
|
253
|
+
if _TEXT_CODEC_ATTR in data.attrs:
|
|
254
|
+
if name not in _TEXT_GROUPS or var not in _TEXT_VARS:
|
|
255
|
+
raise ValueError(f"unknown diagnostic text codec for {name}.{var}")
|
|
256
|
+
group[var] = _decode_text(data, name, var)
|
|
257
|
+
return idata
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def save_inference_data(idata, path: str | Path, *, attrs: dict | None = None) -> None:
|
|
261
|
+
"""Save a snapshot while leaving every caller-owned attribute unchanged."""
|
|
262
|
+
snapshot = sanitize_inference_data(idata, copy=True)
|
|
263
|
+
if attrs:
|
|
264
|
+
# A root attr is not written by ArviZ 0.x; posterior group is portable.
|
|
265
|
+
group = get_group(snapshot, "posterior")
|
|
266
|
+
if group is None:
|
|
267
|
+
raise ValueError("cannot persist an inference result without a posterior")
|
|
268
|
+
group.attrs.update(attrs)
|
|
269
|
+
snapshot.to_netcdf(str(path), engine="h5netcdf")
|
pymc_cache/naming.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Human-readable, path-safe names; correctness is provided by full cache keys."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def slugify(value: str) -> str:
|
|
8
|
+
return re.sub(r"[^a-z0-9]+", "_", str(value).lower()).strip("_")[:80] or "unnamed"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def model_slug(model) -> str:
|
|
12
|
+
"""A descriptive directory name, never part of the machine identity."""
|
|
13
|
+
name = getattr(model, "name", None)
|
|
14
|
+
if not name:
|
|
15
|
+
variables = getattr(model, "free_RVs", ())
|
|
16
|
+
name = "_".join(str(rv.name).split("::")[-1] for rv in variables[:4])
|
|
17
|
+
if len(variables) > 4:
|
|
18
|
+
name += f"_plus{len(variables) - 4}"
|
|
19
|
+
return slugify(name or "unnamed")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def params_label(kwargs: dict | None) -> str:
|
|
23
|
+
"""Short, sanitized parameter hint for browsing disk entries."""
|
|
24
|
+
kwargs = kwargs or {}
|
|
25
|
+
parts = [f"{name}{kwargs[name]}" for name in (
|
|
26
|
+
"draws", "tune", "chains", "cores", "nuts_sampler", "target_accept"
|
|
27
|
+
) if kwargs.get(name) is not None and not (name == "nuts_sampler" and kwargs[name] == "pymc")]
|
|
28
|
+
return slugify("_".join(parts) if parts else "default")
|
pymc_cache/policy.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Result eligibility: never turn an interrupted or diagnostically failed run into a hit."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from pymc_cache.io import get_group
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CacheWarning(UserWarning):
|
|
11
|
+
"""A cache was bypassed or an entry could not be persisted."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class CachePolicy:
|
|
16
|
+
"""Select whether unseeded sampling may reuse a previous run."""
|
|
17
|
+
|
|
18
|
+
reuse_unseeded: bool = True
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def report_from_idata(idata) -> dict | None:
|
|
22
|
+
post = get_group(idata, "posterior")
|
|
23
|
+
attrs = getattr(post, "attrs", {}) if post is not None else {}
|
|
24
|
+
for name in ("pymc_monitor_convergence", "cloudposterior_convergence"):
|
|
25
|
+
payload = attrs.get(name)
|
|
26
|
+
if not payload:
|
|
27
|
+
continue
|
|
28
|
+
try:
|
|
29
|
+
result = json.loads(payload) if isinstance(payload, str) else payload
|
|
30
|
+
if isinstance(result, dict):
|
|
31
|
+
return result
|
|
32
|
+
except (TypeError, ValueError):
|
|
33
|
+
return None
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def eligible(idata, *, status: str, kwargs: dict, operation="sample",
|
|
38
|
+
effective_sampler=None, sampler_identity=None) -> bool:
|
|
39
|
+
"""Check posterior completeness and report before publishing a result."""
|
|
40
|
+
if status not in ("complete", "met"):
|
|
41
|
+
return False
|
|
42
|
+
post = get_group(idata, "posterior")
|
|
43
|
+
if post is None or not all(dim in post.sizes for dim in ("chain", "draw")):
|
|
44
|
+
return False
|
|
45
|
+
requested_chains = kwargs.get("chains")
|
|
46
|
+
if requested_chains is not None and post.sizes["chain"] != requested_chains:
|
|
47
|
+
return False
|
|
48
|
+
if effective_sampler is not None and sampler_identity is not None:
|
|
49
|
+
if effective_sampler != sampler_identity:
|
|
50
|
+
return False
|
|
51
|
+
report = report_from_idata(idata)
|
|
52
|
+
adaptive = kwargs.get("until") not in (None, False)
|
|
53
|
+
if report:
|
|
54
|
+
actual = report.get("status")
|
|
55
|
+
if actual not in ("complete", "met"):
|
|
56
|
+
return False
|
|
57
|
+
if adaptive and (actual != "met" or report.get("met") is not True):
|
|
58
|
+
return False
|
|
59
|
+
elif adaptive or status == "met":
|
|
60
|
+
return False
|
|
61
|
+
expected_draws = kwargs.get("draws")
|
|
62
|
+
if status == "complete" and expected_draws is not None:
|
|
63
|
+
if post.sizes["draw"] != expected_draws:
|
|
64
|
+
return False
|
|
65
|
+
return True
|
pymc_cache/py.typed
ADDED
|
File without changes
|