stepcache 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stepcache/__init__.py +7 -0
- stepcache/cache.py +434 -0
- stepcache/step.py +192 -0
- stepcache/store.py +471 -0
- stepcache-0.1.0.dist-info/METADATA +235 -0
- stepcache-0.1.0.dist-info/RECORD +8 -0
- stepcache-0.1.0.dist-info/WHEEL +4 -0
- stepcache-0.1.0.dist-info/licenses/LICENSE +202 -0
stepcache/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Content-addressed cache for multi-step processing pipelines."""
|
|
2
|
+
|
|
3
|
+
from .cache import Reason, Result, StepCache, Subject
|
|
4
|
+
from .step import Blob, Refused, Step, Version
|
|
5
|
+
from .store import Manifest, SchemaMismatch, SqliteStore
|
|
6
|
+
|
|
7
|
+
__all__ = ['Blob', 'Manifest', 'Reason', 'Refused', 'Result', 'SchemaMismatch', 'SqliteStore', 'Step', 'StepCache', 'Subject', 'Version']
|
stepcache/cache.py
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import socket
|
|
3
|
+
import threading
|
|
4
|
+
import time
|
|
5
|
+
from contextlib import suppress
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
8
|
+
from itertools import groupby
|
|
9
|
+
from operator import itemgetter
|
|
10
|
+
|
|
11
|
+
from .step import KEY_FORMAT, Blob, Refused, Step, check_outputs, digest, encode, encode_outputs
|
|
12
|
+
from .store import Manifest, Record, SqliteStore
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
_VERSION = version('stepcache')
|
|
16
|
+
except PackageNotFoundError:
|
|
17
|
+
_VERSION = 'unknown'
|
|
18
|
+
|
|
19
|
+
_PRODUCER = {'host': socket.gethostname(), 'stepcache': _VERSION, 'imported': False}
|
|
20
|
+
|
|
21
|
+
WAIT_SECONDS = 0.5
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class Result:
|
|
26
|
+
"""The result of one run.
|
|
27
|
+
|
|
28
|
+
`hit` is False only when this call ran the step. `status` is 'ok', 'refused', or, only in a
|
|
29
|
+
`run_many` result, 'error' with the exception in `error`. `outputs` maps names to a Blob or
|
|
30
|
+
a JSON value and is empty unless the status is 'ok'. A `cache=False` step's result has no
|
|
31
|
+
`key`, and only an 'ok' or 'refused' result of a cached step has a `result_sha`.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
hit: bool
|
|
35
|
+
status: str
|
|
36
|
+
outputs: dict
|
|
37
|
+
refusal: Refused | None
|
|
38
|
+
key: str | None
|
|
39
|
+
result_sha: str | None
|
|
40
|
+
error: BaseException | None = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class Reason:
|
|
45
|
+
"""Why a subject is stale. `step` is a step name.
|
|
46
|
+
|
|
47
|
+
- 'version': the manifest used another version of the step. `detail` is {'old': version_sha,
|
|
48
|
+
'new': version_sha, 'changed': ...}. `changed` holds only the parts that differ: 'code'
|
|
49
|
+
and 'rev' as {'old', 'new'}, 'files' and 'tools' as {'added', 'removed', 'changed'} with
|
|
50
|
+
sorted names. It is None when the old version's components were never stored.
|
|
51
|
+
- 'record': the record of `detail['key']` is gone or has another result than the manifest's.
|
|
52
|
+
- 'failed': the step failed for `detail['key']` in the block that made the manifest. The key is
|
|
53
|
+
None for a `cache=False` step and for inputs that could not make a key.
|
|
54
|
+
- 'missing_step': the manifest has no entry for the step. `detail` is {}.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
kind: str
|
|
58
|
+
step: str
|
|
59
|
+
detail: dict
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class StepCache:
|
|
63
|
+
def __init__(self, store: SqliteStore):
|
|
64
|
+
self.store = store
|
|
65
|
+
self._stored_versions = set()
|
|
66
|
+
|
|
67
|
+
def run(self, step: Step, inputs: dict) -> Result:
|
|
68
|
+
"""Return the stored result for these inputs, or run the step and store its result.
|
|
69
|
+
|
|
70
|
+
While another worker computes the same key, this waits for its result. A step that raises
|
|
71
|
+
stores nothing, and the exception propagates. When another writer stored a result for the
|
|
72
|
+
key first, that result is returned.
|
|
73
|
+
"""
|
|
74
|
+
_check_sync(step)
|
|
75
|
+
if not step.cache:
|
|
76
|
+
return _uncached(step.run(inputs))
|
|
77
|
+
key, encoded_inputs = _key(step, inputs)
|
|
78
|
+
record = _wait(self._record_or_claim(key))
|
|
79
|
+
if record is not None:
|
|
80
|
+
return self._result(record, hit=True)
|
|
81
|
+
return self._compute(step, key, encoded_inputs, inputs)
|
|
82
|
+
|
|
83
|
+
async def async_run(self, step: Step, inputs: dict) -> Result:
|
|
84
|
+
"""`run` for async code. It awaits an async step and calls a sync one directly.
|
|
85
|
+
|
|
86
|
+
While another worker computes the same key, this waits without blocking the event loop.
|
|
87
|
+
"""
|
|
88
|
+
if not step.cache:
|
|
89
|
+
out = step.run(inputs)
|
|
90
|
+
return _uncached(await out if step.is_async else out)
|
|
91
|
+
key, encoded_inputs = _key(step, inputs)
|
|
92
|
+
record = await _async_wait(self._record_or_claim(key))
|
|
93
|
+
if record is not None:
|
|
94
|
+
return self._result(record, hit=True)
|
|
95
|
+
try:
|
|
96
|
+
started = time.monotonic()
|
|
97
|
+
out = step.run(inputs)
|
|
98
|
+
if step.is_async:
|
|
99
|
+
out = await out
|
|
100
|
+
return self._write(step, key, encoded_inputs, out, _ms_since(started))
|
|
101
|
+
finally:
|
|
102
|
+
self.store.release(key)
|
|
103
|
+
|
|
104
|
+
def run_many(self, step: Step, inputs_list: list[dict]) -> list[Result]:
|
|
105
|
+
"""Run the step on each inputs dict and return the results in input order.
|
|
106
|
+
|
|
107
|
+
Hits are filled at once. The misses this caller claims run in one `run_batch` call, or in
|
|
108
|
+
one `run` call each when the step has no `run_batch`, and identical inputs run once.
|
|
109
|
+
Misses that another worker holds are waited for after that. A miss whose step raised, or
|
|
110
|
+
whose `run_batch` item is an exception, stores nothing and gets an 'error' result.
|
|
111
|
+
"""
|
|
112
|
+
_check_sync(step)
|
|
113
|
+
if not step.cache:
|
|
114
|
+
return [_settle(None, out, _uncached) for out in _batch(step, inputs_list)]
|
|
115
|
+
keys = []
|
|
116
|
+
distinct = {}
|
|
117
|
+
for inputs in inputs_list:
|
|
118
|
+
key, encoded_inputs = _key(step, inputs)
|
|
119
|
+
keys.append(key)
|
|
120
|
+
distinct.setdefault(key, (encoded_inputs, inputs))
|
|
121
|
+
results, claimed, waiting = {}, [], {}
|
|
122
|
+
try:
|
|
123
|
+
for key in distinct:
|
|
124
|
+
protocol = self._record_or_claim(key)
|
|
125
|
+
try:
|
|
126
|
+
next(protocol)
|
|
127
|
+
waiting[key] = protocol
|
|
128
|
+
except StopIteration as done:
|
|
129
|
+
if done.value is None:
|
|
130
|
+
claimed.append(key)
|
|
131
|
+
else:
|
|
132
|
+
results[key] = self._result(done.value, hit=True)
|
|
133
|
+
if claimed:
|
|
134
|
+
started = time.monotonic()
|
|
135
|
+
outs = _batch(step, [distinct[key][1] for key in claimed])
|
|
136
|
+
duration_ms = round(_ms_since(started) / len(claimed))
|
|
137
|
+
for key, out in zip(claimed, outs):
|
|
138
|
+
results[key] = _settle(
|
|
139
|
+
key, out, lambda out: self._write(step, key, distinct[key][0], out, duration_ms)
|
|
140
|
+
)
|
|
141
|
+
finally:
|
|
142
|
+
for key in claimed:
|
|
143
|
+
self.store.release(key)
|
|
144
|
+
for key, protocol in waiting.items():
|
|
145
|
+
record = _wait(protocol)
|
|
146
|
+
if record is not None:
|
|
147
|
+
results[key] = self._result(record, hit=True)
|
|
148
|
+
continue
|
|
149
|
+
try:
|
|
150
|
+
results[key] = self._compute(step, key, *distinct[key])
|
|
151
|
+
except Exception as e:
|
|
152
|
+
results[key] = _failed(key, e)
|
|
153
|
+
return [results[key] for key in keys]
|
|
154
|
+
|
|
155
|
+
def invalidate(self, step: Step, key: str):
|
|
156
|
+
"""Delete the record of `key`, so the next run computes it again."""
|
|
157
|
+
self.store.delete_record(key)
|
|
158
|
+
|
|
159
|
+
def subject(self, name: str, source: str | None = None) -> 'Subject':
|
|
160
|
+
"""Start recording the manifest of the subject `name`. Use the result in a `with` block."""
|
|
161
|
+
return Subject(self, name, source)
|
|
162
|
+
|
|
163
|
+
def manifest(self, subject: str) -> Manifest | None:
|
|
164
|
+
"""Return the latest manifest of `subject`, or None when it has none."""
|
|
165
|
+
return self.store.get_manifest(subject)
|
|
166
|
+
|
|
167
|
+
def stale_subjects(self, pipeline: list[Step], complete: bool = False) -> list[tuple[str, list[Reason]]]:
|
|
168
|
+
"""Return each subject to process again with its reasons, sorted by subject.
|
|
169
|
+
|
|
170
|
+
Pipeline steps are matched to manifest entries by name, and entries of other steps are
|
|
171
|
+
ignored. A pipeline step that a manifest lacks is a reason only with `complete`.
|
|
172
|
+
"""
|
|
173
|
+
steps = {step.name: step for step in pipeline}
|
|
174
|
+
rows = self.store.stale_reasons({name: step.version.sha for name, step in steps.items()}, complete)
|
|
175
|
+
return [
|
|
176
|
+
(subject, [_reason(steps[name], kind, value, old) for _, kind, name, value, old in group])
|
|
177
|
+
for subject, group in groupby(rows, key=itemgetter(0))
|
|
178
|
+
]
|
|
179
|
+
|
|
180
|
+
def _record_or_claim(self, key: str):
|
|
181
|
+
held = False
|
|
182
|
+
while True:
|
|
183
|
+
try:
|
|
184
|
+
record = self._lookup(key)
|
|
185
|
+
except BaseException:
|
|
186
|
+
if held:
|
|
187
|
+
self.store.release(key)
|
|
188
|
+
raise
|
|
189
|
+
if held:
|
|
190
|
+
if record is not None:
|
|
191
|
+
self.store.release(key)
|
|
192
|
+
return record
|
|
193
|
+
if record is not None:
|
|
194
|
+
return record
|
|
195
|
+
owner = self.store.claim_owner(key)
|
|
196
|
+
if owner is None:
|
|
197
|
+
held = self.store.claim(key)
|
|
198
|
+
elif not self.store.owner_alive(owner):
|
|
199
|
+
held = self.store.take_over(key, owner)
|
|
200
|
+
if not held:
|
|
201
|
+
yield
|
|
202
|
+
|
|
203
|
+
def _lookup(self, key: str) -> Record | None:
|
|
204
|
+
record = self.store.get_record(key)
|
|
205
|
+
if record is None:
|
|
206
|
+
return None
|
|
207
|
+
shas = [sha for kind, sha in record.outputs.values() if kind == 'blob']
|
|
208
|
+
if not all(map(self.store.has, shas)):
|
|
209
|
+
self.store.delete_record(key, record.created_at)
|
|
210
|
+
return None
|
|
211
|
+
self.store.touch_record(key)
|
|
212
|
+
for sha in shas:
|
|
213
|
+
self.store.touch(sha)
|
|
214
|
+
return record
|
|
215
|
+
|
|
216
|
+
def _compute(self, step: Step, key: str, encoded_inputs: dict, inputs: dict) -> Result:
|
|
217
|
+
try:
|
|
218
|
+
started = time.monotonic()
|
|
219
|
+
out = step.run(inputs)
|
|
220
|
+
return self._write(step, key, encoded_inputs, out, _ms_since(started))
|
|
221
|
+
finally:
|
|
222
|
+
self.store.release(key)
|
|
223
|
+
|
|
224
|
+
def _write(self, step: Step, key: str, encoded_inputs: dict, out, duration_ms: int) -> Result:
|
|
225
|
+
if isinstance(out, Refused):
|
|
226
|
+
status, outputs, refusal = 'refused', {}, {'reason': out.reason, 'detail': out.detail}
|
|
227
|
+
else:
|
|
228
|
+
status, outputs, refusal = 'ok', encode_outputs(out), None
|
|
229
|
+
result_sha = digest([status, outputs, refusal])
|
|
230
|
+
if status == 'ok':
|
|
231
|
+
for value in out.values():
|
|
232
|
+
if isinstance(value, Blob):
|
|
233
|
+
self.store.put(value.read())
|
|
234
|
+
self.store.put_version(step.version.sha, step.version.components)
|
|
235
|
+
record = self.store.put_record_if_absent(Record(
|
|
236
|
+
key=key,
|
|
237
|
+
step=step.name,
|
|
238
|
+
version_sha=step.version.sha,
|
|
239
|
+
inputs=encoded_inputs,
|
|
240
|
+
status=status,
|
|
241
|
+
outputs=outputs,
|
|
242
|
+
refusal=refusal,
|
|
243
|
+
result_sha=result_sha,
|
|
244
|
+
duration_ms=duration_ms,
|
|
245
|
+
producer=_PRODUCER,
|
|
246
|
+
input_key=digest([KEY_FORMAT, step.name, encoded_inputs]),
|
|
247
|
+
))
|
|
248
|
+
return self._result(record, hit=False)
|
|
249
|
+
|
|
250
|
+
def _result(self, record: Record, hit: bool) -> Result:
|
|
251
|
+
outputs = {
|
|
252
|
+
name: Blob._stored(value, self.store.get) if kind == 'blob' else value
|
|
253
|
+
for name, (kind, value) in record.outputs.items()
|
|
254
|
+
}
|
|
255
|
+
refusal = None if record.refusal is None else Refused(**record.refusal)
|
|
256
|
+
return Result(hit, record.status, outputs, refusal, record.key, record.result_sha)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class Subject:
|
|
260
|
+
"""The manifest of one subject while it is processed. Threads may share one Subject.
|
|
261
|
+
|
|
262
|
+
`run`, `run_many` and `async_run` return what the `StepCache` methods return, and record
|
|
263
|
+
(step, version_sha, key) -> (result_sha, failed) for each result. A `cache=False` step has no
|
|
264
|
+
key or result_sha. A failure is a `run_many` result with status 'error' or a call that raises,
|
|
265
|
+
and is recorded as (None, True). A call that raises records a failure for each inputs dict it
|
|
266
|
+
was given and re-raises. Its key is None for a `cache=False` step and for inputs that cannot
|
|
267
|
+
make a key.
|
|
268
|
+
Each (step, version_sha, key) is recorded once. For a key, a success replaces a failure and a
|
|
269
|
+
failure never replaces a success, so a failure retried successfully in the block leaves only
|
|
270
|
+
the success. For the key None it is the other way round, so any failure of a `cache=False`
|
|
271
|
+
step in the block leaves the subject stale, even when a later run of the step succeeds.
|
|
272
|
+
When the `with` block ends normally, the recorded entries replace the subject's manifest.
|
|
273
|
+
When it raises, the previous manifest stays.
|
|
274
|
+
"""
|
|
275
|
+
|
|
276
|
+
def __init__(self, cache: StepCache, name: str, source: str | None):
|
|
277
|
+
self.cache = cache
|
|
278
|
+
self.name = name
|
|
279
|
+
self.source = source
|
|
280
|
+
self._entries = {}
|
|
281
|
+
self._lock = threading.Lock()
|
|
282
|
+
|
|
283
|
+
def __enter__(self) -> 'Subject':
|
|
284
|
+
return self
|
|
285
|
+
|
|
286
|
+
def __exit__(self, exc_type, exc, tb):
|
|
287
|
+
if exc_type is None:
|
|
288
|
+
self.cache.store.put_manifest(self.name, self.source, [(*k, *v) for k, v in self._entries.items()])
|
|
289
|
+
|
|
290
|
+
def run(self, step: Step, inputs: dict) -> Result:
|
|
291
|
+
try:
|
|
292
|
+
res = self.cache.run(step, inputs)
|
|
293
|
+
except BaseException as e:
|
|
294
|
+
with suppress(Exception):
|
|
295
|
+
self._add(step, [_failed(_key_or_none(step, inputs), e)])
|
|
296
|
+
raise
|
|
297
|
+
self._add(step, [res])
|
|
298
|
+
return res
|
|
299
|
+
|
|
300
|
+
def run_many(self, step: Step, inputs_list: list[dict]) -> list[Result]:
|
|
301
|
+
try:
|
|
302
|
+
results = self.cache.run_many(step, inputs_list)
|
|
303
|
+
except BaseException as e:
|
|
304
|
+
with suppress(Exception):
|
|
305
|
+
self._add(step, [_failed(_key_or_none(step, inputs), e) for inputs in inputs_list])
|
|
306
|
+
raise
|
|
307
|
+
self._add(step, results)
|
|
308
|
+
return results
|
|
309
|
+
|
|
310
|
+
async def async_run(self, step: Step, inputs: dict) -> Result:
|
|
311
|
+
try:
|
|
312
|
+
res = await self.cache.async_run(step, inputs)
|
|
313
|
+
except BaseException as e:
|
|
314
|
+
with suppress(Exception):
|
|
315
|
+
self._add(step, [_failed(_key_or_none(step, inputs), e)])
|
|
316
|
+
raise
|
|
317
|
+
self._add(step, [res])
|
|
318
|
+
return res
|
|
319
|
+
|
|
320
|
+
def _add(self, step: Step, results: list[Result]):
|
|
321
|
+
with self._lock:
|
|
322
|
+
for res in results:
|
|
323
|
+
entry = (step.name, step.version.sha, res.key)
|
|
324
|
+
failed = res.status == 'error'
|
|
325
|
+
# A None key cannot tell a retry from another input, so its failure sticks.
|
|
326
|
+
wins = failed if res.key is None else not failed
|
|
327
|
+
if wins or entry not in self._entries:
|
|
328
|
+
self._entries[entry] = (res.result_sha, failed)
|
|
329
|
+
if step.version.sha not in self.cache._stored_versions:
|
|
330
|
+
self.cache.store.put_version(step.version.sha, step.version.components)
|
|
331
|
+
self.cache._stored_versions.add(step.version.sha)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _reason(step: Step, kind: str, value: str | None, old: dict | None) -> Reason:
|
|
335
|
+
if kind == 'version':
|
|
336
|
+
changed = None if old is None else _changed(old, step.version.components)
|
|
337
|
+
return Reason(kind, step.name, {'old': value, 'new': step.version.sha, 'changed': changed})
|
|
338
|
+
return Reason(kind, step.name, {} if kind == 'missing_step' else {'key': value})
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _changed(old: dict, new: dict) -> dict:
|
|
342
|
+
changed = {}
|
|
343
|
+
for part in ('code', 'files', 'tools', 'rev'):
|
|
344
|
+
a, b = old[part], new[part]
|
|
345
|
+
if a == b:
|
|
346
|
+
continue
|
|
347
|
+
if part in ('files', 'tools'):
|
|
348
|
+
changed[part] = {
|
|
349
|
+
'added': sorted(b.keys() - a.keys()),
|
|
350
|
+
'removed': sorted(a.keys() - b.keys()),
|
|
351
|
+
'changed': sorted(name for name in a.keys() & b.keys() if a[name] != b[name]),
|
|
352
|
+
}
|
|
353
|
+
else:
|
|
354
|
+
changed[part] = {'old': a, 'new': b}
|
|
355
|
+
return changed
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _key(step: Step, inputs: dict) -> tuple[str, dict]:
|
|
359
|
+
encoded_inputs = encode(inputs)
|
|
360
|
+
return digest([KEY_FORMAT, step.name, step.version.sha, encoded_inputs]), encoded_inputs
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _key_or_none(step: Step, inputs: dict) -> str | None:
|
|
364
|
+
if step.cache:
|
|
365
|
+
with suppress(Exception):
|
|
366
|
+
return _key(step, inputs)[0]
|
|
367
|
+
return None
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _check_sync(step: Step):
|
|
371
|
+
if step.is_async:
|
|
372
|
+
raise TypeError(f'step {step.name!r} is async; call it with async_run')
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _wait(protocol):
|
|
376
|
+
while True:
|
|
377
|
+
try:
|
|
378
|
+
next(protocol)
|
|
379
|
+
except StopIteration as done:
|
|
380
|
+
return done.value
|
|
381
|
+
time.sleep(WAIT_SECONDS)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
async def _async_wait(protocol):
|
|
385
|
+
while True:
|
|
386
|
+
try:
|
|
387
|
+
next(protocol)
|
|
388
|
+
except StopIteration as done:
|
|
389
|
+
return done.value
|
|
390
|
+
await asyncio.sleep(WAIT_SECONDS)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _batch(step: Step, inputs_list: list[dict]) -> list:
|
|
394
|
+
if step.run_batch is None or not inputs_list:
|
|
395
|
+
return [_call(step.run, inputs) for inputs in inputs_list]
|
|
396
|
+
try:
|
|
397
|
+
outs = step.run_batch(inputs_list)
|
|
398
|
+
except Exception as e:
|
|
399
|
+
return [e] * len(inputs_list)
|
|
400
|
+
if len(outs) != len(inputs_list):
|
|
401
|
+
raise ValueError(
|
|
402
|
+
f'run_batch of step {step.name!r} returned {len(outs)} items for {len(inputs_list)} inputs'
|
|
403
|
+
)
|
|
404
|
+
return outs
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _call(run, inputs):
|
|
408
|
+
try:
|
|
409
|
+
return run(inputs)
|
|
410
|
+
except Exception as e:
|
|
411
|
+
return e
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _settle(key: str | None, out, finish) -> Result:
|
|
415
|
+
if not isinstance(out, Exception):
|
|
416
|
+
try:
|
|
417
|
+
return finish(out)
|
|
418
|
+
except Exception as e:
|
|
419
|
+
out = e
|
|
420
|
+
return _failed(key, out)
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _failed(key: str | None, error: Exception) -> Result:
|
|
424
|
+
return Result(False, 'error', {}, None, key, None, error)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _uncached(out) -> Result:
|
|
428
|
+
if isinstance(out, Refused):
|
|
429
|
+
return Result(False, 'refused', {}, out, None, None)
|
|
430
|
+
return Result(False, 'ok', check_outputs(out), None, None, None)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _ms_since(started: float) -> int:
|
|
434
|
+
return round((time.monotonic() - started) * 1000)
|
stepcache/step.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import glob
|
|
3
|
+
import hashlib
|
|
4
|
+
import inspect
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from functools import cached_property
|
|
9
|
+
from pathlib import Path, PurePath
|
|
10
|
+
from typing import Any, Callable
|
|
11
|
+
|
|
12
|
+
KEY_FORMAT = 'stepcache/1'
|
|
13
|
+
JSON_OUTPUT_MAX = 4096
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def canonical(value) -> bytes:
|
|
17
|
+
# json.dumps would turn a tuple into a list and the key 1 into "1", so two different values could share a key.
|
|
18
|
+
_check_json(value)
|
|
19
|
+
return json.dumps(
|
|
20
|
+
value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False
|
|
21
|
+
).encode()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _check_json(value):
|
|
25
|
+
if isinstance(value, dict):
|
|
26
|
+
for k, v in value.items():
|
|
27
|
+
if not isinstance(k, str):
|
|
28
|
+
raise ValueError(f'JSON object keys must be str, not {k!r}')
|
|
29
|
+
_check_json(v)
|
|
30
|
+
elif isinstance(value, list):
|
|
31
|
+
for v in value:
|
|
32
|
+
_check_json(v)
|
|
33
|
+
elif value is not None and not isinstance(value, (str, int, float)):
|
|
34
|
+
raise ValueError(f'{type(value).__name__} is not a JSON value')
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def digest(value) -> str:
|
|
38
|
+
return hashlib.sha256(canonical(value)).hexdigest()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def encode(values: dict) -> dict:
|
|
42
|
+
return {
|
|
43
|
+
name: ['blob', v.sha256] if isinstance(v, Blob) else ['json', v]
|
|
44
|
+
for name, v in values.items()
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def check_outputs(outputs) -> dict:
|
|
49
|
+
if not isinstance(outputs, dict):
|
|
50
|
+
raise TypeError(f'a step returns a dict of outputs or Refused, not {type(outputs).__name__}')
|
|
51
|
+
return outputs
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def encode_outputs(outputs) -> dict:
|
|
55
|
+
for name, v in check_outputs(outputs).items():
|
|
56
|
+
if not isinstance(v, Blob) and len(canonical(v)) > JSON_OUTPUT_MAX:
|
|
57
|
+
raise ValueError(f'JSON output {name!r} is over {JSON_OUTPUT_MAX} bytes; return a Blob')
|
|
58
|
+
return encode(outputs)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Blob:
|
|
62
|
+
"""Bytes addressed by their sha256. Two blobs are equal when their sha256 is equal."""
|
|
63
|
+
|
|
64
|
+
def __init__(self, data: bytes):
|
|
65
|
+
if not isinstance(data, bytes):
|
|
66
|
+
raise TypeError(f'Blob takes bytes, not {type(data).__name__}')
|
|
67
|
+
self.sha256 = hashlib.sha256(data).hexdigest()
|
|
68
|
+
self._data = data
|
|
69
|
+
self._load = None
|
|
70
|
+
|
|
71
|
+
@classmethod
|
|
72
|
+
def _stored(cls, sha256: str, load: Callable[[str], bytes]) -> 'Blob':
|
|
73
|
+
blob = cls.__new__(cls)
|
|
74
|
+
blob.sha256, blob._data, blob._load = sha256, None, load
|
|
75
|
+
return blob
|
|
76
|
+
|
|
77
|
+
def read(self) -> bytes:
|
|
78
|
+
return self._data if self._load is None else self._load(self.sha256)
|
|
79
|
+
|
|
80
|
+
def __eq__(self, other):
|
|
81
|
+
return self.sha256 == other.sha256 if isinstance(other, Blob) else NotImplemented
|
|
82
|
+
|
|
83
|
+
def __hash__(self):
|
|
84
|
+
return hash(self.sha256)
|
|
85
|
+
|
|
86
|
+
def __repr__(self):
|
|
87
|
+
return f'Blob({self.sha256[:12]})'
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True)
|
|
91
|
+
class Refused:
|
|
92
|
+
"""A step's deterministic "cannot process this input". It is cached like a result.
|
|
93
|
+
|
|
94
|
+
`detail` is a JSON value.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
reason: str
|
|
98
|
+
detail: Any = None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class Version:
|
|
102
|
+
"""What a step computes with.
|
|
103
|
+
|
|
104
|
+
`components` is {'code': sha | None, 'files': {relpath: sha}, 'tools': {name: str},
|
|
105
|
+
'rev': str}. `sha` hashes it, and keys use that hash.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
def __init__(self, components: dict):
|
|
109
|
+
self.components = components
|
|
110
|
+
|
|
111
|
+
@cached_property
|
|
112
|
+
def sha(self) -> str:
|
|
113
|
+
return digest(self.components)
|
|
114
|
+
|
|
115
|
+
@classmethod
|
|
116
|
+
def of(cls, files=None, tools=None, rev='0', code=True) -> 'Version':
|
|
117
|
+
"""Build a version spec.
|
|
118
|
+
|
|
119
|
+
With `code=True`, `Step` puts the hash of the module that defines `run` in it.
|
|
120
|
+
`code=False` leaves the code out.
|
|
121
|
+
"""
|
|
122
|
+
return cls({'code': True if code else None, 'files': files or {}, 'tools': tools or {}, 'rev': rev})
|
|
123
|
+
|
|
124
|
+
@staticmethod
|
|
125
|
+
def files(root, patterns) -> dict[str, str]:
|
|
126
|
+
"""Hash the files under `root` that match the glob patterns.
|
|
127
|
+
|
|
128
|
+
Returns {relative path: sha256} sorted by path, with `/` separators, so the same tree
|
|
129
|
+
under another root gives the same result. `**` matches any depth. As in `glob`, a name
|
|
130
|
+
starting with a dot matches only a pattern that spells the dot. A pattern that matches no
|
|
131
|
+
file raises ValueError, so a wrong root cannot leave the files out unnoticed.
|
|
132
|
+
"""
|
|
133
|
+
found = {}
|
|
134
|
+
for pattern in patterns:
|
|
135
|
+
paths = [
|
|
136
|
+
p for p in glob.glob(pattern, root_dir=root, recursive=True)
|
|
137
|
+
if os.path.isfile(os.path.join(root, p))
|
|
138
|
+
]
|
|
139
|
+
if not paths:
|
|
140
|
+
raise ValueError(f'{pattern!r} matches no file under {root}')
|
|
141
|
+
for p in paths:
|
|
142
|
+
found[PurePath(p).as_posix()] = _file_sha(Path(root, p))
|
|
143
|
+
return dict(sorted(found.items()))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class Step:
|
|
147
|
+
"""A named computation and the version it computes with.
|
|
148
|
+
|
|
149
|
+
`run(inputs) -> outputs | Refused`, where inputs and outputs map names to a Blob or a
|
|
150
|
+
JSON value. An `async def` run is called only through `StepCache.async_run`. A
|
|
151
|
+
`cache=False` step runs every time and stores nothing.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
def __init__(self, name: str, version: Version, run, run_batch=None, cache=True):
|
|
155
|
+
if version.components['code'] is True:
|
|
156
|
+
version = Version({**version.components, 'code': _code_sha(run, run_batch)})
|
|
157
|
+
self.name = name
|
|
158
|
+
self.version = version
|
|
159
|
+
self.run = run
|
|
160
|
+
self.run_batch = run_batch
|
|
161
|
+
self.cache = cache
|
|
162
|
+
self.is_async = inspect.iscoroutinefunction(run)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _code_sha(run, run_batch) -> str:
|
|
166
|
+
paths = dict.fromkeys(_source_file(f) for f in (run, run_batch) if f is not None)
|
|
167
|
+
shas = [_file_sha(p) for p in paths]
|
|
168
|
+
return shas[0] if len(shas) == 1 else digest(shas)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _source_file(fn) -> str:
|
|
172
|
+
target = fn
|
|
173
|
+
while isinstance(target, functools.partial):
|
|
174
|
+
target = target.func
|
|
175
|
+
target = inspect.unwrap(target)
|
|
176
|
+
if not (inspect.isfunction(target) or inspect.ismethod(target) or inspect.isclass(target)):
|
|
177
|
+
target = type(target)
|
|
178
|
+
try:
|
|
179
|
+
path = inspect.getsourcefile(target)
|
|
180
|
+
except (TypeError, OSError):
|
|
181
|
+
path = None
|
|
182
|
+
if path is None or not os.path.isfile(path):
|
|
183
|
+
raise ValueError(
|
|
184
|
+
f'cannot find the source file of {fn!r}; pass Version.of(code=False) '
|
|
185
|
+
'and declare what the step depends on in files, tools or rev'
|
|
186
|
+
)
|
|
187
|
+
return path
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _file_sha(path) -> str:
|
|
191
|
+
with open(path, 'rb') as f:
|
|
192
|
+
return hashlib.file_digest(f, 'sha256').hexdigest()
|