stepcache 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stepcache/__init__.py ADDED
@@ -0,0 +1,7 @@
1
+ """Content-addressed cache for multi-step processing pipelines."""
2
+
3
+ from .cache import Reason, Result, StepCache, Subject
4
+ from .step import Blob, Refused, Step, Version
5
+ from .store import Manifest, SchemaMismatch, SqliteStore
6
+
7
+ __all__ = ['Blob', 'Manifest', 'Reason', 'Refused', 'Result', 'SchemaMismatch', 'SqliteStore', 'Step', 'StepCache', 'Subject', 'Version']
stepcache/cache.py ADDED
@@ -0,0 +1,434 @@
1
+ import asyncio
2
+ import socket
3
+ import threading
4
+ import time
5
+ from contextlib import suppress
6
+ from dataclasses import dataclass
7
+ from importlib.metadata import PackageNotFoundError, version
8
+ from itertools import groupby
9
+ from operator import itemgetter
10
+
11
+ from .step import KEY_FORMAT, Blob, Refused, Step, check_outputs, digest, encode, encode_outputs
12
+ from .store import Manifest, Record, SqliteStore
13
+
14
+ try:
15
+ _VERSION = version('stepcache')
16
+ except PackageNotFoundError:
17
+ _VERSION = 'unknown'
18
+
19
+ _PRODUCER = {'host': socket.gethostname(), 'stepcache': _VERSION, 'imported': False}
20
+
21
+ WAIT_SECONDS = 0.5
22
+
23
+
24
+ @dataclass(frozen=True)
25
+ class Result:
26
+ """The result of one run.
27
+
28
+ `hit` is False only when this call ran the step. `status` is 'ok', 'refused', or, only in a
29
+ `run_many` result, 'error' with the exception in `error`. `outputs` maps names to a Blob or
30
+ a JSON value and is empty unless the status is 'ok'. A `cache=False` step's result has no
31
+ `key`, and only an 'ok' or 'refused' result of a cached step has a `result_sha`.
32
+ """
33
+
34
+ hit: bool
35
+ status: str
36
+ outputs: dict
37
+ refusal: Refused | None
38
+ key: str | None
39
+ result_sha: str | None
40
+ error: BaseException | None = None
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class Reason:
45
+ """Why a subject is stale. `step` is a step name.
46
+
47
+ - 'version': the manifest used another version of the step. `detail` is {'old': version_sha,
48
+ 'new': version_sha, 'changed': ...}. `changed` holds only the parts that differ: 'code'
49
+ and 'rev' as {'old', 'new'}, 'files' and 'tools' as {'added', 'removed', 'changed'} with
50
+ sorted names. It is None when the old version's components were never stored.
51
+ - 'record': the record of `detail['key']` is gone or has another result than the manifest's.
52
+ - 'failed': the step failed for `detail['key']` in the block that made the manifest. The key is
53
+ None for a `cache=False` step and for inputs that could not make a key.
54
+ - 'missing_step': the manifest has no entry for the step. `detail` is {}.
55
+ """
56
+
57
+ kind: str
58
+ step: str
59
+ detail: dict
60
+
61
+
62
+ class StepCache:
63
+ def __init__(self, store: SqliteStore):
64
+ self.store = store
65
+ self._stored_versions = set()
66
+
67
+ def run(self, step: Step, inputs: dict) -> Result:
68
+ """Return the stored result for these inputs, or run the step and store its result.
69
+
70
+ While another worker computes the same key, this waits for its result. A step that raises
71
+ stores nothing, and the exception propagates. When another writer stored a result for the
72
+ key first, that result is returned.
73
+ """
74
+ _check_sync(step)
75
+ if not step.cache:
76
+ return _uncached(step.run(inputs))
77
+ key, encoded_inputs = _key(step, inputs)
78
+ record = _wait(self._record_or_claim(key))
79
+ if record is not None:
80
+ return self._result(record, hit=True)
81
+ return self._compute(step, key, encoded_inputs, inputs)
82
+
83
+ async def async_run(self, step: Step, inputs: dict) -> Result:
84
+ """`run` for async code. It awaits an async step and calls a sync one directly.
85
+
86
+ While another worker computes the same key, this waits without blocking the event loop.
87
+ """
88
+ if not step.cache:
89
+ out = step.run(inputs)
90
+ return _uncached(await out if step.is_async else out)
91
+ key, encoded_inputs = _key(step, inputs)
92
+ record = await _async_wait(self._record_or_claim(key))
93
+ if record is not None:
94
+ return self._result(record, hit=True)
95
+ try:
96
+ started = time.monotonic()
97
+ out = step.run(inputs)
98
+ if step.is_async:
99
+ out = await out
100
+ return self._write(step, key, encoded_inputs, out, _ms_since(started))
101
+ finally:
102
+ self.store.release(key)
103
+
104
+ def run_many(self, step: Step, inputs_list: list[dict]) -> list[Result]:
105
+ """Run the step on each inputs dict and return the results in input order.
106
+
107
+ Hits are filled at once. The misses this caller claims run in one `run_batch` call, or in
108
+ one `run` call each when the step has no `run_batch`, and identical inputs run once.
109
+ Misses that another worker holds are waited for after that. A miss whose step raised, or
110
+ whose `run_batch` item is an exception, stores nothing and gets an 'error' result.
111
+ """
112
+ _check_sync(step)
113
+ if not step.cache:
114
+ return [_settle(None, out, _uncached) for out in _batch(step, inputs_list)]
115
+ keys = []
116
+ distinct = {}
117
+ for inputs in inputs_list:
118
+ key, encoded_inputs = _key(step, inputs)
119
+ keys.append(key)
120
+ distinct.setdefault(key, (encoded_inputs, inputs))
121
+ results, claimed, waiting = {}, [], {}
122
+ try:
123
+ for key in distinct:
124
+ protocol = self._record_or_claim(key)
125
+ try:
126
+ next(protocol)
127
+ waiting[key] = protocol
128
+ except StopIteration as done:
129
+ if done.value is None:
130
+ claimed.append(key)
131
+ else:
132
+ results[key] = self._result(done.value, hit=True)
133
+ if claimed:
134
+ started = time.monotonic()
135
+ outs = _batch(step, [distinct[key][1] for key in claimed])
136
+ duration_ms = round(_ms_since(started) / len(claimed))
137
+ for key, out in zip(claimed, outs):
138
+ results[key] = _settle(
139
+ key, out, lambda out: self._write(step, key, distinct[key][0], out, duration_ms)
140
+ )
141
+ finally:
142
+ for key in claimed:
143
+ self.store.release(key)
144
+ for key, protocol in waiting.items():
145
+ record = _wait(protocol)
146
+ if record is not None:
147
+ results[key] = self._result(record, hit=True)
148
+ continue
149
+ try:
150
+ results[key] = self._compute(step, key, *distinct[key])
151
+ except Exception as e:
152
+ results[key] = _failed(key, e)
153
+ return [results[key] for key in keys]
154
+
155
+ def invalidate(self, step: Step, key: str):
156
+ """Delete the record of `key`, so the next run computes it again."""
157
+ self.store.delete_record(key)
158
+
159
+ def subject(self, name: str, source: str | None = None) -> 'Subject':
160
+ """Start recording the manifest of the subject `name`. Use the result in a `with` block."""
161
+ return Subject(self, name, source)
162
+
163
+ def manifest(self, subject: str) -> Manifest | None:
164
+ """Return the latest manifest of `subject`, or None when it has none."""
165
+ return self.store.get_manifest(subject)
166
+
167
+ def stale_subjects(self, pipeline: list[Step], complete: bool = False) -> list[tuple[str, list[Reason]]]:
168
+ """Return each subject to process again with its reasons, sorted by subject.
169
+
170
+ Pipeline steps are matched to manifest entries by name, and entries of other steps are
171
+ ignored. A pipeline step that a manifest lacks is a reason only with `complete`.
172
+ """
173
+ steps = {step.name: step for step in pipeline}
174
+ rows = self.store.stale_reasons({name: step.version.sha for name, step in steps.items()}, complete)
175
+ return [
176
+ (subject, [_reason(steps[name], kind, value, old) for _, kind, name, value, old in group])
177
+ for subject, group in groupby(rows, key=itemgetter(0))
178
+ ]
179
+
180
+ def _record_or_claim(self, key: str):
181
+ held = False
182
+ while True:
183
+ try:
184
+ record = self._lookup(key)
185
+ except BaseException:
186
+ if held:
187
+ self.store.release(key)
188
+ raise
189
+ if held:
190
+ if record is not None:
191
+ self.store.release(key)
192
+ return record
193
+ if record is not None:
194
+ return record
195
+ owner = self.store.claim_owner(key)
196
+ if owner is None:
197
+ held = self.store.claim(key)
198
+ elif not self.store.owner_alive(owner):
199
+ held = self.store.take_over(key, owner)
200
+ if not held:
201
+ yield
202
+
203
+ def _lookup(self, key: str) -> Record | None:
204
+ record = self.store.get_record(key)
205
+ if record is None:
206
+ return None
207
+ shas = [sha for kind, sha in record.outputs.values() if kind == 'blob']
208
+ if not all(map(self.store.has, shas)):
209
+ self.store.delete_record(key, record.created_at)
210
+ return None
211
+ self.store.touch_record(key)
212
+ for sha in shas:
213
+ self.store.touch(sha)
214
+ return record
215
+
216
+ def _compute(self, step: Step, key: str, encoded_inputs: dict, inputs: dict) -> Result:
217
+ try:
218
+ started = time.monotonic()
219
+ out = step.run(inputs)
220
+ return self._write(step, key, encoded_inputs, out, _ms_since(started))
221
+ finally:
222
+ self.store.release(key)
223
+
224
+ def _write(self, step: Step, key: str, encoded_inputs: dict, out, duration_ms: int) -> Result:
225
+ if isinstance(out, Refused):
226
+ status, outputs, refusal = 'refused', {}, {'reason': out.reason, 'detail': out.detail}
227
+ else:
228
+ status, outputs, refusal = 'ok', encode_outputs(out), None
229
+ result_sha = digest([status, outputs, refusal])
230
+ if status == 'ok':
231
+ for value in out.values():
232
+ if isinstance(value, Blob):
233
+ self.store.put(value.read())
234
+ self.store.put_version(step.version.sha, step.version.components)
235
+ record = self.store.put_record_if_absent(Record(
236
+ key=key,
237
+ step=step.name,
238
+ version_sha=step.version.sha,
239
+ inputs=encoded_inputs,
240
+ status=status,
241
+ outputs=outputs,
242
+ refusal=refusal,
243
+ result_sha=result_sha,
244
+ duration_ms=duration_ms,
245
+ producer=_PRODUCER,
246
+ input_key=digest([KEY_FORMAT, step.name, encoded_inputs]),
247
+ ))
248
+ return self._result(record, hit=False)
249
+
250
+ def _result(self, record: Record, hit: bool) -> Result:
251
+ outputs = {
252
+ name: Blob._stored(value, self.store.get) if kind == 'blob' else value
253
+ for name, (kind, value) in record.outputs.items()
254
+ }
255
+ refusal = None if record.refusal is None else Refused(**record.refusal)
256
+ return Result(hit, record.status, outputs, refusal, record.key, record.result_sha)
257
+
258
+
259
+ class Subject:
260
+ """The manifest of one subject while it is processed. Threads may share one Subject.
261
+
262
+ `run`, `run_many` and `async_run` return what the `StepCache` methods return, and record
263
+ (step, version_sha, key) -> (result_sha, failed) for each result. A `cache=False` step has no
264
+ key or result_sha. A failure is a `run_many` result with status 'error' or a call that raises,
265
+ and is recorded as (None, True). A call that raises records a failure for each inputs dict it
266
+ was given and re-raises. Its key is None for a `cache=False` step and for inputs that cannot
267
+ make a key.
268
+ Each (step, version_sha, key) is recorded once. For a key, a success replaces a failure and a
269
+ failure never replaces a success, so a failure retried successfully in the block leaves only
270
+ the success. For the key None it is the other way round, so any failure of a `cache=False`
271
+ step in the block leaves the subject stale, even when a later run of the step succeeds.
272
+ When the `with` block ends normally, the recorded entries replace the subject's manifest.
273
+ When it raises, the previous manifest stays.
274
+ """
275
+
276
+ def __init__(self, cache: StepCache, name: str, source: str | None):
277
+ self.cache = cache
278
+ self.name = name
279
+ self.source = source
280
+ self._entries = {}
281
+ self._lock = threading.Lock()
282
+
283
+ def __enter__(self) -> 'Subject':
284
+ return self
285
+
286
+ def __exit__(self, exc_type, exc, tb):
287
+ if exc_type is None:
288
+ self.cache.store.put_manifest(self.name, self.source, [(*k, *v) for k, v in self._entries.items()])
289
+
290
+ def run(self, step: Step, inputs: dict) -> Result:
291
+ try:
292
+ res = self.cache.run(step, inputs)
293
+ except BaseException as e:
294
+ with suppress(Exception):
295
+ self._add(step, [_failed(_key_or_none(step, inputs), e)])
296
+ raise
297
+ self._add(step, [res])
298
+ return res
299
+
300
+ def run_many(self, step: Step, inputs_list: list[dict]) -> list[Result]:
301
+ try:
302
+ results = self.cache.run_many(step, inputs_list)
303
+ except BaseException as e:
304
+ with suppress(Exception):
305
+ self._add(step, [_failed(_key_or_none(step, inputs), e) for inputs in inputs_list])
306
+ raise
307
+ self._add(step, results)
308
+ return results
309
+
310
+ async def async_run(self, step: Step, inputs: dict) -> Result:
311
+ try:
312
+ res = await self.cache.async_run(step, inputs)
313
+ except BaseException as e:
314
+ with suppress(Exception):
315
+ self._add(step, [_failed(_key_or_none(step, inputs), e)])
316
+ raise
317
+ self._add(step, [res])
318
+ return res
319
+
320
+ def _add(self, step: Step, results: list[Result]):
321
+ with self._lock:
322
+ for res in results:
323
+ entry = (step.name, step.version.sha, res.key)
324
+ failed = res.status == 'error'
325
+ # A None key cannot tell a retry from another input, so its failure sticks.
326
+ wins = failed if res.key is None else not failed
327
+ if wins or entry not in self._entries:
328
+ self._entries[entry] = (res.result_sha, failed)
329
+ if step.version.sha not in self.cache._stored_versions:
330
+ self.cache.store.put_version(step.version.sha, step.version.components)
331
+ self.cache._stored_versions.add(step.version.sha)
332
+
333
+
334
+ def _reason(step: Step, kind: str, value: str | None, old: dict | None) -> Reason:
335
+ if kind == 'version':
336
+ changed = None if old is None else _changed(old, step.version.components)
337
+ return Reason(kind, step.name, {'old': value, 'new': step.version.sha, 'changed': changed})
338
+ return Reason(kind, step.name, {} if kind == 'missing_step' else {'key': value})
339
+
340
+
341
+ def _changed(old: dict, new: dict) -> dict:
342
+ changed = {}
343
+ for part in ('code', 'files', 'tools', 'rev'):
344
+ a, b = old[part], new[part]
345
+ if a == b:
346
+ continue
347
+ if part in ('files', 'tools'):
348
+ changed[part] = {
349
+ 'added': sorted(b.keys() - a.keys()),
350
+ 'removed': sorted(a.keys() - b.keys()),
351
+ 'changed': sorted(name for name in a.keys() & b.keys() if a[name] != b[name]),
352
+ }
353
+ else:
354
+ changed[part] = {'old': a, 'new': b}
355
+ return changed
356
+
357
+
358
+ def _key(step: Step, inputs: dict) -> tuple[str, dict]:
359
+ encoded_inputs = encode(inputs)
360
+ return digest([KEY_FORMAT, step.name, step.version.sha, encoded_inputs]), encoded_inputs
361
+
362
+
363
+ def _key_or_none(step: Step, inputs: dict) -> str | None:
364
+ if step.cache:
365
+ with suppress(Exception):
366
+ return _key(step, inputs)[0]
367
+ return None
368
+
369
+
370
+ def _check_sync(step: Step):
371
+ if step.is_async:
372
+ raise TypeError(f'step {step.name!r} is async; call it with async_run')
373
+
374
+
375
+ def _wait(protocol):
376
+ while True:
377
+ try:
378
+ next(protocol)
379
+ except StopIteration as done:
380
+ return done.value
381
+ time.sleep(WAIT_SECONDS)
382
+
383
+
384
+ async def _async_wait(protocol):
385
+ while True:
386
+ try:
387
+ next(protocol)
388
+ except StopIteration as done:
389
+ return done.value
390
+ await asyncio.sleep(WAIT_SECONDS)
391
+
392
+
393
+ def _batch(step: Step, inputs_list: list[dict]) -> list:
394
+ if step.run_batch is None or not inputs_list:
395
+ return [_call(step.run, inputs) for inputs in inputs_list]
396
+ try:
397
+ outs = step.run_batch(inputs_list)
398
+ except Exception as e:
399
+ return [e] * len(inputs_list)
400
+ if len(outs) != len(inputs_list):
401
+ raise ValueError(
402
+ f'run_batch of step {step.name!r} returned {len(outs)} items for {len(inputs_list)} inputs'
403
+ )
404
+ return outs
405
+
406
+
407
+ def _call(run, inputs):
408
+ try:
409
+ return run(inputs)
410
+ except Exception as e:
411
+ return e
412
+
413
+
414
+ def _settle(key: str | None, out, finish) -> Result:
415
+ if not isinstance(out, Exception):
416
+ try:
417
+ return finish(out)
418
+ except Exception as e:
419
+ out = e
420
+ return _failed(key, out)
421
+
422
+
423
+ def _failed(key: str | None, error: Exception) -> Result:
424
+ return Result(False, 'error', {}, None, key, None, error)
425
+
426
+
427
+ def _uncached(out) -> Result:
428
+ if isinstance(out, Refused):
429
+ return Result(False, 'refused', {}, out, None, None)
430
+ return Result(False, 'ok', check_outputs(out), None, None, None)
431
+
432
+
433
+ def _ms_since(started: float) -> int:
434
+ return round((time.monotonic() - started) * 1000)
stepcache/step.py ADDED
@@ -0,0 +1,192 @@
1
+ import functools
2
+ import glob
3
+ import hashlib
4
+ import inspect
5
+ import json
6
+ import os
7
+ from dataclasses import dataclass
8
+ from functools import cached_property
9
+ from pathlib import Path, PurePath
10
+ from typing import Any, Callable
11
+
12
+ KEY_FORMAT = 'stepcache/1'
13
+ JSON_OUTPUT_MAX = 4096
14
+
15
+
16
+ def canonical(value) -> bytes:
17
+ # json.dumps would turn a tuple into a list and the key 1 into "1", so two different values could share a key.
18
+ _check_json(value)
19
+ return json.dumps(
20
+ value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False
21
+ ).encode()
22
+
23
+
24
+ def _check_json(value):
25
+ if isinstance(value, dict):
26
+ for k, v in value.items():
27
+ if not isinstance(k, str):
28
+ raise ValueError(f'JSON object keys must be str, not {k!r}')
29
+ _check_json(v)
30
+ elif isinstance(value, list):
31
+ for v in value:
32
+ _check_json(v)
33
+ elif value is not None and not isinstance(value, (str, int, float)):
34
+ raise ValueError(f'{type(value).__name__} is not a JSON value')
35
+
36
+
37
+ def digest(value) -> str:
38
+ return hashlib.sha256(canonical(value)).hexdigest()
39
+
40
+
41
+ def encode(values: dict) -> dict:
42
+ return {
43
+ name: ['blob', v.sha256] if isinstance(v, Blob) else ['json', v]
44
+ for name, v in values.items()
45
+ }
46
+
47
+
48
+ def check_outputs(outputs) -> dict:
49
+ if not isinstance(outputs, dict):
50
+ raise TypeError(f'a step returns a dict of outputs or Refused, not {type(outputs).__name__}')
51
+ return outputs
52
+
53
+
54
+ def encode_outputs(outputs) -> dict:
55
+ for name, v in check_outputs(outputs).items():
56
+ if not isinstance(v, Blob) and len(canonical(v)) > JSON_OUTPUT_MAX:
57
+ raise ValueError(f'JSON output {name!r} is over {JSON_OUTPUT_MAX} bytes; return a Blob')
58
+ return encode(outputs)
59
+
60
+
61
+ class Blob:
62
+ """Bytes addressed by their sha256. Two blobs are equal when their sha256 is equal."""
63
+
64
+ def __init__(self, data: bytes):
65
+ if not isinstance(data, bytes):
66
+ raise TypeError(f'Blob takes bytes, not {type(data).__name__}')
67
+ self.sha256 = hashlib.sha256(data).hexdigest()
68
+ self._data = data
69
+ self._load = None
70
+
71
+ @classmethod
72
+ def _stored(cls, sha256: str, load: Callable[[str], bytes]) -> 'Blob':
73
+ blob = cls.__new__(cls)
74
+ blob.sha256, blob._data, blob._load = sha256, None, load
75
+ return blob
76
+
77
+ def read(self) -> bytes:
78
+ return self._data if self._load is None else self._load(self.sha256)
79
+
80
+ def __eq__(self, other):
81
+ return self.sha256 == other.sha256 if isinstance(other, Blob) else NotImplemented
82
+
83
+ def __hash__(self):
84
+ return hash(self.sha256)
85
+
86
+ def __repr__(self):
87
+ return f'Blob({self.sha256[:12]})'
88
+
89
+
90
+ @dataclass(frozen=True)
91
+ class Refused:
92
+ """A step's deterministic "cannot process this input". It is cached like a result.
93
+
94
+ `detail` is a JSON value.
95
+ """
96
+
97
+ reason: str
98
+ detail: Any = None
99
+
100
+
101
+ class Version:
102
+ """What a step computes with.
103
+
104
+ `components` is {'code': sha | None, 'files': {relpath: sha}, 'tools': {name: str},
105
+ 'rev': str}. `sha` hashes it, and keys use that hash.
106
+ """
107
+
108
+ def __init__(self, components: dict):
109
+ self.components = components
110
+
111
+ @cached_property
112
+ def sha(self) -> str:
113
+ return digest(self.components)
114
+
115
+ @classmethod
116
+ def of(cls, files=None, tools=None, rev='0', code=True) -> 'Version':
117
+ """Build a version spec.
118
+
119
+ With `code=True`, `Step` puts the hash of the module that defines `run` in it.
120
+ `code=False` leaves the code out.
121
+ """
122
+ return cls({'code': True if code else None, 'files': files or {}, 'tools': tools or {}, 'rev': rev})
123
+
124
+ @staticmethod
125
+ def files(root, patterns) -> dict[str, str]:
126
+ """Hash the files under `root` that match the glob patterns.
127
+
128
+ Returns {relative path: sha256} sorted by path, with `/` separators, so the same tree
129
+ under another root gives the same result. `**` matches any depth. As in `glob`, a name
130
+ starting with a dot matches only a pattern that spells the dot. A pattern that matches no
131
+ file raises ValueError, so a wrong root cannot leave the files out unnoticed.
132
+ """
133
+ found = {}
134
+ for pattern in patterns:
135
+ paths = [
136
+ p for p in glob.glob(pattern, root_dir=root, recursive=True)
137
+ if os.path.isfile(os.path.join(root, p))
138
+ ]
139
+ if not paths:
140
+ raise ValueError(f'{pattern!r} matches no file under {root}')
141
+ for p in paths:
142
+ found[PurePath(p).as_posix()] = _file_sha(Path(root, p))
143
+ return dict(sorted(found.items()))
144
+
145
+
146
+ class Step:
147
+ """A named computation and the version it computes with.
148
+
149
+ `run(inputs) -> outputs | Refused`, where inputs and outputs map names to a Blob or a
150
+ JSON value. An `async def` run is called only through `StepCache.async_run`. A
151
+ `cache=False` step runs every time and stores nothing.
152
+ """
153
+
154
+ def __init__(self, name: str, version: Version, run, run_batch=None, cache=True):
155
+ if version.components['code'] is True:
156
+ version = Version({**version.components, 'code': _code_sha(run, run_batch)})
157
+ self.name = name
158
+ self.version = version
159
+ self.run = run
160
+ self.run_batch = run_batch
161
+ self.cache = cache
162
+ self.is_async = inspect.iscoroutinefunction(run)
163
+
164
+
165
+ def _code_sha(run, run_batch) -> str:
166
+ paths = dict.fromkeys(_source_file(f) for f in (run, run_batch) if f is not None)
167
+ shas = [_file_sha(p) for p in paths]
168
+ return shas[0] if len(shas) == 1 else digest(shas)
169
+
170
+
171
+ def _source_file(fn) -> str:
172
+ target = fn
173
+ while isinstance(target, functools.partial):
174
+ target = target.func
175
+ target = inspect.unwrap(target)
176
+ if not (inspect.isfunction(target) or inspect.ismethod(target) or inspect.isclass(target)):
177
+ target = type(target)
178
+ try:
179
+ path = inspect.getsourcefile(target)
180
+ except (TypeError, OSError):
181
+ path = None
182
+ if path is None or not os.path.isfile(path):
183
+ raise ValueError(
184
+ f'cannot find the source file of {fn!r}; pass Version.of(code=False) '
185
+ 'and declare what the step depends on in files, tools or rev'
186
+ )
187
+ return path
188
+
189
+
190
+ def _file_sha(path) -> str:
191
+ with open(path, 'rb') as f:
192
+ return hashlib.file_digest(f, 'sha256').hexdigest()