stepcache 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stepcache/store.py ADDED
@@ -0,0 +1,471 @@
1
+ import atexit
2
+ import fcntl
3
+ import hashlib
4
+ import json
5
+ import os
6
+ import re
7
+ import secrets
8
+ import sqlite3
9
+ import threading
10
+ import time
11
+ from contextlib import contextmanager, suppress
12
+ from dataclasses import dataclass, fields, replace
13
+ from datetime import datetime, timezone
14
+ from pathlib import Path
15
+
16
+ SCHEMA_VERSION = 2
17
+ INLINE_BLOB_MAX = 256 * 1024
18
+ BUSY_TIMEOUT_SECONDS = 60
19
+
20
+ _SCHEMA = [
21
+ 'CREATE TABLE meta (name TEXT PRIMARY KEY, value NOT NULL)',
22
+ 'CREATE TABLE versions (version_sha TEXT PRIMARY KEY, components TEXT NOT NULL)',
23
+ """CREATE TABLE records (
24
+ key TEXT PRIMARY KEY,
25
+ step TEXT NOT NULL,
26
+ version_sha TEXT NOT NULL,
27
+ inputs TEXT NOT NULL,
28
+ status TEXT NOT NULL CHECK (status IN ('ok', 'refused')),
29
+ outputs TEXT NOT NULL,
30
+ refusal TEXT,
31
+ result_sha TEXT NOT NULL,
32
+ duration_ms INTEGER NOT NULL,
33
+ producer TEXT NOT NULL,
34
+ input_key TEXT NOT NULL,
35
+ created_at TEXT NOT NULL,
36
+ last_used_at TEXT NOT NULL
37
+ )""",
38
+ 'CREATE INDEX records_input_key ON records (input_key)',
39
+ """CREATE TABLE blobs (
40
+ sha TEXT PRIMARY KEY,
41
+ size INTEGER NOT NULL,
42
+ last_used_at TEXT NOT NULL,
43
+ data BLOB
44
+ )""",
45
+ 'CREATE TABLE manifests (subject TEXT PRIMARY KEY, produced_at TEXT NOT NULL, source TEXT)',
46
+ """CREATE TABLE manifest_entries (
47
+ subject TEXT NOT NULL,
48
+ step TEXT NOT NULL,
49
+ version_sha TEXT NOT NULL,
50
+ key TEXT,
51
+ result_sha TEXT,
52
+ failed INTEGER NOT NULL DEFAULT 0 CHECK (failed IN (0, 1))
53
+ )""",
54
+ 'CREATE INDEX manifest_entries_subject ON manifest_entries (subject)',
55
+ 'CREATE TABLE claims (key TEXT PRIMARY KEY, owner TEXT NOT NULL)',
56
+ ]
57
+
58
+ # Connections inherited across fork. SQLite must not touch them in the child, not even to close.
59
+ _inherited = []
60
+
61
+ _LOCK_NAME = re.compile('[0-9a-f]{32}')
62
+
63
+ _held_locks: dict[Path, tuple[int, str]] = {}
64
+ _held_locks_guard = threading.Lock()
65
+
66
+
67
+ def _drop_inherited_locks():
68
+ # An inherited descriptor shares the parent's lock, so a child that never touches stepcache
69
+ # would keep a dead parent's claims looking alive. Closing it leaves the parent's lock held.
70
+ global _held_locks_guard
71
+ _held_locks_guard = threading.Lock()
72
+ for fd, _ in _held_locks.values():
73
+ os.close(fd)
74
+ _held_locks.clear()
75
+
76
+
77
+ os.register_at_fork(after_in_child=_drop_inherited_locks)
78
+
79
+
80
+ @atexit.register
81
+ def _remove_held_locks():
82
+ for folder, (_, name) in _held_locks.items():
83
+ (folder / name).unlink(missing_ok=True)
84
+
85
+
86
+ class SchemaMismatch(Exception):
87
+ """The database was not made by this schema version of stepcache."""
88
+
89
+
90
+ def now() -> datetime:
91
+ return datetime.now(timezone.utc)
92
+
93
+
94
+ def _stamp() -> str:
95
+ return now().isoformat(timespec='microseconds')
96
+
97
+
98
+ @dataclass(frozen=True)
99
+ class Record:
100
+ """The stored result of one key.
101
+
102
+ `inputs` and `outputs` map names to ['blob', sha] or ['json', value]. The store sets
103
+ `created_at` and `last_used_at` when it inserts the record.
104
+ """
105
+
106
+ key: str
107
+ step: str
108
+ version_sha: str
109
+ inputs: dict
110
+ status: str
111
+ outputs: dict
112
+ refusal: dict | None
113
+ result_sha: str
114
+ duration_ms: int
115
+ producer: dict
116
+ input_key: str
117
+ created_at: str | None = None
118
+ last_used_at: str | None = None
119
+
120
+
121
+ @dataclass(frozen=True)
122
+ class Manifest:
123
+ """The latest manifest of one subject.
124
+
125
+ `produced_at` is when it was stored, as an ISO 8601 UTC time. `source` is what the
126
+ application passed to `subject`. Each entry is (step, version_sha, key, result_sha, failed).
127
+ """
128
+
129
+ subject: str
130
+ produced_at: str
131
+ source: str | None
132
+ entries: frozenset
133
+
134
+
135
+ _RECORD_COLUMNS = [f.name for f in fields(Record)]
136
+ _RECORD_JSON = {'inputs', 'outputs', 'refusal', 'producer'}
137
+ _SELECT_RECORD = f'SELECT {", ".join(_RECORD_COLUMNS)} FROM records WHERE key = ?'
138
+ _INSERT_RECORD = (
139
+ f'INSERT INTO records ({", ".join(_RECORD_COLUMNS)}) '
140
+ f'VALUES ({", ".join("?" * len(_RECORD_COLUMNS))}) ON CONFLICT (key) DO NOTHING'
141
+ )
142
+
143
+
144
+ def _to_column(name: str, value):
145
+ return json.dumps(value, ensure_ascii=False) if name in _RECORD_JSON and value is not None else value
146
+
147
+
148
+ def _from_column(name: str, value):
149
+ return json.loads(value) if name in _RECORD_JSON and value is not None else value
150
+
151
+
152
+ def _record_row(record: Record) -> list:
153
+ return [_to_column(name, getattr(record, name)) for name in _RECORD_COLUMNS]
154
+
155
+
156
+ def _row_record(row) -> Record:
157
+ return Record(*map(_from_column, _RECORD_COLUMNS, row))
158
+
159
+
160
+ _STALE = """
161
+ WITH pipeline (position, step, version_sha) AS (
162
+ SELECT key, json_extract(value, '$[0]'), json_extract(value, '$[1]') FROM json_each(?)
163
+ ), reasons (subject, position, kind_order, kind, step, value, components) AS (
164
+ SELECT DISTINCT e.subject, p.position, 0, 'version', e.step, e.version_sha, v.components
165
+ FROM manifest_entries e
166
+ JOIN pipeline p ON p.step = e.step AND p.version_sha != e.version_sha
167
+ LEFT JOIN versions v ON v.version_sha = e.version_sha
168
+ UNION ALL
169
+ SELECT DISTINCT e.subject, p.position, 1, 'record', e.step, e.key, NULL
170
+ FROM manifest_entries e
171
+ JOIN pipeline p ON p.step = e.step
172
+ LEFT JOIN records r ON r.key = e.key
173
+ WHERE e.key IS NOT NULL AND NOT e.failed AND r.result_sha IS NOT e.result_sha
174
+ UNION ALL
175
+ SELECT DISTINCT e.subject, p.position, 2, 'failed', e.step, e.key, NULL
176
+ FROM manifest_entries e
177
+ JOIN pipeline p ON p.step = e.step
178
+ WHERE e.failed
179
+ UNION ALL
180
+ SELECT m.subject, p.position, 3, 'missing_step', p.step, NULL, NULL
181
+ FROM manifests m CROSS JOIN pipeline p
182
+ WHERE ? AND NOT EXISTS (SELECT 1 FROM manifest_entries e WHERE e.subject = m.subject AND e.step = p.step)
183
+ )
184
+ SELECT subject, kind, step, value, components FROM reasons ORDER BY subject, position, kind_order, value
185
+ """
186
+
187
+
188
+ class SqliteStore:
189
+ """A store on one machine's local disk: a SQLite database and a folder for large blobs.
190
+
191
+ Blobs up to INLINE_BLOB_MAX bytes live in the database, larger ones in
192
+ `files/<sha[:2]>/<sha[2:]>`. Processes on the same machine may share the store.
193
+
194
+ A claim names its owner by a lock file in `<db file name>.locks/` that the claiming process
195
+ holds with `flock` until it exits. An owner whose file is gone or can be locked is dead, and
196
+ so is an owner that is not a lock file name: the database may be written by other users, so
197
+ its owner values never become other paths.
198
+ """
199
+
200
+ def __init__(self, path, files):
201
+ self.path = Path(path)
202
+ self.files = Path(files)
203
+ self._locks = self.path.with_name(f'{self.path.name}.locks').resolve()
204
+ self._local = threading.local()
205
+ self.path.parent.mkdir(parents=True, exist_ok=True)
206
+ with self._write() as c:
207
+ version = _schema_version(c, self.path)
208
+ if version is None:
209
+ for statement in _SCHEMA:
210
+ c.execute(statement)
211
+ c.execute("INSERT INTO meta VALUES ('schema_version', ?)", (SCHEMA_VERSION,))
212
+ version = SCHEMA_VERSION
213
+ if version != SCHEMA_VERSION:
214
+ raise SchemaMismatch(f'{self.path} has schema version {version}, not {SCHEMA_VERSION}')
215
+
216
+ def _conn(self) -> sqlite3.Connection:
217
+ local = self._local
218
+ if getattr(local, 'pid', None) != os.getpid():
219
+ if hasattr(local, 'conn'):
220
+ _inherited.append(local.conn)
221
+ local.conn = sqlite3.connect(self.path, timeout=BUSY_TIMEOUT_SECONDS, isolation_level=None)
222
+ _use_wal(local.conn)
223
+ local.conn.execute('PRAGMA synchronous = NORMAL')
224
+ local.pid = os.getpid()
225
+ return local.conn
226
+
227
+ @contextmanager
228
+ def _write(self):
229
+ c = self._conn()
230
+ with c:
231
+ c.execute('BEGIN IMMEDIATE')
232
+ yield c
233
+
234
+ def _touch(self, table: str, column: str, value: str):
235
+ stamp = _stamp()
236
+ today = stamp[:10]
237
+ where = f'WHERE {column} = ? AND last_used_at < ?'
238
+ args = (value, today)
239
+ stale = self._conn().execute(f'SELECT 1 FROM {table} {where}', args).fetchone()
240
+ if stale:
241
+ with self._write() as c:
242
+ c.execute(f'UPDATE {table} SET last_used_at = ? {where}', (stamp, *args))
243
+
244
+ def get_record(self, key: str) -> Record | None:
245
+ row = self._conn().execute(_SELECT_RECORD, (key,)).fetchone()
246
+ return None if row is None else _row_record(row)
247
+
248
+ def put_record_if_absent(self, record: Record) -> Record:
249
+ """Insert the record unless its key has one, and return the record the store holds."""
250
+ stamp = _stamp()
251
+ record = replace(record, created_at=stamp, last_used_at=stamp)
252
+ with self._write() as c:
253
+ c.execute(_INSERT_RECORD, _record_row(record))
254
+ row = c.execute(_SELECT_RECORD, (record.key,)).fetchone()
255
+ return _row_record(row)
256
+
257
+ def delete_record(self, key: str, created_at: str | None = None):
258
+ """Delete the record of `key`, or with `created_at` only the record created then."""
259
+ with self._write() as c:
260
+ c.execute(
261
+ 'DELETE FROM records WHERE key = ? AND created_at = coalesce(?, created_at)', (key, created_at)
262
+ )
263
+
264
+ def touch_record(self, key: str):
265
+ """Mark the record used now, writing at most once per UTC day."""
266
+ self._touch('records', 'key', key)
267
+
268
+ def _blob_path(self, sha: str) -> Path:
269
+ return self.files / sha[:2] / sha[2:]
270
+
271
+ def put(self, data: bytes) -> str:
272
+ sha = hashlib.sha256(data).hexdigest()
273
+ if self.has(sha):
274
+ self.touch(sha)
275
+ return sha
276
+ inline = len(data) <= INLINE_BLOB_MAX
277
+ if not inline:
278
+ self._write_file(sha, data)
279
+ with self._write() as c:
280
+ c.execute(
281
+ 'INSERT INTO blobs (sha, size, last_used_at, data) VALUES (?, ?, ?, ?) '
282
+ 'ON CONFLICT (sha) DO NOTHING',
283
+ (sha, len(data), _stamp(), data if inline else None),
284
+ )
285
+ return sha
286
+
287
+ def _write_file(self, sha: str, data: bytes):
288
+ path = self._blob_path(sha)
289
+ path.parent.mkdir(parents=True, exist_ok=True)
290
+ # Not tempfile.mkstemp: its 0600 mode would hide the blob from other users of the store.
291
+ tmp = path.with_name(f'{path.name}.{secrets.token_hex(8)}.tmp')
292
+ try:
293
+ with open(tmp, 'xb') as f:
294
+ f.write(data)
295
+ f.flush()
296
+ os.fsync(f.fileno())
297
+ os.replace(tmp, path)
298
+ except BaseException:
299
+ tmp.unlink(missing_ok=True)
300
+ raise
301
+
302
+ def get(self, sha: str) -> bytes:
303
+ row = self._conn().execute('SELECT data FROM blobs WHERE sha = ?', (sha,)).fetchone()
304
+ if row is None:
305
+ raise KeyError(sha)
306
+ return self._blob_path(sha).read_bytes() if row[0] is None else row[0]
307
+
308
+ def has(self, sha: str) -> bool:
309
+ row = self._conn().execute('SELECT data IS NULL FROM blobs WHERE sha = ?', (sha,)).fetchone()
310
+ return row is not None and (not row[0] or self._blob_path(sha).exists())
311
+
312
+ def touch(self, sha: str):
313
+ """Mark the blob used now, writing at most once per UTC day."""
314
+ self._touch('blobs', 'sha', sha)
315
+
316
+ def delete(self, sha: str):
317
+ with self._write() as c:
318
+ c.execute('DELETE FROM blobs WHERE sha = ?', (sha,))
319
+ self._blob_path(sha).unlink(missing_ok=True)
320
+
321
+ def claim(self, key: str) -> bool:
322
+ """Claim `key` for this process unless it is claimed, and say whether this call claimed it."""
323
+ owner = self._lock_name()
324
+ with self._write() as c:
325
+ return c.execute(
326
+ 'INSERT INTO claims (key, owner) VALUES (?, ?) ON CONFLICT (key) DO NOTHING', (key, owner)
327
+ ).rowcount == 1
328
+
329
+ def claim_owner(self, key: str) -> str | None:
330
+ row = self._conn().execute('SELECT owner FROM claims WHERE key = ?', (key,)).fetchone()
331
+ return None if row is None else row[0]
332
+
333
+ def owner_alive(self, owner: str) -> bool:
334
+ """Whether the process that holds the lock file named `owner` is alive.
335
+
336
+ A dead owner's lock file is deleted.
337
+ """
338
+ held = _held_locks.get(self._locks)
339
+ if held is not None and held[1] == owner:
340
+ return True
341
+ if not _LOCK_NAME.fullmatch(owner):
342
+ return False
343
+ path = self._locks / owner
344
+ try:
345
+ fd = os.open(path, os.O_RDONLY)
346
+ except FileNotFoundError:
347
+ return False
348
+ try:
349
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
350
+ except BlockingIOError:
351
+ return True
352
+ else:
353
+ # Deleting is tidying only. Another user's file in a shared folder may not be ours to delete.
354
+ with suppress(OSError):
355
+ path.unlink()
356
+ return False
357
+ finally:
358
+ os.close(fd)
359
+
360
+ def take_over(self, key: str, dead_owner: str) -> bool:
361
+ """Move the claim on `key` from `dead_owner` to this process, and say whether it moved."""
362
+ owner = self._lock_name()
363
+ with self._write() as c:
364
+ return c.execute(
365
+ 'UPDATE claims SET owner = ? WHERE key = ? AND owner = ?', (owner, key, dead_owner)
366
+ ).rowcount == 1
367
+
368
+ def release(self, key: str):
369
+ """Delete this process's claim on `key`."""
370
+ owner = self._lock_name()
371
+ with self._write() as c:
372
+ c.execute('DELETE FROM claims WHERE key = ? AND owner = ?', (key, owner))
373
+
374
+ def _lock_name(self) -> str:
375
+ with _held_locks_guard:
376
+ if self._locks not in _held_locks:
377
+ self._locks.mkdir(exist_ok=True)
378
+ name = secrets.token_hex(16)
379
+ # The file gets a lock file name only once it is locked, so a sweep never deletes it.
380
+ tmp = self._locks / f'{name}.tmp'
381
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666)
382
+ fcntl.flock(fd, fcntl.LOCK_EX)
383
+ os.rename(tmp, self._locks / name)
384
+ _held_locks[self._locks] = (fd, name)
385
+ for path in self._locks.iterdir():
386
+ self.owner_alive(path.name)
387
+ return _held_locks[self._locks][1]
388
+
389
+ def put_version(self, sha: str, components: dict):
390
+ with self._write() as c:
391
+ c.execute(
392
+ 'INSERT INTO versions (version_sha, components) VALUES (?, ?) '
393
+ 'ON CONFLICT (version_sha) DO NOTHING',
394
+ (sha, json.dumps(components, ensure_ascii=False)),
395
+ )
396
+
397
+ def get_version(self, sha: str) -> dict | None:
398
+ row = self._conn().execute(
399
+ 'SELECT components FROM versions WHERE version_sha = ?', (sha,)
400
+ ).fetchone()
401
+ return None if row is None else json.loads(row[0])
402
+
403
+ def put_manifest(self, subject: str, source: str | None, entries):
404
+ """Replace the manifest of `subject`. Each entry is (step, version_sha, key, result_sha, failed)."""
405
+ with self._write() as c:
406
+ c.execute(
407
+ 'INSERT INTO manifests (subject, produced_at, source) VALUES (?, ?, ?) ON CONFLICT (subject) '
408
+ 'DO UPDATE SET produced_at = excluded.produced_at, source = excluded.source',
409
+ (subject, _stamp(), source),
410
+ )
411
+ c.execute('DELETE FROM manifest_entries WHERE subject = ?', (subject,))
412
+ c.executemany(
413
+ 'INSERT INTO manifest_entries (subject, step, version_sha, key, result_sha, failed) '
414
+ 'VALUES (?, ?, ?, ?, ?, ?)',
415
+ [(subject, *entry) for entry in entries],
416
+ )
417
+
418
+ def get_manifest(self, subject: str) -> Manifest | None:
419
+ rows = self._conn().execute(
420
+ 'SELECT m.produced_at, m.source, e.step, e.version_sha, e.key, e.result_sha, e.failed FROM manifests m '
421
+ 'LEFT JOIN manifest_entries e ON e.subject = m.subject WHERE m.subject = ?',
422
+ (subject,),
423
+ ).fetchall()
424
+ if not rows:
425
+ return None
426
+ entries = frozenset((*row[2:6], bool(row[6])) for row in rows if row[2] is not None)
427
+ return Manifest(subject, rows[0][0], rows[0][1], entries)
428
+
429
+ def stale_reasons(self, pipeline: dict[str, str], complete: bool) -> list[tuple]:
430
+ """Compare every manifest with `pipeline`, which maps step names to their current version_sha.
431
+
432
+ Returns (subject, kind, step, value, components) rows, sorted by subject, then pipeline
433
+ order, then kind, then value. Entries of steps not in `pipeline` are ignored.
434
+
435
+ - 'version': one row per subject, step and old version_sha. `value` is the old version_sha
436
+ and `components` its stored components, or None when `versions` lacks it.
437
+ - 'record': one row per subject and key of an entry that did not fail and whose record is
438
+ gone or has another result_sha. `value` is the key.
439
+ - 'failed': one row per subject, step and key of a failed entry. `value` is the key, which
440
+ may be None.
441
+ - 'missing_step': only with `complete`, a pipeline step the manifest has no entry for.
442
+ """
443
+ rows = self._conn().execute(_STALE, (json.dumps(list(pipeline.items())), complete)).fetchall()
444
+ return [
445
+ (subject, kind, step, value, None if components is None else json.loads(components))
446
+ for subject, kind, step, value, components in rows
447
+ ]
448
+
449
+
450
+ def _use_wal(c: sqlite3.Connection):
451
+ # Switching a new database to WAL needs an exclusive lock, and SQLite reports SQLITE_BUSY
452
+ # at once instead of waiting on busy_timeout when other processes are opening it too.
453
+ deadline = time.monotonic() + BUSY_TIMEOUT_SECONDS
454
+ while True:
455
+ try:
456
+ c.execute('PRAGMA journal_mode = WAL')
457
+ return
458
+ except sqlite3.OperationalError as e:
459
+ if e.sqlite_errorcode & 0xff != sqlite3.SQLITE_BUSY or time.monotonic() > deadline:
460
+ raise
461
+ time.sleep(0.01)
462
+
463
+
464
+ def _schema_version(c: sqlite3.Connection, path: Path) -> int | None:
465
+ tables = {name for (name,) in c.execute("SELECT name FROM sqlite_master WHERE type = 'table'")}
466
+ if not tables:
467
+ return None
468
+ row = c.execute("SELECT value FROM meta WHERE name = 'schema_version'").fetchone() if 'meta' in tables else None
469
+ if row is None:
470
+ raise SchemaMismatch(f'{path} is not a stepcache database')
471
+ return row[0]