qplotter 1.6.0b3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. qplot/__init__.py +46 -0
  2. qplot/__main__.py +232 -0
  3. qplot/_auxiliary_paths.py +63 -0
  4. qplot/_metadata.py +12 -0
  5. qplot/_repair.py +14 -0
  6. qplot/_shutdown_supervisor.py +4049 -0
  7. qplot/_version.py +12 -0
  8. qplot/_windows_shutdown_job.py +1166 -0
  9. qplot/configuration/__init__.py +3 -0
  10. qplot/configuration/config.py +573 -0
  11. qplot/configuration/config_schema.json +319 -0
  12. qplot/configuration/scripts.py +255 -0
  13. qplot/configuration/themes/__init__.py +13 -0
  14. qplot/configuration/themes/_base.py +579 -0
  15. qplot/configuration/themes/blank.py +17 -0
  16. qplot/configuration/themes/dark.py +54 -0
  17. qplot/configuration/themes/light.py +54 -0
  18. qplot/datahandling/LoadFromDB.py +224 -0
  19. qplot/datahandling/__init__.py +47 -0
  20. qplot/datahandling/_trusted_live_helper.py +812 -0
  21. qplot/datahandling/_trusted_live_protocol.py +1745 -0
  22. qplot/datahandling/database.py +2330 -0
  23. qplot/datahandling/dimensions.py +43 -0
  24. qplot/datahandling/file_identity.py +461 -0
  25. qplot/datahandling/parameter_data.py +149 -0
  26. qplot/datahandling/plot_progress.py +17 -0
  27. qplot/datahandling/qcodes_cache.py +277 -0
  28. qplot/datahandling/qcodes_compat.py +14 -0
  29. qplot/datahandling/readSQL.py +2442 -0
  30. qplot/datahandling/readonly.py +2561 -0
  31. qplot/datahandling/source_activity.py +21 -0
  32. qplot/datahandling/trusted_array_preview.py +127 -0
  33. qplot/datahandling/trusted_derived_cache.py +1605 -0
  34. qplot/datahandling/trusted_derived_rendering.py +868 -0
  35. qplot/datahandling/trusted_heatmap.py +241 -0
  36. qplot/datahandling/trusted_live.py +2808 -0
  37. qplot/datahandling/trusted_live_queries.py +5907 -0
  38. qplot/datahandling/trusted_live_service.py +1894 -0
  39. qplot/datahandling/trusted_live_supervisor.py +2161 -0
  40. qplot/datahandling/trusted_plot.py +283 -0
  41. qplot/datahandling/trusted_presentation.py +1065 -0
  42. qplot/datahandling/trusted_snapshot.py +1132 -0
  43. qplot/datahandling/trusted_work_coordinator.py +2157 -0
  44. qplot/datahandling/trusted_work_scheduler.py +1204 -0
  45. qplot/diagnostics.py +241 -0
  46. qplot/resources/testdata/qplot_test_db_01_10mb.csv +11 -0
  47. qplot/resources/testdata/qplot_test_db_02_25mb.csv +14 -0
  48. qplot/resources/testdata/qplot_test_db_03_50mb.csv +17 -0
  49. qplot/resources/testdata/qplot_test_db_04_100mb.csv +20 -0
  50. qplot/resources/testdata/qplot_test_db_05_250mb.csv +23 -0
  51. qplot/resources/testdata/qplot_test_db_06_500mb.csv +26 -0
  52. qplot/resources/testdata/qplot_test_db_07_1gb.csv +29 -0
  53. qplot/resources/testdata/qplot_test_db_08_5gb.csv +32 -0
  54. qplot/resources/testdata/qplot_test_db_09_10gb.csv +35 -0
  55. qplot/resources/testdata/qplot_test_db_10_30gb.csv +38 -0
  56. qplot/testdata.py +1733 -0
  57. qplot/tools/__init__.py +13 -0
  58. qplot/tools/finite_means.py +114 -0
  59. qplot/tools/general.py +134 -0
  60. qplot/tools/heatmap_geometry.py +381 -0
  61. qplot/tools/operation_registry.py +179 -0
  62. qplot/tools/plot_tools.py +625 -0
  63. qplot/tools/sample_statistics.py +56 -0
  64. qplot/tools/worker.py +2787 -0
  65. qplot/windows/__init__.py +9 -0
  66. qplot/windows/_colorbar.py +422 -0
  67. qplot/windows/_commands.py +523 -0
  68. qplot/windows/_config_persistence.py +123 -0
  69. qplot/windows/_database_actions.py +4904 -0
  70. qplot/windows/_dataset_handle.py +221 -0
  71. qplot/windows/_dragdrop.py +71 -0
  72. qplot/windows/_export_paths.py +1167 -0
  73. qplot/windows/_help.py +169 -0
  74. qplot/windows/_native_averaging.py +173 -0
  75. qplot/windows/_native_transforms.py +389 -0
  76. qplot/windows/_plot1d_snap.py +608 -0
  77. qplot/windows/_plot1d_traces.py +2183 -0
  78. qplot/windows/_plot2d_colorbar.py +1126 -0
  79. qplot/windows/_plot2d_colorbar_dialog.py +786 -0
  80. qplot/windows/_plot2d_layers.py +2197 -0
  81. qplot/windows/_plot2d_sweeps.py +712 -0
  82. qplot/windows/_plotWin.py +1993 -0
  83. qplot/windows/_plot_actions.py +2695 -0
  84. qplot/windows/_plot_appearance.py +276 -0
  85. qplot/windows/_plot_axis_scaling.py +1485 -0
  86. qplot/windows/_plot_export.py +1445 -0
  87. qplot/windows/_plot_feedback.py +110 -0
  88. qplot/windows/_plot_marquee.py +656 -0
  89. qplot/windows/_plot_refresh.py +942 -0
  90. qplot/windows/_plot_state.py +239 -0
  91. qplot/windows/_plot_transform_labels.py +60 -0
  92. qplot/windows/_preferences.py +635 -0
  93. qplot/windows/_refresh_interval.py +70 -0
  94. qplot/windows/_run_controls.py +828 -0
  95. qplot/windows/_shortcuts.py +47 -0
  96. qplot/windows/_subplots/__init__.py +8 -0
  97. qplot/windows/_subplots/subplot1d.py +531 -0
  98. qplot/windows/_subplots/subplot2d.py +918 -0
  99. qplot/windows/_svg_export.py +209 -0
  100. qplot/windows/_trusted_derived_qt.py +1759 -0
  101. qplot/windows/_widgets/__init__.py +23 -0
  102. qplot/windows/_widgets/_run_formatting.py +387 -0
  103. qplot/windows/_widgets/details_tables.py +1119 -0
  104. qplot/windows/_widgets/dropbox.py +290 -0
  105. qplot/windows/_widgets/operations.py +460 -0
  106. qplot/windows/_widgets/preview.py +2997 -0
  107. qplot/windows/_widgets/run_list_items.py +488 -0
  108. qplot/windows/_widgets/toolbar.py +111 -0
  109. qplot/windows/_widgets/treeWidgets.py +2572 -0
  110. qplot/windows/_window_controls.py +413 -0
  111. qplot/windows/main.py +1635 -0
  112. qplot/windows/plot1d.py +311 -0
  113. qplot/windows/plot2d.py +1901 -0
  114. qplotter-1.6.0b3.dist-info/METADATA +318 -0
  115. qplotter-1.6.0b3.dist-info/RECORD +119 -0
  116. qplotter-1.6.0b3.dist-info/WHEEL +5 -0
  117. qplotter-1.6.0b3.dist-info/entry_points.txt +4 -0
  118. qplotter-1.6.0b3.dist-info/licenses/LICENSE +21 -0
  119. qplotter-1.6.0b3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1605 @@
1
+ """Bounded, non-executable disk cache for trusted Stage 5B payloads."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import binascii
7
+ import hashlib
8
+ import json
9
+ import ntpath
10
+ import os
11
+ import re
12
+ import secrets
13
+ import sqlite3
14
+ import stat
15
+ import struct
16
+ import threading
17
+ import time
18
+ from collections.abc import Callable
19
+ from contextlib import contextmanager
20
+ from dataclasses import dataclass
21
+ from itertools import islice
22
+ from pathlib import Path
23
+ from typing import cast
24
+
25
+ from qplot.datahandling.file_identity import DatabaseInstance
26
+ from qplot.datahandling.trusted_derived_rendering import (
27
+ TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES,
28
+ DerivedPayload,
29
+ TrustedDerivedRenderingError,
30
+ validate_trusted_derived_payload,
31
+ )
32
+ from qplot.datahandling.trusted_work_scheduler import (
33
+ TrustedCacheWorkKey,
34
+ trusted_derived_cache_root,
35
+ )
36
+
37
+ TRUSTED_DERIVED_CACHE_FORMAT_VERSION = 3
38
+ TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES = 16 * 1024 * 1024
39
+ TRUSTED_DERIVED_CACHE_MAX_TOTAL_BYTES = 256 * 1024 * 1024
40
+ TRUSTED_DERIVED_CACHE_MAX_ENTRIES = 2_048
41
+ TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES = 4_096
42
+ TRUSTED_DERIVED_CACHE_MAX_HEADER_BYTES = 64 * 1024
43
+ TRUSTED_DERIVED_CACHE_MAX_TEMP_FILES = 16
44
+ TRUSTED_DERIVED_CACHE_MAX_JSON_DEPTH = 32
45
+ TRUSTED_DERIVED_CACHE_MAX_JSON_NODES = 131_072
46
+ TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS = 65_536
47
+ TRUSTED_DERIVED_CACHE_MAX_TEXT_BYTES = 1024 * 1024
48
+
49
+ _MAGIC = b"QPLTDCB1"
50
+ _PREFIX = struct.Struct(">8sIQ")
51
+ _ENTRY_PATTERN = re.compile(r"^[0-9a-f]{64}\.qdc$")
52
+ _TEMP_PATTERN = re.compile(r"^[0-9a-f]{64}\.[0-9a-f]{32}\.tmp$")
53
+ _INDEX_NAME = ".qplot-derived-cache-index.sqlite3"
54
+ _LOCK_NAME = ".qplot-derived-cache.lock"
55
+ _INDEX_SCHEMA_VERSION = "1"
56
+ _INDEX_MAX_VALIDATION_ROWS = TRUSTED_DERIVED_CACHE_MAX_ENTRIES + 1
57
+
58
+
59
+ class _CacheLockUnavailable(OSError):
60
+ pass
61
+
62
+
63
+ class _CacheEpochChanged(RuntimeError):
64
+ pass
65
+
66
+
67
+ class _CacheIndexCorrupt(ValueError):
68
+ pass
69
+
70
+
71
+ class _CacheOwnershipError(ValueError):
72
+ pass
73
+
74
+
75
+ @dataclass(frozen=True)
76
+ class _ArtifactProof:
77
+ path: Path
78
+ device: int
79
+ inode: int
80
+ temporary: bool
81
+ canonical_key: bytes
82
+
83
+
84
+ @dataclass(frozen=True)
85
+ class _IndexProof:
86
+ device: int
87
+ inode: int
88
+ size: int
89
+ modified: int
90
+ changed: int
91
+ created: bool
92
+
93
+
94
+ def _rollback_index_best_effort(connection: object) -> None:
95
+ try:
96
+ cast(sqlite3.Connection, connection).rollback()
97
+ except (sqlite3.Error, OSError):
98
+ pass
99
+
100
+
101
+ def _close_index_best_effort(connection: object) -> None:
102
+ try:
103
+ cast(sqlite3.Connection, connection).close()
104
+ except (sqlite3.Error, OSError):
105
+ pass
106
+
107
+
108
+ def _open_owned_cache_lock(path: Path) -> int:
109
+ """Open only a new lock or the exact empty/one-zero-byte qPlot lock."""
110
+
111
+ flags = getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
112
+ flags |= getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOINHERIT", 0)
113
+ try:
114
+ status = path.stat(follow_symlinks=False)
115
+ except FileNotFoundError:
116
+ try:
117
+ return os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL | flags, 0o600)
118
+ except FileExistsError:
119
+ # Another cache process may have created the lock first.
120
+ status = path.stat(follow_symlinks=False)
121
+ if (
122
+ not stat.S_ISREG(status.st_mode)
123
+ or status.st_nlink != 1
124
+ or status.st_size not in (0, 1)
125
+ ):
126
+ raise _CacheOwnershipError("The cache lock is not owned by qPlot.")
127
+ with path.open("rb") as stream:
128
+ opened = os.fstat(stream.fileno())
129
+ if (
130
+ (opened.st_dev, opened.st_ino) != (status.st_dev, status.st_ino)
131
+ or stream.read(2) not in (b"", b"\0")
132
+ ):
133
+ raise _CacheOwnershipError("The cache lock is not owned by qPlot.")
134
+ current = path.stat(follow_symlinks=False)
135
+ if (
136
+ not stat.S_ISREG(current.st_mode)
137
+ or current.st_nlink != 1
138
+ or (current.st_dev, current.st_ino) != (status.st_dev, status.st_ino)
139
+ or current.st_size not in (0, 1)
140
+ ):
141
+ raise _CacheOwnershipError("The cache lock changed during validation.")
142
+ # Existing locks are never created, truncated or replaced. The cache
143
+ # namespace is trusted; no hostile path substitution is assumed here.
144
+ return os.open(path, os.O_RDWR | flags)
145
+
146
+
147
+ @contextmanager
148
+ def _cross_process_cache_lock(
149
+ path: Path,
150
+ *,
151
+ cancel_check: Callable[[], None],
152
+ ):
153
+ """Acquire one bounded unprivileged root lock on POSIX or Windows."""
154
+
155
+ descriptor = _open_owned_cache_lock(path)
156
+ acquired = False
157
+ deadline = time.monotonic() + 1.0
158
+ try:
159
+ if os.name == "posix":
160
+ import fcntl
161
+
162
+ os.chmod(descriptor, 0o600)
163
+ while True:
164
+ cancel_check()
165
+ try:
166
+ fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
167
+ acquired = True
168
+ break
169
+ except BlockingIOError:
170
+ if time.monotonic() >= deadline:
171
+ raise _CacheLockUnavailable(
172
+ "The cache root lock is busy."
173
+ ) from None
174
+ time.sleep(0.005)
175
+ elif os.name == "nt":
176
+ import msvcrt
177
+
178
+ locking = msvcrt.locking # type: ignore[attr-defined]
179
+ lock_nonblocking = msvcrt.LK_NBLCK # type: ignore[attr-defined]
180
+ if os.fstat(descriptor).st_size == 0:
181
+ os.write(descriptor, b"\0")
182
+ while True:
183
+ cancel_check()
184
+ os.lseek(descriptor, 0, os.SEEK_SET)
185
+ try:
186
+ locking(descriptor, lock_nonblocking, 1)
187
+ acquired = True
188
+ break
189
+ except OSError:
190
+ if time.monotonic() >= deadline:
191
+ raise _CacheLockUnavailable(
192
+ "The cache root lock is busy."
193
+ ) from None
194
+ time.sleep(0.005)
195
+ else:
196
+ raise _CacheLockUnavailable("No cache-root lock is available.")
197
+ yield
198
+ finally:
199
+ if acquired:
200
+ if os.name == "posix":
201
+ import fcntl
202
+
203
+ fcntl.flock(descriptor, fcntl.LOCK_UN)
204
+ elif os.name == "nt":
205
+ import msvcrt
206
+
207
+ os.lseek(descriptor, 0, os.SEEK_SET)
208
+ msvcrt.locking( # type: ignore[attr-defined]
209
+ descriptor,
210
+ msvcrt.LK_UNLCK, # type: ignore[attr-defined]
211
+ 1,
212
+ )
213
+ os.close(descriptor)
214
+
215
+
216
+ def _valid_text(value: str, *, maximum: int) -> str:
217
+ encoded = value.encode("utf-8")
218
+ if len(encoded) > maximum:
219
+ raise ValueError("A cache text value is oversized.")
220
+ return value
221
+
222
+
223
+ def _canonical_value(value: object) -> object:
224
+ if value is None:
225
+ return ["none"]
226
+ if isinstance(value, bool):
227
+ return ["bool", value]
228
+ if type(value) is int:
229
+ return ["int", str(value)]
230
+ if isinstance(value, float):
231
+ return ["float", value.hex()]
232
+ if isinstance(value, str):
233
+ return ["str", _valid_text(value, maximum=TRUSTED_DERIVED_CACHE_MAX_TEXT_BYTES)]
234
+ if isinstance(value, bytes):
235
+ return ["bytes", base64.b64encode(value).decode("ascii")]
236
+ if isinstance(value, tuple):
237
+ return ["tuple", [_canonical_value(item) for item in value]]
238
+ if isinstance(value, dict):
239
+ if any(not isinstance(name, str) for name in value):
240
+ raise TypeError("Derived payload dictionary keys must be text.")
241
+ return [
242
+ "dict",
243
+ [
244
+ [name, _canonical_value(value[name])]
245
+ for name in sorted(cast(dict[str, object], value))
246
+ ],
247
+ ]
248
+ raise TypeError(f"Unsupported canonical cache value {type(value).__name__}.")
249
+
250
+
251
+ def _restore_value(
252
+ value: object,
253
+ *,
254
+ depth: int = 0,
255
+ budget: list[int] | None = None,
256
+ ) -> object:
257
+ if budget is None:
258
+ budget = [0]
259
+ budget[0] += 1
260
+ if budget[0] > TRUSTED_DERIVED_CACHE_MAX_JSON_NODES:
261
+ raise ValueError("A cache value has too many nodes.")
262
+ if depth > 16 or not isinstance(value, list) or not value:
263
+ raise ValueError("A cache value has an invalid canonical structure.")
264
+ tag = value[0]
265
+ if tag == "none" and len(value) == 1:
266
+ return None
267
+ if tag == "bool" and len(value) == 2 and isinstance(value[1], bool):
268
+ return value[1]
269
+ if tag == "int" and len(value) == 2 and isinstance(value[1], str):
270
+ _valid_text(value[1], maximum=128)
271
+ return int(value[1])
272
+ if tag == "float" and len(value) == 2 and isinstance(value[1], str):
273
+ _valid_text(value[1], maximum=128)
274
+ restored = float.fromhex(value[1])
275
+ if not (restored == restored and abs(restored) != float("inf")):
276
+ raise ValueError("A cache float is not finite.")
277
+ return restored
278
+ if tag == "str" and len(value) == 2 and isinstance(value[1], str):
279
+ return _valid_text(value[1], maximum=TRUSTED_DERIVED_CACHE_MAX_TEXT_BYTES)
280
+ if tag == "bytes" and len(value) == 2 and isinstance(value[1], str):
281
+ maximum_encoded = 4 * ((TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES + 2) // 3)
282
+ if len(value[1]) > maximum_encoded:
283
+ raise ValueError("A cache byte scalar is oversized.")
284
+ restored_bytes = base64.b64decode(value[1], validate=True)
285
+ if len(restored_bytes) > TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES:
286
+ raise ValueError("A cache byte scalar is oversized.")
287
+ return restored_bytes
288
+ if tag == "tuple" and len(value) == 2 and isinstance(value[1], list):
289
+ if len(value[1]) > TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS:
290
+ raise ValueError("A cache tuple has too many items.")
291
+ return tuple(
292
+ _restore_value(item, depth=depth + 1, budget=budget) for item in value[1]
293
+ )
294
+ if tag == "dict" and len(value) == 2 and isinstance(value[1], list):
295
+ if len(value[1]) > TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS:
296
+ raise ValueError("A cache dictionary has too many items.")
297
+ restored_dict: dict[str, object] = {}
298
+ for item in value[1]:
299
+ if (
300
+ not isinstance(item, list)
301
+ or len(item) != 2
302
+ or not isinstance(item[0], str)
303
+ or item[0] in restored_dict
304
+ ):
305
+ raise ValueError("A cache dictionary item is invalid.")
306
+ _valid_text(item[0], maximum=256)
307
+ restored_dict[item[0]] = _restore_value(
308
+ item[1], depth=depth + 1, budget=budget
309
+ )
310
+ return restored_dict
311
+ raise ValueError("A cache value has an unknown canonical tag.")
312
+
313
+
314
+ def canonical_trusted_cache_key(key: TrustedCacheWorkKey) -> bytes:
315
+ """Return a type-preserving deterministic cache-key encoding."""
316
+
317
+ if not isinstance(key, TrustedCacheWorkKey):
318
+ raise TypeError("key must be TrustedCacheWorkKey.")
319
+ value = (
320
+ "qplot-trusted-cache-key-v1",
321
+ (
322
+ key.database_instance.logical_path,
323
+ key.database_instance.resolved_path,
324
+ cast(tuple[object, ...], key.database_instance.identity),
325
+ ),
326
+ key.run_guid,
327
+ int(key.kind),
328
+ key.source_revision.fingerprint,
329
+ key.renderer_version,
330
+ key.rendering_options.canonical_values,
331
+ )
332
+ return json.dumps(
333
+ _canonical_value(value),
334
+ ensure_ascii=False,
335
+ separators=(",", ":"),
336
+ ).encode("utf-8")
337
+
338
+
339
+ def trusted_cache_filename(key: TrustedCacheWorkKey) -> str:
340
+ return f"{hashlib.sha256(canonical_trusted_cache_key(key)).hexdigest()}.qdc"
341
+
342
+
343
+ class _JsonPreflightParser:
344
+ """Count JSON structure without materialising its container graph."""
345
+
346
+ def __init__(self, text: str) -> None:
347
+ self._text = text
348
+ self._length = len(text)
349
+ self._position = 0
350
+ self._nodes = 0
351
+
352
+ def parse(self) -> None:
353
+ self._skip_whitespace()
354
+ self._parse_value(depth=0)
355
+ self._skip_whitespace()
356
+ if self._position != self._length:
357
+ raise ValueError("A cache JSON document has trailing content.")
358
+
359
+ def _skip_whitespace(self) -> None:
360
+ while self._position < self._length and self._text[self._position] in " \t\r\n":
361
+ self._position += 1
362
+
363
+ def _count_node(self) -> None:
364
+ self._nodes += 1
365
+ if self._nodes > TRUSTED_DERIVED_CACHE_MAX_JSON_NODES:
366
+ raise ValueError("A cache JSON document has too many nodes.")
367
+
368
+ def _parse_value(self, *, depth: int, binary: bool = False) -> str | None:
369
+ self._skip_whitespace()
370
+ if self._position >= self._length:
371
+ raise ValueError("A cache JSON document is incomplete.")
372
+ self._count_node()
373
+ character = self._text[self._position]
374
+ if character == '"':
375
+ return self._parse_string(binary=binary)
376
+ if character == "[":
377
+ self._parse_array(depth=depth + 1)
378
+ return None
379
+ if character == "{":
380
+ self._parse_object(depth=depth + 1)
381
+ return None
382
+ start = self._position
383
+ while (
384
+ self._position < self._length
385
+ and self._text[self._position] not in " \t\r\n,]}"
386
+ ):
387
+ self._position += 1
388
+ if self._position == start:
389
+ raise ValueError("A cache JSON scalar is invalid.")
390
+ return None
391
+
392
+ def _parse_array(self, *, depth: int) -> None:
393
+ if depth > TRUSTED_DERIVED_CACHE_MAX_JSON_DEPTH:
394
+ raise ValueError("A cache JSON document is too deeply nested.")
395
+ self._position += 1
396
+ self._skip_whitespace()
397
+ count = 0
398
+ first_value: str | None = None
399
+ while self._position >= self._length or self._text[self._position] != "]":
400
+ count += 1
401
+ if count > TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS:
402
+ raise ValueError("A cache JSON container has too many items.")
403
+ value = self._parse_value(
404
+ depth=depth,
405
+ binary=count == 2 and first_value == "bytes",
406
+ )
407
+ if count == 1:
408
+ first_value = value
409
+ self._skip_whitespace()
410
+ if self._position >= self._length:
411
+ raise ValueError("A cache JSON document is incomplete.")
412
+ if self._text[self._position] == "]":
413
+ break
414
+ if self._text[self._position] != ",":
415
+ raise ValueError("A cache JSON array is invalid.")
416
+ self._position += 1
417
+ self._skip_whitespace()
418
+ self._position += 1
419
+
420
+ def _parse_object(self, *, depth: int) -> None:
421
+ if depth > TRUSTED_DERIVED_CACHE_MAX_JSON_DEPTH:
422
+ raise ValueError("A cache JSON document is too deeply nested.")
423
+ self._position += 1
424
+ self._skip_whitespace()
425
+ count = 0
426
+ while self._position >= self._length or self._text[self._position] != "}":
427
+ count += 1
428
+ if count > TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS:
429
+ raise ValueError("A cache JSON container has too many items.")
430
+ if self._position >= self._length or self._text[self._position] != '"':
431
+ raise ValueError("A cache JSON object key is invalid.")
432
+ self._parse_string(binary=False)
433
+ self._skip_whitespace()
434
+ if self._position >= self._length or self._text[self._position] != ":":
435
+ raise ValueError("A cache JSON object is invalid.")
436
+ self._position += 1
437
+ self._parse_value(depth=depth)
438
+ self._skip_whitespace()
439
+ if self._position >= self._length:
440
+ raise ValueError("A cache JSON document is incomplete.")
441
+ if self._text[self._position] == "}":
442
+ break
443
+ if self._text[self._position] != ",":
444
+ raise ValueError("A cache JSON object is invalid.")
445
+ self._position += 1
446
+ self._skip_whitespace()
447
+ self._position += 1
448
+
449
+ def _parse_string(self, *, binary: bool) -> str | None:
450
+ start = self._position + 1
451
+ self._position = start
452
+ escaped = False
453
+ byte_count = 0
454
+ while self._position < self._length:
455
+ character = self._text[self._position]
456
+ if not escaped and character == '"':
457
+ break
458
+ if binary:
459
+ if escaped or character == "\\" or not character.isascii():
460
+ raise ValueError("A cache byte scalar is invalid.")
461
+ if not (character.isalnum() or character in "+/="):
462
+ raise ValueError("A cache byte scalar is invalid.")
463
+ byte_count += 1
464
+ else:
465
+ byte_count += len(character.encode("utf-8"))
466
+ if byte_count > TRUSTED_DERIVED_CACHE_MAX_TEXT_BYTES:
467
+ raise ValueError("A cache JSON string is oversized.")
468
+ if escaped:
469
+ escaped = False
470
+ elif character == "\\":
471
+ escaped = True
472
+ self._position += 1
473
+ if self._position >= self._length or escaped:
474
+ raise ValueError("A cache JSON document is incomplete.")
475
+ end = self._position
476
+ self._position += 1
477
+ if binary:
478
+ maximum_encoded = 4 * ((TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES + 2) // 3)
479
+ if byte_count > maximum_encoded or byte_count % 4:
480
+ raise ValueError("A cache byte scalar is oversized.")
481
+ padding = int(byte_count > 0 and self._text[end - 1] == "=")
482
+ padding += int(byte_count > 1 and self._text[end - 2] == "=")
483
+ if self._text.find("=", start, end - padding) != -1:
484
+ raise ValueError("A cache byte scalar is invalid.")
485
+ decoded_bytes = byte_count // 4 * 3 - padding
486
+ if decoded_bytes > TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES:
487
+ raise ValueError("A cache byte scalar is oversized.")
488
+ return None
489
+ if end - start == 5 and self._text.startswith("bytes", start):
490
+ return "bytes"
491
+ return None
492
+
493
+
494
+ def _preflight_json(data: bytes) -> None:
495
+ """Bound JSON structure and canonical binary strings before ``json.loads``."""
496
+
497
+ if len(data) > TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES:
498
+ raise ValueError("A cache JSON document is oversized.")
499
+ _JsonPreflightParser(data.decode("utf-8")).parse()
500
+
501
+
502
+ class TrustedDerivedDiskCache:
503
+ """Best-effort qPlot-owned cache; every failure degrades to an uncached result."""
504
+
505
+ def __init__(
506
+ self,
507
+ root: str | os.PathLike[str] | None = None,
508
+ *,
509
+ enabled: bool = True,
510
+ max_entry_bytes: int = TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES,
511
+ max_total_bytes: int = TRUSTED_DERIVED_CACHE_MAX_TOTAL_BYTES,
512
+ max_entries: int = TRUSTED_DERIVED_CACHE_MAX_ENTRIES,
513
+ ) -> None:
514
+ selected_root = Path(
515
+ trusted_derived_cache_root() if root is None else os.fspath(root)
516
+ )
517
+ if not selected_root.is_absolute():
518
+ raise ValueError("The trusted derived cache root must be absolute.")
519
+ if (
520
+ type(max_entry_bytes) is not int
521
+ or type(max_total_bytes) is not int
522
+ or type(max_entries) is not int
523
+ or not 1 <= max_entry_bytes <= TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES
524
+ or not max_entry_bytes
525
+ <= max_total_bytes
526
+ <= TRUSTED_DERIVED_CACHE_MAX_TOTAL_BYTES
527
+ or not 1 <= max_entries <= TRUSTED_DERIVED_CACHE_MAX_ENTRIES
528
+ ):
529
+ raise ValueError("Trusted derived cache bounds are invalid.")
530
+ self.root = selected_root
531
+ self._requested_enabled = bool(enabled)
532
+ self.enabled = bool(enabled)
533
+ self.max_entry_bytes = max_entry_bytes
534
+ self.max_total_bytes = max_total_bytes
535
+ self.max_entries = max_entries
536
+ self._state_lock = threading.RLock()
537
+ self._state_changed = threading.Condition(self._state_lock)
538
+ self._epoch = 0
539
+ self._active_puts = 0
540
+ self._protected_database_family: frozenset[Path] = frozenset()
541
+
542
+ def configure_for_database(self, database_instance: DatabaseInstance) -> None:
543
+ """Disable disk writes unless cache and database directories are disjoint."""
544
+
545
+ if not isinstance(database_instance, DatabaseInstance):
546
+ raise TypeError("database_instance must be a DatabaseInstance.")
547
+ database_path = Path(database_instance.resolved_path)
548
+ protected_family = frozenset(
549
+ Path(f"{database_path}{suffix}")
550
+ for suffix in ("", "-wal", "-journal", "-shm")
551
+ )
552
+ try:
553
+ root = self.root.resolve(strict=False)
554
+ database_directory = database_path.parent.resolve(strict=False)
555
+ unsafe = (
556
+ root == database_directory
557
+ or root.is_relative_to(database_directory)
558
+ or database_directory.is_relative_to(root)
559
+ )
560
+ except (OSError, RuntimeError, ValueError):
561
+ unsafe = True
562
+ with self._state_changed:
563
+ self._epoch += 1
564
+ self._protected_database_family = protected_family
565
+ self.enabled = self._requested_enabled and not unsafe
566
+ self._state_changed.notify_all()
567
+
568
+ def get(
569
+ self,
570
+ key: TrustedCacheWorkKey,
571
+ *,
572
+ cancel_check: Callable[[], None] = lambda: None,
573
+ ) -> DerivedPayload | None:
574
+ """Read and verify one exact entry without creating or touching files."""
575
+
576
+ if not self.enabled:
577
+ return None
578
+ try:
579
+ cancel_check()
580
+ canonical_key = canonical_trusted_cache_key(key)
581
+ path = self.root / trusted_cache_filename(key)
582
+ status = path.stat()
583
+ if not path.is_file() or status.st_size > self.max_entry_bytes:
584
+ return None
585
+ data = path.read_bytes()
586
+ except (InterruptedError, TimeoutError):
587
+ raise
588
+ except (
589
+ OSError,
590
+ UnicodeError,
591
+ ValueError,
592
+ TypeError,
593
+ OverflowError,
594
+ RecursionError,
595
+ ):
596
+ return None
597
+ try:
598
+ if len(data) < _PREFIX.size:
599
+ return None
600
+ magic, header_length, payload_length = _PREFIX.unpack_from(data)
601
+ if (
602
+ magic != _MAGIC
603
+ or header_length > TRUSTED_DERIVED_CACHE_MAX_HEADER_BYTES
604
+ or payload_length > self.max_entry_bytes
605
+ or _PREFIX.size + header_length + payload_length != len(data)
606
+ ):
607
+ return None
608
+ cancel_check()
609
+ header_start = _PREFIX.size
610
+ payload_start = header_start + header_length
611
+ _preflight_json(data[header_start:payload_start])
612
+ header = json.loads(data[header_start:payload_start].decode("utf-8"))
613
+ if not isinstance(header, dict):
614
+ return None
615
+ if header.get("format_version") != TRUSTED_DERIVED_CACHE_FORMAT_VERSION:
616
+ return None
617
+ expected_key = base64.b64encode(canonical_key).decode("ascii")
618
+ if (
619
+ header.get("key") != expected_key
620
+ or header.get("payload_length") != payload_length
621
+ ):
622
+ return None
623
+ payload_bytes = data[payload_start:]
624
+ if (
625
+ header.get("payload_sha256")
626
+ != hashlib.sha256(payload_bytes).hexdigest()
627
+ ):
628
+ return None
629
+ _preflight_json(payload_bytes)
630
+ decoded = json.loads(payload_bytes.decode("utf-8"))
631
+ restored = _restore_value(decoded)
632
+ if not isinstance(restored, dict):
633
+ return None
634
+ payload = cast(DerivedPayload, restored)
635
+ validate_trusted_derived_payload(payload)
636
+ cancel_check()
637
+ return payload
638
+ except (InterruptedError, TimeoutError):
639
+ raise
640
+ except (
641
+ binascii.Error,
642
+ UnicodeError,
643
+ ValueError,
644
+ TypeError,
645
+ OverflowError,
646
+ RecursionError,
647
+ json.JSONDecodeError,
648
+ OSError,
649
+ TrustedDerivedRenderingError,
650
+ ):
651
+ return None
652
+
653
+ def put(
654
+ self,
655
+ key: TrustedCacheWorkKey,
656
+ payload: DerivedPayload,
657
+ *,
658
+ cancel_check: Callable[[], None] = lambda: None,
659
+ ) -> bool:
660
+ """Atomically store one entry; never turn a cache fault into DB failure."""
661
+
662
+ with self._state_changed:
663
+ if not self.enabled:
664
+ return False
665
+ epoch = self._epoch
666
+ self._active_puts += 1
667
+ temporary: Path | None = None
668
+ temporary_proof: _ArtifactProof | None = None
669
+ destination: Path | None = None
670
+ published: Path | None = None
671
+ published_proof: _ArtifactProof | None = None
672
+ successful = False
673
+ index_started = False
674
+ disable_after_cleanup = False
675
+
676
+ def cache_cancel_check() -> None:
677
+ cancel_check()
678
+ with self._state_lock:
679
+ if not self.enabled or epoch != self._epoch:
680
+ raise _CacheEpochChanged("The cache database selection changed.")
681
+
682
+ try:
683
+ cache_cancel_check()
684
+ validate_trusted_derived_payload(payload)
685
+ canonical_key = canonical_trusted_cache_key(key)
686
+ payload_bytes = json.dumps(
687
+ _canonical_value(payload),
688
+ ensure_ascii=False,
689
+ separators=(",", ":"),
690
+ ).encode("utf-8")
691
+ header = json.dumps(
692
+ {
693
+ "format_version": TRUSTED_DERIVED_CACHE_FORMAT_VERSION,
694
+ "key": base64.b64encode(canonical_key).decode("ascii"),
695
+ "payload_length": len(payload_bytes),
696
+ "payload_sha256": hashlib.sha256(payload_bytes).hexdigest(),
697
+ },
698
+ sort_keys=True,
699
+ separators=(",", ":"),
700
+ ).encode("utf-8")
701
+ total = _PREFIX.size + len(header) + len(payload_bytes)
702
+ if (
703
+ len(header) > TRUSTED_DERIVED_CACHE_MAX_HEADER_BYTES
704
+ or total > self.max_entry_bytes
705
+ ):
706
+ return False
707
+ self._ensure_root()
708
+ filename = trusted_cache_filename(key)
709
+ destination = self.root / filename
710
+ temporary = self.root / f"{filename[:-4]}.{secrets.token_hex(16)}.tmp"
711
+ cache_cancel_check()
712
+ descriptor = os.open(
713
+ temporary,
714
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL,
715
+ 0o600,
716
+ )
717
+ created_status = os.fstat(descriptor)
718
+ temporary_proof = _ArtifactProof(
719
+ temporary,
720
+ created_status.st_dev,
721
+ created_status.st_ino,
722
+ True,
723
+ canonical_key,
724
+ )
725
+ with os.fdopen(descriptor, "wb") as stream:
726
+ stream.write(_PREFIX.pack(_MAGIC, len(header), len(payload_bytes)))
727
+ stream.write(header)
728
+ stream.write(payload_bytes)
729
+ stream.flush()
730
+ os.fsync(stream.fileno())
731
+ cache_cancel_check()
732
+ self._before_publish()
733
+ cache_cancel_check()
734
+ with _cross_process_cache_lock(
735
+ self.root / _LOCK_NAME,
736
+ cancel_check=cache_cancel_check,
737
+ ):
738
+ index_started = True
739
+ connection = self._open_index()
740
+ try:
741
+ if not self._ensure_global_inventory(
742
+ connection,
743
+ cancel_check=cache_cancel_check,
744
+ expected_epoch=epoch,
745
+ ):
746
+ return False
747
+ cache_cancel_check()
748
+ try:
749
+ existing_proof = self._prove_framed_artifact(
750
+ destination,
751
+ temporary=False,
752
+ )
753
+ except FileNotFoundError:
754
+ existing_proof = None
755
+ except (OSError, ValueError):
756
+ disable_after_cleanup = True
757
+ return False
758
+ if existing_proof is not None:
759
+ if existing_proof.canonical_key != canonical_key:
760
+ disable_after_cleanup = True
761
+ return False
762
+ self._reuse_existing_entry(
763
+ connection,
764
+ filename,
765
+ existing_proof,
766
+ expected_epoch=epoch,
767
+ )
768
+ cache_cancel_check()
769
+ successful = True
770
+ return True
771
+ if not self._reserve_and_evict(
772
+ connection,
773
+ filename,
774
+ total,
775
+ cancel_check=cache_cancel_check,
776
+ expected_epoch=epoch,
777
+ ):
778
+ disable_after_cleanup = True
779
+ return False
780
+ try:
781
+ published_proof = self._publish_no_clobber(
782
+ temporary_proof,
783
+ destination,
784
+ expected_epoch=epoch,
785
+ )
786
+ except FileExistsError:
787
+ try:
788
+ existing_proof = self._prove_framed_artifact(
789
+ destination,
790
+ temporary=False,
791
+ )
792
+ except (OSError, ValueError):
793
+ disable_after_cleanup = True
794
+ return False
795
+ if existing_proof.canonical_key != canonical_key:
796
+ disable_after_cleanup = True
797
+ return False
798
+ self._reuse_existing_entry(
799
+ connection,
800
+ filename,
801
+ existing_proof,
802
+ expected_epoch=epoch,
803
+ )
804
+ cache_cancel_check()
805
+ successful = True
806
+ return True
807
+ else:
808
+ published = destination
809
+ status = destination.stat()
810
+ connection.execute("BEGIN IMMEDIATE")
811
+ connection.execute(
812
+ "UPDATE entries SET modified = ?, size = ?, ready = 1 "
813
+ "WHERE name = ?",
814
+ (status.st_mtime_ns, status.st_size, filename),
815
+ )
816
+ connection.commit()
817
+ cache_cancel_check()
818
+ except BaseException:
819
+ _rollback_index_best_effort(connection)
820
+ raise
821
+ finally:
822
+ _close_index_best_effort(connection)
823
+ successful = True
824
+ return True
825
+ except (InterruptedError, TimeoutError):
826
+ raise
827
+ except _CacheEpochChanged:
828
+ return False
829
+ except (sqlite3.Error, _CacheOwnershipError):
830
+ disable_after_cleanup = True
831
+ return False
832
+ except (
833
+ OSError,
834
+ ValueError,
835
+ TypeError,
836
+ OverflowError,
837
+ RecursionError,
838
+ TrustedDerivedRenderingError,
839
+ ):
840
+ if index_started:
841
+ disable_after_cleanup = True
842
+ return False
843
+ finally:
844
+ if published is not None and published_proof is not None and not successful:
845
+ try:
846
+ self._unlink_proven_artifact(
847
+ published_proof,
848
+ expected_epoch=epoch,
849
+ operation="failed-publication",
850
+ )
851
+ except (OSError, ValueError, _CacheEpochChanged):
852
+ pass
853
+ if temporary is not None and temporary_proof is not None:
854
+ try:
855
+ with self._state_lock:
856
+ operation_invalidated = not self.enabled or self._epoch != epoch
857
+ self._unlink_proven_artifact(
858
+ temporary_proof,
859
+ expected_epoch=epoch,
860
+ operation=(
861
+ "invalidated-operation-cleanup"
862
+ if operation_invalidated
863
+ else "temporary-cleanup"
864
+ ),
865
+ )
866
+ except (OSError, ValueError, _CacheEpochChanged):
867
+ pass
868
+ with self._state_changed:
869
+ if disable_after_cleanup:
870
+ self.enabled = False
871
+ self._active_puts -= 1
872
+ self._state_changed.notify_all()
873
+
874
+ def _ensure_root(self) -> None:
875
+ self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
876
+ if os.name == "posix":
877
+ os.chmod(self.root, 0o700)
878
+
879
+ def _before_publish(self) -> None:
880
+ """Test seam immediately before the epoch/publication lock."""
881
+
882
+ def _before_artifact_proof(self, path: Path, operation: str) -> None:
883
+ """Test seam after candidate selection and before ownership proof."""
884
+
885
+ def _before_destructive_cache_file(self, path: Path, operation: str) -> None:
886
+ """Test seam after ownership proof and before the atomic state guard."""
887
+
888
+ @staticmethod
889
+ def _is_protected_database_artifact(
890
+ path: Path,
891
+ protected_family: frozenset[Path],
892
+ ) -> bool:
893
+ if path in protected_family:
894
+ return True
895
+ try:
896
+ resolved = path.resolve(strict=False)
897
+ return any(
898
+ resolved == protected.resolve(strict=False)
899
+ for protected in protected_family
900
+ )
901
+ except (OSError, RuntimeError, ValueError):
902
+ return True
903
+
904
+ def _validated_artifact_path(
905
+ self,
906
+ stored_name: object,
907
+ *,
908
+ temporary: bool,
909
+ index_derived: bool,
910
+ protected_family: frozenset[Path] | None = None,
911
+ ) -> Path:
912
+ """Return one syntactically safe direct-child candidate."""
913
+
914
+ def reject() -> None:
915
+ error = "The cache inventory contains an unsafe deletion target."
916
+ if index_derived:
917
+ raise _CacheIndexCorrupt(error)
918
+ raise ValueError(error)
919
+
920
+ if type(stored_name) is not str:
921
+ reject()
922
+ name = cast(str, stored_name)
923
+ drive, _tail = ntpath.splitdrive(name)
924
+ if (
925
+ not name
926
+ or name in {".", ".."}
927
+ or drive
928
+ or name.startswith(("/", "\\"))
929
+ or "/" in name
930
+ or "\\" in name
931
+ or Path(name).is_absolute()
932
+ or Path(name).name != name
933
+ ):
934
+ reject()
935
+ pattern = _TEMP_PATTERN if temporary else _ENTRY_PATTERN
936
+ if pattern.fullmatch(name) is None:
937
+ reject()
938
+ candidate = self.root / name
939
+ if candidate.parent != self.root:
940
+ reject()
941
+ try:
942
+ root = self.root.resolve(strict=False)
943
+ resolved = candidate.resolve(strict=False)
944
+ except (OSError, RuntimeError, ValueError):
945
+ reject()
946
+ if protected_family is None:
947
+ with self._state_lock:
948
+ protected_family = self._protected_database_family
949
+ if resolved.parent != root or self._is_protected_database_artifact(
950
+ candidate,
951
+ protected_family,
952
+ ):
953
+ reject()
954
+ return candidate
955
+
956
+ def _prove_framed_artifact(
957
+ self,
958
+ path: Path,
959
+ *,
960
+ temporary: bool,
961
+ ) -> _ArtifactProof:
962
+ """Prove a persistent artifact from bounded qPlot framing and integrity."""
963
+
964
+ candidate = self._validated_artifact_path(
965
+ path.name,
966
+ temporary=temporary,
967
+ index_derived=False,
968
+ )
969
+ if candidate != path:
970
+ raise _CacheOwnershipError("The cache artifact path is not canonical.")
971
+ status = path.stat(follow_symlinks=False)
972
+ if (
973
+ not stat.S_ISREG(status.st_mode)
974
+ or status.st_size < _PREFIX.size
975
+ or status.st_size > self.max_entry_bytes
976
+ ):
977
+ raise _CacheOwnershipError("The cache artifact is not a bounded file.")
978
+ with path.open("rb") as stream:
979
+ data = stream.read(self.max_entry_bytes + 1)
980
+ final_status = path.stat(follow_symlinks=False)
981
+ if (final_status.st_dev, final_status.st_ino) != (
982
+ status.st_dev,
983
+ status.st_ino,
984
+ ) or len(data) != status.st_size:
985
+ raise _CacheOwnershipError("The cache artifact identity changed.")
986
+ magic, header_length, payload_length = _PREFIX.unpack_from(data)
987
+ if (
988
+ magic != _MAGIC
989
+ or header_length > TRUSTED_DERIVED_CACHE_MAX_HEADER_BYTES
990
+ or payload_length > self.max_entry_bytes
991
+ or _PREFIX.size + header_length + payload_length != len(data)
992
+ ):
993
+ raise _CacheOwnershipError("The cache artifact framing is invalid.")
994
+ header_start = _PREFIX.size
995
+ payload_start = header_start + header_length
996
+ header_bytes = data[header_start:payload_start]
997
+ _preflight_json(header_bytes)
998
+ header = json.loads(header_bytes.decode("utf-8"))
999
+ if not isinstance(header, dict) or set(header) != {
1000
+ "format_version",
1001
+ "key",
1002
+ "payload_length",
1003
+ "payload_sha256",
1004
+ }:
1005
+ raise _CacheOwnershipError("The cache artifact header is invalid.")
1006
+ encoded_key = header.get("key")
1007
+ digest = header.get("payload_sha256")
1008
+ if (
1009
+ header.get("format_version") != TRUSTED_DERIVED_CACHE_FORMAT_VERSION
1010
+ or header.get("payload_length") != payload_length
1011
+ or not isinstance(encoded_key, str)
1012
+ or not isinstance(digest, str)
1013
+ or re.fullmatch(r"[0-9a-f]{64}", digest) is None
1014
+ ):
1015
+ raise _CacheOwnershipError("The cache artifact header is invalid.")
1016
+ canonical_key = base64.b64decode(encoded_key, validate=True)
1017
+ expected_stem = hashlib.sha256(canonical_key).hexdigest()
1018
+ actual_stem = path.name[:64]
1019
+ payload_bytes = data[payload_start:]
1020
+ if (
1021
+ actual_stem != expected_stem
1022
+ or digest != hashlib.sha256(payload_bytes).hexdigest()
1023
+ ):
1024
+ raise _CacheOwnershipError("The cache artifact integrity is invalid.")
1025
+ return _ArtifactProof(
1026
+ path,
1027
+ final_status.st_dev,
1028
+ final_status.st_ino,
1029
+ temporary,
1030
+ canonical_key,
1031
+ )
1032
+
1033
+ def _recheck_proof_locked(
1034
+ self,
1035
+ proof: _ArtifactProof,
1036
+ *,
1037
+ expected_epoch: int,
1038
+ ) -> Path:
1039
+ if not self.enabled or self._epoch != expected_epoch:
1040
+ raise _CacheEpochChanged("The cache database selection changed.")
1041
+ path = self._validated_artifact_path(
1042
+ proof.path.name,
1043
+ temporary=proof.temporary,
1044
+ index_derived=False,
1045
+ protected_family=self._protected_database_family,
1046
+ )
1047
+ if path != proof.path:
1048
+ raise _CacheOwnershipError("The cache artifact path changed.")
1049
+ status = path.stat(follow_symlinks=False)
1050
+ if not stat.S_ISREG(status.st_mode) or (status.st_dev, status.st_ino) != (
1051
+ proof.device,
1052
+ proof.inode,
1053
+ ):
1054
+ raise _CacheOwnershipError("The cache artifact identity changed.")
1055
+ return path
1056
+
1057
+ def _unlink_proven_artifact(
1058
+ self,
1059
+ proof: _ArtifactProof,
1060
+ *,
1061
+ expected_epoch: int,
1062
+ operation: str,
1063
+ ) -> None:
1064
+ """Perform the only cache-artifact unlink under the configuration lock."""
1065
+
1066
+ self._before_destructive_cache_file(proof.path, operation)
1067
+ with self._state_lock:
1068
+ path = self._recheck_proof_locked(proof, expected_epoch=expected_epoch)
1069
+ path.unlink()
1070
+
1071
+ def _publish_no_clobber(
1072
+ self,
1073
+ temporary_proof: _ArtifactProof,
1074
+ destination: Path,
1075
+ *,
1076
+ expected_epoch: int,
1077
+ ) -> _ArtifactProof:
1078
+ """Atomically publish by hard link, never replacing an existing path."""
1079
+
1080
+ with self._state_lock:
1081
+ temporary = self._recheck_proof_locked(
1082
+ temporary_proof,
1083
+ expected_epoch=expected_epoch,
1084
+ )
1085
+ destination = self._validated_artifact_path(
1086
+ destination.name,
1087
+ temporary=False,
1088
+ index_derived=False,
1089
+ protected_family=self._protected_database_family,
1090
+ )
1091
+ os.link(temporary, destination)
1092
+ return _ArtifactProof(
1093
+ destination,
1094
+ temporary_proof.device,
1095
+ temporary_proof.inode,
1096
+ False,
1097
+ temporary_proof.canonical_key,
1098
+ )
1099
+
1100
+ def _reuse_existing_entry(
1101
+ self,
1102
+ connection: sqlite3.Connection,
1103
+ filename: str,
1104
+ proof: _ArtifactProof,
1105
+ *,
1106
+ expected_epoch: int,
1107
+ ) -> None:
1108
+ with self._state_lock:
1109
+ path = self._recheck_proof_locked(proof, expected_epoch=expected_epoch)
1110
+ status = path.stat(follow_symlinks=False)
1111
+ connection.execute("BEGIN IMMEDIATE")
1112
+ connection.execute(
1113
+ "INSERT OR REPLACE INTO entries(name, modified, size, ready) "
1114
+ "VALUES(?, ?, ?, 1)",
1115
+ (filename, status.st_mtime_ns, status.st_size),
1116
+ )
1117
+ connection.commit()
1118
+
1119
+ def _open_index(self) -> sqlite3.Connection:
1120
+ index_path = self.root / _INDEX_NAME
1121
+ # Ownership of the main inventory cannot establish ownership of its
1122
+ # SQLite sidecars. Even an unrelated database may occupy these names.
1123
+ # Decline crash recovery and continue without disk caching instead.
1124
+ for suffix in ("-journal", "-wal", "-shm"):
1125
+ try:
1126
+ Path(f"{index_path}{suffix}").stat(follow_symlinks=False)
1127
+ except FileNotFoundError:
1128
+ continue
1129
+ raise _CacheOwnershipError(
1130
+ "The cache inventory has an unowned SQLite sidecar."
1131
+ )
1132
+ proof = self._establish_index_ownership(index_path)
1133
+ self._recheck_index_proof(index_path, proof)
1134
+ connection = sqlite3.connect(index_path, timeout=0.0, isolation_level=None)
1135
+ try:
1136
+ if os.name == "posix":
1137
+ os.chmod(index_path, 0o600)
1138
+ connection.execute("PRAGMA trusted_schema = OFF")
1139
+ connection.execute("PRAGMA journal_mode = DELETE")
1140
+ if proof.created:
1141
+ connection.execute("BEGIN IMMEDIATE")
1142
+ connection.execute(
1143
+ "CREATE TABLE cache_meta ("
1144
+ "key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID"
1145
+ )
1146
+ connection.execute(
1147
+ "CREATE TABLE entries ("
1148
+ "name TEXT PRIMARY KEY, modified INTEGER NOT NULL, "
1149
+ "size INTEGER NOT NULL, ready INTEGER NOT NULL) WITHOUT ROWID"
1150
+ )
1151
+ connection.execute(
1152
+ "CREATE INDEX entries_oldest ON entries(modified, name)"
1153
+ )
1154
+ connection.execute(
1155
+ "INSERT INTO cache_meta(key, value) VALUES('schema', ?)",
1156
+ (_INDEX_SCHEMA_VERSION,),
1157
+ )
1158
+ connection.commit()
1159
+ schema = connection.execute(
1160
+ "SELECT value FROM cache_meta WHERE key = 'schema'"
1161
+ ).fetchone()
1162
+ if schema != (_INDEX_SCHEMA_VERSION,):
1163
+ raise ValueError("The cache inventory schema is incompatible.")
1164
+ return connection
1165
+ except BaseException:
1166
+ _close_index_best_effort(connection)
1167
+ raise
1168
+
1169
+ def _establish_index_ownership(self, index_path: Path) -> _IndexProof:
1170
+ """Claim a missing inventory or prove an existing one without writes."""
1171
+
1172
+ flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
1173
+ flags |= getattr(os, "O_BINARY", 0)
1174
+ flags |= getattr(os, "O_CLOEXEC", 0)
1175
+ flags |= getattr(os, "O_NOINHERIT", 0)
1176
+ flags |= getattr(os, "O_NOFOLLOW", 0)
1177
+ try:
1178
+ descriptor = os.open(index_path, flags, 0o600)
1179
+ except FileExistsError:
1180
+ return self._prove_existing_index_read_only(index_path)
1181
+ try:
1182
+ status = os.fstat(descriptor)
1183
+ if not stat.S_ISREG(status.st_mode):
1184
+ raise _CacheOwnershipError(
1185
+ "The newly claimed cache inventory is not a regular file."
1186
+ )
1187
+ return _IndexProof(
1188
+ status.st_dev,
1189
+ status.st_ino,
1190
+ status.st_size,
1191
+ status.st_mtime_ns,
1192
+ status.st_ctime_ns,
1193
+ True,
1194
+ )
1195
+ finally:
1196
+ os.close(descriptor)
1197
+
1198
+ @staticmethod
1199
+ def _recheck_index_proof(index_path: Path, proof: _IndexProof) -> None:
1200
+ status = index_path.stat(follow_symlinks=False)
1201
+ if (
1202
+ not stat.S_ISREG(status.st_mode)
1203
+ or (status.st_dev, status.st_ino) != (proof.device, proof.inode)
1204
+ or status.st_size != proof.size
1205
+ or status.st_mtime_ns != proof.modified
1206
+ or status.st_ctime_ns != proof.changed
1207
+ ):
1208
+ raise _CacheOwnershipError(
1209
+ "The cache inventory changed after ownership validation."
1210
+ )
1211
+
1212
+ def _prove_existing_index_read_only(self, index_path: Path) -> _IndexProof:
1213
+ """Recognise the exact current qPlot inventory via immutable SQLite."""
1214
+
1215
+ initial_status = index_path.stat(follow_symlinks=False)
1216
+ if not stat.S_ISREG(initial_status.st_mode):
1217
+ raise _CacheOwnershipError(
1218
+ "The cache inventory is not a regular qPlot file."
1219
+ )
1220
+ connection = sqlite3.connect(
1221
+ f"{index_path.as_uri()}?mode=ro&immutable=1",
1222
+ timeout=0.0,
1223
+ isolation_level=None,
1224
+ uri=True,
1225
+ )
1226
+ try:
1227
+ connection.execute("PRAGMA query_only = ON")
1228
+ connection.execute("PRAGMA trusted_schema = OFF")
1229
+ objects = connection.execute(
1230
+ "SELECT type, name, tbl_name FROM sqlite_schema "
1231
+ "WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name"
1232
+ ).fetchall()
1233
+ if objects != [
1234
+ ("index", "entries_oldest", "entries"),
1235
+ ("table", "cache_meta", "cache_meta"),
1236
+ ("table", "entries", "entries"),
1237
+ ]:
1238
+ raise _CacheOwnershipError(
1239
+ "The cache inventory schema is not owned by qPlot."
1240
+ )
1241
+ if connection.execute("PRAGMA table_info(cache_meta)").fetchall() != [
1242
+ (0, "key", "TEXT", 1, None, 1),
1243
+ (1, "value", "TEXT", 1, None, 0),
1244
+ ]:
1245
+ raise _CacheOwnershipError(
1246
+ "The cache inventory metadata table is incompatible."
1247
+ )
1248
+ if connection.execute("PRAGMA table_info(entries)").fetchall() != [
1249
+ (0, "name", "TEXT", 1, None, 1),
1250
+ (1, "modified", "INTEGER", 1, None, 0),
1251
+ (2, "size", "INTEGER", 1, None, 0),
1252
+ (3, "ready", "INTEGER", 1, None, 0),
1253
+ ]:
1254
+ raise _CacheOwnershipError(
1255
+ "The cache inventory entries table is incompatible."
1256
+ )
1257
+ if connection.execute("PRAGMA index_info(entries_oldest)").fetchall() != [
1258
+ (0, 1, "modified"),
1259
+ (1, 0, "name"),
1260
+ ]:
1261
+ raise _CacheOwnershipError(
1262
+ "The cache inventory ordering index is incompatible."
1263
+ )
1264
+ metadata = connection.execute(
1265
+ "SELECT key, value FROM cache_meta ORDER BY key"
1266
+ ).fetchall()
1267
+ if metadata not in [
1268
+ [("schema", _INDEX_SCHEMA_VERSION)],
1269
+ [
1270
+ ("inventory_complete", "1"),
1271
+ ("schema", _INDEX_SCHEMA_VERSION),
1272
+ ],
1273
+ ]:
1274
+ raise _CacheOwnershipError(
1275
+ "The cache inventory metadata is incompatible."
1276
+ )
1277
+ rows = connection.execute(
1278
+ "SELECT name, modified, size, ready FROM entries "
1279
+ "ORDER BY name LIMIT ?",
1280
+ (_INDEX_MAX_VALIDATION_ROWS,),
1281
+ ).fetchall()
1282
+ if len(rows) >= _INDEX_MAX_VALIDATION_ROWS:
1283
+ raise _CacheOwnershipError(
1284
+ "The cache inventory contains too many entries."
1285
+ )
1286
+ for name, modified, size, ready in rows:
1287
+ if (
1288
+ type(name) is not str
1289
+ or _ENTRY_PATTERN.fullmatch(name) is None
1290
+ or type(modified) is not int
1291
+ or type(size) is not int
1292
+ or not 0 <= size <= TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES
1293
+ or type(ready) is not int
1294
+ or ready not in (0, 1)
1295
+ ):
1296
+ raise _CacheOwnershipError(
1297
+ "The cache inventory contains an incompatible entry."
1298
+ )
1299
+ finally:
1300
+ _close_index_best_effort(connection)
1301
+ final_status = index_path.stat(follow_symlinks=False)
1302
+ if (
1303
+ not stat.S_ISREG(final_status.st_mode)
1304
+ or (final_status.st_dev, final_status.st_ino)
1305
+ != (initial_status.st_dev, initial_status.st_ino)
1306
+ or final_status.st_size != initial_status.st_size
1307
+ or final_status.st_mtime_ns != initial_status.st_mtime_ns
1308
+ or final_status.st_ctime_ns != initial_status.st_ctime_ns
1309
+ ):
1310
+ raise _CacheOwnershipError(
1311
+ "The cache inventory changed during ownership validation."
1312
+ )
1313
+ return _IndexProof(
1314
+ final_status.st_dev,
1315
+ final_status.st_ino,
1316
+ final_status.st_size,
1317
+ final_status.st_mtime_ns,
1318
+ final_status.st_ctime_ns,
1319
+ False,
1320
+ )
1321
+
1322
+ def _ensure_global_inventory(
1323
+ self,
1324
+ connection: sqlite3.Connection,
1325
+ *,
1326
+ cancel_check: Callable[[], None],
1327
+ expected_epoch: int,
1328
+ ) -> bool:
1329
+ complete = connection.execute(
1330
+ "SELECT value FROM cache_meta WHERE key = 'inventory_complete'"
1331
+ ).fetchone()
1332
+ if complete == ("1",):
1333
+ return self._recover_pending(connection, cancel_check=cancel_check)
1334
+
1335
+ scanned: list[os.DirEntry[str]] = []
1336
+ with os.scandir(self.root) as iterator:
1337
+ for index, entry in enumerate(
1338
+ islice(iterator, TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES)
1339
+ ):
1340
+ if index % 128 == 0:
1341
+ cancel_check()
1342
+ scanned.append(entry)
1343
+ if len(scanned) >= TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES:
1344
+ deletion_targets: list[_ArtifactProof] = []
1345
+ for entry in scanned:
1346
+ if not (
1347
+ _ENTRY_PATTERN.fullmatch(entry.name)
1348
+ or _TEMP_PATTERN.fullmatch(entry.name)
1349
+ ):
1350
+ continue
1351
+ cancel_check()
1352
+ if not entry.is_file(follow_symlinks=False):
1353
+ continue
1354
+ is_temporary = _TEMP_PATTERN.fullmatch(entry.name) is not None
1355
+ path = self._validated_artifact_path(
1356
+ entry.name,
1357
+ temporary=is_temporary,
1358
+ index_derived=True,
1359
+ )
1360
+ self._before_artifact_proof(path, "inventory-overflow")
1361
+ deletion_targets.append(
1362
+ self._prove_framed_artifact(path, temporary=is_temporary)
1363
+ )
1364
+ for proof in deletion_targets:
1365
+ cancel_check()
1366
+ self._unlink_proven_artifact(
1367
+ proof,
1368
+ expected_epoch=expected_epoch,
1369
+ operation="inventory-overflow",
1370
+ )
1371
+ return False
1372
+
1373
+ connection.execute("BEGIN IMMEDIATE")
1374
+ try:
1375
+ connection.execute("DELETE FROM entries")
1376
+ stale_temporaries: list[tuple[int, _ArtifactProof]] = []
1377
+ for entry in scanned:
1378
+ cancel_check()
1379
+ try:
1380
+ status = entry.stat(follow_symlinks=False)
1381
+ except OSError:
1382
+ continue
1383
+ if not entry.is_file(follow_symlinks=False):
1384
+ continue
1385
+ if _ENTRY_PATTERN.fullmatch(entry.name):
1386
+ path = self._validated_artifact_path(
1387
+ entry.name,
1388
+ temporary=False,
1389
+ index_derived=True,
1390
+ )
1391
+ self._before_artifact_proof(path, "inventory-entry")
1392
+ self._prove_framed_artifact(path, temporary=False)
1393
+ connection.execute(
1394
+ "INSERT OR REPLACE INTO entries "
1395
+ "(name, modified, size, ready) VALUES(?, ?, ?, 1)",
1396
+ (entry.name, status.st_mtime_ns, status.st_size),
1397
+ )
1398
+ elif _TEMP_PATTERN.fullmatch(entry.name):
1399
+ path = self._validated_artifact_path(
1400
+ entry.name,
1401
+ temporary=True,
1402
+ index_derived=True,
1403
+ )
1404
+ self._before_artifact_proof(path, "stale-temporary")
1405
+ stale_temporaries.append(
1406
+ (
1407
+ status.st_mtime_ns,
1408
+ self._prove_framed_artifact(path, temporary=True),
1409
+ )
1410
+ )
1411
+ stale_temporaries.sort(key=lambda item: (item[0], item[1].path.name))
1412
+ for _modified, proof in stale_temporaries[
1413
+ :-TRUSTED_DERIVED_CACHE_MAX_TEMP_FILES
1414
+ ]:
1415
+ cancel_check()
1416
+ self._unlink_proven_artifact(
1417
+ proof,
1418
+ expected_epoch=expected_epoch,
1419
+ operation="stale-temporary",
1420
+ )
1421
+ connection.execute(
1422
+ "INSERT OR REPLACE INTO cache_meta(key, value) "
1423
+ "VALUES('inventory_complete', '1')"
1424
+ )
1425
+ connection.commit()
1426
+ except BaseException:
1427
+ _rollback_index_best_effort(connection)
1428
+ raise
1429
+ within_limits = self._enforce_global_limits(
1430
+ connection,
1431
+ exclude=None,
1432
+ cancel_check=cancel_check,
1433
+ expected_epoch=expected_epoch,
1434
+ )
1435
+ if not within_limits:
1436
+ with self._state_lock:
1437
+ self.enabled = False
1438
+ return within_limits
1439
+
1440
+ def _recover_pending(
1441
+ self,
1442
+ connection: sqlite3.Connection,
1443
+ *,
1444
+ cancel_check: Callable[[], None],
1445
+ ) -> bool:
1446
+ pending = connection.execute(
1447
+ "SELECT name FROM entries WHERE ready = 0 ORDER BY modified, name LIMIT ?",
1448
+ (TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES,),
1449
+ ).fetchall()
1450
+ validated_pending = []
1451
+ for (name,) in pending:
1452
+ path = self._validated_artifact_path(
1453
+ name,
1454
+ temporary=False,
1455
+ index_derived=True,
1456
+ )
1457
+ try:
1458
+ proof = self._prove_framed_artifact(path, temporary=False)
1459
+ except FileNotFoundError:
1460
+ proof = None
1461
+ validated_pending.append((name, path, proof))
1462
+ connection.execute("BEGIN IMMEDIATE")
1463
+ try:
1464
+ for name, path, proof in validated_pending:
1465
+ cancel_check()
1466
+ try:
1467
+ status = path.stat()
1468
+ except FileNotFoundError:
1469
+ connection.execute("DELETE FROM entries WHERE name = ?", (name,))
1470
+ except OSError:
1471
+ connection.rollback()
1472
+ with self._state_lock:
1473
+ self.enabled = False
1474
+ return False
1475
+ else:
1476
+ if proof is None:
1477
+ raise _CacheOwnershipError(
1478
+ "A pending cache artifact lacks ownership proof."
1479
+ )
1480
+ connection.execute(
1481
+ "UPDATE entries SET modified = ?, size = ?, ready = 1 "
1482
+ "WHERE name = ?",
1483
+ (status.st_mtime_ns, status.st_size, name),
1484
+ )
1485
+ connection.commit()
1486
+ except BaseException:
1487
+ _rollback_index_best_effort(connection)
1488
+ raise
1489
+ return len(pending) < TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES
1490
+
1491
+ def _reserve_and_evict(
1492
+ self,
1493
+ connection: sqlite3.Connection,
1494
+ name: str,
1495
+ size: int,
1496
+ *,
1497
+ cancel_check: Callable[[], None],
1498
+ expected_epoch: int,
1499
+ ) -> bool:
1500
+ connection.execute("BEGIN IMMEDIATE")
1501
+ try:
1502
+ prior = connection.execute(
1503
+ "SELECT size FROM entries WHERE name = ?", (name,)
1504
+ ).fetchone()
1505
+ reserved = max(size, prior[0] if prior is not None else 0)
1506
+ connection.execute(
1507
+ "INSERT OR REPLACE INTO entries(name, modified, size, ready) "
1508
+ "VALUES(?, ?, ?, 0)",
1509
+ (name, time.time_ns(), reserved),
1510
+ )
1511
+ if not self._enforce_global_limits(
1512
+ connection,
1513
+ exclude=name,
1514
+ cancel_check=cancel_check,
1515
+ expected_epoch=expected_epoch,
1516
+ in_transaction=True,
1517
+ ):
1518
+ connection.rollback()
1519
+ return False
1520
+ connection.commit()
1521
+ return True
1522
+ except BaseException:
1523
+ _rollback_index_best_effort(connection)
1524
+ raise
1525
+
1526
+ def _enforce_global_limits(
1527
+ self,
1528
+ connection: sqlite3.Connection,
1529
+ *,
1530
+ exclude: str | None,
1531
+ cancel_check: Callable[[], None],
1532
+ expected_epoch: int,
1533
+ in_transaction: bool = False,
1534
+ ) -> bool:
1535
+ if not in_transaction:
1536
+ connection.execute("BEGIN IMMEDIATE")
1537
+ try:
1538
+ count, total = connection.execute(
1539
+ "SELECT COUNT(*), COALESCE(SUM(size), 0) FROM entries"
1540
+ ).fetchone()
1541
+ inventory = connection.execute(
1542
+ "SELECT name, size FROM entries ORDER BY modified, name LIMIT ?",
1543
+ (TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES,),
1544
+ ).fetchall()
1545
+ if (
1546
+ type(count) is not int
1547
+ or type(total) is not int
1548
+ or count < 0
1549
+ or total < 0
1550
+ or count != len(inventory)
1551
+ ):
1552
+ raise _CacheIndexCorrupt("The cache inventory bounds are invalid.")
1553
+ validated_inventory: list[tuple[str, int, Path, _ArtifactProof | None]] = []
1554
+ for name, entry_size in inventory:
1555
+ cancel_check()
1556
+ if type(entry_size) is not int or entry_size < 0:
1557
+ raise _CacheIndexCorrupt("A cache inventory size is invalid.")
1558
+ path = self._validated_artifact_path(
1559
+ name,
1560
+ temporary=False,
1561
+ index_derived=True,
1562
+ )
1563
+ self._before_artifact_proof(path, "eviction")
1564
+ try:
1565
+ proof = self._prove_framed_artifact(path, temporary=False)
1566
+ except FileNotFoundError:
1567
+ proof = None
1568
+ validated_inventory.append((cast(str, name), entry_size, path, proof))
1569
+ victims: list[tuple[str, Path, _ArtifactProof | None]] = []
1570
+ projected_count = count
1571
+ projected_total = total
1572
+ for name, entry_size, path, proof in validated_inventory:
1573
+ if (
1574
+ projected_count <= self.max_entries
1575
+ and projected_total <= self.max_total_bytes
1576
+ ):
1577
+ break
1578
+ if exclude is not None and name == exclude:
1579
+ continue
1580
+ victims.append((name, path, proof))
1581
+ projected_count -= 1
1582
+ projected_total -= entry_size
1583
+ if (
1584
+ projected_count > self.max_entries
1585
+ or projected_total > self.max_total_bytes
1586
+ ):
1587
+ if not in_transaction:
1588
+ connection.rollback()
1589
+ return False
1590
+ for name, _path, proof in victims:
1591
+ cancel_check()
1592
+ if proof is not None:
1593
+ self._unlink_proven_artifact(
1594
+ proof,
1595
+ expected_epoch=expected_epoch,
1596
+ operation="eviction",
1597
+ )
1598
+ connection.execute("DELETE FROM entries WHERE name = ?", (name,))
1599
+ if not in_transaction:
1600
+ connection.commit()
1601
+ return True
1602
+ except BaseException:
1603
+ if not in_transaction:
1604
+ _rollback_index_best_effort(connection)
1605
+ raise