qplotter 1.6.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qplot/__init__.py +46 -0
- qplot/__main__.py +232 -0
- qplot/_auxiliary_paths.py +63 -0
- qplot/_metadata.py +12 -0
- qplot/_repair.py +14 -0
- qplot/_shutdown_supervisor.py +4049 -0
- qplot/_version.py +12 -0
- qplot/_windows_shutdown_job.py +1166 -0
- qplot/configuration/__init__.py +3 -0
- qplot/configuration/config.py +573 -0
- qplot/configuration/config_schema.json +319 -0
- qplot/configuration/scripts.py +255 -0
- qplot/configuration/themes/__init__.py +13 -0
- qplot/configuration/themes/_base.py +579 -0
- qplot/configuration/themes/blank.py +17 -0
- qplot/configuration/themes/dark.py +54 -0
- qplot/configuration/themes/light.py +54 -0
- qplot/datahandling/LoadFromDB.py +224 -0
- qplot/datahandling/__init__.py +47 -0
- qplot/datahandling/_trusted_live_helper.py +812 -0
- qplot/datahandling/_trusted_live_protocol.py +1745 -0
- qplot/datahandling/database.py +2330 -0
- qplot/datahandling/dimensions.py +43 -0
- qplot/datahandling/file_identity.py +461 -0
- qplot/datahandling/parameter_data.py +149 -0
- qplot/datahandling/plot_progress.py +17 -0
- qplot/datahandling/qcodes_cache.py +277 -0
- qplot/datahandling/qcodes_compat.py +14 -0
- qplot/datahandling/readSQL.py +2442 -0
- qplot/datahandling/readonly.py +2561 -0
- qplot/datahandling/source_activity.py +21 -0
- qplot/datahandling/trusted_array_preview.py +127 -0
- qplot/datahandling/trusted_derived_cache.py +1605 -0
- qplot/datahandling/trusted_derived_rendering.py +868 -0
- qplot/datahandling/trusted_heatmap.py +241 -0
- qplot/datahandling/trusted_live.py +2808 -0
- qplot/datahandling/trusted_live_queries.py +5907 -0
- qplot/datahandling/trusted_live_service.py +1894 -0
- qplot/datahandling/trusted_live_supervisor.py +2161 -0
- qplot/datahandling/trusted_plot.py +283 -0
- qplot/datahandling/trusted_presentation.py +1065 -0
- qplot/datahandling/trusted_snapshot.py +1132 -0
- qplot/datahandling/trusted_work_coordinator.py +2157 -0
- qplot/datahandling/trusted_work_scheduler.py +1204 -0
- qplot/diagnostics.py +241 -0
- qplot/resources/testdata/qplot_test_db_01_10mb.csv +11 -0
- qplot/resources/testdata/qplot_test_db_02_25mb.csv +14 -0
- qplot/resources/testdata/qplot_test_db_03_50mb.csv +17 -0
- qplot/resources/testdata/qplot_test_db_04_100mb.csv +20 -0
- qplot/resources/testdata/qplot_test_db_05_250mb.csv +23 -0
- qplot/resources/testdata/qplot_test_db_06_500mb.csv +26 -0
- qplot/resources/testdata/qplot_test_db_07_1gb.csv +29 -0
- qplot/resources/testdata/qplot_test_db_08_5gb.csv +32 -0
- qplot/resources/testdata/qplot_test_db_09_10gb.csv +35 -0
- qplot/resources/testdata/qplot_test_db_10_30gb.csv +38 -0
- qplot/testdata.py +1733 -0
- qplot/tools/__init__.py +13 -0
- qplot/tools/finite_means.py +114 -0
- qplot/tools/general.py +134 -0
- qplot/tools/heatmap_geometry.py +381 -0
- qplot/tools/operation_registry.py +179 -0
- qplot/tools/plot_tools.py +625 -0
- qplot/tools/sample_statistics.py +56 -0
- qplot/tools/worker.py +2787 -0
- qplot/windows/__init__.py +9 -0
- qplot/windows/_colorbar.py +422 -0
- qplot/windows/_commands.py +523 -0
- qplot/windows/_config_persistence.py +123 -0
- qplot/windows/_database_actions.py +4904 -0
- qplot/windows/_dataset_handle.py +221 -0
- qplot/windows/_dragdrop.py +71 -0
- qplot/windows/_export_paths.py +1167 -0
- qplot/windows/_help.py +169 -0
- qplot/windows/_native_averaging.py +173 -0
- qplot/windows/_native_transforms.py +389 -0
- qplot/windows/_plot1d_snap.py +608 -0
- qplot/windows/_plot1d_traces.py +2183 -0
- qplot/windows/_plot2d_colorbar.py +1126 -0
- qplot/windows/_plot2d_colorbar_dialog.py +786 -0
- qplot/windows/_plot2d_layers.py +2197 -0
- qplot/windows/_plot2d_sweeps.py +712 -0
- qplot/windows/_plotWin.py +1993 -0
- qplot/windows/_plot_actions.py +2695 -0
- qplot/windows/_plot_appearance.py +276 -0
- qplot/windows/_plot_axis_scaling.py +1485 -0
- qplot/windows/_plot_export.py +1445 -0
- qplot/windows/_plot_feedback.py +110 -0
- qplot/windows/_plot_marquee.py +656 -0
- qplot/windows/_plot_refresh.py +942 -0
- qplot/windows/_plot_state.py +239 -0
- qplot/windows/_plot_transform_labels.py +60 -0
- qplot/windows/_preferences.py +635 -0
- qplot/windows/_refresh_interval.py +70 -0
- qplot/windows/_run_controls.py +828 -0
- qplot/windows/_shortcuts.py +47 -0
- qplot/windows/_subplots/__init__.py +8 -0
- qplot/windows/_subplots/subplot1d.py +531 -0
- qplot/windows/_subplots/subplot2d.py +918 -0
- qplot/windows/_svg_export.py +209 -0
- qplot/windows/_trusted_derived_qt.py +1759 -0
- qplot/windows/_widgets/__init__.py +23 -0
- qplot/windows/_widgets/_run_formatting.py +387 -0
- qplot/windows/_widgets/details_tables.py +1119 -0
- qplot/windows/_widgets/dropbox.py +290 -0
- qplot/windows/_widgets/operations.py +460 -0
- qplot/windows/_widgets/preview.py +2997 -0
- qplot/windows/_widgets/run_list_items.py +488 -0
- qplot/windows/_widgets/toolbar.py +111 -0
- qplot/windows/_widgets/treeWidgets.py +2572 -0
- qplot/windows/_window_controls.py +413 -0
- qplot/windows/main.py +1635 -0
- qplot/windows/plot1d.py +311 -0
- qplot/windows/plot2d.py +1901 -0
- qplotter-1.6.0b3.dist-info/METADATA +318 -0
- qplotter-1.6.0b3.dist-info/RECORD +119 -0
- qplotter-1.6.0b3.dist-info/WHEEL +5 -0
- qplotter-1.6.0b3.dist-info/entry_points.txt +4 -0
- qplotter-1.6.0b3.dist-info/licenses/LICENSE +21 -0
- qplotter-1.6.0b3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1605 @@
|
|
|
1
|
+
"""Bounded, non-executable disk cache for trusted Stage 5B payloads."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import binascii
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import ntpath
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import secrets
|
|
13
|
+
import sqlite3
|
|
14
|
+
import stat
|
|
15
|
+
import struct
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
from collections.abc import Callable
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from itertools import islice
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import cast
|
|
24
|
+
|
|
25
|
+
from qplot.datahandling.file_identity import DatabaseInstance
|
|
26
|
+
from qplot.datahandling.trusted_derived_rendering import (
|
|
27
|
+
TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES,
|
|
28
|
+
DerivedPayload,
|
|
29
|
+
TrustedDerivedRenderingError,
|
|
30
|
+
validate_trusted_derived_payload,
|
|
31
|
+
)
|
|
32
|
+
from qplot.datahandling.trusted_work_scheduler import (
|
|
33
|
+
TrustedCacheWorkKey,
|
|
34
|
+
trusted_derived_cache_root,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
TRUSTED_DERIVED_CACHE_FORMAT_VERSION = 3
|
|
38
|
+
TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES = 16 * 1024 * 1024
|
|
39
|
+
TRUSTED_DERIVED_CACHE_MAX_TOTAL_BYTES = 256 * 1024 * 1024
|
|
40
|
+
TRUSTED_DERIVED_CACHE_MAX_ENTRIES = 2_048
|
|
41
|
+
TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES = 4_096
|
|
42
|
+
TRUSTED_DERIVED_CACHE_MAX_HEADER_BYTES = 64 * 1024
|
|
43
|
+
TRUSTED_DERIVED_CACHE_MAX_TEMP_FILES = 16
|
|
44
|
+
TRUSTED_DERIVED_CACHE_MAX_JSON_DEPTH = 32
|
|
45
|
+
TRUSTED_DERIVED_CACHE_MAX_JSON_NODES = 131_072
|
|
46
|
+
TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS = 65_536
|
|
47
|
+
TRUSTED_DERIVED_CACHE_MAX_TEXT_BYTES = 1024 * 1024
|
|
48
|
+
|
|
49
|
+
_MAGIC = b"QPLTDCB1"
|
|
50
|
+
_PREFIX = struct.Struct(">8sIQ")
|
|
51
|
+
_ENTRY_PATTERN = re.compile(r"^[0-9a-f]{64}\.qdc$")
|
|
52
|
+
_TEMP_PATTERN = re.compile(r"^[0-9a-f]{64}\.[0-9a-f]{32}\.tmp$")
|
|
53
|
+
_INDEX_NAME = ".qplot-derived-cache-index.sqlite3"
|
|
54
|
+
_LOCK_NAME = ".qplot-derived-cache.lock"
|
|
55
|
+
_INDEX_SCHEMA_VERSION = "1"
|
|
56
|
+
_INDEX_MAX_VALIDATION_ROWS = TRUSTED_DERIVED_CACHE_MAX_ENTRIES + 1
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class _CacheLockUnavailable(OSError):
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class _CacheEpochChanged(RuntimeError):
|
|
64
|
+
pass
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class _CacheIndexCorrupt(ValueError):
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class _CacheOwnershipError(ValueError):
|
|
72
|
+
pass
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(frozen=True)
|
|
76
|
+
class _ArtifactProof:
|
|
77
|
+
path: Path
|
|
78
|
+
device: int
|
|
79
|
+
inode: int
|
|
80
|
+
temporary: bool
|
|
81
|
+
canonical_key: bytes
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass(frozen=True)
|
|
85
|
+
class _IndexProof:
|
|
86
|
+
device: int
|
|
87
|
+
inode: int
|
|
88
|
+
size: int
|
|
89
|
+
modified: int
|
|
90
|
+
changed: int
|
|
91
|
+
created: bool
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _rollback_index_best_effort(connection: object) -> None:
|
|
95
|
+
try:
|
|
96
|
+
cast(sqlite3.Connection, connection).rollback()
|
|
97
|
+
except (sqlite3.Error, OSError):
|
|
98
|
+
pass
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _close_index_best_effort(connection: object) -> None:
|
|
102
|
+
try:
|
|
103
|
+
cast(sqlite3.Connection, connection).close()
|
|
104
|
+
except (sqlite3.Error, OSError):
|
|
105
|
+
pass
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _open_owned_cache_lock(path: Path) -> int:
|
|
109
|
+
"""Open only a new lock or the exact empty/one-zero-byte qPlot lock."""
|
|
110
|
+
|
|
111
|
+
flags = getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
112
|
+
flags |= getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOINHERIT", 0)
|
|
113
|
+
try:
|
|
114
|
+
status = path.stat(follow_symlinks=False)
|
|
115
|
+
except FileNotFoundError:
|
|
116
|
+
try:
|
|
117
|
+
return os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL | flags, 0o600)
|
|
118
|
+
except FileExistsError:
|
|
119
|
+
# Another cache process may have created the lock first.
|
|
120
|
+
status = path.stat(follow_symlinks=False)
|
|
121
|
+
if (
|
|
122
|
+
not stat.S_ISREG(status.st_mode)
|
|
123
|
+
or status.st_nlink != 1
|
|
124
|
+
or status.st_size not in (0, 1)
|
|
125
|
+
):
|
|
126
|
+
raise _CacheOwnershipError("The cache lock is not owned by qPlot.")
|
|
127
|
+
with path.open("rb") as stream:
|
|
128
|
+
opened = os.fstat(stream.fileno())
|
|
129
|
+
if (
|
|
130
|
+
(opened.st_dev, opened.st_ino) != (status.st_dev, status.st_ino)
|
|
131
|
+
or stream.read(2) not in (b"", b"\0")
|
|
132
|
+
):
|
|
133
|
+
raise _CacheOwnershipError("The cache lock is not owned by qPlot.")
|
|
134
|
+
current = path.stat(follow_symlinks=False)
|
|
135
|
+
if (
|
|
136
|
+
not stat.S_ISREG(current.st_mode)
|
|
137
|
+
or current.st_nlink != 1
|
|
138
|
+
or (current.st_dev, current.st_ino) != (status.st_dev, status.st_ino)
|
|
139
|
+
or current.st_size not in (0, 1)
|
|
140
|
+
):
|
|
141
|
+
raise _CacheOwnershipError("The cache lock changed during validation.")
|
|
142
|
+
# Existing locks are never created, truncated or replaced. The cache
|
|
143
|
+
# namespace is trusted; no hostile path substitution is assumed here.
|
|
144
|
+
return os.open(path, os.O_RDWR | flags)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@contextmanager
|
|
148
|
+
def _cross_process_cache_lock(
|
|
149
|
+
path: Path,
|
|
150
|
+
*,
|
|
151
|
+
cancel_check: Callable[[], None],
|
|
152
|
+
):
|
|
153
|
+
"""Acquire one bounded unprivileged root lock on POSIX or Windows."""
|
|
154
|
+
|
|
155
|
+
descriptor = _open_owned_cache_lock(path)
|
|
156
|
+
acquired = False
|
|
157
|
+
deadline = time.monotonic() + 1.0
|
|
158
|
+
try:
|
|
159
|
+
if os.name == "posix":
|
|
160
|
+
import fcntl
|
|
161
|
+
|
|
162
|
+
os.chmod(descriptor, 0o600)
|
|
163
|
+
while True:
|
|
164
|
+
cancel_check()
|
|
165
|
+
try:
|
|
166
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
167
|
+
acquired = True
|
|
168
|
+
break
|
|
169
|
+
except BlockingIOError:
|
|
170
|
+
if time.monotonic() >= deadline:
|
|
171
|
+
raise _CacheLockUnavailable(
|
|
172
|
+
"The cache root lock is busy."
|
|
173
|
+
) from None
|
|
174
|
+
time.sleep(0.005)
|
|
175
|
+
elif os.name == "nt":
|
|
176
|
+
import msvcrt
|
|
177
|
+
|
|
178
|
+
locking = msvcrt.locking # type: ignore[attr-defined]
|
|
179
|
+
lock_nonblocking = msvcrt.LK_NBLCK # type: ignore[attr-defined]
|
|
180
|
+
if os.fstat(descriptor).st_size == 0:
|
|
181
|
+
os.write(descriptor, b"\0")
|
|
182
|
+
while True:
|
|
183
|
+
cancel_check()
|
|
184
|
+
os.lseek(descriptor, 0, os.SEEK_SET)
|
|
185
|
+
try:
|
|
186
|
+
locking(descriptor, lock_nonblocking, 1)
|
|
187
|
+
acquired = True
|
|
188
|
+
break
|
|
189
|
+
except OSError:
|
|
190
|
+
if time.monotonic() >= deadline:
|
|
191
|
+
raise _CacheLockUnavailable(
|
|
192
|
+
"The cache root lock is busy."
|
|
193
|
+
) from None
|
|
194
|
+
time.sleep(0.005)
|
|
195
|
+
else:
|
|
196
|
+
raise _CacheLockUnavailable("No cache-root lock is available.")
|
|
197
|
+
yield
|
|
198
|
+
finally:
|
|
199
|
+
if acquired:
|
|
200
|
+
if os.name == "posix":
|
|
201
|
+
import fcntl
|
|
202
|
+
|
|
203
|
+
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
204
|
+
elif os.name == "nt":
|
|
205
|
+
import msvcrt
|
|
206
|
+
|
|
207
|
+
os.lseek(descriptor, 0, os.SEEK_SET)
|
|
208
|
+
msvcrt.locking( # type: ignore[attr-defined]
|
|
209
|
+
descriptor,
|
|
210
|
+
msvcrt.LK_UNLCK, # type: ignore[attr-defined]
|
|
211
|
+
1,
|
|
212
|
+
)
|
|
213
|
+
os.close(descriptor)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _valid_text(value: str, *, maximum: int) -> str:
|
|
217
|
+
encoded = value.encode("utf-8")
|
|
218
|
+
if len(encoded) > maximum:
|
|
219
|
+
raise ValueError("A cache text value is oversized.")
|
|
220
|
+
return value
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _canonical_value(value: object) -> object:
|
|
224
|
+
if value is None:
|
|
225
|
+
return ["none"]
|
|
226
|
+
if isinstance(value, bool):
|
|
227
|
+
return ["bool", value]
|
|
228
|
+
if type(value) is int:
|
|
229
|
+
return ["int", str(value)]
|
|
230
|
+
if isinstance(value, float):
|
|
231
|
+
return ["float", value.hex()]
|
|
232
|
+
if isinstance(value, str):
|
|
233
|
+
return ["str", _valid_text(value, maximum=TRUSTED_DERIVED_CACHE_MAX_TEXT_BYTES)]
|
|
234
|
+
if isinstance(value, bytes):
|
|
235
|
+
return ["bytes", base64.b64encode(value).decode("ascii")]
|
|
236
|
+
if isinstance(value, tuple):
|
|
237
|
+
return ["tuple", [_canonical_value(item) for item in value]]
|
|
238
|
+
if isinstance(value, dict):
|
|
239
|
+
if any(not isinstance(name, str) for name in value):
|
|
240
|
+
raise TypeError("Derived payload dictionary keys must be text.")
|
|
241
|
+
return [
|
|
242
|
+
"dict",
|
|
243
|
+
[
|
|
244
|
+
[name, _canonical_value(value[name])]
|
|
245
|
+
for name in sorted(cast(dict[str, object], value))
|
|
246
|
+
],
|
|
247
|
+
]
|
|
248
|
+
raise TypeError(f"Unsupported canonical cache value {type(value).__name__}.")
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _restore_value(
|
|
252
|
+
value: object,
|
|
253
|
+
*,
|
|
254
|
+
depth: int = 0,
|
|
255
|
+
budget: list[int] | None = None,
|
|
256
|
+
) -> object:
|
|
257
|
+
if budget is None:
|
|
258
|
+
budget = [0]
|
|
259
|
+
budget[0] += 1
|
|
260
|
+
if budget[0] > TRUSTED_DERIVED_CACHE_MAX_JSON_NODES:
|
|
261
|
+
raise ValueError("A cache value has too many nodes.")
|
|
262
|
+
if depth > 16 or not isinstance(value, list) or not value:
|
|
263
|
+
raise ValueError("A cache value has an invalid canonical structure.")
|
|
264
|
+
tag = value[0]
|
|
265
|
+
if tag == "none" and len(value) == 1:
|
|
266
|
+
return None
|
|
267
|
+
if tag == "bool" and len(value) == 2 and isinstance(value[1], bool):
|
|
268
|
+
return value[1]
|
|
269
|
+
if tag == "int" and len(value) == 2 and isinstance(value[1], str):
|
|
270
|
+
_valid_text(value[1], maximum=128)
|
|
271
|
+
return int(value[1])
|
|
272
|
+
if tag == "float" and len(value) == 2 and isinstance(value[1], str):
|
|
273
|
+
_valid_text(value[1], maximum=128)
|
|
274
|
+
restored = float.fromhex(value[1])
|
|
275
|
+
if not (restored == restored and abs(restored) != float("inf")):
|
|
276
|
+
raise ValueError("A cache float is not finite.")
|
|
277
|
+
return restored
|
|
278
|
+
if tag == "str" and len(value) == 2 and isinstance(value[1], str):
|
|
279
|
+
return _valid_text(value[1], maximum=TRUSTED_DERIVED_CACHE_MAX_TEXT_BYTES)
|
|
280
|
+
if tag == "bytes" and len(value) == 2 and isinstance(value[1], str):
|
|
281
|
+
maximum_encoded = 4 * ((TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES + 2) // 3)
|
|
282
|
+
if len(value[1]) > maximum_encoded:
|
|
283
|
+
raise ValueError("A cache byte scalar is oversized.")
|
|
284
|
+
restored_bytes = base64.b64decode(value[1], validate=True)
|
|
285
|
+
if len(restored_bytes) > TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES:
|
|
286
|
+
raise ValueError("A cache byte scalar is oversized.")
|
|
287
|
+
return restored_bytes
|
|
288
|
+
if tag == "tuple" and len(value) == 2 and isinstance(value[1], list):
|
|
289
|
+
if len(value[1]) > TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS:
|
|
290
|
+
raise ValueError("A cache tuple has too many items.")
|
|
291
|
+
return tuple(
|
|
292
|
+
_restore_value(item, depth=depth + 1, budget=budget) for item in value[1]
|
|
293
|
+
)
|
|
294
|
+
if tag == "dict" and len(value) == 2 and isinstance(value[1], list):
|
|
295
|
+
if len(value[1]) > TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS:
|
|
296
|
+
raise ValueError("A cache dictionary has too many items.")
|
|
297
|
+
restored_dict: dict[str, object] = {}
|
|
298
|
+
for item in value[1]:
|
|
299
|
+
if (
|
|
300
|
+
not isinstance(item, list)
|
|
301
|
+
or len(item) != 2
|
|
302
|
+
or not isinstance(item[0], str)
|
|
303
|
+
or item[0] in restored_dict
|
|
304
|
+
):
|
|
305
|
+
raise ValueError("A cache dictionary item is invalid.")
|
|
306
|
+
_valid_text(item[0], maximum=256)
|
|
307
|
+
restored_dict[item[0]] = _restore_value(
|
|
308
|
+
item[1], depth=depth + 1, budget=budget
|
|
309
|
+
)
|
|
310
|
+
return restored_dict
|
|
311
|
+
raise ValueError("A cache value has an unknown canonical tag.")
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def canonical_trusted_cache_key(key: TrustedCacheWorkKey) -> bytes:
|
|
315
|
+
"""Return a type-preserving deterministic cache-key encoding."""
|
|
316
|
+
|
|
317
|
+
if not isinstance(key, TrustedCacheWorkKey):
|
|
318
|
+
raise TypeError("key must be TrustedCacheWorkKey.")
|
|
319
|
+
value = (
|
|
320
|
+
"qplot-trusted-cache-key-v1",
|
|
321
|
+
(
|
|
322
|
+
key.database_instance.logical_path,
|
|
323
|
+
key.database_instance.resolved_path,
|
|
324
|
+
cast(tuple[object, ...], key.database_instance.identity),
|
|
325
|
+
),
|
|
326
|
+
key.run_guid,
|
|
327
|
+
int(key.kind),
|
|
328
|
+
key.source_revision.fingerprint,
|
|
329
|
+
key.renderer_version,
|
|
330
|
+
key.rendering_options.canonical_values,
|
|
331
|
+
)
|
|
332
|
+
return json.dumps(
|
|
333
|
+
_canonical_value(value),
|
|
334
|
+
ensure_ascii=False,
|
|
335
|
+
separators=(",", ":"),
|
|
336
|
+
).encode("utf-8")
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def trusted_cache_filename(key: TrustedCacheWorkKey) -> str:
|
|
340
|
+
return f"{hashlib.sha256(canonical_trusted_cache_key(key)).hexdigest()}.qdc"
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
class _JsonPreflightParser:
|
|
344
|
+
"""Count JSON structure without materialising its container graph."""
|
|
345
|
+
|
|
346
|
+
def __init__(self, text: str) -> None:
|
|
347
|
+
self._text = text
|
|
348
|
+
self._length = len(text)
|
|
349
|
+
self._position = 0
|
|
350
|
+
self._nodes = 0
|
|
351
|
+
|
|
352
|
+
def parse(self) -> None:
|
|
353
|
+
self._skip_whitespace()
|
|
354
|
+
self._parse_value(depth=0)
|
|
355
|
+
self._skip_whitespace()
|
|
356
|
+
if self._position != self._length:
|
|
357
|
+
raise ValueError("A cache JSON document has trailing content.")
|
|
358
|
+
|
|
359
|
+
def _skip_whitespace(self) -> None:
|
|
360
|
+
while self._position < self._length and self._text[self._position] in " \t\r\n":
|
|
361
|
+
self._position += 1
|
|
362
|
+
|
|
363
|
+
def _count_node(self) -> None:
|
|
364
|
+
self._nodes += 1
|
|
365
|
+
if self._nodes > TRUSTED_DERIVED_CACHE_MAX_JSON_NODES:
|
|
366
|
+
raise ValueError("A cache JSON document has too many nodes.")
|
|
367
|
+
|
|
368
|
+
def _parse_value(self, *, depth: int, binary: bool = False) -> str | None:
|
|
369
|
+
self._skip_whitespace()
|
|
370
|
+
if self._position >= self._length:
|
|
371
|
+
raise ValueError("A cache JSON document is incomplete.")
|
|
372
|
+
self._count_node()
|
|
373
|
+
character = self._text[self._position]
|
|
374
|
+
if character == '"':
|
|
375
|
+
return self._parse_string(binary=binary)
|
|
376
|
+
if character == "[":
|
|
377
|
+
self._parse_array(depth=depth + 1)
|
|
378
|
+
return None
|
|
379
|
+
if character == "{":
|
|
380
|
+
self._parse_object(depth=depth + 1)
|
|
381
|
+
return None
|
|
382
|
+
start = self._position
|
|
383
|
+
while (
|
|
384
|
+
self._position < self._length
|
|
385
|
+
and self._text[self._position] not in " \t\r\n,]}"
|
|
386
|
+
):
|
|
387
|
+
self._position += 1
|
|
388
|
+
if self._position == start:
|
|
389
|
+
raise ValueError("A cache JSON scalar is invalid.")
|
|
390
|
+
return None
|
|
391
|
+
|
|
392
|
+
def _parse_array(self, *, depth: int) -> None:
|
|
393
|
+
if depth > TRUSTED_DERIVED_CACHE_MAX_JSON_DEPTH:
|
|
394
|
+
raise ValueError("A cache JSON document is too deeply nested.")
|
|
395
|
+
self._position += 1
|
|
396
|
+
self._skip_whitespace()
|
|
397
|
+
count = 0
|
|
398
|
+
first_value: str | None = None
|
|
399
|
+
while self._position >= self._length or self._text[self._position] != "]":
|
|
400
|
+
count += 1
|
|
401
|
+
if count > TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS:
|
|
402
|
+
raise ValueError("A cache JSON container has too many items.")
|
|
403
|
+
value = self._parse_value(
|
|
404
|
+
depth=depth,
|
|
405
|
+
binary=count == 2 and first_value == "bytes",
|
|
406
|
+
)
|
|
407
|
+
if count == 1:
|
|
408
|
+
first_value = value
|
|
409
|
+
self._skip_whitespace()
|
|
410
|
+
if self._position >= self._length:
|
|
411
|
+
raise ValueError("A cache JSON document is incomplete.")
|
|
412
|
+
if self._text[self._position] == "]":
|
|
413
|
+
break
|
|
414
|
+
if self._text[self._position] != ",":
|
|
415
|
+
raise ValueError("A cache JSON array is invalid.")
|
|
416
|
+
self._position += 1
|
|
417
|
+
self._skip_whitespace()
|
|
418
|
+
self._position += 1
|
|
419
|
+
|
|
420
|
+
def _parse_object(self, *, depth: int) -> None:
|
|
421
|
+
if depth > TRUSTED_DERIVED_CACHE_MAX_JSON_DEPTH:
|
|
422
|
+
raise ValueError("A cache JSON document is too deeply nested.")
|
|
423
|
+
self._position += 1
|
|
424
|
+
self._skip_whitespace()
|
|
425
|
+
count = 0
|
|
426
|
+
while self._position >= self._length or self._text[self._position] != "}":
|
|
427
|
+
count += 1
|
|
428
|
+
if count > TRUSTED_DERIVED_CACHE_MAX_CONTAINER_ITEMS:
|
|
429
|
+
raise ValueError("A cache JSON container has too many items.")
|
|
430
|
+
if self._position >= self._length or self._text[self._position] != '"':
|
|
431
|
+
raise ValueError("A cache JSON object key is invalid.")
|
|
432
|
+
self._parse_string(binary=False)
|
|
433
|
+
self._skip_whitespace()
|
|
434
|
+
if self._position >= self._length or self._text[self._position] != ":":
|
|
435
|
+
raise ValueError("A cache JSON object is invalid.")
|
|
436
|
+
self._position += 1
|
|
437
|
+
self._parse_value(depth=depth)
|
|
438
|
+
self._skip_whitespace()
|
|
439
|
+
if self._position >= self._length:
|
|
440
|
+
raise ValueError("A cache JSON document is incomplete.")
|
|
441
|
+
if self._text[self._position] == "}":
|
|
442
|
+
break
|
|
443
|
+
if self._text[self._position] != ",":
|
|
444
|
+
raise ValueError("A cache JSON object is invalid.")
|
|
445
|
+
self._position += 1
|
|
446
|
+
self._skip_whitespace()
|
|
447
|
+
self._position += 1
|
|
448
|
+
|
|
449
|
+
def _parse_string(self, *, binary: bool) -> str | None:
|
|
450
|
+
start = self._position + 1
|
|
451
|
+
self._position = start
|
|
452
|
+
escaped = False
|
|
453
|
+
byte_count = 0
|
|
454
|
+
while self._position < self._length:
|
|
455
|
+
character = self._text[self._position]
|
|
456
|
+
if not escaped and character == '"':
|
|
457
|
+
break
|
|
458
|
+
if binary:
|
|
459
|
+
if escaped or character == "\\" or not character.isascii():
|
|
460
|
+
raise ValueError("A cache byte scalar is invalid.")
|
|
461
|
+
if not (character.isalnum() or character in "+/="):
|
|
462
|
+
raise ValueError("A cache byte scalar is invalid.")
|
|
463
|
+
byte_count += 1
|
|
464
|
+
else:
|
|
465
|
+
byte_count += len(character.encode("utf-8"))
|
|
466
|
+
if byte_count > TRUSTED_DERIVED_CACHE_MAX_TEXT_BYTES:
|
|
467
|
+
raise ValueError("A cache JSON string is oversized.")
|
|
468
|
+
if escaped:
|
|
469
|
+
escaped = False
|
|
470
|
+
elif character == "\\":
|
|
471
|
+
escaped = True
|
|
472
|
+
self._position += 1
|
|
473
|
+
if self._position >= self._length or escaped:
|
|
474
|
+
raise ValueError("A cache JSON document is incomplete.")
|
|
475
|
+
end = self._position
|
|
476
|
+
self._position += 1
|
|
477
|
+
if binary:
|
|
478
|
+
maximum_encoded = 4 * ((TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES + 2) // 3)
|
|
479
|
+
if byte_count > maximum_encoded or byte_count % 4:
|
|
480
|
+
raise ValueError("A cache byte scalar is oversized.")
|
|
481
|
+
padding = int(byte_count > 0 and self._text[end - 1] == "=")
|
|
482
|
+
padding += int(byte_count > 1 and self._text[end - 2] == "=")
|
|
483
|
+
if self._text.find("=", start, end - padding) != -1:
|
|
484
|
+
raise ValueError("A cache byte scalar is invalid.")
|
|
485
|
+
decoded_bytes = byte_count // 4 * 3 - padding
|
|
486
|
+
if decoded_bytes > TRUSTED_DERIVED_MAX_ENCODED_IMAGE_BYTES:
|
|
487
|
+
raise ValueError("A cache byte scalar is oversized.")
|
|
488
|
+
return None
|
|
489
|
+
if end - start == 5 and self._text.startswith("bytes", start):
|
|
490
|
+
return "bytes"
|
|
491
|
+
return None
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _preflight_json(data: bytes) -> None:
|
|
495
|
+
"""Bound JSON structure and canonical binary strings before ``json.loads``."""
|
|
496
|
+
|
|
497
|
+
if len(data) > TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES:
|
|
498
|
+
raise ValueError("A cache JSON document is oversized.")
|
|
499
|
+
_JsonPreflightParser(data.decode("utf-8")).parse()
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
class TrustedDerivedDiskCache:
|
|
503
|
+
"""Best-effort qPlot-owned cache; every failure degrades to an uncached result."""
|
|
504
|
+
|
|
505
|
+
def __init__(
|
|
506
|
+
self,
|
|
507
|
+
root: str | os.PathLike[str] | None = None,
|
|
508
|
+
*,
|
|
509
|
+
enabled: bool = True,
|
|
510
|
+
max_entry_bytes: int = TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES,
|
|
511
|
+
max_total_bytes: int = TRUSTED_DERIVED_CACHE_MAX_TOTAL_BYTES,
|
|
512
|
+
max_entries: int = TRUSTED_DERIVED_CACHE_MAX_ENTRIES,
|
|
513
|
+
) -> None:
|
|
514
|
+
selected_root = Path(
|
|
515
|
+
trusted_derived_cache_root() if root is None else os.fspath(root)
|
|
516
|
+
)
|
|
517
|
+
if not selected_root.is_absolute():
|
|
518
|
+
raise ValueError("The trusted derived cache root must be absolute.")
|
|
519
|
+
if (
|
|
520
|
+
type(max_entry_bytes) is not int
|
|
521
|
+
or type(max_total_bytes) is not int
|
|
522
|
+
or type(max_entries) is not int
|
|
523
|
+
or not 1 <= max_entry_bytes <= TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES
|
|
524
|
+
or not max_entry_bytes
|
|
525
|
+
<= max_total_bytes
|
|
526
|
+
<= TRUSTED_DERIVED_CACHE_MAX_TOTAL_BYTES
|
|
527
|
+
or not 1 <= max_entries <= TRUSTED_DERIVED_CACHE_MAX_ENTRIES
|
|
528
|
+
):
|
|
529
|
+
raise ValueError("Trusted derived cache bounds are invalid.")
|
|
530
|
+
self.root = selected_root
|
|
531
|
+
self._requested_enabled = bool(enabled)
|
|
532
|
+
self.enabled = bool(enabled)
|
|
533
|
+
self.max_entry_bytes = max_entry_bytes
|
|
534
|
+
self.max_total_bytes = max_total_bytes
|
|
535
|
+
self.max_entries = max_entries
|
|
536
|
+
self._state_lock = threading.RLock()
|
|
537
|
+
self._state_changed = threading.Condition(self._state_lock)
|
|
538
|
+
self._epoch = 0
|
|
539
|
+
self._active_puts = 0
|
|
540
|
+
self._protected_database_family: frozenset[Path] = frozenset()
|
|
541
|
+
|
|
542
|
+
def configure_for_database(self, database_instance: DatabaseInstance) -> None:
|
|
543
|
+
"""Disable disk writes unless cache and database directories are disjoint."""
|
|
544
|
+
|
|
545
|
+
if not isinstance(database_instance, DatabaseInstance):
|
|
546
|
+
raise TypeError("database_instance must be a DatabaseInstance.")
|
|
547
|
+
database_path = Path(database_instance.resolved_path)
|
|
548
|
+
protected_family = frozenset(
|
|
549
|
+
Path(f"{database_path}{suffix}")
|
|
550
|
+
for suffix in ("", "-wal", "-journal", "-shm")
|
|
551
|
+
)
|
|
552
|
+
try:
|
|
553
|
+
root = self.root.resolve(strict=False)
|
|
554
|
+
database_directory = database_path.parent.resolve(strict=False)
|
|
555
|
+
unsafe = (
|
|
556
|
+
root == database_directory
|
|
557
|
+
or root.is_relative_to(database_directory)
|
|
558
|
+
or database_directory.is_relative_to(root)
|
|
559
|
+
)
|
|
560
|
+
except (OSError, RuntimeError, ValueError):
|
|
561
|
+
unsafe = True
|
|
562
|
+
with self._state_changed:
|
|
563
|
+
self._epoch += 1
|
|
564
|
+
self._protected_database_family = protected_family
|
|
565
|
+
self.enabled = self._requested_enabled and not unsafe
|
|
566
|
+
self._state_changed.notify_all()
|
|
567
|
+
|
|
568
|
+
def get(
|
|
569
|
+
self,
|
|
570
|
+
key: TrustedCacheWorkKey,
|
|
571
|
+
*,
|
|
572
|
+
cancel_check: Callable[[], None] = lambda: None,
|
|
573
|
+
) -> DerivedPayload | None:
|
|
574
|
+
"""Read and verify one exact entry without creating or touching files."""
|
|
575
|
+
|
|
576
|
+
if not self.enabled:
|
|
577
|
+
return None
|
|
578
|
+
try:
|
|
579
|
+
cancel_check()
|
|
580
|
+
canonical_key = canonical_trusted_cache_key(key)
|
|
581
|
+
path = self.root / trusted_cache_filename(key)
|
|
582
|
+
status = path.stat()
|
|
583
|
+
if not path.is_file() or status.st_size > self.max_entry_bytes:
|
|
584
|
+
return None
|
|
585
|
+
data = path.read_bytes()
|
|
586
|
+
except (InterruptedError, TimeoutError):
|
|
587
|
+
raise
|
|
588
|
+
except (
|
|
589
|
+
OSError,
|
|
590
|
+
UnicodeError,
|
|
591
|
+
ValueError,
|
|
592
|
+
TypeError,
|
|
593
|
+
OverflowError,
|
|
594
|
+
RecursionError,
|
|
595
|
+
):
|
|
596
|
+
return None
|
|
597
|
+
try:
|
|
598
|
+
if len(data) < _PREFIX.size:
|
|
599
|
+
return None
|
|
600
|
+
magic, header_length, payload_length = _PREFIX.unpack_from(data)
|
|
601
|
+
if (
|
|
602
|
+
magic != _MAGIC
|
|
603
|
+
or header_length > TRUSTED_DERIVED_CACHE_MAX_HEADER_BYTES
|
|
604
|
+
or payload_length > self.max_entry_bytes
|
|
605
|
+
or _PREFIX.size + header_length + payload_length != len(data)
|
|
606
|
+
):
|
|
607
|
+
return None
|
|
608
|
+
cancel_check()
|
|
609
|
+
header_start = _PREFIX.size
|
|
610
|
+
payload_start = header_start + header_length
|
|
611
|
+
_preflight_json(data[header_start:payload_start])
|
|
612
|
+
header = json.loads(data[header_start:payload_start].decode("utf-8"))
|
|
613
|
+
if not isinstance(header, dict):
|
|
614
|
+
return None
|
|
615
|
+
if header.get("format_version") != TRUSTED_DERIVED_CACHE_FORMAT_VERSION:
|
|
616
|
+
return None
|
|
617
|
+
expected_key = base64.b64encode(canonical_key).decode("ascii")
|
|
618
|
+
if (
|
|
619
|
+
header.get("key") != expected_key
|
|
620
|
+
or header.get("payload_length") != payload_length
|
|
621
|
+
):
|
|
622
|
+
return None
|
|
623
|
+
payload_bytes = data[payload_start:]
|
|
624
|
+
if (
|
|
625
|
+
header.get("payload_sha256")
|
|
626
|
+
!= hashlib.sha256(payload_bytes).hexdigest()
|
|
627
|
+
):
|
|
628
|
+
return None
|
|
629
|
+
_preflight_json(payload_bytes)
|
|
630
|
+
decoded = json.loads(payload_bytes.decode("utf-8"))
|
|
631
|
+
restored = _restore_value(decoded)
|
|
632
|
+
if not isinstance(restored, dict):
|
|
633
|
+
return None
|
|
634
|
+
payload = cast(DerivedPayload, restored)
|
|
635
|
+
validate_trusted_derived_payload(payload)
|
|
636
|
+
cancel_check()
|
|
637
|
+
return payload
|
|
638
|
+
except (InterruptedError, TimeoutError):
|
|
639
|
+
raise
|
|
640
|
+
except (
|
|
641
|
+
binascii.Error,
|
|
642
|
+
UnicodeError,
|
|
643
|
+
ValueError,
|
|
644
|
+
TypeError,
|
|
645
|
+
OverflowError,
|
|
646
|
+
RecursionError,
|
|
647
|
+
json.JSONDecodeError,
|
|
648
|
+
OSError,
|
|
649
|
+
TrustedDerivedRenderingError,
|
|
650
|
+
):
|
|
651
|
+
return None
|
|
652
|
+
|
|
653
|
+
def put(
|
|
654
|
+
self,
|
|
655
|
+
key: TrustedCacheWorkKey,
|
|
656
|
+
payload: DerivedPayload,
|
|
657
|
+
*,
|
|
658
|
+
cancel_check: Callable[[], None] = lambda: None,
|
|
659
|
+
) -> bool:
|
|
660
|
+
"""Atomically store one entry; never turn a cache fault into DB failure."""
|
|
661
|
+
|
|
662
|
+
with self._state_changed:
|
|
663
|
+
if not self.enabled:
|
|
664
|
+
return False
|
|
665
|
+
epoch = self._epoch
|
|
666
|
+
self._active_puts += 1
|
|
667
|
+
temporary: Path | None = None
|
|
668
|
+
temporary_proof: _ArtifactProof | None = None
|
|
669
|
+
destination: Path | None = None
|
|
670
|
+
published: Path | None = None
|
|
671
|
+
published_proof: _ArtifactProof | None = None
|
|
672
|
+
successful = False
|
|
673
|
+
index_started = False
|
|
674
|
+
disable_after_cleanup = False
|
|
675
|
+
|
|
676
|
+
def cache_cancel_check() -> None:
|
|
677
|
+
cancel_check()
|
|
678
|
+
with self._state_lock:
|
|
679
|
+
if not self.enabled or epoch != self._epoch:
|
|
680
|
+
raise _CacheEpochChanged("The cache database selection changed.")
|
|
681
|
+
|
|
682
|
+
try:
|
|
683
|
+
cache_cancel_check()
|
|
684
|
+
validate_trusted_derived_payload(payload)
|
|
685
|
+
canonical_key = canonical_trusted_cache_key(key)
|
|
686
|
+
payload_bytes = json.dumps(
|
|
687
|
+
_canonical_value(payload),
|
|
688
|
+
ensure_ascii=False,
|
|
689
|
+
separators=(",", ":"),
|
|
690
|
+
).encode("utf-8")
|
|
691
|
+
header = json.dumps(
|
|
692
|
+
{
|
|
693
|
+
"format_version": TRUSTED_DERIVED_CACHE_FORMAT_VERSION,
|
|
694
|
+
"key": base64.b64encode(canonical_key).decode("ascii"),
|
|
695
|
+
"payload_length": len(payload_bytes),
|
|
696
|
+
"payload_sha256": hashlib.sha256(payload_bytes).hexdigest(),
|
|
697
|
+
},
|
|
698
|
+
sort_keys=True,
|
|
699
|
+
separators=(",", ":"),
|
|
700
|
+
).encode("utf-8")
|
|
701
|
+
total = _PREFIX.size + len(header) + len(payload_bytes)
|
|
702
|
+
if (
|
|
703
|
+
len(header) > TRUSTED_DERIVED_CACHE_MAX_HEADER_BYTES
|
|
704
|
+
or total > self.max_entry_bytes
|
|
705
|
+
):
|
|
706
|
+
return False
|
|
707
|
+
self._ensure_root()
|
|
708
|
+
filename = trusted_cache_filename(key)
|
|
709
|
+
destination = self.root / filename
|
|
710
|
+
temporary = self.root / f"{filename[:-4]}.{secrets.token_hex(16)}.tmp"
|
|
711
|
+
cache_cancel_check()
|
|
712
|
+
descriptor = os.open(
|
|
713
|
+
temporary,
|
|
714
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
|
715
|
+
0o600,
|
|
716
|
+
)
|
|
717
|
+
created_status = os.fstat(descriptor)
|
|
718
|
+
temporary_proof = _ArtifactProof(
|
|
719
|
+
temporary,
|
|
720
|
+
created_status.st_dev,
|
|
721
|
+
created_status.st_ino,
|
|
722
|
+
True,
|
|
723
|
+
canonical_key,
|
|
724
|
+
)
|
|
725
|
+
with os.fdopen(descriptor, "wb") as stream:
|
|
726
|
+
stream.write(_PREFIX.pack(_MAGIC, len(header), len(payload_bytes)))
|
|
727
|
+
stream.write(header)
|
|
728
|
+
stream.write(payload_bytes)
|
|
729
|
+
stream.flush()
|
|
730
|
+
os.fsync(stream.fileno())
|
|
731
|
+
cache_cancel_check()
|
|
732
|
+
self._before_publish()
|
|
733
|
+
cache_cancel_check()
|
|
734
|
+
with _cross_process_cache_lock(
|
|
735
|
+
self.root / _LOCK_NAME,
|
|
736
|
+
cancel_check=cache_cancel_check,
|
|
737
|
+
):
|
|
738
|
+
index_started = True
|
|
739
|
+
connection = self._open_index()
|
|
740
|
+
try:
|
|
741
|
+
if not self._ensure_global_inventory(
|
|
742
|
+
connection,
|
|
743
|
+
cancel_check=cache_cancel_check,
|
|
744
|
+
expected_epoch=epoch,
|
|
745
|
+
):
|
|
746
|
+
return False
|
|
747
|
+
cache_cancel_check()
|
|
748
|
+
try:
|
|
749
|
+
existing_proof = self._prove_framed_artifact(
|
|
750
|
+
destination,
|
|
751
|
+
temporary=False,
|
|
752
|
+
)
|
|
753
|
+
except FileNotFoundError:
|
|
754
|
+
existing_proof = None
|
|
755
|
+
except (OSError, ValueError):
|
|
756
|
+
disable_after_cleanup = True
|
|
757
|
+
return False
|
|
758
|
+
if existing_proof is not None:
|
|
759
|
+
if existing_proof.canonical_key != canonical_key:
|
|
760
|
+
disable_after_cleanup = True
|
|
761
|
+
return False
|
|
762
|
+
self._reuse_existing_entry(
|
|
763
|
+
connection,
|
|
764
|
+
filename,
|
|
765
|
+
existing_proof,
|
|
766
|
+
expected_epoch=epoch,
|
|
767
|
+
)
|
|
768
|
+
cache_cancel_check()
|
|
769
|
+
successful = True
|
|
770
|
+
return True
|
|
771
|
+
if not self._reserve_and_evict(
|
|
772
|
+
connection,
|
|
773
|
+
filename,
|
|
774
|
+
total,
|
|
775
|
+
cancel_check=cache_cancel_check,
|
|
776
|
+
expected_epoch=epoch,
|
|
777
|
+
):
|
|
778
|
+
disable_after_cleanup = True
|
|
779
|
+
return False
|
|
780
|
+
try:
|
|
781
|
+
published_proof = self._publish_no_clobber(
|
|
782
|
+
temporary_proof,
|
|
783
|
+
destination,
|
|
784
|
+
expected_epoch=epoch,
|
|
785
|
+
)
|
|
786
|
+
except FileExistsError:
|
|
787
|
+
try:
|
|
788
|
+
existing_proof = self._prove_framed_artifact(
|
|
789
|
+
destination,
|
|
790
|
+
temporary=False,
|
|
791
|
+
)
|
|
792
|
+
except (OSError, ValueError):
|
|
793
|
+
disable_after_cleanup = True
|
|
794
|
+
return False
|
|
795
|
+
if existing_proof.canonical_key != canonical_key:
|
|
796
|
+
disable_after_cleanup = True
|
|
797
|
+
return False
|
|
798
|
+
self._reuse_existing_entry(
|
|
799
|
+
connection,
|
|
800
|
+
filename,
|
|
801
|
+
existing_proof,
|
|
802
|
+
expected_epoch=epoch,
|
|
803
|
+
)
|
|
804
|
+
cache_cancel_check()
|
|
805
|
+
successful = True
|
|
806
|
+
return True
|
|
807
|
+
else:
|
|
808
|
+
published = destination
|
|
809
|
+
status = destination.stat()
|
|
810
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
811
|
+
connection.execute(
|
|
812
|
+
"UPDATE entries SET modified = ?, size = ?, ready = 1 "
|
|
813
|
+
"WHERE name = ?",
|
|
814
|
+
(status.st_mtime_ns, status.st_size, filename),
|
|
815
|
+
)
|
|
816
|
+
connection.commit()
|
|
817
|
+
cache_cancel_check()
|
|
818
|
+
except BaseException:
|
|
819
|
+
_rollback_index_best_effort(connection)
|
|
820
|
+
raise
|
|
821
|
+
finally:
|
|
822
|
+
_close_index_best_effort(connection)
|
|
823
|
+
successful = True
|
|
824
|
+
return True
|
|
825
|
+
except (InterruptedError, TimeoutError):
|
|
826
|
+
raise
|
|
827
|
+
except _CacheEpochChanged:
|
|
828
|
+
return False
|
|
829
|
+
except (sqlite3.Error, _CacheOwnershipError):
|
|
830
|
+
disable_after_cleanup = True
|
|
831
|
+
return False
|
|
832
|
+
except (
|
|
833
|
+
OSError,
|
|
834
|
+
ValueError,
|
|
835
|
+
TypeError,
|
|
836
|
+
OverflowError,
|
|
837
|
+
RecursionError,
|
|
838
|
+
TrustedDerivedRenderingError,
|
|
839
|
+
):
|
|
840
|
+
if index_started:
|
|
841
|
+
disable_after_cleanup = True
|
|
842
|
+
return False
|
|
843
|
+
finally:
|
|
844
|
+
if published is not None and published_proof is not None and not successful:
|
|
845
|
+
try:
|
|
846
|
+
self._unlink_proven_artifact(
|
|
847
|
+
published_proof,
|
|
848
|
+
expected_epoch=epoch,
|
|
849
|
+
operation="failed-publication",
|
|
850
|
+
)
|
|
851
|
+
except (OSError, ValueError, _CacheEpochChanged):
|
|
852
|
+
pass
|
|
853
|
+
if temporary is not None and temporary_proof is not None:
|
|
854
|
+
try:
|
|
855
|
+
with self._state_lock:
|
|
856
|
+
operation_invalidated = not self.enabled or self._epoch != epoch
|
|
857
|
+
self._unlink_proven_artifact(
|
|
858
|
+
temporary_proof,
|
|
859
|
+
expected_epoch=epoch,
|
|
860
|
+
operation=(
|
|
861
|
+
"invalidated-operation-cleanup"
|
|
862
|
+
if operation_invalidated
|
|
863
|
+
else "temporary-cleanup"
|
|
864
|
+
),
|
|
865
|
+
)
|
|
866
|
+
except (OSError, ValueError, _CacheEpochChanged):
|
|
867
|
+
pass
|
|
868
|
+
with self._state_changed:
|
|
869
|
+
if disable_after_cleanup:
|
|
870
|
+
self.enabled = False
|
|
871
|
+
self._active_puts -= 1
|
|
872
|
+
self._state_changed.notify_all()
|
|
873
|
+
|
|
874
|
+
def _ensure_root(self) -> None:
|
|
875
|
+
self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
876
|
+
if os.name == "posix":
|
|
877
|
+
os.chmod(self.root, 0o700)
|
|
878
|
+
|
|
879
|
+
def _before_publish(self) -> None:
|
|
880
|
+
"""Test seam immediately before the epoch/publication lock."""
|
|
881
|
+
|
|
882
|
+
def _before_artifact_proof(self, path: Path, operation: str) -> None:
|
|
883
|
+
"""Test seam after candidate selection and before ownership proof."""
|
|
884
|
+
|
|
885
|
+
def _before_destructive_cache_file(self, path: Path, operation: str) -> None:
|
|
886
|
+
"""Test seam after ownership proof and before the atomic state guard."""
|
|
887
|
+
|
|
888
|
+
@staticmethod
|
|
889
|
+
def _is_protected_database_artifact(
|
|
890
|
+
path: Path,
|
|
891
|
+
protected_family: frozenset[Path],
|
|
892
|
+
) -> bool:
|
|
893
|
+
if path in protected_family:
|
|
894
|
+
return True
|
|
895
|
+
try:
|
|
896
|
+
resolved = path.resolve(strict=False)
|
|
897
|
+
return any(
|
|
898
|
+
resolved == protected.resolve(strict=False)
|
|
899
|
+
for protected in protected_family
|
|
900
|
+
)
|
|
901
|
+
except (OSError, RuntimeError, ValueError):
|
|
902
|
+
return True
|
|
903
|
+
|
|
904
|
+
def _validated_artifact_path(
|
|
905
|
+
self,
|
|
906
|
+
stored_name: object,
|
|
907
|
+
*,
|
|
908
|
+
temporary: bool,
|
|
909
|
+
index_derived: bool,
|
|
910
|
+
protected_family: frozenset[Path] | None = None,
|
|
911
|
+
) -> Path:
|
|
912
|
+
"""Return one syntactically safe direct-child candidate."""
|
|
913
|
+
|
|
914
|
+
def reject() -> None:
|
|
915
|
+
error = "The cache inventory contains an unsafe deletion target."
|
|
916
|
+
if index_derived:
|
|
917
|
+
raise _CacheIndexCorrupt(error)
|
|
918
|
+
raise ValueError(error)
|
|
919
|
+
|
|
920
|
+
if type(stored_name) is not str:
|
|
921
|
+
reject()
|
|
922
|
+
name = cast(str, stored_name)
|
|
923
|
+
drive, _tail = ntpath.splitdrive(name)
|
|
924
|
+
if (
|
|
925
|
+
not name
|
|
926
|
+
or name in {".", ".."}
|
|
927
|
+
or drive
|
|
928
|
+
or name.startswith(("/", "\\"))
|
|
929
|
+
or "/" in name
|
|
930
|
+
or "\\" in name
|
|
931
|
+
or Path(name).is_absolute()
|
|
932
|
+
or Path(name).name != name
|
|
933
|
+
):
|
|
934
|
+
reject()
|
|
935
|
+
pattern = _TEMP_PATTERN if temporary else _ENTRY_PATTERN
|
|
936
|
+
if pattern.fullmatch(name) is None:
|
|
937
|
+
reject()
|
|
938
|
+
candidate = self.root / name
|
|
939
|
+
if candidate.parent != self.root:
|
|
940
|
+
reject()
|
|
941
|
+
try:
|
|
942
|
+
root = self.root.resolve(strict=False)
|
|
943
|
+
resolved = candidate.resolve(strict=False)
|
|
944
|
+
except (OSError, RuntimeError, ValueError):
|
|
945
|
+
reject()
|
|
946
|
+
if protected_family is None:
|
|
947
|
+
with self._state_lock:
|
|
948
|
+
protected_family = self._protected_database_family
|
|
949
|
+
if resolved.parent != root or self._is_protected_database_artifact(
|
|
950
|
+
candidate,
|
|
951
|
+
protected_family,
|
|
952
|
+
):
|
|
953
|
+
reject()
|
|
954
|
+
return candidate
|
|
955
|
+
|
|
956
|
+
def _prove_framed_artifact(
|
|
957
|
+
self,
|
|
958
|
+
path: Path,
|
|
959
|
+
*,
|
|
960
|
+
temporary: bool,
|
|
961
|
+
) -> _ArtifactProof:
|
|
962
|
+
"""Prove a persistent artifact from bounded qPlot framing and integrity."""
|
|
963
|
+
|
|
964
|
+
candidate = self._validated_artifact_path(
|
|
965
|
+
path.name,
|
|
966
|
+
temporary=temporary,
|
|
967
|
+
index_derived=False,
|
|
968
|
+
)
|
|
969
|
+
if candidate != path:
|
|
970
|
+
raise _CacheOwnershipError("The cache artifact path is not canonical.")
|
|
971
|
+
status = path.stat(follow_symlinks=False)
|
|
972
|
+
if (
|
|
973
|
+
not stat.S_ISREG(status.st_mode)
|
|
974
|
+
or status.st_size < _PREFIX.size
|
|
975
|
+
or status.st_size > self.max_entry_bytes
|
|
976
|
+
):
|
|
977
|
+
raise _CacheOwnershipError("The cache artifact is not a bounded file.")
|
|
978
|
+
with path.open("rb") as stream:
|
|
979
|
+
data = stream.read(self.max_entry_bytes + 1)
|
|
980
|
+
final_status = path.stat(follow_symlinks=False)
|
|
981
|
+
if (final_status.st_dev, final_status.st_ino) != (
|
|
982
|
+
status.st_dev,
|
|
983
|
+
status.st_ino,
|
|
984
|
+
) or len(data) != status.st_size:
|
|
985
|
+
raise _CacheOwnershipError("The cache artifact identity changed.")
|
|
986
|
+
magic, header_length, payload_length = _PREFIX.unpack_from(data)
|
|
987
|
+
if (
|
|
988
|
+
magic != _MAGIC
|
|
989
|
+
or header_length > TRUSTED_DERIVED_CACHE_MAX_HEADER_BYTES
|
|
990
|
+
or payload_length > self.max_entry_bytes
|
|
991
|
+
or _PREFIX.size + header_length + payload_length != len(data)
|
|
992
|
+
):
|
|
993
|
+
raise _CacheOwnershipError("The cache artifact framing is invalid.")
|
|
994
|
+
header_start = _PREFIX.size
|
|
995
|
+
payload_start = header_start + header_length
|
|
996
|
+
header_bytes = data[header_start:payload_start]
|
|
997
|
+
_preflight_json(header_bytes)
|
|
998
|
+
header = json.loads(header_bytes.decode("utf-8"))
|
|
999
|
+
if not isinstance(header, dict) or set(header) != {
|
|
1000
|
+
"format_version",
|
|
1001
|
+
"key",
|
|
1002
|
+
"payload_length",
|
|
1003
|
+
"payload_sha256",
|
|
1004
|
+
}:
|
|
1005
|
+
raise _CacheOwnershipError("The cache artifact header is invalid.")
|
|
1006
|
+
encoded_key = header.get("key")
|
|
1007
|
+
digest = header.get("payload_sha256")
|
|
1008
|
+
if (
|
|
1009
|
+
header.get("format_version") != TRUSTED_DERIVED_CACHE_FORMAT_VERSION
|
|
1010
|
+
or header.get("payload_length") != payload_length
|
|
1011
|
+
or not isinstance(encoded_key, str)
|
|
1012
|
+
or not isinstance(digest, str)
|
|
1013
|
+
or re.fullmatch(r"[0-9a-f]{64}", digest) is None
|
|
1014
|
+
):
|
|
1015
|
+
raise _CacheOwnershipError("The cache artifact header is invalid.")
|
|
1016
|
+
canonical_key = base64.b64decode(encoded_key, validate=True)
|
|
1017
|
+
expected_stem = hashlib.sha256(canonical_key).hexdigest()
|
|
1018
|
+
actual_stem = path.name[:64]
|
|
1019
|
+
payload_bytes = data[payload_start:]
|
|
1020
|
+
if (
|
|
1021
|
+
actual_stem != expected_stem
|
|
1022
|
+
or digest != hashlib.sha256(payload_bytes).hexdigest()
|
|
1023
|
+
):
|
|
1024
|
+
raise _CacheOwnershipError("The cache artifact integrity is invalid.")
|
|
1025
|
+
return _ArtifactProof(
|
|
1026
|
+
path,
|
|
1027
|
+
final_status.st_dev,
|
|
1028
|
+
final_status.st_ino,
|
|
1029
|
+
temporary,
|
|
1030
|
+
canonical_key,
|
|
1031
|
+
)
|
|
1032
|
+
|
|
1033
|
+
def _recheck_proof_locked(
|
|
1034
|
+
self,
|
|
1035
|
+
proof: _ArtifactProof,
|
|
1036
|
+
*,
|
|
1037
|
+
expected_epoch: int,
|
|
1038
|
+
) -> Path:
|
|
1039
|
+
if not self.enabled or self._epoch != expected_epoch:
|
|
1040
|
+
raise _CacheEpochChanged("The cache database selection changed.")
|
|
1041
|
+
path = self._validated_artifact_path(
|
|
1042
|
+
proof.path.name,
|
|
1043
|
+
temporary=proof.temporary,
|
|
1044
|
+
index_derived=False,
|
|
1045
|
+
protected_family=self._protected_database_family,
|
|
1046
|
+
)
|
|
1047
|
+
if path != proof.path:
|
|
1048
|
+
raise _CacheOwnershipError("The cache artifact path changed.")
|
|
1049
|
+
status = path.stat(follow_symlinks=False)
|
|
1050
|
+
if not stat.S_ISREG(status.st_mode) or (status.st_dev, status.st_ino) != (
|
|
1051
|
+
proof.device,
|
|
1052
|
+
proof.inode,
|
|
1053
|
+
):
|
|
1054
|
+
raise _CacheOwnershipError("The cache artifact identity changed.")
|
|
1055
|
+
return path
|
|
1056
|
+
|
|
1057
|
+
def _unlink_proven_artifact(
|
|
1058
|
+
self,
|
|
1059
|
+
proof: _ArtifactProof,
|
|
1060
|
+
*,
|
|
1061
|
+
expected_epoch: int,
|
|
1062
|
+
operation: str,
|
|
1063
|
+
) -> None:
|
|
1064
|
+
"""Perform the only cache-artifact unlink under the configuration lock."""
|
|
1065
|
+
|
|
1066
|
+
self._before_destructive_cache_file(proof.path, operation)
|
|
1067
|
+
with self._state_lock:
|
|
1068
|
+
path = self._recheck_proof_locked(proof, expected_epoch=expected_epoch)
|
|
1069
|
+
path.unlink()
|
|
1070
|
+
|
|
1071
|
+
def _publish_no_clobber(
|
|
1072
|
+
self,
|
|
1073
|
+
temporary_proof: _ArtifactProof,
|
|
1074
|
+
destination: Path,
|
|
1075
|
+
*,
|
|
1076
|
+
expected_epoch: int,
|
|
1077
|
+
) -> _ArtifactProof:
|
|
1078
|
+
"""Atomically publish by hard link, never replacing an existing path."""
|
|
1079
|
+
|
|
1080
|
+
with self._state_lock:
|
|
1081
|
+
temporary = self._recheck_proof_locked(
|
|
1082
|
+
temporary_proof,
|
|
1083
|
+
expected_epoch=expected_epoch,
|
|
1084
|
+
)
|
|
1085
|
+
destination = self._validated_artifact_path(
|
|
1086
|
+
destination.name,
|
|
1087
|
+
temporary=False,
|
|
1088
|
+
index_derived=False,
|
|
1089
|
+
protected_family=self._protected_database_family,
|
|
1090
|
+
)
|
|
1091
|
+
os.link(temporary, destination)
|
|
1092
|
+
return _ArtifactProof(
|
|
1093
|
+
destination,
|
|
1094
|
+
temporary_proof.device,
|
|
1095
|
+
temporary_proof.inode,
|
|
1096
|
+
False,
|
|
1097
|
+
temporary_proof.canonical_key,
|
|
1098
|
+
)
|
|
1099
|
+
|
|
1100
|
+
def _reuse_existing_entry(
|
|
1101
|
+
self,
|
|
1102
|
+
connection: sqlite3.Connection,
|
|
1103
|
+
filename: str,
|
|
1104
|
+
proof: _ArtifactProof,
|
|
1105
|
+
*,
|
|
1106
|
+
expected_epoch: int,
|
|
1107
|
+
) -> None:
|
|
1108
|
+
with self._state_lock:
|
|
1109
|
+
path = self._recheck_proof_locked(proof, expected_epoch=expected_epoch)
|
|
1110
|
+
status = path.stat(follow_symlinks=False)
|
|
1111
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
1112
|
+
connection.execute(
|
|
1113
|
+
"INSERT OR REPLACE INTO entries(name, modified, size, ready) "
|
|
1114
|
+
"VALUES(?, ?, ?, 1)",
|
|
1115
|
+
(filename, status.st_mtime_ns, status.st_size),
|
|
1116
|
+
)
|
|
1117
|
+
connection.commit()
|
|
1118
|
+
|
|
1119
|
+
def _open_index(self) -> sqlite3.Connection:
|
|
1120
|
+
index_path = self.root / _INDEX_NAME
|
|
1121
|
+
# Ownership of the main inventory cannot establish ownership of its
|
|
1122
|
+
# SQLite sidecars. Even an unrelated database may occupy these names.
|
|
1123
|
+
# Decline crash recovery and continue without disk caching instead.
|
|
1124
|
+
for suffix in ("-journal", "-wal", "-shm"):
|
|
1125
|
+
try:
|
|
1126
|
+
Path(f"{index_path}{suffix}").stat(follow_symlinks=False)
|
|
1127
|
+
except FileNotFoundError:
|
|
1128
|
+
continue
|
|
1129
|
+
raise _CacheOwnershipError(
|
|
1130
|
+
"The cache inventory has an unowned SQLite sidecar."
|
|
1131
|
+
)
|
|
1132
|
+
proof = self._establish_index_ownership(index_path)
|
|
1133
|
+
self._recheck_index_proof(index_path, proof)
|
|
1134
|
+
connection = sqlite3.connect(index_path, timeout=0.0, isolation_level=None)
|
|
1135
|
+
try:
|
|
1136
|
+
if os.name == "posix":
|
|
1137
|
+
os.chmod(index_path, 0o600)
|
|
1138
|
+
connection.execute("PRAGMA trusted_schema = OFF")
|
|
1139
|
+
connection.execute("PRAGMA journal_mode = DELETE")
|
|
1140
|
+
if proof.created:
|
|
1141
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
1142
|
+
connection.execute(
|
|
1143
|
+
"CREATE TABLE cache_meta ("
|
|
1144
|
+
"key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID"
|
|
1145
|
+
)
|
|
1146
|
+
connection.execute(
|
|
1147
|
+
"CREATE TABLE entries ("
|
|
1148
|
+
"name TEXT PRIMARY KEY, modified INTEGER NOT NULL, "
|
|
1149
|
+
"size INTEGER NOT NULL, ready INTEGER NOT NULL) WITHOUT ROWID"
|
|
1150
|
+
)
|
|
1151
|
+
connection.execute(
|
|
1152
|
+
"CREATE INDEX entries_oldest ON entries(modified, name)"
|
|
1153
|
+
)
|
|
1154
|
+
connection.execute(
|
|
1155
|
+
"INSERT INTO cache_meta(key, value) VALUES('schema', ?)",
|
|
1156
|
+
(_INDEX_SCHEMA_VERSION,),
|
|
1157
|
+
)
|
|
1158
|
+
connection.commit()
|
|
1159
|
+
schema = connection.execute(
|
|
1160
|
+
"SELECT value FROM cache_meta WHERE key = 'schema'"
|
|
1161
|
+
).fetchone()
|
|
1162
|
+
if schema != (_INDEX_SCHEMA_VERSION,):
|
|
1163
|
+
raise ValueError("The cache inventory schema is incompatible.")
|
|
1164
|
+
return connection
|
|
1165
|
+
except BaseException:
|
|
1166
|
+
_close_index_best_effort(connection)
|
|
1167
|
+
raise
|
|
1168
|
+
|
|
1169
|
+
def _establish_index_ownership(self, index_path: Path) -> _IndexProof:
|
|
1170
|
+
"""Claim a missing inventory or prove an existing one without writes."""
|
|
1171
|
+
|
|
1172
|
+
flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
|
|
1173
|
+
flags |= getattr(os, "O_BINARY", 0)
|
|
1174
|
+
flags |= getattr(os, "O_CLOEXEC", 0)
|
|
1175
|
+
flags |= getattr(os, "O_NOINHERIT", 0)
|
|
1176
|
+
flags |= getattr(os, "O_NOFOLLOW", 0)
|
|
1177
|
+
try:
|
|
1178
|
+
descriptor = os.open(index_path, flags, 0o600)
|
|
1179
|
+
except FileExistsError:
|
|
1180
|
+
return self._prove_existing_index_read_only(index_path)
|
|
1181
|
+
try:
|
|
1182
|
+
status = os.fstat(descriptor)
|
|
1183
|
+
if not stat.S_ISREG(status.st_mode):
|
|
1184
|
+
raise _CacheOwnershipError(
|
|
1185
|
+
"The newly claimed cache inventory is not a regular file."
|
|
1186
|
+
)
|
|
1187
|
+
return _IndexProof(
|
|
1188
|
+
status.st_dev,
|
|
1189
|
+
status.st_ino,
|
|
1190
|
+
status.st_size,
|
|
1191
|
+
status.st_mtime_ns,
|
|
1192
|
+
status.st_ctime_ns,
|
|
1193
|
+
True,
|
|
1194
|
+
)
|
|
1195
|
+
finally:
|
|
1196
|
+
os.close(descriptor)
|
|
1197
|
+
|
|
1198
|
+
@staticmethod
|
|
1199
|
+
def _recheck_index_proof(index_path: Path, proof: _IndexProof) -> None:
|
|
1200
|
+
status = index_path.stat(follow_symlinks=False)
|
|
1201
|
+
if (
|
|
1202
|
+
not stat.S_ISREG(status.st_mode)
|
|
1203
|
+
or (status.st_dev, status.st_ino) != (proof.device, proof.inode)
|
|
1204
|
+
or status.st_size != proof.size
|
|
1205
|
+
or status.st_mtime_ns != proof.modified
|
|
1206
|
+
or status.st_ctime_ns != proof.changed
|
|
1207
|
+
):
|
|
1208
|
+
raise _CacheOwnershipError(
|
|
1209
|
+
"The cache inventory changed after ownership validation."
|
|
1210
|
+
)
|
|
1211
|
+
|
|
1212
|
+
def _prove_existing_index_read_only(self, index_path: Path) -> _IndexProof:
|
|
1213
|
+
"""Recognise the exact current qPlot inventory via immutable SQLite."""
|
|
1214
|
+
|
|
1215
|
+
initial_status = index_path.stat(follow_symlinks=False)
|
|
1216
|
+
if not stat.S_ISREG(initial_status.st_mode):
|
|
1217
|
+
raise _CacheOwnershipError(
|
|
1218
|
+
"The cache inventory is not a regular qPlot file."
|
|
1219
|
+
)
|
|
1220
|
+
connection = sqlite3.connect(
|
|
1221
|
+
f"{index_path.as_uri()}?mode=ro&immutable=1",
|
|
1222
|
+
timeout=0.0,
|
|
1223
|
+
isolation_level=None,
|
|
1224
|
+
uri=True,
|
|
1225
|
+
)
|
|
1226
|
+
try:
|
|
1227
|
+
connection.execute("PRAGMA query_only = ON")
|
|
1228
|
+
connection.execute("PRAGMA trusted_schema = OFF")
|
|
1229
|
+
objects = connection.execute(
|
|
1230
|
+
"SELECT type, name, tbl_name FROM sqlite_schema "
|
|
1231
|
+
"WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name"
|
|
1232
|
+
).fetchall()
|
|
1233
|
+
if objects != [
|
|
1234
|
+
("index", "entries_oldest", "entries"),
|
|
1235
|
+
("table", "cache_meta", "cache_meta"),
|
|
1236
|
+
("table", "entries", "entries"),
|
|
1237
|
+
]:
|
|
1238
|
+
raise _CacheOwnershipError(
|
|
1239
|
+
"The cache inventory schema is not owned by qPlot."
|
|
1240
|
+
)
|
|
1241
|
+
if connection.execute("PRAGMA table_info(cache_meta)").fetchall() != [
|
|
1242
|
+
(0, "key", "TEXT", 1, None, 1),
|
|
1243
|
+
(1, "value", "TEXT", 1, None, 0),
|
|
1244
|
+
]:
|
|
1245
|
+
raise _CacheOwnershipError(
|
|
1246
|
+
"The cache inventory metadata table is incompatible."
|
|
1247
|
+
)
|
|
1248
|
+
if connection.execute("PRAGMA table_info(entries)").fetchall() != [
|
|
1249
|
+
(0, "name", "TEXT", 1, None, 1),
|
|
1250
|
+
(1, "modified", "INTEGER", 1, None, 0),
|
|
1251
|
+
(2, "size", "INTEGER", 1, None, 0),
|
|
1252
|
+
(3, "ready", "INTEGER", 1, None, 0),
|
|
1253
|
+
]:
|
|
1254
|
+
raise _CacheOwnershipError(
|
|
1255
|
+
"The cache inventory entries table is incompatible."
|
|
1256
|
+
)
|
|
1257
|
+
if connection.execute("PRAGMA index_info(entries_oldest)").fetchall() != [
|
|
1258
|
+
(0, 1, "modified"),
|
|
1259
|
+
(1, 0, "name"),
|
|
1260
|
+
]:
|
|
1261
|
+
raise _CacheOwnershipError(
|
|
1262
|
+
"The cache inventory ordering index is incompatible."
|
|
1263
|
+
)
|
|
1264
|
+
metadata = connection.execute(
|
|
1265
|
+
"SELECT key, value FROM cache_meta ORDER BY key"
|
|
1266
|
+
).fetchall()
|
|
1267
|
+
if metadata not in [
|
|
1268
|
+
[("schema", _INDEX_SCHEMA_VERSION)],
|
|
1269
|
+
[
|
|
1270
|
+
("inventory_complete", "1"),
|
|
1271
|
+
("schema", _INDEX_SCHEMA_VERSION),
|
|
1272
|
+
],
|
|
1273
|
+
]:
|
|
1274
|
+
raise _CacheOwnershipError(
|
|
1275
|
+
"The cache inventory metadata is incompatible."
|
|
1276
|
+
)
|
|
1277
|
+
rows = connection.execute(
|
|
1278
|
+
"SELECT name, modified, size, ready FROM entries "
|
|
1279
|
+
"ORDER BY name LIMIT ?",
|
|
1280
|
+
(_INDEX_MAX_VALIDATION_ROWS,),
|
|
1281
|
+
).fetchall()
|
|
1282
|
+
if len(rows) >= _INDEX_MAX_VALIDATION_ROWS:
|
|
1283
|
+
raise _CacheOwnershipError(
|
|
1284
|
+
"The cache inventory contains too many entries."
|
|
1285
|
+
)
|
|
1286
|
+
for name, modified, size, ready in rows:
|
|
1287
|
+
if (
|
|
1288
|
+
type(name) is not str
|
|
1289
|
+
or _ENTRY_PATTERN.fullmatch(name) is None
|
|
1290
|
+
or type(modified) is not int
|
|
1291
|
+
or type(size) is not int
|
|
1292
|
+
or not 0 <= size <= TRUSTED_DERIVED_CACHE_MAX_ENTRY_BYTES
|
|
1293
|
+
or type(ready) is not int
|
|
1294
|
+
or ready not in (0, 1)
|
|
1295
|
+
):
|
|
1296
|
+
raise _CacheOwnershipError(
|
|
1297
|
+
"The cache inventory contains an incompatible entry."
|
|
1298
|
+
)
|
|
1299
|
+
finally:
|
|
1300
|
+
_close_index_best_effort(connection)
|
|
1301
|
+
final_status = index_path.stat(follow_symlinks=False)
|
|
1302
|
+
if (
|
|
1303
|
+
not stat.S_ISREG(final_status.st_mode)
|
|
1304
|
+
or (final_status.st_dev, final_status.st_ino)
|
|
1305
|
+
!= (initial_status.st_dev, initial_status.st_ino)
|
|
1306
|
+
or final_status.st_size != initial_status.st_size
|
|
1307
|
+
or final_status.st_mtime_ns != initial_status.st_mtime_ns
|
|
1308
|
+
or final_status.st_ctime_ns != initial_status.st_ctime_ns
|
|
1309
|
+
):
|
|
1310
|
+
raise _CacheOwnershipError(
|
|
1311
|
+
"The cache inventory changed during ownership validation."
|
|
1312
|
+
)
|
|
1313
|
+
return _IndexProof(
|
|
1314
|
+
final_status.st_dev,
|
|
1315
|
+
final_status.st_ino,
|
|
1316
|
+
final_status.st_size,
|
|
1317
|
+
final_status.st_mtime_ns,
|
|
1318
|
+
final_status.st_ctime_ns,
|
|
1319
|
+
False,
|
|
1320
|
+
)
|
|
1321
|
+
|
|
1322
|
+
def _ensure_global_inventory(
|
|
1323
|
+
self,
|
|
1324
|
+
connection: sqlite3.Connection,
|
|
1325
|
+
*,
|
|
1326
|
+
cancel_check: Callable[[], None],
|
|
1327
|
+
expected_epoch: int,
|
|
1328
|
+
) -> bool:
|
|
1329
|
+
complete = connection.execute(
|
|
1330
|
+
"SELECT value FROM cache_meta WHERE key = 'inventory_complete'"
|
|
1331
|
+
).fetchone()
|
|
1332
|
+
if complete == ("1",):
|
|
1333
|
+
return self._recover_pending(connection, cancel_check=cancel_check)
|
|
1334
|
+
|
|
1335
|
+
scanned: list[os.DirEntry[str]] = []
|
|
1336
|
+
with os.scandir(self.root) as iterator:
|
|
1337
|
+
for index, entry in enumerate(
|
|
1338
|
+
islice(iterator, TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES)
|
|
1339
|
+
):
|
|
1340
|
+
if index % 128 == 0:
|
|
1341
|
+
cancel_check()
|
|
1342
|
+
scanned.append(entry)
|
|
1343
|
+
if len(scanned) >= TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES:
|
|
1344
|
+
deletion_targets: list[_ArtifactProof] = []
|
|
1345
|
+
for entry in scanned:
|
|
1346
|
+
if not (
|
|
1347
|
+
_ENTRY_PATTERN.fullmatch(entry.name)
|
|
1348
|
+
or _TEMP_PATTERN.fullmatch(entry.name)
|
|
1349
|
+
):
|
|
1350
|
+
continue
|
|
1351
|
+
cancel_check()
|
|
1352
|
+
if not entry.is_file(follow_symlinks=False):
|
|
1353
|
+
continue
|
|
1354
|
+
is_temporary = _TEMP_PATTERN.fullmatch(entry.name) is not None
|
|
1355
|
+
path = self._validated_artifact_path(
|
|
1356
|
+
entry.name,
|
|
1357
|
+
temporary=is_temporary,
|
|
1358
|
+
index_derived=True,
|
|
1359
|
+
)
|
|
1360
|
+
self._before_artifact_proof(path, "inventory-overflow")
|
|
1361
|
+
deletion_targets.append(
|
|
1362
|
+
self._prove_framed_artifact(path, temporary=is_temporary)
|
|
1363
|
+
)
|
|
1364
|
+
for proof in deletion_targets:
|
|
1365
|
+
cancel_check()
|
|
1366
|
+
self._unlink_proven_artifact(
|
|
1367
|
+
proof,
|
|
1368
|
+
expected_epoch=expected_epoch,
|
|
1369
|
+
operation="inventory-overflow",
|
|
1370
|
+
)
|
|
1371
|
+
return False
|
|
1372
|
+
|
|
1373
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
1374
|
+
try:
|
|
1375
|
+
connection.execute("DELETE FROM entries")
|
|
1376
|
+
stale_temporaries: list[tuple[int, _ArtifactProof]] = []
|
|
1377
|
+
for entry in scanned:
|
|
1378
|
+
cancel_check()
|
|
1379
|
+
try:
|
|
1380
|
+
status = entry.stat(follow_symlinks=False)
|
|
1381
|
+
except OSError:
|
|
1382
|
+
continue
|
|
1383
|
+
if not entry.is_file(follow_symlinks=False):
|
|
1384
|
+
continue
|
|
1385
|
+
if _ENTRY_PATTERN.fullmatch(entry.name):
|
|
1386
|
+
path = self._validated_artifact_path(
|
|
1387
|
+
entry.name,
|
|
1388
|
+
temporary=False,
|
|
1389
|
+
index_derived=True,
|
|
1390
|
+
)
|
|
1391
|
+
self._before_artifact_proof(path, "inventory-entry")
|
|
1392
|
+
self._prove_framed_artifact(path, temporary=False)
|
|
1393
|
+
connection.execute(
|
|
1394
|
+
"INSERT OR REPLACE INTO entries "
|
|
1395
|
+
"(name, modified, size, ready) VALUES(?, ?, ?, 1)",
|
|
1396
|
+
(entry.name, status.st_mtime_ns, status.st_size),
|
|
1397
|
+
)
|
|
1398
|
+
elif _TEMP_PATTERN.fullmatch(entry.name):
|
|
1399
|
+
path = self._validated_artifact_path(
|
|
1400
|
+
entry.name,
|
|
1401
|
+
temporary=True,
|
|
1402
|
+
index_derived=True,
|
|
1403
|
+
)
|
|
1404
|
+
self._before_artifact_proof(path, "stale-temporary")
|
|
1405
|
+
stale_temporaries.append(
|
|
1406
|
+
(
|
|
1407
|
+
status.st_mtime_ns,
|
|
1408
|
+
self._prove_framed_artifact(path, temporary=True),
|
|
1409
|
+
)
|
|
1410
|
+
)
|
|
1411
|
+
stale_temporaries.sort(key=lambda item: (item[0], item[1].path.name))
|
|
1412
|
+
for _modified, proof in stale_temporaries[
|
|
1413
|
+
:-TRUSTED_DERIVED_CACHE_MAX_TEMP_FILES
|
|
1414
|
+
]:
|
|
1415
|
+
cancel_check()
|
|
1416
|
+
self._unlink_proven_artifact(
|
|
1417
|
+
proof,
|
|
1418
|
+
expected_epoch=expected_epoch,
|
|
1419
|
+
operation="stale-temporary",
|
|
1420
|
+
)
|
|
1421
|
+
connection.execute(
|
|
1422
|
+
"INSERT OR REPLACE INTO cache_meta(key, value) "
|
|
1423
|
+
"VALUES('inventory_complete', '1')"
|
|
1424
|
+
)
|
|
1425
|
+
connection.commit()
|
|
1426
|
+
except BaseException:
|
|
1427
|
+
_rollback_index_best_effort(connection)
|
|
1428
|
+
raise
|
|
1429
|
+
within_limits = self._enforce_global_limits(
|
|
1430
|
+
connection,
|
|
1431
|
+
exclude=None,
|
|
1432
|
+
cancel_check=cancel_check,
|
|
1433
|
+
expected_epoch=expected_epoch,
|
|
1434
|
+
)
|
|
1435
|
+
if not within_limits:
|
|
1436
|
+
with self._state_lock:
|
|
1437
|
+
self.enabled = False
|
|
1438
|
+
return within_limits
|
|
1439
|
+
|
|
1440
|
+
def _recover_pending(
|
|
1441
|
+
self,
|
|
1442
|
+
connection: sqlite3.Connection,
|
|
1443
|
+
*,
|
|
1444
|
+
cancel_check: Callable[[], None],
|
|
1445
|
+
) -> bool:
|
|
1446
|
+
pending = connection.execute(
|
|
1447
|
+
"SELECT name FROM entries WHERE ready = 0 ORDER BY modified, name LIMIT ?",
|
|
1448
|
+
(TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES,),
|
|
1449
|
+
).fetchall()
|
|
1450
|
+
validated_pending = []
|
|
1451
|
+
for (name,) in pending:
|
|
1452
|
+
path = self._validated_artifact_path(
|
|
1453
|
+
name,
|
|
1454
|
+
temporary=False,
|
|
1455
|
+
index_derived=True,
|
|
1456
|
+
)
|
|
1457
|
+
try:
|
|
1458
|
+
proof = self._prove_framed_artifact(path, temporary=False)
|
|
1459
|
+
except FileNotFoundError:
|
|
1460
|
+
proof = None
|
|
1461
|
+
validated_pending.append((name, path, proof))
|
|
1462
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
1463
|
+
try:
|
|
1464
|
+
for name, path, proof in validated_pending:
|
|
1465
|
+
cancel_check()
|
|
1466
|
+
try:
|
|
1467
|
+
status = path.stat()
|
|
1468
|
+
except FileNotFoundError:
|
|
1469
|
+
connection.execute("DELETE FROM entries WHERE name = ?", (name,))
|
|
1470
|
+
except OSError:
|
|
1471
|
+
connection.rollback()
|
|
1472
|
+
with self._state_lock:
|
|
1473
|
+
self.enabled = False
|
|
1474
|
+
return False
|
|
1475
|
+
else:
|
|
1476
|
+
if proof is None:
|
|
1477
|
+
raise _CacheOwnershipError(
|
|
1478
|
+
"A pending cache artifact lacks ownership proof."
|
|
1479
|
+
)
|
|
1480
|
+
connection.execute(
|
|
1481
|
+
"UPDATE entries SET modified = ?, size = ?, ready = 1 "
|
|
1482
|
+
"WHERE name = ?",
|
|
1483
|
+
(status.st_mtime_ns, status.st_size, name),
|
|
1484
|
+
)
|
|
1485
|
+
connection.commit()
|
|
1486
|
+
except BaseException:
|
|
1487
|
+
_rollback_index_best_effort(connection)
|
|
1488
|
+
raise
|
|
1489
|
+
return len(pending) < TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES
|
|
1490
|
+
|
|
1491
|
+
def _reserve_and_evict(
|
|
1492
|
+
self,
|
|
1493
|
+
connection: sqlite3.Connection,
|
|
1494
|
+
name: str,
|
|
1495
|
+
size: int,
|
|
1496
|
+
*,
|
|
1497
|
+
cancel_check: Callable[[], None],
|
|
1498
|
+
expected_epoch: int,
|
|
1499
|
+
) -> bool:
|
|
1500
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
1501
|
+
try:
|
|
1502
|
+
prior = connection.execute(
|
|
1503
|
+
"SELECT size FROM entries WHERE name = ?", (name,)
|
|
1504
|
+
).fetchone()
|
|
1505
|
+
reserved = max(size, prior[0] if prior is not None else 0)
|
|
1506
|
+
connection.execute(
|
|
1507
|
+
"INSERT OR REPLACE INTO entries(name, modified, size, ready) "
|
|
1508
|
+
"VALUES(?, ?, ?, 0)",
|
|
1509
|
+
(name, time.time_ns(), reserved),
|
|
1510
|
+
)
|
|
1511
|
+
if not self._enforce_global_limits(
|
|
1512
|
+
connection,
|
|
1513
|
+
exclude=name,
|
|
1514
|
+
cancel_check=cancel_check,
|
|
1515
|
+
expected_epoch=expected_epoch,
|
|
1516
|
+
in_transaction=True,
|
|
1517
|
+
):
|
|
1518
|
+
connection.rollback()
|
|
1519
|
+
return False
|
|
1520
|
+
connection.commit()
|
|
1521
|
+
return True
|
|
1522
|
+
except BaseException:
|
|
1523
|
+
_rollback_index_best_effort(connection)
|
|
1524
|
+
raise
|
|
1525
|
+
|
|
1526
|
+
def _enforce_global_limits(
|
|
1527
|
+
self,
|
|
1528
|
+
connection: sqlite3.Connection,
|
|
1529
|
+
*,
|
|
1530
|
+
exclude: str | None,
|
|
1531
|
+
cancel_check: Callable[[], None],
|
|
1532
|
+
expected_epoch: int,
|
|
1533
|
+
in_transaction: bool = False,
|
|
1534
|
+
) -> bool:
|
|
1535
|
+
if not in_transaction:
|
|
1536
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
1537
|
+
try:
|
|
1538
|
+
count, total = connection.execute(
|
|
1539
|
+
"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM entries"
|
|
1540
|
+
).fetchone()
|
|
1541
|
+
inventory = connection.execute(
|
|
1542
|
+
"SELECT name, size FROM entries ORDER BY modified, name LIMIT ?",
|
|
1543
|
+
(TRUSTED_DERIVED_CACHE_MAX_CLEANUP_FILES,),
|
|
1544
|
+
).fetchall()
|
|
1545
|
+
if (
|
|
1546
|
+
type(count) is not int
|
|
1547
|
+
or type(total) is not int
|
|
1548
|
+
or count < 0
|
|
1549
|
+
or total < 0
|
|
1550
|
+
or count != len(inventory)
|
|
1551
|
+
):
|
|
1552
|
+
raise _CacheIndexCorrupt("The cache inventory bounds are invalid.")
|
|
1553
|
+
validated_inventory: list[tuple[str, int, Path, _ArtifactProof | None]] = []
|
|
1554
|
+
for name, entry_size in inventory:
|
|
1555
|
+
cancel_check()
|
|
1556
|
+
if type(entry_size) is not int or entry_size < 0:
|
|
1557
|
+
raise _CacheIndexCorrupt("A cache inventory size is invalid.")
|
|
1558
|
+
path = self._validated_artifact_path(
|
|
1559
|
+
name,
|
|
1560
|
+
temporary=False,
|
|
1561
|
+
index_derived=True,
|
|
1562
|
+
)
|
|
1563
|
+
self._before_artifact_proof(path, "eviction")
|
|
1564
|
+
try:
|
|
1565
|
+
proof = self._prove_framed_artifact(path, temporary=False)
|
|
1566
|
+
except FileNotFoundError:
|
|
1567
|
+
proof = None
|
|
1568
|
+
validated_inventory.append((cast(str, name), entry_size, path, proof))
|
|
1569
|
+
victims: list[tuple[str, Path, _ArtifactProof | None]] = []
|
|
1570
|
+
projected_count = count
|
|
1571
|
+
projected_total = total
|
|
1572
|
+
for name, entry_size, path, proof in validated_inventory:
|
|
1573
|
+
if (
|
|
1574
|
+
projected_count <= self.max_entries
|
|
1575
|
+
and projected_total <= self.max_total_bytes
|
|
1576
|
+
):
|
|
1577
|
+
break
|
|
1578
|
+
if exclude is not None and name == exclude:
|
|
1579
|
+
continue
|
|
1580
|
+
victims.append((name, path, proof))
|
|
1581
|
+
projected_count -= 1
|
|
1582
|
+
projected_total -= entry_size
|
|
1583
|
+
if (
|
|
1584
|
+
projected_count > self.max_entries
|
|
1585
|
+
or projected_total > self.max_total_bytes
|
|
1586
|
+
):
|
|
1587
|
+
if not in_transaction:
|
|
1588
|
+
connection.rollback()
|
|
1589
|
+
return False
|
|
1590
|
+
for name, _path, proof in victims:
|
|
1591
|
+
cancel_check()
|
|
1592
|
+
if proof is not None:
|
|
1593
|
+
self._unlink_proven_artifact(
|
|
1594
|
+
proof,
|
|
1595
|
+
expected_epoch=expected_epoch,
|
|
1596
|
+
operation="eviction",
|
|
1597
|
+
)
|
|
1598
|
+
connection.execute("DELETE FROM entries WHERE name = ?", (name,))
|
|
1599
|
+
if not in_transaction:
|
|
1600
|
+
connection.commit()
|
|
1601
|
+
return True
|
|
1602
|
+
except BaseException:
|
|
1603
|
+
if not in_transaction:
|
|
1604
|
+
_rollback_index_best_effort(connection)
|
|
1605
|
+
raise
|