qplotter 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qplot/__init__.py +46 -0
- qplot/__main__.py +232 -0
- qplot/_auxiliary_paths.py +67 -0
- qplot/_build_info.json +1 -0
- qplot/_metadata.py +75 -0
- qplot/_repair.py +14 -0
- qplot/_shutdown_supervisor.py +4049 -0
- qplot/_version.py +12 -0
- qplot/_windows_shutdown_job.py +1166 -0
- qplot/configuration/__init__.py +3 -0
- qplot/configuration/config.py +578 -0
- qplot/configuration/config_schema.json +357 -0
- qplot/configuration/fonts.py +3 -0
- qplot/configuration/scripts.py +255 -0
- qplot/configuration/themes/__init__.py +13 -0
- qplot/configuration/themes/_base.py +592 -0
- qplot/configuration/themes/blank.py +17 -0
- qplot/configuration/themes/dark.py +54 -0
- qplot/configuration/themes/light.py +54 -0
- qplot/datahandling/LoadFromDB.py +224 -0
- qplot/datahandling/__init__.py +47 -0
- qplot/datahandling/_trusted_live_helper.py +812 -0
- qplot/datahandling/_trusted_live_protocol.py +1745 -0
- qplot/datahandling/database.py +2340 -0
- qplot/datahandling/dimensions.py +43 -0
- qplot/datahandling/file_identity.py +461 -0
- qplot/datahandling/parameter_data.py +149 -0
- qplot/datahandling/plot_progress.py +17 -0
- qplot/datahandling/qcodes_cache.py +277 -0
- qplot/datahandling/qcodes_compat.py +14 -0
- qplot/datahandling/readSQL.py +2442 -0
- qplot/datahandling/readonly.py +2566 -0
- qplot/datahandling/source_activity.py +21 -0
- qplot/datahandling/trusted_array_preview.py +148 -0
- qplot/datahandling/trusted_derived_cache.py +1605 -0
- qplot/datahandling/trusted_derived_rendering.py +876 -0
- qplot/datahandling/trusted_heatmap.py +246 -0
- qplot/datahandling/trusted_live.py +2808 -0
- qplot/datahandling/trusted_live_queries.py +5907 -0
- qplot/datahandling/trusted_live_service.py +1919 -0
- qplot/datahandling/trusted_live_supervisor.py +2161 -0
- qplot/datahandling/trusted_plot.py +283 -0
- qplot/datahandling/trusted_presentation.py +1065 -0
- qplot/datahandling/trusted_snapshot.py +1132 -0
- qplot/datahandling/trusted_work_coordinator.py +2157 -0
- qplot/datahandling/trusted_work_scheduler.py +1204 -0
- qplot/diagnostics.py +241 -0
- qplot/resources/testdata/qplot_test_db_01_10mb.csv +11 -0
- qplot/resources/testdata/qplot_test_db_02_25mb.csv +14 -0
- qplot/resources/testdata/qplot_test_db_03_50mb.csv +17 -0
- qplot/resources/testdata/qplot_test_db_04_100mb.csv +20 -0
- qplot/resources/testdata/qplot_test_db_05_250mb.csv +23 -0
- qplot/resources/testdata/qplot_test_db_06_500mb.csv +26 -0
- qplot/resources/testdata/qplot_test_db_07_1gb.csv +29 -0
- qplot/resources/testdata/qplot_test_db_08_5gb.csv +32 -0
- qplot/resources/testdata/qplot_test_db_09_10gb.csv +35 -0
- qplot/resources/testdata/qplot_test_db_10_30gb.csv +38 -0
- qplot/testdata.py +1804 -0
- qplot/tools/__init__.py +13 -0
- qplot/tools/finite_means.py +123 -0
- qplot/tools/general.py +139 -0
- qplot/tools/heatmap_geometry.py +391 -0
- qplot/tools/operation_registry.py +179 -0
- qplot/tools/plot_tools.py +650 -0
- qplot/tools/sample_statistics.py +111 -0
- qplot/tools/worker.py +2838 -0
- qplot/windows/__init__.py +9 -0
- qplot/windows/_colorbar.py +422 -0
- qplot/windows/_commands.py +523 -0
- qplot/windows/_config_persistence.py +123 -0
- qplot/windows/_database_actions.py +4933 -0
- qplot/windows/_dataset_handle.py +221 -0
- qplot/windows/_display_precision.py +48 -0
- qplot/windows/_dragdrop.py +70 -0
- qplot/windows/_export_paths.py +1167 -0
- qplot/windows/_help.py +210 -0
- qplot/windows/_mixed_appearance.py +303 -0
- qplot/windows/_mixed_plots.py +461 -0
- qplot/windows/_native_averaging.py +323 -0
- qplot/windows/_native_transforms.py +551 -0
- qplot/windows/_plot1d_snap.py +612 -0
- qplot/windows/_plot1d_traces.py +2236 -0
- qplot/windows/_plot2d_colorbar.py +1127 -0
- qplot/windows/_plot2d_colorbar_dialog.py +786 -0
- qplot/windows/_plot2d_layers.py +2291 -0
- qplot/windows/_plot2d_sweeps.py +712 -0
- qplot/windows/_plotWin.py +2082 -0
- qplot/windows/_plot_actions.py +2703 -0
- qplot/windows/_plot_annotation.py +186 -0
- qplot/windows/_plot_appearance.py +283 -0
- qplot/windows/_plot_axis_scaling.py +1521 -0
- qplot/windows/_plot_data_viewbox.py +94 -0
- qplot/windows/_plot_drop_axes.py +224 -0
- qplot/windows/_plot_export.py +1445 -0
- qplot/windows/_plot_feedback.py +112 -0
- qplot/windows/_plot_marquee.py +659 -0
- qplot/windows/_plot_refresh.py +950 -0
- qplot/windows/_plot_source_promotion.py +178 -0
- qplot/windows/_plot_state.py +237 -0
- qplot/windows/_plot_status.py +115 -0
- qplot/windows/_plot_tiling.py +35 -0
- qplot/windows/_plot_transform_labels.py +60 -0
- qplot/windows/_preferences.py +724 -0
- qplot/windows/_refresh_interval.py +172 -0
- qplot/windows/_run_controls.py +789 -0
- qplot/windows/_shortcuts.py +47 -0
- qplot/windows/_subplots/__init__.py +8 -0
- qplot/windows/_subplots/subplot1d.py +532 -0
- qplot/windows/_subplots/subplot2d.py +919 -0
- qplot/windows/_svg_export.py +209 -0
- qplot/windows/_trusted_derived_qt.py +1759 -0
- qplot/windows/_widgets/__init__.py +23 -0
- qplot/windows/_widgets/_run_formatting.py +387 -0
- qplot/windows/_widgets/details_tables.py +1119 -0
- qplot/windows/_widgets/dropbox.py +290 -0
- qplot/windows/_widgets/operations.py +460 -0
- qplot/windows/_widgets/preview.py +2997 -0
- qplot/windows/_widgets/run_list_items.py +488 -0
- qplot/windows/_widgets/toolbar.py +111 -0
- qplot/windows/_widgets/treeWidgets.py +2572 -0
- qplot/windows/_window_controls.py +414 -0
- qplot/windows/main.py +1632 -0
- qplot/windows/plot1d.py +313 -0
- qplot/windows/plot2d.py +1935 -0
- qplotter-1.6.0.dist-info/METADATA +322 -0
- qplotter-1.6.0.dist-info/RECORD +130 -0
- qplotter-1.6.0.dist-info/WHEEL +5 -0
- qplotter-1.6.0.dist-info/entry_points.txt +4 -0
- qplotter-1.6.0.dist-info/licenses/LICENSE +21 -0
- qplotter-1.6.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2566 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sqlite3
|
|
3
|
+
import stat
|
|
4
|
+
import struct
|
|
5
|
+
import sys
|
|
6
|
+
import tempfile
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
import weakref
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from qplot.datahandling.file_identity import (
|
|
15
|
+
QPLOT_GENERATED_DATABASE_APPLICATION_ID,
|
|
16
|
+
QPLOT_GENERATION_LINEAGE_FORMAT_VERSION,
|
|
17
|
+
QPLOT_GENERATION_LINEAGE_NONCE_BYTES,
|
|
18
|
+
QPLOT_GENERATION_LINEAGE_RING_TABLE,
|
|
19
|
+
QPLOT_GENERATION_LINEAGE_STATE_TABLE,
|
|
20
|
+
QPLOT_GENERATION_LINEAGE_WINDOW,
|
|
21
|
+
QPLOT_GENERATION_PROVENANCE_TABLE,
|
|
22
|
+
QPLOT_GENERATION_PROVENANCE_TOKEN_BYTES,
|
|
23
|
+
DatabaseFileIdentity,
|
|
24
|
+
database_file_identity,
|
|
25
|
+
database_has_qplot_generation_marker,
|
|
26
|
+
database_publication_guard_path,
|
|
27
|
+
generation_provenance_trigger,
|
|
28
|
+
open_file_identity,
|
|
29
|
+
path_bound_file_identity,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def connect(
|
|
34
|
+
name,
|
|
35
|
+
debug=False,
|
|
36
|
+
version=-1,
|
|
37
|
+
read_only=False,
|
|
38
|
+
*,
|
|
39
|
+
cancelled_callback=None,
|
|
40
|
+
deadline=None,
|
|
41
|
+
):
|
|
42
|
+
"""Open QCoDeS connections while keeping viewer reads under qPlot control."""
|
|
43
|
+
from qcodes.dataset.sqlite.database import connect as qcodes_connect
|
|
44
|
+
|
|
45
|
+
if not read_only:
|
|
46
|
+
return qcodes_connect(
|
|
47
|
+
name,
|
|
48
|
+
debug=debug,
|
|
49
|
+
version=version,
|
|
50
|
+
read_only=False,
|
|
51
|
+
)
|
|
52
|
+
return _qcodes_read_only_atomic_connection(
|
|
53
|
+
name,
|
|
54
|
+
debug=debug,
|
|
55
|
+
cancelled_callback=cancelled_callback,
|
|
56
|
+
deadline=deadline,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_DB_debug():
|
|
61
|
+
"""Return QCoDeS' database debug setting on demand."""
|
|
62
|
+
from qcodes.dataset.sqlite.database import get_DB_debug as qcodes_get_DB_debug
|
|
63
|
+
|
|
64
|
+
return qcodes_get_DB_debug()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_DB_location():
|
|
68
|
+
"""Return QCoDeS' configured database path on demand."""
|
|
69
|
+
from qcodes.dataset.sqlite.database import (
|
|
70
|
+
get_DB_location as qcodes_get_DB_location,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
return qcodes_get_DB_location()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def load_by_guid(*args, **kwargs):
|
|
77
|
+
"""Forward to QCoDeS' GUID loader on demand."""
|
|
78
|
+
from qcodes.dataset import load_by_guid as qcodes_load_by_guid
|
|
79
|
+
|
|
80
|
+
return qcodes_load_by_guid(*args, **kwargs)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def load_by_id(*args, **kwargs):
|
|
84
|
+
"""Forward to QCoDeS' run-ID loader on demand."""
|
|
85
|
+
from qcodes.dataset import load_by_id as qcodes_load_by_id
|
|
86
|
+
|
|
87
|
+
return qcodes_load_by_id(*args, **kwargs)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def result_owns_supplied_connection(result):
|
|
91
|
+
"""Check QCoDeS connection ownership without importing it for probes."""
|
|
92
|
+
from qplot.datahandling.qcodes_compat import (
|
|
93
|
+
result_owns_supplied_connection as qcodes_result_owns_connection,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
return qcodes_result_owns_connection(result)
|
|
97
|
+
|
|
98
|
+
SQLITE_READ_ONLY_CACHE_KIB = 16 * 1024
|
|
99
|
+
SNAPSHOT_COPY_CHUNK_BYTES = 1024 * 1024
|
|
100
|
+
QCODE_SQLITE_BUSY_QUANTUM_SECONDS = 0.05
|
|
101
|
+
QCODE_SQLITE_OPEN_TIMEOUT_SECONDS = 5.0
|
|
102
|
+
WAL_SNAPSHOT_ATTEMPTS = 5
|
|
103
|
+
_ROLLBACK_JOURNAL_MAGIC = b"\xd9\xd5\x05\xf9 \xa1c\xd7"
|
|
104
|
+
_ROLLBACK_JOURNAL_PREFIX_BYTES = 4096
|
|
105
|
+
_WAL_HEADER_BYTES = 32
|
|
106
|
+
_WAL_FRAME_HEADER_BYTES = 24
|
|
107
|
+
_WAL_FORMAT_VERSION = 3_007_000
|
|
108
|
+
_WAL_MAGIC_LITTLE_ENDIAN_CHECKSUMS = 0x377F0682
|
|
109
|
+
_WAL_MAGIC_BIG_ENDIAN_CHECKSUMS = 0x377F0683
|
|
110
|
+
_SQLITE_PENDING_BYTE = 0x40000000
|
|
111
|
+
_SQLITE_SHARED_FIRST = _SQLITE_PENDING_BYTE + 2
|
|
112
|
+
_SQLITE_SHARED_SIZE = 510
|
|
113
|
+
_DATABASE_INSTANCE_REGISTRY: dict[
|
|
114
|
+
Path,
|
|
115
|
+
tuple[DatabaseFileIdentity | None, bool],
|
|
116
|
+
] = {}
|
|
117
|
+
_DATABASE_INSTANCE_REGISTRY_LOCK = threading.Lock()
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class ReadOnlyDatabaseAccessError(RuntimeError):
|
|
121
|
+
"""Raised when qPlot cannot take a non-mutating view of a database."""
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class ReadOnlyDatabaseCancelledError(InterruptedError):
|
|
125
|
+
"""Raised when a caller cancels read-only database preparation."""
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class DatabaseInstanceChangedError(ReadOnlyDatabaseAccessError):
|
|
129
|
+
"""Raised when a database was atomically replaced during a requested read."""
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class UnverifiableDatabaseWalError(ReadOnlyDatabaseAccessError):
|
|
133
|
+
"""Raised when a WAL cannot be proven to descend from its selected main."""
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _raise_if_read_interrupted(cancelled_callback=None, deadline=None):
|
|
137
|
+
"""Stop cooperative work before it can publish a partial read target."""
|
|
138
|
+
if cancelled_callback is not None and cancelled_callback():
|
|
139
|
+
raise ReadOnlyDatabaseCancelledError("Database read cancelled.")
|
|
140
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
141
|
+
raise TimeoutError("Timed out while preparing a read-only database view.")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _read_control_progress_handler(cancelled_callback=None, deadline=None):
|
|
145
|
+
"""Return a SQLite progress handler for the same cooperative controls."""
|
|
146
|
+
if cancelled_callback is None and deadline is None:
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
def interrupted():
|
|
150
|
+
if cancelled_callback is not None and cancelled_callback():
|
|
151
|
+
return 1
|
|
152
|
+
return int(deadline is not None and time.monotonic() >= deadline)
|
|
153
|
+
|
|
154
|
+
return interrupted
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _install_read_control_progress_handler(
|
|
158
|
+
connection,
|
|
159
|
+
cancelled_callback=None,
|
|
160
|
+
deadline=None,
|
|
161
|
+
):
|
|
162
|
+
progress_handler = _read_control_progress_handler(
|
|
163
|
+
cancelled_callback,
|
|
164
|
+
deadline,
|
|
165
|
+
)
|
|
166
|
+
if progress_handler is not None:
|
|
167
|
+
set_progress_handler = getattr(connection, "set_progress_handler", None)
|
|
168
|
+
if callable(set_progress_handler):
|
|
169
|
+
set_progress_handler(progress_handler, 1000)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _sqlite_timeout_before_deadline(timeout, deadline):
|
|
173
|
+
"""Limit SQLite's busy wait to the remaining cooperative deadline."""
|
|
174
|
+
if deadline is None:
|
|
175
|
+
return timeout
|
|
176
|
+
remaining = deadline - time.monotonic()
|
|
177
|
+
if remaining <= 0:
|
|
178
|
+
raise TimeoutError("Timed out while preparing a read-only database view.")
|
|
179
|
+
return min(float(timeout), remaining)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _register_qcodes_sqlite_types(qcodes_database):
|
|
183
|
+
"""Register the adapters and converters used by QCoDeS connections."""
|
|
184
|
+
sqlite3.register_adapter(
|
|
185
|
+
qcodes_database.np.ndarray,
|
|
186
|
+
qcodes_database._adapt_array,
|
|
187
|
+
)
|
|
188
|
+
sqlite3.register_converter("array", qcodes_database._convert_array)
|
|
189
|
+
for numpy_int in qcodes_database.numpy_ints:
|
|
190
|
+
sqlite3.register_adapter(numpy_int, int)
|
|
191
|
+
sqlite3.register_converter("numeric", qcodes_database._convert_numeric)
|
|
192
|
+
for numpy_float in (float, *qcodes_database.numpy_floats):
|
|
193
|
+
sqlite3.register_adapter(numpy_float, qcodes_database._adapt_float)
|
|
194
|
+
for complex_type in qcodes_database.complex_types:
|
|
195
|
+
sqlite3.register_adapter(
|
|
196
|
+
complex_type,
|
|
197
|
+
qcodes_database._adapt_complex,
|
|
198
|
+
)
|
|
199
|
+
sqlite3.register_converter("complex", qcodes_database._convert_complex)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _sqlite_error_is_busy(error):
|
|
203
|
+
"""Return whether SQLite reported a retryable lock or busy condition."""
|
|
204
|
+
error_code = getattr(error, "sqlite_errorcode", None)
|
|
205
|
+
if isinstance(error_code, int):
|
|
206
|
+
primary_code = error_code & 0xFF
|
|
207
|
+
return primary_code in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED)
|
|
208
|
+
message = str(error).lower()
|
|
209
|
+
return "locked" in message or "busy" in message
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _qcodes_read_only_atomic_connection(
|
|
213
|
+
name,
|
|
214
|
+
*,
|
|
215
|
+
debug=False,
|
|
216
|
+
cancelled_callback=None,
|
|
217
|
+
deadline=None,
|
|
218
|
+
):
|
|
219
|
+
"""Build a latest-schema QCoDeS connection without viewer-side upgrades."""
|
|
220
|
+
from qcodes.dataset.sqlite import database as qcodes_database
|
|
221
|
+
|
|
222
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
223
|
+
_register_qcodes_sqlite_types(qcodes_database)
|
|
224
|
+
connection = None
|
|
225
|
+
busy_deadline = time.monotonic() + QCODE_SQLITE_OPEN_TIMEOUT_SECONDS
|
|
226
|
+
try:
|
|
227
|
+
connection = sqlite3.connect(
|
|
228
|
+
f"file:{name!s}?mode=ro",
|
|
229
|
+
detect_types=sqlite3.PARSE_DECLTYPES,
|
|
230
|
+
check_same_thread=True,
|
|
231
|
+
timeout=_sqlite_timeout_before_deadline(
|
|
232
|
+
QCODE_SQLITE_BUSY_QUANTUM_SECONDS,
|
|
233
|
+
deadline,
|
|
234
|
+
),
|
|
235
|
+
uri=True,
|
|
236
|
+
factory=qcodes_database.AtomicConnection,
|
|
237
|
+
)
|
|
238
|
+
_install_read_control_progress_handler(
|
|
239
|
+
connection,
|
|
240
|
+
cancelled_callback,
|
|
241
|
+
deadline,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
while True:
|
|
245
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
246
|
+
try:
|
|
247
|
+
version_row = connection.execute("PRAGMA user_version").fetchone()
|
|
248
|
+
connection.execute(
|
|
249
|
+
"SELECT 1 FROM sqlite_schema LIMIT 1"
|
|
250
|
+
).fetchone()
|
|
251
|
+
except sqlite3.OperationalError as error:
|
|
252
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
253
|
+
if (
|
|
254
|
+
not _sqlite_error_is_busy(error)
|
|
255
|
+
or time.monotonic() >= busy_deadline
|
|
256
|
+
):
|
|
257
|
+
raise
|
|
258
|
+
continue
|
|
259
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
260
|
+
break
|
|
261
|
+
|
|
262
|
+
if (
|
|
263
|
+
version_row is None
|
|
264
|
+
or len(version_row) != 1
|
|
265
|
+
or not isinstance(version_row[0], int)
|
|
266
|
+
):
|
|
267
|
+
raise RuntimeError(
|
|
268
|
+
f"Database {name} has an invalid SQLite user_version."
|
|
269
|
+
)
|
|
270
|
+
database_version = version_row[0]
|
|
271
|
+
latest_version = qcodes_database._latest_available_version()
|
|
272
|
+
if database_version != latest_version:
|
|
273
|
+
raise RuntimeError(
|
|
274
|
+
f"Database {name} has schema version {database_version}, but "
|
|
275
|
+
f"qPlot requires the latest supported QCoDeS schema version "
|
|
276
|
+
f"{latest_version}. qPlot did not upgrade the input database."
|
|
277
|
+
)
|
|
278
|
+
if debug:
|
|
279
|
+
connection.set_trace_callback(print)
|
|
280
|
+
return connection
|
|
281
|
+
except Exception:
|
|
282
|
+
if connection is not None:
|
|
283
|
+
try:
|
|
284
|
+
connection.close()
|
|
285
|
+
except sqlite3.Error:
|
|
286
|
+
pass
|
|
287
|
+
raise
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
@contextmanager
|
|
291
|
+
def _posix_sqlite_shared_lock_bytes(
|
|
292
|
+
database_path,
|
|
293
|
+
*,
|
|
294
|
+
cancelled_callback=None,
|
|
295
|
+
deadline=None,
|
|
296
|
+
):
|
|
297
|
+
"""Hold SQLite's rollback-mode shared-lock bytes without opening SQLite."""
|
|
298
|
+
import errno
|
|
299
|
+
import fcntl
|
|
300
|
+
|
|
301
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
302
|
+
try:
|
|
303
|
+
database_file = open(database_path, "rb")
|
|
304
|
+
except OSError as error:
|
|
305
|
+
raise ReadOnlyDatabaseAccessError(
|
|
306
|
+
f"Could not open {database_path} for a read-only lock check: {error}"
|
|
307
|
+
) from error
|
|
308
|
+
|
|
309
|
+
pending_locked = False
|
|
310
|
+
shared_locked = False
|
|
311
|
+
try:
|
|
312
|
+
try:
|
|
313
|
+
fcntl.lockf(
|
|
314
|
+
database_file.fileno(),
|
|
315
|
+
fcntl.LOCK_SH | fcntl.LOCK_NB,
|
|
316
|
+
1,
|
|
317
|
+
_SQLITE_PENDING_BYTE,
|
|
318
|
+
os.SEEK_SET,
|
|
319
|
+
)
|
|
320
|
+
pending_locked = True
|
|
321
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
322
|
+
fcntl.lockf(
|
|
323
|
+
database_file.fileno(),
|
|
324
|
+
fcntl.LOCK_SH | fcntl.LOCK_NB,
|
|
325
|
+
_SQLITE_SHARED_SIZE,
|
|
326
|
+
_SQLITE_SHARED_FIRST,
|
|
327
|
+
os.SEEK_SET,
|
|
328
|
+
)
|
|
329
|
+
shared_locked = True
|
|
330
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
331
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
332
|
+
raise
|
|
333
|
+
except OSError as error:
|
|
334
|
+
if error.errno in (errno.EACCES, errno.EAGAIN):
|
|
335
|
+
raise sqlite3.OperationalError("database is locked") from error
|
|
336
|
+
raise ReadOnlyDatabaseAccessError(
|
|
337
|
+
"Could not prove that SQLite shared-lock bytes are available "
|
|
338
|
+
f"for {database_path}: {error}"
|
|
339
|
+
) from error
|
|
340
|
+
|
|
341
|
+
yield database_file
|
|
342
|
+
finally:
|
|
343
|
+
if shared_locked:
|
|
344
|
+
try:
|
|
345
|
+
fcntl.lockf(
|
|
346
|
+
database_file.fileno(),
|
|
347
|
+
fcntl.LOCK_UN,
|
|
348
|
+
_SQLITE_SHARED_SIZE,
|
|
349
|
+
_SQLITE_SHARED_FIRST,
|
|
350
|
+
os.SEEK_SET,
|
|
351
|
+
)
|
|
352
|
+
except OSError:
|
|
353
|
+
pass
|
|
354
|
+
if pending_locked:
|
|
355
|
+
try:
|
|
356
|
+
fcntl.lockf(
|
|
357
|
+
database_file.fileno(),
|
|
358
|
+
fcntl.LOCK_UN,
|
|
359
|
+
1,
|
|
360
|
+
_SQLITE_PENDING_BYTE,
|
|
361
|
+
os.SEEK_SET,
|
|
362
|
+
)
|
|
363
|
+
except OSError:
|
|
364
|
+
pass
|
|
365
|
+
database_file.close()
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
@contextmanager
|
|
369
|
+
def _windows_sqlite_shared_lock_bytes(
|
|
370
|
+
database_path,
|
|
371
|
+
*,
|
|
372
|
+
cancelled_callback=None,
|
|
373
|
+
deadline=None,
|
|
374
|
+
):
|
|
375
|
+
"""Hold SQLite's Windows shared-lock ranges using nonblocking LockFileEx."""
|
|
376
|
+
import ctypes
|
|
377
|
+
import msvcrt
|
|
378
|
+
from ctypes import wintypes
|
|
379
|
+
|
|
380
|
+
class _Overlapped(ctypes.Structure):
|
|
381
|
+
_fields_ = (
|
|
382
|
+
("Internal", ctypes.c_size_t),
|
|
383
|
+
("InternalHigh", ctypes.c_size_t),
|
|
384
|
+
("Offset", wintypes.DWORD),
|
|
385
|
+
("OffsetHigh", wintypes.DWORD),
|
|
386
|
+
("hEvent", wintypes.HANDLE),
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
390
|
+
try:
|
|
391
|
+
database_file = open(database_path, "rb")
|
|
392
|
+
except OSError as error:
|
|
393
|
+
raise ReadOnlyDatabaseAccessError(
|
|
394
|
+
f"Could not open {database_path} for a read-only lock check: {error}"
|
|
395
|
+
) from error
|
|
396
|
+
|
|
397
|
+
kernel32 = ctypes.WinDLL( # type: ignore[attr-defined]
|
|
398
|
+
"kernel32",
|
|
399
|
+
use_last_error=True,
|
|
400
|
+
)
|
|
401
|
+
lock_file_ex = kernel32.LockFileEx
|
|
402
|
+
lock_file_ex.argtypes = (
|
|
403
|
+
wintypes.HANDLE,
|
|
404
|
+
wintypes.DWORD,
|
|
405
|
+
wintypes.DWORD,
|
|
406
|
+
wintypes.DWORD,
|
|
407
|
+
wintypes.DWORD,
|
|
408
|
+
ctypes.POINTER(_Overlapped),
|
|
409
|
+
)
|
|
410
|
+
lock_file_ex.restype = wintypes.BOOL
|
|
411
|
+
unlock_file_ex = kernel32.UnlockFileEx
|
|
412
|
+
unlock_file_ex.argtypes = (
|
|
413
|
+
wintypes.HANDLE,
|
|
414
|
+
wintypes.DWORD,
|
|
415
|
+
wintypes.DWORD,
|
|
416
|
+
wintypes.DWORD,
|
|
417
|
+
ctypes.POINTER(_Overlapped),
|
|
418
|
+
)
|
|
419
|
+
unlock_file_ex.restype = wintypes.BOOL
|
|
420
|
+
windows_handle = wintypes.HANDLE(
|
|
421
|
+
msvcrt.get_osfhandle( # type: ignore[attr-defined]
|
|
422
|
+
database_file.fileno()
|
|
423
|
+
)
|
|
424
|
+
)
|
|
425
|
+
locked_ranges = []
|
|
426
|
+
|
|
427
|
+
def lock_shared_range(offset, length):
|
|
428
|
+
overlapped = _Overlapped()
|
|
429
|
+
overlapped.Offset = offset & 0xFFFFFFFF
|
|
430
|
+
overlapped.OffsetHigh = (offset >> 32) & 0xFFFFFFFF
|
|
431
|
+
if not lock_file_ex(
|
|
432
|
+
windows_handle,
|
|
433
|
+
0x00000001,
|
|
434
|
+
0,
|
|
435
|
+
length & 0xFFFFFFFF,
|
|
436
|
+
(length >> 32) & 0xFFFFFFFF,
|
|
437
|
+
ctypes.byref(overlapped),
|
|
438
|
+
):
|
|
439
|
+
error_code = ctypes.get_last_error() # type: ignore[attr-defined]
|
|
440
|
+
if error_code in (32, 33, 997):
|
|
441
|
+
raise sqlite3.OperationalError("database is locked")
|
|
442
|
+
raise ReadOnlyDatabaseAccessError(
|
|
443
|
+
"Could not prove that SQLite shared-lock bytes are available "
|
|
444
|
+
f"for {database_path}: Windows error {error_code}."
|
|
445
|
+
)
|
|
446
|
+
locked_ranges.append((overlapped, length))
|
|
447
|
+
|
|
448
|
+
try:
|
|
449
|
+
lock_shared_range(_SQLITE_PENDING_BYTE, 1)
|
|
450
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
451
|
+
lock_shared_range(_SQLITE_SHARED_FIRST, _SQLITE_SHARED_SIZE)
|
|
452
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
453
|
+
yield database_file
|
|
454
|
+
finally:
|
|
455
|
+
for overlapped, length in reversed(locked_ranges):
|
|
456
|
+
unlock_file_ex(
|
|
457
|
+
windows_handle,
|
|
458
|
+
0,
|
|
459
|
+
length & 0xFFFFFFFF,
|
|
460
|
+
(length >> 32) & 0xFFFFFFFF,
|
|
461
|
+
ctypes.byref(overlapped),
|
|
462
|
+
)
|
|
463
|
+
database_file.close()
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
@contextmanager
|
|
467
|
+
def _sqlite_shared_lock_bytes(
|
|
468
|
+
database_path,
|
|
469
|
+
*,
|
|
470
|
+
cancelled_callback=None,
|
|
471
|
+
deadline=None,
|
|
472
|
+
):
|
|
473
|
+
"""Hold the platform's SQLite shared-lock ranges or fail closed."""
|
|
474
|
+
if os.name == "posix":
|
|
475
|
+
lock_context = _posix_sqlite_shared_lock_bytes(
|
|
476
|
+
database_path,
|
|
477
|
+
cancelled_callback=cancelled_callback,
|
|
478
|
+
deadline=deadline,
|
|
479
|
+
)
|
|
480
|
+
elif os.name == "nt":
|
|
481
|
+
lock_context = _windows_sqlite_shared_lock_bytes(
|
|
482
|
+
database_path,
|
|
483
|
+
cancelled_callback=cancelled_callback,
|
|
484
|
+
deadline=deadline,
|
|
485
|
+
)
|
|
486
|
+
else:
|
|
487
|
+
raise ReadOnlyDatabaseAccessError(
|
|
488
|
+
"qPlot cannot perform a non-mutating SQLite lock check on this "
|
|
489
|
+
"platform. Database access was rejected without opening a mutable "
|
|
490
|
+
"SQLite view."
|
|
491
|
+
)
|
|
492
|
+
with lock_context as database_file:
|
|
493
|
+
yield database_file
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _source_matches_under_sqlite_lock(
|
|
497
|
+
database_path,
|
|
498
|
+
expected_source,
|
|
499
|
+
database_file,
|
|
500
|
+
*,
|
|
501
|
+
cancelled_callback=None,
|
|
502
|
+
deadline=None,
|
|
503
|
+
):
|
|
504
|
+
"""Validate a source while never reopening its locked main file."""
|
|
505
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
506
|
+
try:
|
|
507
|
+
descriptor_signature_before = _stat_signature(
|
|
508
|
+
os.fstat(database_file.fileno())
|
|
509
|
+
)
|
|
510
|
+
descriptor_identity_before = open_file_identity(database_file.fileno())
|
|
511
|
+
path_signature_before = _file_signature(database_path)
|
|
512
|
+
path_identity_before = path_bound_file_identity(database_path)
|
|
513
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
514
|
+
raise
|
|
515
|
+
except OSError as error:
|
|
516
|
+
raise ReadOnlyDatabaseAccessError(
|
|
517
|
+
"Could not validate the database main file under its SQLite "
|
|
518
|
+
f"shared lock: {error}"
|
|
519
|
+
) from error
|
|
520
|
+
|
|
521
|
+
try:
|
|
522
|
+
wal_signature, wal_identity, wal_stable = _wal_observation(
|
|
523
|
+
database_path,
|
|
524
|
+
cancelled_callback=cancelled_callback,
|
|
525
|
+
deadline=deadline,
|
|
526
|
+
)
|
|
527
|
+
journal = _rollback_journal_observation(
|
|
528
|
+
database_path,
|
|
529
|
+
cancelled_callback=cancelled_callback,
|
|
530
|
+
deadline=deadline,
|
|
531
|
+
)
|
|
532
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
533
|
+
raise
|
|
534
|
+
except OSError as error:
|
|
535
|
+
raise ReadOnlyDatabaseAccessError(
|
|
536
|
+
"Could not validate SQLite sidecars while holding the database "
|
|
537
|
+
f"shared lock: {error}"
|
|
538
|
+
) from error
|
|
539
|
+
|
|
540
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
541
|
+
try:
|
|
542
|
+
descriptor_signature_after = _stat_signature(
|
|
543
|
+
os.fstat(database_file.fileno())
|
|
544
|
+
)
|
|
545
|
+
descriptor_identity_after = open_file_identity(database_file.fileno())
|
|
546
|
+
path_signature_after = _file_signature(database_path)
|
|
547
|
+
path_identity_after = path_bound_file_identity(database_path)
|
|
548
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
549
|
+
raise
|
|
550
|
+
except OSError as error:
|
|
551
|
+
raise ReadOnlyDatabaseAccessError(
|
|
552
|
+
"Could not finish validating the database main file under its "
|
|
553
|
+
f"SQLite shared lock: {error}"
|
|
554
|
+
) from error
|
|
555
|
+
|
|
556
|
+
main_file_stable = _path_and_descriptor_observations_match(
|
|
557
|
+
path_signature_before,
|
|
558
|
+
path_identity_before,
|
|
559
|
+
descriptor_signature_before,
|
|
560
|
+
descriptor_identity_before,
|
|
561
|
+
descriptor_signature_after,
|
|
562
|
+
descriptor_identity_after,
|
|
563
|
+
path_signature_after,
|
|
564
|
+
path_identity_after,
|
|
565
|
+
)
|
|
566
|
+
return (
|
|
567
|
+
expected_source.database is not None
|
|
568
|
+
and main_file_stable
|
|
569
|
+
and path_signature_before
|
|
570
|
+
== path_signature_after
|
|
571
|
+
== expected_source.database
|
|
572
|
+
and path_identity_before
|
|
573
|
+
== path_identity_after
|
|
574
|
+
== expected_source.database_identity
|
|
575
|
+
and wal_stable
|
|
576
|
+
and wal_signature == expected_source.wal
|
|
577
|
+
and wal_identity == expected_source.wal_identity
|
|
578
|
+
and journal == expected_source.journal
|
|
579
|
+
and (journal is None or journal.stable)
|
|
580
|
+
)
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def _copy_file_cooperatively(
|
|
584
|
+
source,
|
|
585
|
+
destination,
|
|
586
|
+
*,
|
|
587
|
+
cancelled_callback=None,
|
|
588
|
+
deadline=None,
|
|
589
|
+
):
|
|
590
|
+
"""Copy one snapshot artifact in bounded, cooperatively checked chunks."""
|
|
591
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
592
|
+
if _clone_file_if_supported(source, destination):
|
|
593
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
594
|
+
return destination
|
|
595
|
+
with open(source, "rb") as source_file, open(destination, "wb") as destination_file:
|
|
596
|
+
while True:
|
|
597
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
598
|
+
chunk = source_file.read(SNAPSHOT_COPY_CHUNK_BYTES)
|
|
599
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
600
|
+
if not chunk:
|
|
601
|
+
break
|
|
602
|
+
written = destination_file.write(chunk)
|
|
603
|
+
if written != len(chunk):
|
|
604
|
+
raise OSError(
|
|
605
|
+
f"Short write while copying {source} to a private snapshot."
|
|
606
|
+
)
|
|
607
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
608
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
609
|
+
return destination
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def _clone_file_if_supported(source, destination):
|
|
613
|
+
"""Create a copy-on-write clone without weakening snapshot isolation."""
|
|
614
|
+
destination_path = Path(destination)
|
|
615
|
+
destination_existed = destination_path.exists()
|
|
616
|
+
if sys.platform == "darwin":
|
|
617
|
+
import ctypes
|
|
618
|
+
|
|
619
|
+
clonefile = ctypes.CDLL(None, use_errno=True).clonefile
|
|
620
|
+
clonefile.argtypes = (
|
|
621
|
+
ctypes.c_char_p,
|
|
622
|
+
ctypes.c_char_p,
|
|
623
|
+
ctypes.c_int,
|
|
624
|
+
)
|
|
625
|
+
clonefile.restype = ctypes.c_int
|
|
626
|
+
result = clonefile(
|
|
627
|
+
os.fsencode(source),
|
|
628
|
+
os.fsencode(destination),
|
|
629
|
+
0,
|
|
630
|
+
)
|
|
631
|
+
if result == 0:
|
|
632
|
+
return True
|
|
633
|
+
elif sys.platform.startswith("linux"):
|
|
634
|
+
import fcntl
|
|
635
|
+
|
|
636
|
+
try:
|
|
637
|
+
with (
|
|
638
|
+
open(source, "rb") as source_file,
|
|
639
|
+
open(destination, "xb") as destination_file,
|
|
640
|
+
):
|
|
641
|
+
fcntl.ioctl(
|
|
642
|
+
destination_file.fileno(),
|
|
643
|
+
0x40049409, # Linux FICLONE
|
|
644
|
+
source_file.fileno(),
|
|
645
|
+
)
|
|
646
|
+
return True
|
|
647
|
+
except OSError:
|
|
648
|
+
pass
|
|
649
|
+
else:
|
|
650
|
+
return False
|
|
651
|
+
|
|
652
|
+
if not destination_existed:
|
|
653
|
+
try:
|
|
654
|
+
destination_path.unlink()
|
|
655
|
+
except FileNotFoundError:
|
|
656
|
+
pass
|
|
657
|
+
return False
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
@dataclass(frozen=True, slots=True)
|
|
661
|
+
class _RollbackJournalObservation:
|
|
662
|
+
"""One path-bound rollback-journal identity and state observation."""
|
|
663
|
+
|
|
664
|
+
file_signature: tuple[int, int, int, int, int]
|
|
665
|
+
file_identity: DatabaseFileIdentity | None
|
|
666
|
+
prefix: bytes
|
|
667
|
+
trailer: bytes
|
|
668
|
+
stable: bool
|
|
669
|
+
|
|
670
|
+
@property
|
|
671
|
+
def potentially_hot(self):
|
|
672
|
+
return (
|
|
673
|
+
self.file_signature[2] > 512
|
|
674
|
+
and self.prefix.startswith(_ROLLBACK_JOURNAL_MAGIC)
|
|
675
|
+
)
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
@dataclass(frozen=True, slots=True)
|
|
679
|
+
class _SourceSignature:
|
|
680
|
+
"""Ephemeral state binding one private snapshot to its source files."""
|
|
681
|
+
|
|
682
|
+
database: tuple[int, int, int, int, int] | None
|
|
683
|
+
database_identity: DatabaseFileIdentity | None
|
|
684
|
+
database_header: bytes
|
|
685
|
+
wal: tuple[int, int, int, int, int] | None
|
|
686
|
+
wal_identity: DatabaseFileIdentity | None
|
|
687
|
+
journal: _RollbackJournalObservation | None
|
|
688
|
+
stable: bool
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
@dataclass(frozen=True, slots=True)
|
|
692
|
+
class _SourceReadPolicy:
|
|
693
|
+
"""Sidecar policy derived from one stable source observation."""
|
|
694
|
+
|
|
695
|
+
journal: _RollbackJournalObservation | None
|
|
696
|
+
wal_present: bool
|
|
697
|
+
quarantined: bool
|
|
698
|
+
ignore_wal: bool
|
|
699
|
+
include_wal: bool
|
|
700
|
+
database_is_wal_format: bool
|
|
701
|
+
|
|
702
|
+
@property
|
|
703
|
+
def requires_private_snapshot(self):
|
|
704
|
+
"""Classify states unsuitable for the probe's direct lock-byte check.
|
|
705
|
+
|
|
706
|
+
Full viewer openers snapshot every source state. This narrower flag is
|
|
707
|
+
retained only so the isolated access probe never treats a journal- or
|
|
708
|
+
WAL-bearing source as a rollback-mode direct-lock candidate.
|
|
709
|
+
"""
|
|
710
|
+
return (
|
|
711
|
+
self.journal is not None
|
|
712
|
+
or self.wal_present
|
|
713
|
+
or self.database_is_wal_format
|
|
714
|
+
)
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
class _ManagedSQLiteConnection(sqlite3.Connection):
|
|
718
|
+
"""SQLite connection that owns its temporary database snapshot."""
|
|
719
|
+
|
|
720
|
+
_qplot_snapshot: tempfile.TemporaryDirectory | None = None
|
|
721
|
+
|
|
722
|
+
def attach_snapshot(self, snapshot):
|
|
723
|
+
self._qplot_snapshot = snapshot
|
|
724
|
+
|
|
725
|
+
def close(self):
|
|
726
|
+
try:
|
|
727
|
+
super().close()
|
|
728
|
+
finally:
|
|
729
|
+
if self._qplot_snapshot is not None:
|
|
730
|
+
self._qplot_snapshot.cleanup()
|
|
731
|
+
self._qplot_snapshot = None
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def set_qcodes_database_location(database_path):
|
|
735
|
+
"""Point QCoDeS at a database without initialising or upgrading it."""
|
|
736
|
+
import qcodes
|
|
737
|
+
|
|
738
|
+
qcodes.config.core.db_location = str(database_path)
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def quarantine_wal_for_replaced_database(database_path):
|
|
742
|
+
"""Keep an unpaired WAL out of a newly replaced database view.
|
|
743
|
+
|
|
744
|
+
Atomic replacement changes the main database file but can leave an old
|
|
745
|
+
writer's ``-wal`` sidecar behind. SQLite can otherwise combine those two
|
|
746
|
+
unrelated files and expose the old database. Once a replacement is
|
|
747
|
+
detected, qPlot captures a private main-only view while the retained
|
|
748
|
+
sidecar is unpaired. A sidecar can vanish and later be recreated by the old
|
|
749
|
+
writer, so a transient absence is not enough to prove a later WAL belongs
|
|
750
|
+
to the replacement. A generated database's matching token and nonce-linked
|
|
751
|
+
history can establish that a later WAL strictly descends from the
|
|
752
|
+
replacement and release this quarantine. qPlot never changes the source
|
|
753
|
+
database or any sidecar to make that happen.
|
|
754
|
+
|
|
755
|
+
A changed WAL identity is not enough to prove it belongs to the replacement
|
|
756
|
+
main file: an old writer can rotate or recreate its WAL after the main file
|
|
757
|
+
has been atomically replaced. Therefore the whole sidecar lifetime is
|
|
758
|
+
treated conservatively as ambiguous unless embedded lineage proves it.
|
|
759
|
+
"""
|
|
760
|
+
|
|
761
|
+
source_path = _resolved_database_path(database_path)
|
|
762
|
+
database_identity = database_file_identity(source_path)
|
|
763
|
+
with _DATABASE_INSTANCE_REGISTRY_LOCK:
|
|
764
|
+
_DATABASE_INSTANCE_REGISTRY[source_path] = (database_identity, True)
|
|
765
|
+
return database_identity is not None
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
def _release_wal_quarantine(database_path, expected_database_identity):
|
|
769
|
+
"""Trust a WAL whose embedded lineage proves it belongs to this main."""
|
|
770
|
+
source_path = _resolved_database_path(database_path)
|
|
771
|
+
with _DATABASE_INSTANCE_REGISTRY_LOCK:
|
|
772
|
+
database_identity = database_file_identity(source_path)
|
|
773
|
+
if database_identity != expected_database_identity:
|
|
774
|
+
return False
|
|
775
|
+
_DATABASE_INSTANCE_REGISTRY[source_path] = (database_identity, False)
|
|
776
|
+
return True
|
|
777
|
+
|
|
778
|
+
|
|
779
|
+
def _observe_database_instance(database_path):
|
|
780
|
+
"""Record replacement history for every source opened by qPlot.
|
|
781
|
+
|
|
782
|
+
The quarantine intentionally survives a momentary missing WAL and later
|
|
783
|
+
main-file replacements. An old writer can recreate either sidecar after a
|
|
784
|
+
transient gap, so absence alone cannot prove that a future WAL belongs to
|
|
785
|
+
the current main file.
|
|
786
|
+
"""
|
|
787
|
+
|
|
788
|
+
source_path = _resolved_database_path(database_path)
|
|
789
|
+
database_identity = database_file_identity(source_path)
|
|
790
|
+
with _DATABASE_INSTANCE_REGISTRY_LOCK:
|
|
791
|
+
prior = _DATABASE_INSTANCE_REGISTRY.get(source_path)
|
|
792
|
+
if prior is None:
|
|
793
|
+
quarantined = False
|
|
794
|
+
else:
|
|
795
|
+
prior_identity, quarantined = prior
|
|
796
|
+
if prior_identity is not None and prior_identity != database_identity:
|
|
797
|
+
quarantined = True
|
|
798
|
+
_DATABASE_INSTANCE_REGISTRY[source_path] = (
|
|
799
|
+
database_identity,
|
|
800
|
+
quarantined,
|
|
801
|
+
)
|
|
802
|
+
return database_identity, quarantined
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _require_expected_database_instance(database_path, expected_database_identity):
|
|
806
|
+
"""Reject a read whose source identity no longer matches its dataset key."""
|
|
807
|
+
|
|
808
|
+
if expected_database_identity is None:
|
|
809
|
+
return
|
|
810
|
+
database_identity, _quarantined = _observe_database_instance(database_path)
|
|
811
|
+
if database_identity == expected_database_identity:
|
|
812
|
+
return
|
|
813
|
+
quarantine_wal_for_replaced_database(database_path)
|
|
814
|
+
raise DatabaseInstanceChangedError(
|
|
815
|
+
"The database was replaced while qPlot was preparing a read."
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _require_prepared_database_instance(database_path, prepared_database_identity):
|
|
820
|
+
"""Bind publication-marker and sidecar decisions to one main-file identity."""
|
|
821
|
+
if database_file_identity(database_path) == prepared_database_identity:
|
|
822
|
+
return
|
|
823
|
+
quarantine_wal_for_replaced_database(database_path)
|
|
824
|
+
raise DatabaseInstanceChangedError(
|
|
825
|
+
"The database was replaced while qPlot was selecting a read-only view. "
|
|
826
|
+
"Refresh to retry; qPlot did not combine the replacement with a prior "
|
|
827
|
+
"SQLite sidecar."
|
|
828
|
+
)
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def configure_read_only_sqlite_connection(conn):
|
|
832
|
+
"""Keep read-only SQLite scans from growing qPlot's memory footprint."""
|
|
833
|
+
pragmas = (
|
|
834
|
+
"PRAGMA query_only = ON",
|
|
835
|
+
f"PRAGMA cache_size = -{SQLITE_READ_ONLY_CACHE_KIB}",
|
|
836
|
+
"PRAGMA temp_store = FILE",
|
|
837
|
+
)
|
|
838
|
+
for pragma in pragmas:
|
|
839
|
+
try:
|
|
840
|
+
conn.execute(pragma)
|
|
841
|
+
except Exception:
|
|
842
|
+
pass
|
|
843
|
+
return conn
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def _cleanup_failed_connection_open(conn, snapshot):
|
|
847
|
+
"""Release both provisional owners after any open/attachment failure."""
|
|
848
|
+
if conn is not None:
|
|
849
|
+
try:
|
|
850
|
+
conn.close()
|
|
851
|
+
except Exception:
|
|
852
|
+
pass
|
|
853
|
+
if snapshot is not None:
|
|
854
|
+
try:
|
|
855
|
+
snapshot.cleanup()
|
|
856
|
+
except Exception:
|
|
857
|
+
pass
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def qcodes_read_only_connection(
|
|
861
|
+
database_path,
|
|
862
|
+
*,
|
|
863
|
+
ignore_unpaired_wal=False,
|
|
864
|
+
expected_database_identity=None,
|
|
865
|
+
cancelled_callback=None,
|
|
866
|
+
deadline=None,
|
|
867
|
+
):
|
|
868
|
+
"""Open an AtomicConnection-compatible private database snapshot.
|
|
869
|
+
|
|
870
|
+
Every source state is copied under the system temporary directory before
|
|
871
|
+
SQLite is invoked. Immutable access, WAL handling, and rollback recovery
|
|
872
|
+
therefore occur only on qPlot's private files.
|
|
873
|
+
"""
|
|
874
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
875
|
+
source_path = _resolved_database_path(database_path)
|
|
876
|
+
for _attempt in range(2):
|
|
877
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
878
|
+
(
|
|
879
|
+
target_path,
|
|
880
|
+
immutable,
|
|
881
|
+
snapshot,
|
|
882
|
+
ignore_wal,
|
|
883
|
+
prepared_source,
|
|
884
|
+
) = _prepare_read_target(
|
|
885
|
+
source_path,
|
|
886
|
+
ignore_unpaired_wal=ignore_unpaired_wal,
|
|
887
|
+
expected_database_identity=expected_database_identity,
|
|
888
|
+
cancelled_callback=cancelled_callback,
|
|
889
|
+
deadline=deadline,
|
|
890
|
+
)
|
|
891
|
+
conn = None
|
|
892
|
+
try:
|
|
893
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
894
|
+
if not _prepared_source_is_current(
|
|
895
|
+
source_path,
|
|
896
|
+
prepared_source,
|
|
897
|
+
direct=snapshot is None,
|
|
898
|
+
ignore_wal=ignore_wal,
|
|
899
|
+
cancelled_callback=cancelled_callback,
|
|
900
|
+
deadline=deadline,
|
|
901
|
+
):
|
|
902
|
+
if snapshot is not None:
|
|
903
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
904
|
+
continue
|
|
905
|
+
_require_publication_complete(source_path)
|
|
906
|
+
conn = connect(
|
|
907
|
+
_qcodes_uri_path(target_path, immutable=immutable),
|
|
908
|
+
get_DB_debug(),
|
|
909
|
+
read_only=True,
|
|
910
|
+
cancelled_callback=cancelled_callback,
|
|
911
|
+
deadline=deadline,
|
|
912
|
+
)
|
|
913
|
+
_install_read_control_progress_handler(
|
|
914
|
+
conn,
|
|
915
|
+
cancelled_callback,
|
|
916
|
+
deadline,
|
|
917
|
+
)
|
|
918
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
919
|
+
_require_publication_complete(source_path)
|
|
920
|
+
_require_expected_database_instance(
|
|
921
|
+
source_path,
|
|
922
|
+
expected_database_identity,
|
|
923
|
+
)
|
|
924
|
+
if not _prepared_source_is_current(
|
|
925
|
+
source_path,
|
|
926
|
+
prepared_source,
|
|
927
|
+
direct=snapshot is None,
|
|
928
|
+
ignore_wal=ignore_wal,
|
|
929
|
+
cancelled_callback=cancelled_callback,
|
|
930
|
+
deadline=deadline,
|
|
931
|
+
):
|
|
932
|
+
_cleanup_failed_connection_open(conn, snapshot)
|
|
933
|
+
continue
|
|
934
|
+
_require_publication_complete(source_path)
|
|
935
|
+
conn.path_to_dbfile = str(source_path)
|
|
936
|
+
conn._qplot_source_activity_timestamp = _prepared_activity_timestamp(
|
|
937
|
+
prepared_source, ignore_wal=ignore_wal
|
|
938
|
+
)
|
|
939
|
+
if snapshot is not None:
|
|
940
|
+
_attach_snapshot_cleanup(conn, snapshot)
|
|
941
|
+
snapshot = None
|
|
942
|
+
configured = configure_read_only_sqlite_connection(conn)
|
|
943
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
944
|
+
return configured
|
|
945
|
+
except Exception:
|
|
946
|
+
_cleanup_failed_connection_open(conn, snapshot)
|
|
947
|
+
raise
|
|
948
|
+
|
|
949
|
+
raise ReadOnlyDatabaseAccessError(
|
|
950
|
+
"The database became busy while qPlot was opening a transaction-consistent "
|
|
951
|
+
"view. It is temporarily unavailable; refresh to retry. qPlot did not "
|
|
952
|
+
"modify the source database or its SQLite sidecars."
|
|
953
|
+
)
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def load_by_guid_read_only(
|
|
957
|
+
guid,
|
|
958
|
+
database_path=None,
|
|
959
|
+
*,
|
|
960
|
+
expected_database_identity=None,
|
|
961
|
+
cancelled_callback=None,
|
|
962
|
+
deadline=None,
|
|
963
|
+
):
|
|
964
|
+
"""Load a QCoDeS dataset by GUID through a read-only connection."""
|
|
965
|
+
if database_path is None:
|
|
966
|
+
database_path = get_DB_location()
|
|
967
|
+
conn = qcodes_read_only_connection(
|
|
968
|
+
database_path,
|
|
969
|
+
expected_database_identity=expected_database_identity,
|
|
970
|
+
cancelled_callback=cancelled_callback,
|
|
971
|
+
deadline=deadline,
|
|
972
|
+
)
|
|
973
|
+
connection_transferred = False
|
|
974
|
+
try:
|
|
975
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
976
|
+
result = load_by_guid(guid, conn=conn)
|
|
977
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
978
|
+
connection_transferred = result_owns_supplied_connection(result)
|
|
979
|
+
return result
|
|
980
|
+
except Exception:
|
|
981
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
982
|
+
raise
|
|
983
|
+
finally:
|
|
984
|
+
if not connection_transferred:
|
|
985
|
+
conn.close()
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def load_by_id_read_only(
|
|
989
|
+
run_id,
|
|
990
|
+
database_path=None,
|
|
991
|
+
*,
|
|
992
|
+
expected_database_identity=None,
|
|
993
|
+
cancelled_callback=None,
|
|
994
|
+
deadline=None,
|
|
995
|
+
):
|
|
996
|
+
"""Load a QCoDeS dataset by run ID through a read-only connection."""
|
|
997
|
+
if database_path is None:
|
|
998
|
+
database_path = get_DB_location()
|
|
999
|
+
conn = qcodes_read_only_connection(
|
|
1000
|
+
database_path,
|
|
1001
|
+
expected_database_identity=expected_database_identity,
|
|
1002
|
+
cancelled_callback=cancelled_callback,
|
|
1003
|
+
deadline=deadline,
|
|
1004
|
+
)
|
|
1005
|
+
connection_transferred = False
|
|
1006
|
+
try:
|
|
1007
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1008
|
+
result = load_by_id(run_id, conn=conn)
|
|
1009
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1010
|
+
connection_transferred = result_owns_supplied_connection(result)
|
|
1011
|
+
return result
|
|
1012
|
+
except Exception:
|
|
1013
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1014
|
+
raise
|
|
1015
|
+
finally:
|
|
1016
|
+
if not connection_transferred:
|
|
1017
|
+
conn.close()
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
def _resolved_database_path(database_path):
|
|
1021
|
+
return Path(database_path).resolve()
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
def _require_publication_complete(database_path):
|
|
1025
|
+
"""Reject a view while qPlot is replacing the database main file."""
|
|
1026
|
+
guard_path = database_publication_guard_path(database_path)
|
|
1027
|
+
try:
|
|
1028
|
+
guard_path.lstat()
|
|
1029
|
+
except FileNotFoundError:
|
|
1030
|
+
return
|
|
1031
|
+
except OSError as error:
|
|
1032
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1033
|
+
f"Could not prove that database publication is complete: {error}"
|
|
1034
|
+
) from error
|
|
1035
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1036
|
+
"Test-database publication is still in progress or needs recovery. "
|
|
1037
|
+
"If generation has stopped, follow its recovery instructions before "
|
|
1038
|
+
"removing the publication guard. qPlot did not open the database or "
|
|
1039
|
+
"its SQLite sidecars."
|
|
1040
|
+
)
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
def _sqlite_uri_path(database_path):
|
|
1044
|
+
"""Return an absolute, URI-escaped SQLite path without the ``file:`` prefix."""
|
|
1045
|
+
return _resolved_database_path(database_path).as_uri().removeprefix("file:")
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
def sqlite_read_only_uri(database_path, *, immutable=False):
|
|
1049
|
+
"""Build an exact-path SQLite URI with enforced read-only access."""
|
|
1050
|
+
uri = f"file:{_sqlite_uri_path(database_path)}?mode=ro"
|
|
1051
|
+
if immutable:
|
|
1052
|
+
uri += "&immutable=1"
|
|
1053
|
+
return uri
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def sqlite_read_only_connection(
|
|
1057
|
+
database_path,
|
|
1058
|
+
timeout=10,
|
|
1059
|
+
*,
|
|
1060
|
+
ignore_unpaired_wal=False,
|
|
1061
|
+
expected_database_identity=None,
|
|
1062
|
+
cancelled_callback=None,
|
|
1063
|
+
deadline=None,
|
|
1064
|
+
**kwargs,
|
|
1065
|
+
):
|
|
1066
|
+
"""Open a source-preserving SQLite connection to a private snapshot.
|
|
1067
|
+
|
|
1068
|
+
Every source state uses a consistency-checked private copy. SQLite can
|
|
1069
|
+
therefore use immutable access, follow a WAL, or recover a hot journal
|
|
1070
|
+
without opening or changing the source database and its sidecars.
|
|
1071
|
+
"""
|
|
1072
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1073
|
+
source_path = _resolved_database_path(database_path)
|
|
1074
|
+
kwargs.setdefault("factory", _ManagedSQLiteConnection)
|
|
1075
|
+
|
|
1076
|
+
for _attempt in range(2):
|
|
1077
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1078
|
+
(
|
|
1079
|
+
target_path,
|
|
1080
|
+
immutable,
|
|
1081
|
+
snapshot,
|
|
1082
|
+
ignore_wal,
|
|
1083
|
+
prepared_source,
|
|
1084
|
+
) = _prepare_read_target(
|
|
1085
|
+
source_path,
|
|
1086
|
+
ignore_unpaired_wal=ignore_unpaired_wal,
|
|
1087
|
+
expected_database_identity=expected_database_identity,
|
|
1088
|
+
cancelled_callback=cancelled_callback,
|
|
1089
|
+
deadline=deadline,
|
|
1090
|
+
)
|
|
1091
|
+
conn = None
|
|
1092
|
+
try:
|
|
1093
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1094
|
+
if not _prepared_source_is_current(
|
|
1095
|
+
source_path,
|
|
1096
|
+
prepared_source,
|
|
1097
|
+
direct=snapshot is None,
|
|
1098
|
+
ignore_wal=ignore_wal,
|
|
1099
|
+
cancelled_callback=cancelled_callback,
|
|
1100
|
+
deadline=deadline,
|
|
1101
|
+
):
|
|
1102
|
+
if snapshot is not None:
|
|
1103
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
1104
|
+
continue
|
|
1105
|
+
_require_publication_complete(source_path)
|
|
1106
|
+
conn = sqlite3.connect(
|
|
1107
|
+
sqlite_read_only_uri(target_path, immutable=immutable),
|
|
1108
|
+
timeout=_sqlite_timeout_before_deadline(timeout, deadline),
|
|
1109
|
+
uri=True,
|
|
1110
|
+
**kwargs,
|
|
1111
|
+
)
|
|
1112
|
+
_install_read_control_progress_handler(
|
|
1113
|
+
conn,
|
|
1114
|
+
cancelled_callback,
|
|
1115
|
+
deadline,
|
|
1116
|
+
)
|
|
1117
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1118
|
+
_require_publication_complete(source_path)
|
|
1119
|
+
_require_expected_database_instance(
|
|
1120
|
+
source_path,
|
|
1121
|
+
expected_database_identity,
|
|
1122
|
+
)
|
|
1123
|
+
if not _prepared_source_is_current(
|
|
1124
|
+
source_path,
|
|
1125
|
+
prepared_source,
|
|
1126
|
+
direct=snapshot is None,
|
|
1127
|
+
ignore_wal=ignore_wal,
|
|
1128
|
+
cancelled_callback=cancelled_callback,
|
|
1129
|
+
deadline=deadline,
|
|
1130
|
+
):
|
|
1131
|
+
_cleanup_failed_connection_open(conn, snapshot)
|
|
1132
|
+
continue
|
|
1133
|
+
_require_publication_complete(source_path)
|
|
1134
|
+
if snapshot is not None:
|
|
1135
|
+
attach_snapshot = getattr(conn, "attach_snapshot", None)
|
|
1136
|
+
if not callable(attach_snapshot):
|
|
1137
|
+
raise TypeError(
|
|
1138
|
+
"A custom SQLite connection factory used for a live "
|
|
1139
|
+
"database snapshot must provide attach_snapshot()."
|
|
1140
|
+
)
|
|
1141
|
+
attach_snapshot(snapshot)
|
|
1142
|
+
snapshot = None
|
|
1143
|
+
conn._qplot_source_activity_timestamp = _prepared_activity_timestamp(
|
|
1144
|
+
prepared_source, ignore_wal=ignore_wal
|
|
1145
|
+
)
|
|
1146
|
+
configured = configure_read_only_sqlite_connection(conn)
|
|
1147
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1148
|
+
return configured
|
|
1149
|
+
except Exception:
|
|
1150
|
+
_cleanup_failed_connection_open(conn, snapshot)
|
|
1151
|
+
raise
|
|
1152
|
+
|
|
1153
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1154
|
+
"The database became busy while qPlot was opening a transaction-consistent "
|
|
1155
|
+
"view. It is temporarily unavailable; refresh to retry. qPlot did not "
|
|
1156
|
+
"modify the source database or its SQLite sidecars."
|
|
1157
|
+
)
|
|
1158
|
+
|
|
1159
|
+
|
|
1160
|
+
def _probe_sqlite_user_version(
|
|
1161
|
+
database_path,
|
|
1162
|
+
*,
|
|
1163
|
+
cancelled_callback=None,
|
|
1164
|
+
deadline=None,
|
|
1165
|
+
):
|
|
1166
|
+
"""Perform one bounded immutable read without ever following sidecars.
|
|
1167
|
+
|
|
1168
|
+
Lock availability is checked separately through the platform lock-byte
|
|
1169
|
+
protocol when the isolated access-probe child requests it. Keeping this
|
|
1170
|
+
SQLite connection immutable prevents a DELETE-to-WAL race from creating a
|
|
1171
|
+
source ``-shm`` file between signature observation and connection opening.
|
|
1172
|
+
"""
|
|
1173
|
+
connection = None
|
|
1174
|
+
try:
|
|
1175
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1176
|
+
connection = sqlite3.connect(
|
|
1177
|
+
sqlite_read_only_uri(database_path, immutable=True),
|
|
1178
|
+
timeout=_sqlite_timeout_before_deadline(1, deadline),
|
|
1179
|
+
uri=True,
|
|
1180
|
+
)
|
|
1181
|
+
_install_read_control_progress_handler(
|
|
1182
|
+
connection,
|
|
1183
|
+
cancelled_callback,
|
|
1184
|
+
deadline,
|
|
1185
|
+
)
|
|
1186
|
+
connection.execute("SELECT 1 FROM sqlite_schema LIMIT 1").fetchone()
|
|
1187
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1188
|
+
except sqlite3.Error:
|
|
1189
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1190
|
+
raise
|
|
1191
|
+
finally:
|
|
1192
|
+
if connection is not None:
|
|
1193
|
+
connection.close()
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
def probe_read_only_database(
|
|
1197
|
+
database_path,
|
|
1198
|
+
*,
|
|
1199
|
+
ignore_unpaired_wal=False,
|
|
1200
|
+
expected_database_identity=None,
|
|
1201
|
+
cancelled_callback=None,
|
|
1202
|
+
deadline=None,
|
|
1203
|
+
_check_sqlite_lock_bytes=False,
|
|
1204
|
+
):
|
|
1205
|
+
"""Promptly check access and identity without building a full snapshot.
|
|
1206
|
+
|
|
1207
|
+
The real opener remains responsible for transaction-consistent snapshot
|
|
1208
|
+
creation and generated-WAL provenance. This probe deliberately touches
|
|
1209
|
+
only bounded regions of source artifacts and opens SQLite immutable, so a
|
|
1210
|
+
concurrent transition to WAL cannot create source sidecars. The isolated
|
|
1211
|
+
access-check child can additionally request a platform lock-byte check for
|
|
1212
|
+
a sidecar-free rollback-mode source.
|
|
1213
|
+
"""
|
|
1214
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1215
|
+
source_path = _resolved_database_path(database_path)
|
|
1216
|
+
_require_publication_complete(source_path)
|
|
1217
|
+
if not source_path.is_file():
|
|
1218
|
+
raise FileNotFoundError(source_path)
|
|
1219
|
+
|
|
1220
|
+
_require_expected_database_instance(
|
|
1221
|
+
source_path,
|
|
1222
|
+
expected_database_identity,
|
|
1223
|
+
)
|
|
1224
|
+
prepared_database_identity = database_file_identity(source_path)
|
|
1225
|
+
if prepared_database_identity is None:
|
|
1226
|
+
raise FileNotFoundError(source_path)
|
|
1227
|
+
|
|
1228
|
+
try:
|
|
1229
|
+
generation_marker = database_has_qplot_generation_marker(source_path)
|
|
1230
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
1231
|
+
raise
|
|
1232
|
+
except OSError as error:
|
|
1233
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1234
|
+
f"Could not inspect the database publication marker: {error}"
|
|
1235
|
+
) from error
|
|
1236
|
+
|
|
1237
|
+
for _attempt in range(WAL_SNAPSHOT_ATTEMPTS):
|
|
1238
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1239
|
+
_require_expected_database_instance(
|
|
1240
|
+
source_path,
|
|
1241
|
+
expected_database_identity,
|
|
1242
|
+
)
|
|
1243
|
+
try:
|
|
1244
|
+
before = _source_signature_with_controls(
|
|
1245
|
+
source_path,
|
|
1246
|
+
cancelled_callback,
|
|
1247
|
+
deadline,
|
|
1248
|
+
)
|
|
1249
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
1250
|
+
raise
|
|
1251
|
+
except OSError as error:
|
|
1252
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1253
|
+
f"Could not inspect a read-only view of {source_path}: {error}"
|
|
1254
|
+
) from error
|
|
1255
|
+
if before.database is None:
|
|
1256
|
+
raise FileNotFoundError(source_path)
|
|
1257
|
+
if not before.stable:
|
|
1258
|
+
continue
|
|
1259
|
+
|
|
1260
|
+
policy = _source_read_policy(
|
|
1261
|
+
source_path,
|
|
1262
|
+
before,
|
|
1263
|
+
generation_marker=generation_marker,
|
|
1264
|
+
ignore_unpaired_wal=ignore_unpaired_wal,
|
|
1265
|
+
)
|
|
1266
|
+
if policy.include_wal and not generation_marker:
|
|
1267
|
+
_require_expected_database_instance(
|
|
1268
|
+
source_path,
|
|
1269
|
+
expected_database_identity,
|
|
1270
|
+
)
|
|
1271
|
+
_require_prepared_database_instance(
|
|
1272
|
+
source_path,
|
|
1273
|
+
prepared_database_identity,
|
|
1274
|
+
)
|
|
1275
|
+
after = _source_signature_for_validation(
|
|
1276
|
+
source_path,
|
|
1277
|
+
cancelled_callback=cancelled_callback,
|
|
1278
|
+
deadline=deadline,
|
|
1279
|
+
)
|
|
1280
|
+
if after != before or not after.stable:
|
|
1281
|
+
continue
|
|
1282
|
+
raise _unverifiable_unmarked_wal_error(source_path)
|
|
1283
|
+
|
|
1284
|
+
if not _bounded_artifact_is_current(
|
|
1285
|
+
source_path,
|
|
1286
|
+
before.database,
|
|
1287
|
+
before.database_identity,
|
|
1288
|
+
cancelled_callback=cancelled_callback,
|
|
1289
|
+
deadline=deadline,
|
|
1290
|
+
):
|
|
1291
|
+
continue
|
|
1292
|
+
if (
|
|
1293
|
+
before.wal is not None
|
|
1294
|
+
and not _bounded_artifact_is_current(
|
|
1295
|
+
_wal_path(source_path),
|
|
1296
|
+
before.wal,
|
|
1297
|
+
before.wal_identity,
|
|
1298
|
+
cancelled_callback=cancelled_callback,
|
|
1299
|
+
deadline=deadline,
|
|
1300
|
+
)
|
|
1301
|
+
):
|
|
1302
|
+
continue
|
|
1303
|
+
|
|
1304
|
+
def source_is_still_current(expected_source=before):
|
|
1305
|
+
_require_publication_complete(source_path)
|
|
1306
|
+
_require_expected_database_instance(
|
|
1307
|
+
source_path,
|
|
1308
|
+
expected_database_identity,
|
|
1309
|
+
)
|
|
1310
|
+
_require_prepared_database_instance(
|
|
1311
|
+
source_path,
|
|
1312
|
+
prepared_database_identity,
|
|
1313
|
+
)
|
|
1314
|
+
after = _source_signature_for_validation(
|
|
1315
|
+
source_path,
|
|
1316
|
+
cancelled_callback=cancelled_callback,
|
|
1317
|
+
deadline=deadline,
|
|
1318
|
+
)
|
|
1319
|
+
return after == expected_source and after.stable
|
|
1320
|
+
|
|
1321
|
+
sqlite_error = None
|
|
1322
|
+
try:
|
|
1323
|
+
_probe_sqlite_user_version(
|
|
1324
|
+
source_path,
|
|
1325
|
+
cancelled_callback=cancelled_callback,
|
|
1326
|
+
deadline=deadline,
|
|
1327
|
+
)
|
|
1328
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
1329
|
+
raise
|
|
1330
|
+
except sqlite3.Error as error:
|
|
1331
|
+
sqlite_error = error
|
|
1332
|
+
|
|
1333
|
+
# This is deliberately the last full validation before a platform
|
|
1334
|
+
# lock-byte check. On POSIX, closing *any* descriptor for an inode drops
|
|
1335
|
+
# all process-owned fcntl locks for that inode, so no helper that opens
|
|
1336
|
+
# the main file may run after the lock context is entered.
|
|
1337
|
+
if not source_is_still_current():
|
|
1338
|
+
continue
|
|
1339
|
+
if sqlite_error is not None:
|
|
1340
|
+
raise sqlite_error
|
|
1341
|
+
|
|
1342
|
+
if (
|
|
1343
|
+
not _check_sqlite_lock_bytes
|
|
1344
|
+
or policy.requires_private_snapshot
|
|
1345
|
+
):
|
|
1346
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1347
|
+
return
|
|
1348
|
+
|
|
1349
|
+
try:
|
|
1350
|
+
with _sqlite_shared_lock_bytes(
|
|
1351
|
+
source_path,
|
|
1352
|
+
cancelled_callback=cancelled_callback,
|
|
1353
|
+
deadline=deadline,
|
|
1354
|
+
) as database_file:
|
|
1355
|
+
_require_publication_complete(source_path)
|
|
1356
|
+
_require_expected_database_instance(
|
|
1357
|
+
source_path,
|
|
1358
|
+
expected_database_identity,
|
|
1359
|
+
)
|
|
1360
|
+
_require_prepared_database_instance(
|
|
1361
|
+
source_path,
|
|
1362
|
+
prepared_database_identity,
|
|
1363
|
+
)
|
|
1364
|
+
if not _source_matches_under_sqlite_lock(
|
|
1365
|
+
source_path,
|
|
1366
|
+
before,
|
|
1367
|
+
database_file,
|
|
1368
|
+
cancelled_callback=cancelled_callback,
|
|
1369
|
+
deadline=deadline,
|
|
1370
|
+
):
|
|
1371
|
+
continue
|
|
1372
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1373
|
+
return
|
|
1374
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
1375
|
+
raise
|
|
1376
|
+
except sqlite3.Error:
|
|
1377
|
+
# A concurrent journal-mode or file transition can look exactly
|
|
1378
|
+
# like lock contention. Revalidate after releasing any partial OS
|
|
1379
|
+
# locks; reject the lock only when the source is still identical.
|
|
1380
|
+
if not source_is_still_current():
|
|
1381
|
+
continue
|
|
1382
|
+
raise
|
|
1383
|
+
|
|
1384
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1385
|
+
"The database main file or SQLite sidecars changed continuously while "
|
|
1386
|
+
"qPlot checked read-only access. The database is busy or temporarily "
|
|
1387
|
+
"unavailable; refresh to retry. qPlot did not modify the source database "
|
|
1388
|
+
"or its SQLite sidecars."
|
|
1389
|
+
)
|
|
1390
|
+
|
|
1391
|
+
|
|
1392
|
+
def _qcodes_uri_path(database_path, *, immutable):
|
|
1393
|
+
"""Build the URI path expected by QCoDeS' URI-constructing helper.
|
|
1394
|
+
|
|
1395
|
+
QCoDeS adds ``file:`` and a final ``?mode=ro`` itself. For immutable
|
|
1396
|
+
access, the dummy final parameter absorbs that second question mark while
|
|
1397
|
+
retaining the earlier, valid ``mode=ro`` and ``immutable=1`` options.
|
|
1398
|
+
"""
|
|
1399
|
+
path = _sqlite_uri_path(database_path)
|
|
1400
|
+
if immutable:
|
|
1401
|
+
return f"{path}?mode=ro&immutable=1&qplot="
|
|
1402
|
+
return path
|
|
1403
|
+
|
|
1404
|
+
|
|
1405
|
+
def _wal_path(database_path):
|
|
1406
|
+
return Path(f"{database_path}-wal")
|
|
1407
|
+
|
|
1408
|
+
|
|
1409
|
+
def _journal_path(database_path):
|
|
1410
|
+
return Path(f"{database_path}-journal")
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
def replacement_wal_is_quarantined(database_path):
|
|
1414
|
+
"""Return whether an unpaired WAL must be omitted from this read.
|
|
1415
|
+
|
|
1416
|
+
Replacement history is path-local and intentionally carries forward to a
|
|
1417
|
+
later main-file identity. A changed sidecar or a transient missing sidecar
|
|
1418
|
+
cannot prove it belongs to the current main file.
|
|
1419
|
+
"""
|
|
1420
|
+
|
|
1421
|
+
_database_identity, quarantined = _observe_database_instance(database_path)
|
|
1422
|
+
return bool(quarantined)
|
|
1423
|
+
|
|
1424
|
+
|
|
1425
|
+
def _prepared_activity_timestamp(source, *, ignore_wal):
|
|
1426
|
+
"""Keep source activity bound to the validated copy, before SQLite replay.
|
|
1427
|
+
|
|
1428
|
+
Snapshot creation/recovery timestamps are never acquisition timestamps.
|
|
1429
|
+
Omitted WAL files do not contribute evidence for the copied database.
|
|
1430
|
+
"""
|
|
1431
|
+
signatures = (source.database, None if ignore_wal else source.wal)
|
|
1432
|
+
return max(
|
|
1433
|
+
(signature[3] / 1_000_000_000 for signature in signatures
|
|
1434
|
+
if signature is not None and signature[2]),
|
|
1435
|
+
default=None,
|
|
1436
|
+
)
|
|
1437
|
+
|
|
1438
|
+
|
|
1439
|
+
def _stat_signature(stat_result):
|
|
1440
|
+
return (
|
|
1441
|
+
stat_result.st_dev,
|
|
1442
|
+
stat_result.st_ino,
|
|
1443
|
+
stat_result.st_size,
|
|
1444
|
+
stat_result.st_mtime_ns,
|
|
1445
|
+
stat_result.st_ctime_ns,
|
|
1446
|
+
)
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
def _file_signature(path):
|
|
1450
|
+
try:
|
|
1451
|
+
stat_result = path.stat()
|
|
1452
|
+
except FileNotFoundError:
|
|
1453
|
+
return None
|
|
1454
|
+
# Snapshot artifacts are files, never streams. Inspect before any header
|
|
1455
|
+
# read or copy so a FIFO sidecar cannot block past the read deadline.
|
|
1456
|
+
if not stat.S_ISREG(stat_result.st_mode):
|
|
1457
|
+
raise OSError(f"The database artifact is not a regular file: {path}")
|
|
1458
|
+
return _stat_signature(stat_result)
|
|
1459
|
+
|
|
1460
|
+
|
|
1461
|
+
def _file_prefix_observation(
|
|
1462
|
+
path,
|
|
1463
|
+
byte_count,
|
|
1464
|
+
*,
|
|
1465
|
+
include_trailer=False,
|
|
1466
|
+
cancelled_callback=None,
|
|
1467
|
+
deadline=None,
|
|
1468
|
+
):
|
|
1469
|
+
"""Read identifying bytes while proving which path instance supplied them."""
|
|
1470
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1471
|
+
path_identity_before = path_bound_file_identity(path)
|
|
1472
|
+
path_signature_before = _file_signature(path)
|
|
1473
|
+
if path_signature_before is None:
|
|
1474
|
+
return None, None, b"", b"", True
|
|
1475
|
+
try:
|
|
1476
|
+
with path.open("rb") as handle:
|
|
1477
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1478
|
+
descriptor_identity_before = open_file_identity(handle.fileno())
|
|
1479
|
+
descriptor_before = _stat_signature(os.fstat(handle.fileno()))
|
|
1480
|
+
prefix = handle.read(byte_count)
|
|
1481
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1482
|
+
trailer = b""
|
|
1483
|
+
if include_trailer and descriptor_before[2] >= 16:
|
|
1484
|
+
handle.seek(-16, os.SEEK_END)
|
|
1485
|
+
trailer = handle.read(16)
|
|
1486
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1487
|
+
descriptor_after = _stat_signature(os.fstat(handle.fileno()))
|
|
1488
|
+
descriptor_identity_after = open_file_identity(handle.fileno())
|
|
1489
|
+
except FileNotFoundError:
|
|
1490
|
+
return path_signature_before, path_identity_before, b"", b"", False
|
|
1491
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1492
|
+
path_signature_after = _file_signature(path)
|
|
1493
|
+
path_identity_after = path_bound_file_identity(path)
|
|
1494
|
+
stable = _path_and_descriptor_observations_match(
|
|
1495
|
+
path_signature_before,
|
|
1496
|
+
path_identity_before,
|
|
1497
|
+
descriptor_before,
|
|
1498
|
+
descriptor_identity_before,
|
|
1499
|
+
descriptor_after,
|
|
1500
|
+
descriptor_identity_after,
|
|
1501
|
+
path_signature_after,
|
|
1502
|
+
path_identity_after,
|
|
1503
|
+
)
|
|
1504
|
+
observed_signature = (
|
|
1505
|
+
path_signature_after
|
|
1506
|
+
if path_signature_after is not None
|
|
1507
|
+
else path_signature_before
|
|
1508
|
+
)
|
|
1509
|
+
return observed_signature, path_identity_after, prefix, trailer, stable
|
|
1510
|
+
|
|
1511
|
+
|
|
1512
|
+
def _path_and_descriptor_observations_match(
|
|
1513
|
+
path_signature_before,
|
|
1514
|
+
path_identity_before,
|
|
1515
|
+
descriptor_signature_before,
|
|
1516
|
+
descriptor_identity_before,
|
|
1517
|
+
descriptor_signature_after,
|
|
1518
|
+
descriptor_identity_after,
|
|
1519
|
+
path_signature_after,
|
|
1520
|
+
path_identity_after,
|
|
1521
|
+
):
|
|
1522
|
+
"""Compare path and descriptor observations without mixing stat domains."""
|
|
1523
|
+
signatures = (
|
|
1524
|
+
path_signature_before,
|
|
1525
|
+
descriptor_signature_before,
|
|
1526
|
+
descriptor_signature_after,
|
|
1527
|
+
path_signature_after,
|
|
1528
|
+
)
|
|
1529
|
+
if any(signature is None for signature in signatures):
|
|
1530
|
+
return False
|
|
1531
|
+
|
|
1532
|
+
stable_metadata = (
|
|
1533
|
+
path_signature_before == path_signature_after
|
|
1534
|
+
and descriptor_signature_before == descriptor_signature_after
|
|
1535
|
+
and path_signature_before[2:4] == descriptor_signature_before[2:4]
|
|
1536
|
+
and path_signature_after[2:4] == descriptor_signature_after[2:4]
|
|
1537
|
+
)
|
|
1538
|
+
identities = (
|
|
1539
|
+
path_identity_before,
|
|
1540
|
+
descriptor_identity_before,
|
|
1541
|
+
descriptor_identity_after,
|
|
1542
|
+
path_identity_after,
|
|
1543
|
+
)
|
|
1544
|
+
if all(identity is not None for identity in identities):
|
|
1545
|
+
return stable_metadata and len(set(identities)) == 1
|
|
1546
|
+
if any(identity is not None for identity in identities):
|
|
1547
|
+
return False
|
|
1548
|
+
return len(set(signatures)) == 1
|
|
1549
|
+
|
|
1550
|
+
|
|
1551
|
+
def _bounded_artifact_is_current(
|
|
1552
|
+
artifact_path,
|
|
1553
|
+
expected_signature,
|
|
1554
|
+
expected_identity,
|
|
1555
|
+
*,
|
|
1556
|
+
cancelled_callback=None,
|
|
1557
|
+
deadline=None,
|
|
1558
|
+
):
|
|
1559
|
+
"""Touch an artifact's beginning and end while retaining its identity."""
|
|
1560
|
+
signature, identity, _prefix, _trailer, stable = _file_prefix_observation(
|
|
1561
|
+
artifact_path,
|
|
1562
|
+
_WAL_HEADER_BYTES,
|
|
1563
|
+
include_trailer=True,
|
|
1564
|
+
cancelled_callback=cancelled_callback,
|
|
1565
|
+
deadline=deadline,
|
|
1566
|
+
)
|
|
1567
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1568
|
+
return (
|
|
1569
|
+
stable
|
|
1570
|
+
and signature == expected_signature
|
|
1571
|
+
and identity == expected_identity
|
|
1572
|
+
)
|
|
1573
|
+
|
|
1574
|
+
|
|
1575
|
+
def _rollback_journal_observation(
|
|
1576
|
+
database_path,
|
|
1577
|
+
*,
|
|
1578
|
+
cancelled_callback=None,
|
|
1579
|
+
deadline=None,
|
|
1580
|
+
):
|
|
1581
|
+
journal_path = _journal_path(database_path)
|
|
1582
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1583
|
+
signature, identity, prefix, trailer, stable = _file_prefix_observation(
|
|
1584
|
+
journal_path,
|
|
1585
|
+
_ROLLBACK_JOURNAL_PREFIX_BYTES,
|
|
1586
|
+
include_trailer=True,
|
|
1587
|
+
cancelled_callback=cancelled_callback,
|
|
1588
|
+
deadline=deadline,
|
|
1589
|
+
)
|
|
1590
|
+
if signature is None:
|
|
1591
|
+
return None
|
|
1592
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1593
|
+
return _RollbackJournalObservation(
|
|
1594
|
+
file_signature=signature,
|
|
1595
|
+
file_identity=identity,
|
|
1596
|
+
prefix=prefix,
|
|
1597
|
+
trailer=trailer,
|
|
1598
|
+
stable=stable,
|
|
1599
|
+
)
|
|
1600
|
+
|
|
1601
|
+
|
|
1602
|
+
def _wal_observation(
|
|
1603
|
+
database_path,
|
|
1604
|
+
*,
|
|
1605
|
+
cancelled_callback=None,
|
|
1606
|
+
deadline=None,
|
|
1607
|
+
):
|
|
1608
|
+
"""Observe one path-bound WAL instance without opening it through SQLite."""
|
|
1609
|
+
wal_path = _wal_path(database_path)
|
|
1610
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1611
|
+
identity_before = path_bound_file_identity(wal_path)
|
|
1612
|
+
signature_before = _file_signature(wal_path)
|
|
1613
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1614
|
+
signature_after = _file_signature(wal_path)
|
|
1615
|
+
identity_after = path_bound_file_identity(wal_path)
|
|
1616
|
+
return (
|
|
1617
|
+
signature_after,
|
|
1618
|
+
identity_after,
|
|
1619
|
+
identity_before == identity_after and signature_before == signature_after,
|
|
1620
|
+
)
|
|
1621
|
+
|
|
1622
|
+
|
|
1623
|
+
def _source_signature(
|
|
1624
|
+
database_path,
|
|
1625
|
+
*,
|
|
1626
|
+
cancelled_callback=None,
|
|
1627
|
+
deadline=None,
|
|
1628
|
+
):
|
|
1629
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1630
|
+
(
|
|
1631
|
+
database_signature,
|
|
1632
|
+
database_identity,
|
|
1633
|
+
database_header,
|
|
1634
|
+
_trailer,
|
|
1635
|
+
database_stable,
|
|
1636
|
+
) = (
|
|
1637
|
+
_file_prefix_observation(
|
|
1638
|
+
database_path,
|
|
1639
|
+
100,
|
|
1640
|
+
cancelled_callback=cancelled_callback,
|
|
1641
|
+
deadline=deadline,
|
|
1642
|
+
)
|
|
1643
|
+
)
|
|
1644
|
+
wal_signature, wal_identity, wal_stable = _wal_observation(
|
|
1645
|
+
database_path,
|
|
1646
|
+
cancelled_callback=cancelled_callback,
|
|
1647
|
+
deadline=deadline,
|
|
1648
|
+
)
|
|
1649
|
+
journal = _rollback_journal_observation(
|
|
1650
|
+
database_path,
|
|
1651
|
+
cancelled_callback=cancelled_callback,
|
|
1652
|
+
deadline=deadline,
|
|
1653
|
+
)
|
|
1654
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1655
|
+
return _SourceSignature(
|
|
1656
|
+
database=database_signature,
|
|
1657
|
+
database_identity=database_identity,
|
|
1658
|
+
database_header=database_header,
|
|
1659
|
+
wal=wal_signature,
|
|
1660
|
+
wal_identity=wal_identity,
|
|
1661
|
+
journal=journal,
|
|
1662
|
+
stable=(
|
|
1663
|
+
database_stable
|
|
1664
|
+
and wal_stable
|
|
1665
|
+
and (journal is None or journal.stable)
|
|
1666
|
+
),
|
|
1667
|
+
)
|
|
1668
|
+
|
|
1669
|
+
|
|
1670
|
+
def _source_signature_with_controls(
|
|
1671
|
+
database_path,
|
|
1672
|
+
cancelled_callback=None,
|
|
1673
|
+
deadline=None,
|
|
1674
|
+
):
|
|
1675
|
+
"""Keep no-control instrumentation compatible with the legacy call shape."""
|
|
1676
|
+
if cancelled_callback is None and deadline is None:
|
|
1677
|
+
return _source_signature(database_path)
|
|
1678
|
+
return _source_signature(
|
|
1679
|
+
database_path,
|
|
1680
|
+
cancelled_callback=cancelled_callback,
|
|
1681
|
+
deadline=deadline,
|
|
1682
|
+
)
|
|
1683
|
+
|
|
1684
|
+
|
|
1685
|
+
def _signature_requires_private_snapshot(signature, *, ignore_wal):
|
|
1686
|
+
return (
|
|
1687
|
+
not signature.stable
|
|
1688
|
+
or signature.journal is not None
|
|
1689
|
+
or (signature.wal is not None and not ignore_wal)
|
|
1690
|
+
)
|
|
1691
|
+
|
|
1692
|
+
|
|
1693
|
+
def _source_read_policy(
|
|
1694
|
+
database_path,
|
|
1695
|
+
signature,
|
|
1696
|
+
*,
|
|
1697
|
+
generation_marker,
|
|
1698
|
+
ignore_unpaired_wal,
|
|
1699
|
+
):
|
|
1700
|
+
"""Apply the shared fail-closed sidecar policy to one observation."""
|
|
1701
|
+
journal = signature.journal
|
|
1702
|
+
wal_present = signature.wal is not None
|
|
1703
|
+
empty_wal = wal_present and signature.wal[2] == 0
|
|
1704
|
+
quarantined = (
|
|
1705
|
+
replacement_wal_is_quarantined(database_path)
|
|
1706
|
+
if wal_present or journal is not None
|
|
1707
|
+
else False
|
|
1708
|
+
)
|
|
1709
|
+
ignore_wal = wal_present and (
|
|
1710
|
+
empty_wal
|
|
1711
|
+
or (
|
|
1712
|
+
not generation_marker
|
|
1713
|
+
and (ignore_unpaired_wal or quarantined)
|
|
1714
|
+
)
|
|
1715
|
+
)
|
|
1716
|
+
include_wal = wal_present and not ignore_wal
|
|
1717
|
+
|
|
1718
|
+
if journal is not None and journal.potentially_hot and include_wal:
|
|
1719
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1720
|
+
"The database has both an active-looking rollback journal and "
|
|
1721
|
+
"a WAL, so qPlot cannot prove which transaction state is valid. "
|
|
1722
|
+
"The database is busy or temporarily unavailable; finish the "
|
|
1723
|
+
"writer transaction and refresh. qPlot did not modify the source "
|
|
1724
|
+
"database or its SQLite sidecars."
|
|
1725
|
+
)
|
|
1726
|
+
if journal is not None and journal.potentially_hot and quarantined:
|
|
1727
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1728
|
+
"The rollback journal cannot be paired with the database instance "
|
|
1729
|
+
"that replaced the previously loaded file. The database is busy "
|
|
1730
|
+
"or temporarily unavailable; close the old writer and refresh. "
|
|
1731
|
+
"qPlot did not modify the source database or its SQLite sidecars."
|
|
1732
|
+
)
|
|
1733
|
+
if journal is not None and _journal_names_super_journal(journal):
|
|
1734
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1735
|
+
"The rollback journal belongs to a multi-database transaction, "
|
|
1736
|
+
"which qPlot cannot recover without following files outside its "
|
|
1737
|
+
"private snapshot. The database is temporarily unavailable; "
|
|
1738
|
+
"finish that transaction and refresh. qPlot did not modify the "
|
|
1739
|
+
"source database or its SQLite sidecars."
|
|
1740
|
+
)
|
|
1741
|
+
|
|
1742
|
+
return _SourceReadPolicy(
|
|
1743
|
+
journal=journal,
|
|
1744
|
+
wal_present=wal_present,
|
|
1745
|
+
quarantined=quarantined,
|
|
1746
|
+
ignore_wal=ignore_wal,
|
|
1747
|
+
include_wal=include_wal,
|
|
1748
|
+
database_is_wal_format=(
|
|
1749
|
+
signature.database_header[18:20] == b"\x02\x02"
|
|
1750
|
+
),
|
|
1751
|
+
)
|
|
1752
|
+
|
|
1753
|
+
|
|
1754
|
+
def _prepared_source_is_current(
|
|
1755
|
+
database_path,
|
|
1756
|
+
prepared_source,
|
|
1757
|
+
*,
|
|
1758
|
+
direct,
|
|
1759
|
+
ignore_wal,
|
|
1760
|
+
cancelled_callback=None,
|
|
1761
|
+
deadline=None,
|
|
1762
|
+
):
|
|
1763
|
+
"""Validate the exact main/journal instance through connection opening."""
|
|
1764
|
+
try:
|
|
1765
|
+
current_source = _source_signature_with_controls(
|
|
1766
|
+
database_path,
|
|
1767
|
+
cancelled_callback,
|
|
1768
|
+
deadline,
|
|
1769
|
+
)
|
|
1770
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
1771
|
+
raise
|
|
1772
|
+
except OSError as error:
|
|
1773
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1774
|
+
f"Could not validate the database instance and sidecars: {error}"
|
|
1775
|
+
) from error
|
|
1776
|
+
if current_source.database_identity != prepared_source.database_identity:
|
|
1777
|
+
quarantine_wal_for_replaced_database(database_path)
|
|
1778
|
+
raise DatabaseInstanceChangedError(
|
|
1779
|
+
"The database was replaced while qPlot was opening its read-only view."
|
|
1780
|
+
)
|
|
1781
|
+
return (
|
|
1782
|
+
current_source == prepared_source
|
|
1783
|
+
and (
|
|
1784
|
+
not direct
|
|
1785
|
+
or not _signature_requires_private_snapshot(
|
|
1786
|
+
current_source,
|
|
1787
|
+
ignore_wal=ignore_wal,
|
|
1788
|
+
)
|
|
1789
|
+
)
|
|
1790
|
+
)
|
|
1791
|
+
|
|
1792
|
+
|
|
1793
|
+
def _source_signature_for_validation(
|
|
1794
|
+
database_path,
|
|
1795
|
+
*,
|
|
1796
|
+
cancelled_callback=None,
|
|
1797
|
+
deadline=None,
|
|
1798
|
+
):
|
|
1799
|
+
"""Read a validation signature while keeping filesystem errors explicit."""
|
|
1800
|
+
try:
|
|
1801
|
+
return _source_signature_with_controls(
|
|
1802
|
+
database_path,
|
|
1803
|
+
cancelled_callback,
|
|
1804
|
+
deadline,
|
|
1805
|
+
)
|
|
1806
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
1807
|
+
raise
|
|
1808
|
+
except OSError as error:
|
|
1809
|
+
raise ReadOnlyDatabaseAccessError(
|
|
1810
|
+
f"Could not validate a read-only copy of {database_path}: {error}"
|
|
1811
|
+
) from error
|
|
1812
|
+
|
|
1813
|
+
|
|
1814
|
+
def _generation_provenance(connection):
|
|
1815
|
+
application_id = connection.execute("PRAGMA application_id").fetchone()
|
|
1816
|
+
row = connection.execute(
|
|
1817
|
+
f"SELECT generation_token, write_epoch "
|
|
1818
|
+
f"FROM {QPLOT_GENERATION_PROVENANCE_TABLE} WHERE singleton = 1"
|
|
1819
|
+
).fetchone()
|
|
1820
|
+
extra_row = connection.execute(
|
|
1821
|
+
f"SELECT 1 FROM {QPLOT_GENERATION_PROVENANCE_TABLE} "
|
|
1822
|
+
"WHERE singleton != 1 LIMIT 1"
|
|
1823
|
+
).fetchone()
|
|
1824
|
+
try:
|
|
1825
|
+
token_bytes = (
|
|
1826
|
+
bytes.fromhex(row[0])
|
|
1827
|
+
if row is not None and isinstance(row[0], str)
|
|
1828
|
+
else b""
|
|
1829
|
+
)
|
|
1830
|
+
except ValueError:
|
|
1831
|
+
token_bytes = b""
|
|
1832
|
+
if (
|
|
1833
|
+
application_id != (QPLOT_GENERATED_DATABASE_APPLICATION_ID,)
|
|
1834
|
+
or row is None
|
|
1835
|
+
or extra_row is not None
|
|
1836
|
+
or len(token_bytes) != QPLOT_GENERATION_PROVENANCE_TOKEN_BYTES
|
|
1837
|
+
or not isinstance(row[1], int)
|
|
1838
|
+
or row[1] < 0
|
|
1839
|
+
):
|
|
1840
|
+
raise ValueError("invalid qPlot generation provenance")
|
|
1841
|
+
return row
|
|
1842
|
+
|
|
1843
|
+
|
|
1844
|
+
def _generation_lineage_state(connection):
|
|
1845
|
+
"""Read and structurally validate the branch-bound provenance head."""
|
|
1846
|
+
state = connection.execute(
|
|
1847
|
+
f"SELECT singleton, format_version, generation_token, head_sequence, "
|
|
1848
|
+
f"head_nonce, window_size FROM {QPLOT_GENERATION_LINEAGE_STATE_TABLE}"
|
|
1849
|
+
).fetchall()
|
|
1850
|
+
if len(state) != 1:
|
|
1851
|
+
raise ValueError("invalid qPlot generation lineage state")
|
|
1852
|
+
singleton, version, token, sequence, nonce, window = state[0]
|
|
1853
|
+
tip = connection.execute(
|
|
1854
|
+
f"SELECT slot, parent_nonce, nonce FROM "
|
|
1855
|
+
f"{QPLOT_GENERATION_LINEAGE_RING_TABLE} WHERE sequence = ?",
|
|
1856
|
+
(sequence,),
|
|
1857
|
+
).fetchone()
|
|
1858
|
+
if (
|
|
1859
|
+
singleton != 1
|
|
1860
|
+
or version != QPLOT_GENERATION_LINEAGE_FORMAT_VERSION
|
|
1861
|
+
or not isinstance(token, str)
|
|
1862
|
+
or not isinstance(sequence, int)
|
|
1863
|
+
or sequence < 0
|
|
1864
|
+
or not isinstance(nonce, bytes)
|
|
1865
|
+
or len(nonce) != QPLOT_GENERATION_LINEAGE_NONCE_BYTES
|
|
1866
|
+
or window != QPLOT_GENERATION_LINEAGE_WINDOW
|
|
1867
|
+
or tip is None
|
|
1868
|
+
or tip[0] != sequence % QPLOT_GENERATION_LINEAGE_WINDOW
|
|
1869
|
+
or tip[2] != nonce
|
|
1870
|
+
):
|
|
1871
|
+
raise ValueError("invalid qPlot generation lineage state")
|
|
1872
|
+
return token, sequence, nonce, window
|
|
1873
|
+
|
|
1874
|
+
|
|
1875
|
+
def _wal_checksum(data, byte_order, state=(0, 0)):
|
|
1876
|
+
"""Return SQLite's cumulative two-word WAL checksum."""
|
|
1877
|
+
if len(data) % 8:
|
|
1878
|
+
raise ValueError("invalid WAL checksum input length")
|
|
1879
|
+
checksum_0, checksum_1 = state
|
|
1880
|
+
for word_0, word_1 in struct.iter_unpack(f"{byte_order}II", data):
|
|
1881
|
+
checksum_0 = (checksum_0 + word_0 + checksum_1) & 0xFFFFFFFF
|
|
1882
|
+
checksum_1 = (checksum_1 + word_1 + checksum_0) & 0xFFFFFFFF
|
|
1883
|
+
return checksum_0, checksum_1
|
|
1884
|
+
|
|
1885
|
+
|
|
1886
|
+
def _committed_wal_transaction_pages(
|
|
1887
|
+
wal_path,
|
|
1888
|
+
*,
|
|
1889
|
+
cancelled_callback=None,
|
|
1890
|
+
deadline=None,
|
|
1891
|
+
):
|
|
1892
|
+
"""Return page sets for SQLite's valid committed private-WAL prefix.
|
|
1893
|
+
|
|
1894
|
+
WAL files can retain invalid frames from an earlier checkpoint cycle. The
|
|
1895
|
+
checksum scan mirrors SQLite recovery: it stops at the first invalid frame
|
|
1896
|
+
and ignores valid but uncommitted tail frames after the last commit marker.
|
|
1897
|
+
"""
|
|
1898
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1899
|
+
with open(wal_path, "rb") as wal_file:
|
|
1900
|
+
header = wal_file.read(_WAL_HEADER_BYTES)
|
|
1901
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1902
|
+
if len(header) != _WAL_HEADER_BYTES:
|
|
1903
|
+
raise ValueError("missing or truncated WAL header")
|
|
1904
|
+
magic = int.from_bytes(header[:4], "big")
|
|
1905
|
+
if magic == _WAL_MAGIC_LITTLE_ENDIAN_CHECKSUMS:
|
|
1906
|
+
checksum_byte_order = "<"
|
|
1907
|
+
elif magic == _WAL_MAGIC_BIG_ENDIAN_CHECKSUMS:
|
|
1908
|
+
checksum_byte_order = ">"
|
|
1909
|
+
else:
|
|
1910
|
+
raise ValueError("invalid WAL magic")
|
|
1911
|
+
if int.from_bytes(header[4:8], "big") != _WAL_FORMAT_VERSION:
|
|
1912
|
+
raise ValueError("unsupported WAL format")
|
|
1913
|
+
page_size = int.from_bytes(header[8:12], "big")
|
|
1914
|
+
if (
|
|
1915
|
+
page_size < 512
|
|
1916
|
+
or page_size > 65_536
|
|
1917
|
+
or page_size & (page_size - 1)
|
|
1918
|
+
):
|
|
1919
|
+
raise ValueError("invalid WAL page size")
|
|
1920
|
+
checksum = _wal_checksum(header[:24], checksum_byte_order)
|
|
1921
|
+
stored_header_checksum = (
|
|
1922
|
+
int.from_bytes(header[24:28], "big"),
|
|
1923
|
+
int.from_bytes(header[28:32], "big"),
|
|
1924
|
+
)
|
|
1925
|
+
if checksum != stored_header_checksum:
|
|
1926
|
+
raise ValueError("invalid WAL header checksum")
|
|
1927
|
+
|
|
1928
|
+
salts = header[16:24]
|
|
1929
|
+
current_transaction_pages = set()
|
|
1930
|
+
committed_transactions = []
|
|
1931
|
+
while True:
|
|
1932
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1933
|
+
frame_header = wal_file.read(_WAL_FRAME_HEADER_BYTES)
|
|
1934
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1935
|
+
if not frame_header:
|
|
1936
|
+
break
|
|
1937
|
+
if len(frame_header) != _WAL_FRAME_HEADER_BYTES:
|
|
1938
|
+
break
|
|
1939
|
+
page = wal_file.read(page_size)
|
|
1940
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1941
|
+
if len(page) != page_size or frame_header[8:16] != salts:
|
|
1942
|
+
break
|
|
1943
|
+
next_checksum = _wal_checksum(
|
|
1944
|
+
frame_header[:8],
|
|
1945
|
+
checksum_byte_order,
|
|
1946
|
+
checksum,
|
|
1947
|
+
)
|
|
1948
|
+
next_checksum = _wal_checksum(
|
|
1949
|
+
page,
|
|
1950
|
+
checksum_byte_order,
|
|
1951
|
+
next_checksum,
|
|
1952
|
+
)
|
|
1953
|
+
stored_frame_checksum = (
|
|
1954
|
+
int.from_bytes(frame_header[16:20], "big"),
|
|
1955
|
+
int.from_bytes(frame_header[20:24], "big"),
|
|
1956
|
+
)
|
|
1957
|
+
if next_checksum != stored_frame_checksum:
|
|
1958
|
+
break
|
|
1959
|
+
page_number = int.from_bytes(frame_header[:4], "big")
|
|
1960
|
+
if page_number == 0:
|
|
1961
|
+
break
|
|
1962
|
+
checksum = next_checksum
|
|
1963
|
+
current_transaction_pages.add(page_number)
|
|
1964
|
+
if int.from_bytes(frame_header[4:8], "big"):
|
|
1965
|
+
committed_transactions.append(
|
|
1966
|
+
frozenset(current_transaction_pages)
|
|
1967
|
+
)
|
|
1968
|
+
current_transaction_pages.clear()
|
|
1969
|
+
|
|
1970
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
1971
|
+
if not committed_transactions:
|
|
1972
|
+
raise ValueError("the WAL has no valid committed transaction")
|
|
1973
|
+
return tuple(committed_transactions)
|
|
1974
|
+
|
|
1975
|
+
|
|
1976
|
+
def _lineage_state_root_page(connection):
|
|
1977
|
+
rows = connection.execute(
|
|
1978
|
+
"SELECT rootpage FROM sqlite_schema WHERE type = 'table' AND name = ?",
|
|
1979
|
+
(QPLOT_GENERATION_LINEAGE_STATE_TABLE,),
|
|
1980
|
+
).fetchall()
|
|
1981
|
+
if len(rows) != 1 or not isinstance(rows[0][0], int) or rows[0][0] <= 0:
|
|
1982
|
+
raise ValueError("invalid qPlot generation lineage root page")
|
|
1983
|
+
return rows[0][0]
|
|
1984
|
+
|
|
1985
|
+
|
|
1986
|
+
def _require_generation_lineage_trigger_inventory(connection):
|
|
1987
|
+
"""Require every current user table to carry the exact v2 trigger trio."""
|
|
1988
|
+
excluded_tables = (
|
|
1989
|
+
QPLOT_GENERATION_PROVENANCE_TABLE,
|
|
1990
|
+
QPLOT_GENERATION_LINEAGE_STATE_TABLE,
|
|
1991
|
+
QPLOT_GENERATION_LINEAGE_RING_TABLE,
|
|
1992
|
+
)
|
|
1993
|
+
table_names = [
|
|
1994
|
+
row[0]
|
|
1995
|
+
for row in connection.execute(
|
|
1996
|
+
"SELECT name FROM sqlite_schema "
|
|
1997
|
+
"WHERE type = 'table' AND name NOT LIKE 'sqlite_%' "
|
|
1998
|
+
"AND name NOT IN (?, ?, ?) ORDER BY name",
|
|
1999
|
+
excluded_tables,
|
|
2000
|
+
).fetchall()
|
|
2001
|
+
]
|
|
2002
|
+
triggers = {
|
|
2003
|
+
row[0]: (row[1], row[2])
|
|
2004
|
+
for row in connection.execute(
|
|
2005
|
+
"SELECT name, tbl_name, sql FROM sqlite_schema WHERE type = 'trigger'"
|
|
2006
|
+
).fetchall()
|
|
2007
|
+
}
|
|
2008
|
+
for table_name in table_names:
|
|
2009
|
+
for operation in ("INSERT", "UPDATE", "DELETE"):
|
|
2010
|
+
trigger_name, statement = generation_provenance_trigger(
|
|
2011
|
+
table_name,
|
|
2012
|
+
operation,
|
|
2013
|
+
)
|
|
2014
|
+
if triggers.get(trigger_name) != (table_name, statement):
|
|
2015
|
+
raise ValueError(
|
|
2016
|
+
f"missing or invalid lineage trigger for {table_name!r}"
|
|
2017
|
+
)
|
|
2018
|
+
|
|
2019
|
+
|
|
2020
|
+
def _require_generation_lineage_descendant(
|
|
2021
|
+
connection,
|
|
2022
|
+
main_sequence,
|
|
2023
|
+
main_nonce,
|
|
2024
|
+
snapshot_sequence,
|
|
2025
|
+
snapshot_nonce,
|
|
2026
|
+
window,
|
|
2027
|
+
):
|
|
2028
|
+
"""Prove the WAL head is linked directly to the selected main's head."""
|
|
2029
|
+
transition_count = snapshot_sequence - main_sequence
|
|
2030
|
+
if transition_count <= 0:
|
|
2031
|
+
raise ValueError("non-advancing lineage")
|
|
2032
|
+
if transition_count > window:
|
|
2033
|
+
raise OverflowError("lineage proof window exceeded")
|
|
2034
|
+
rows = connection.execute(
|
|
2035
|
+
f"SELECT slot, sequence, parent_nonce, nonce FROM "
|
|
2036
|
+
f"{QPLOT_GENERATION_LINEAGE_RING_TABLE} "
|
|
2037
|
+
"WHERE sequence > ? AND sequence <= ? ORDER BY sequence",
|
|
2038
|
+
(main_sequence, snapshot_sequence),
|
|
2039
|
+
).fetchall()
|
|
2040
|
+
if len(rows) != transition_count:
|
|
2041
|
+
raise ValueError("lineage branch has missing transitions")
|
|
2042
|
+
|
|
2043
|
+
expected_sequence = main_sequence + 1
|
|
2044
|
+
expected_parent = main_nonce
|
|
2045
|
+
for slot, sequence, parent_nonce, nonce in rows:
|
|
2046
|
+
if (
|
|
2047
|
+
sequence != expected_sequence
|
|
2048
|
+
or slot != sequence % window
|
|
2049
|
+
or parent_nonce != expected_parent
|
|
2050
|
+
or not isinstance(nonce, bytes)
|
|
2051
|
+
or len(nonce) != QPLOT_GENERATION_LINEAGE_NONCE_BYTES
|
|
2052
|
+
):
|
|
2053
|
+
raise ValueError("lineage branch does not descend from the main")
|
|
2054
|
+
expected_sequence += 1
|
|
2055
|
+
expected_parent = nonce
|
|
2056
|
+
if expected_parent != snapshot_nonce:
|
|
2057
|
+
raise ValueError("lineage branch does not reach the WAL head")
|
|
2058
|
+
|
|
2059
|
+
|
|
2060
|
+
def _unverifiable_generated_wal_error(database_path, reason):
|
|
2061
|
+
writer_guidance = ""
|
|
2062
|
+
if (
|
|
2063
|
+
"lineage event" in reason
|
|
2064
|
+
or "older qPlot" in reason
|
|
2065
|
+
or "proof window" in reason
|
|
2066
|
+
):
|
|
2067
|
+
writer_guidance = (
|
|
2068
|
+
" If the owner will create later QCoDeS runs, call "
|
|
2069
|
+
"qplot.testdata.enable_generation_provenance_for_writer on the "
|
|
2070
|
+
"quiescent writable QCoDeS connection before creating the "
|
|
2071
|
+
"Measurement. The writer API refuses to bless a nonempty WAL; "
|
|
2072
|
+
"checkpoint it with TRUNCATE first. SQLite cannot prove an "
|
|
2073
|
+
"already-written uninstrumented WAL after the fact."
|
|
2074
|
+
)
|
|
2075
|
+
return UnverifiableDatabaseWalError(
|
|
2076
|
+
f"qPlot cannot prove that {database_path}-wal belongs to the current "
|
|
2077
|
+
"generated database main file, so it refused to show a possibly stale "
|
|
2078
|
+
f"view ({reason}).{writer_guidance} Close the QCoDeS/SQLite writer "
|
|
2079
|
+
"cleanly so it can "
|
|
2080
|
+
"checkpoint and remove the WAL, then refresh. If the database is being "
|
|
2081
|
+
"moved or copied, stop the writer and keep the main, -wal, and -shm "
|
|
2082
|
+
"files together. qPlot did not modify the database or its sidecars."
|
|
2083
|
+
)
|
|
2084
|
+
|
|
2085
|
+
|
|
2086
|
+
def _unverifiable_unmarked_wal_error(database_path):
|
|
2087
|
+
return UnverifiableDatabaseWalError(
|
|
2088
|
+
f"qPlot cannot prove that {database_path}-wal belongs to the selected "
|
|
2089
|
+
"database main file. Standard SQLite WAL files contain no identifier "
|
|
2090
|
+
"that binds them to a particular main file, and this database has no "
|
|
2091
|
+
"qPlot generation provenance. qPlot therefore refused to show possibly "
|
|
2092
|
+
"unrelated or stale data. Close every QCoDeS/SQLite connection that "
|
|
2093
|
+
"owns the database cleanly so the owning writer checkpoints the WAL, "
|
|
2094
|
+
"then refresh or retry loading. Do not delete, rename, or move the WAL "
|
|
2095
|
+
"by hand. qPlot did not modify the database or any SQLite sidecar."
|
|
2096
|
+
)
|
|
2097
|
+
|
|
2098
|
+
|
|
2099
|
+
def _require_matching_generated_wal_provenance(
|
|
2100
|
+
database_path,
|
|
2101
|
+
main_snapshot_path,
|
|
2102
|
+
snapshot_path,
|
|
2103
|
+
committed_transactions=None,
|
|
2104
|
+
*,
|
|
2105
|
+
cancelled_callback=None,
|
|
2106
|
+
deadline=None,
|
|
2107
|
+
):
|
|
2108
|
+
"""Require a private WAL view to carry this generated main's lineage."""
|
|
2109
|
+
main_connection = None
|
|
2110
|
+
snapshot_connection = None
|
|
2111
|
+
try:
|
|
2112
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2113
|
+
if committed_transactions is None:
|
|
2114
|
+
committed_transactions = _committed_wal_transaction_pages(
|
|
2115
|
+
_wal_path(snapshot_path),
|
|
2116
|
+
cancelled_callback=cancelled_callback,
|
|
2117
|
+
deadline=deadline,
|
|
2118
|
+
)
|
|
2119
|
+
main_connection = sqlite3.connect(
|
|
2120
|
+
sqlite_read_only_uri(main_snapshot_path, immutable=True),
|
|
2121
|
+
uri=True,
|
|
2122
|
+
)
|
|
2123
|
+
_install_read_control_progress_handler(
|
|
2124
|
+
main_connection,
|
|
2125
|
+
cancelled_callback,
|
|
2126
|
+
deadline,
|
|
2127
|
+
)
|
|
2128
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2129
|
+
main_provenance = _generation_provenance(main_connection)
|
|
2130
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2131
|
+
main_lineage = _generation_lineage_state(main_connection)
|
|
2132
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2133
|
+
main_lineage_root = _lineage_state_root_page(main_connection)
|
|
2134
|
+
_require_generation_lineage_trigger_inventory(main_connection)
|
|
2135
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2136
|
+
snapshot_connection = sqlite3.connect(snapshot_path)
|
|
2137
|
+
_install_read_control_progress_handler(
|
|
2138
|
+
snapshot_connection,
|
|
2139
|
+
cancelled_callback,
|
|
2140
|
+
deadline,
|
|
2141
|
+
)
|
|
2142
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2143
|
+
snapshot_provenance = _generation_provenance(snapshot_connection)
|
|
2144
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2145
|
+
snapshot_lineage = _generation_lineage_state(snapshot_connection)
|
|
2146
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2147
|
+
snapshot_lineage_root = _lineage_state_root_page(snapshot_connection)
|
|
2148
|
+
_require_generation_lineage_trigger_inventory(snapshot_connection)
|
|
2149
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2150
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
2151
|
+
raise
|
|
2152
|
+
except (sqlite3.Error, ValueError) as error:
|
|
2153
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2154
|
+
raise _unverifiable_generated_wal_error(
|
|
2155
|
+
database_path,
|
|
2156
|
+
"the lineage history or trigger coverage is missing, invalid, or "
|
|
2157
|
+
"from an older qPlot",
|
|
2158
|
+
) from error
|
|
2159
|
+
else:
|
|
2160
|
+
if (
|
|
2161
|
+
snapshot_lineage_root != main_lineage_root
|
|
2162
|
+
or any(
|
|
2163
|
+
main_lineage_root not in transaction_pages
|
|
2164
|
+
for transaction_pages in committed_transactions
|
|
2165
|
+
)
|
|
2166
|
+
):
|
|
2167
|
+
raise _unverifiable_generated_wal_error(
|
|
2168
|
+
database_path,
|
|
2169
|
+
"one or more WAL transactions carry no verified lineage event",
|
|
2170
|
+
)
|
|
2171
|
+
main_token, main_epoch = main_provenance
|
|
2172
|
+
snapshot_token, snapshot_epoch = snapshot_provenance
|
|
2173
|
+
main_lineage_token, main_sequence, main_nonce, main_window = main_lineage
|
|
2174
|
+
(
|
|
2175
|
+
snapshot_lineage_token,
|
|
2176
|
+
snapshot_sequence,
|
|
2177
|
+
snapshot_nonce,
|
|
2178
|
+
snapshot_window,
|
|
2179
|
+
) = snapshot_lineage
|
|
2180
|
+
if (
|
|
2181
|
+
snapshot_token != main_token
|
|
2182
|
+
or snapshot_lineage_token != main_lineage_token
|
|
2183
|
+
or main_lineage_token != main_token
|
|
2184
|
+
):
|
|
2185
|
+
raise _unverifiable_generated_wal_error(
|
|
2186
|
+
database_path,
|
|
2187
|
+
"the WAL carries a different generation token",
|
|
2188
|
+
)
|
|
2189
|
+
if main_epoch != main_sequence or snapshot_epoch != snapshot_sequence:
|
|
2190
|
+
raise _unverifiable_generated_wal_error(
|
|
2191
|
+
database_path,
|
|
2192
|
+
"the lineage counters disagree with their branch heads",
|
|
2193
|
+
)
|
|
2194
|
+
if main_window != snapshot_window:
|
|
2195
|
+
raise _unverifiable_generated_wal_error(
|
|
2196
|
+
database_path,
|
|
2197
|
+
"the WAL carries a different lineage proof window",
|
|
2198
|
+
)
|
|
2199
|
+
if snapshot_sequence <= main_sequence:
|
|
2200
|
+
raise _unverifiable_generated_wal_error(
|
|
2201
|
+
database_path,
|
|
2202
|
+
"the WAL does not carry a later verified lineage event",
|
|
2203
|
+
)
|
|
2204
|
+
try:
|
|
2205
|
+
_require_generation_lineage_descendant(
|
|
2206
|
+
snapshot_connection,
|
|
2207
|
+
main_sequence,
|
|
2208
|
+
main_nonce,
|
|
2209
|
+
snapshot_sequence,
|
|
2210
|
+
snapshot_nonce,
|
|
2211
|
+
main_window,
|
|
2212
|
+
)
|
|
2213
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2214
|
+
except OverflowError as error:
|
|
2215
|
+
raise _unverifiable_generated_wal_error(
|
|
2216
|
+
database_path,
|
|
2217
|
+
"the WAL exceeded the retained lineage proof window",
|
|
2218
|
+
) from error
|
|
2219
|
+
except ValueError as error:
|
|
2220
|
+
raise _unverifiable_generated_wal_error(
|
|
2221
|
+
database_path,
|
|
2222
|
+
"the WAL lineage branch does not descend from the selected main",
|
|
2223
|
+
) from error
|
|
2224
|
+
except sqlite3.Error as error:
|
|
2225
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2226
|
+
raise _unverifiable_generated_wal_error(
|
|
2227
|
+
database_path,
|
|
2228
|
+
"the WAL lineage branch could not be validated",
|
|
2229
|
+
) from error
|
|
2230
|
+
finally:
|
|
2231
|
+
if snapshot_connection is not None:
|
|
2232
|
+
try:
|
|
2233
|
+
snapshot_connection.close()
|
|
2234
|
+
except sqlite3.Error:
|
|
2235
|
+
pass
|
|
2236
|
+
if main_connection is not None:
|
|
2237
|
+
try:
|
|
2238
|
+
main_connection.close()
|
|
2239
|
+
except sqlite3.Error:
|
|
2240
|
+
pass
|
|
2241
|
+
|
|
2242
|
+
|
|
2243
|
+
def _journal_names_super_journal(journal):
|
|
2244
|
+
"""Reject recovery that could follow a path outside the private snapshot."""
|
|
2245
|
+
if not journal.potentially_hot or len(journal.trailer) != 16:
|
|
2246
|
+
return False
|
|
2247
|
+
if journal.trailer[-8:] != _ROLLBACK_JOURNAL_MAGIC:
|
|
2248
|
+
return False
|
|
2249
|
+
name_length = int.from_bytes(journal.trailer[:4], "big")
|
|
2250
|
+
journal_size = journal.file_signature[2]
|
|
2251
|
+
return 0 < name_length <= journal_size - 20
|
|
2252
|
+
|
|
2253
|
+
|
|
2254
|
+
def _recover_private_rollback_journal(
|
|
2255
|
+
database_path,
|
|
2256
|
+
snapshot_path,
|
|
2257
|
+
*,
|
|
2258
|
+
cancelled_callback=None,
|
|
2259
|
+
deadline=None,
|
|
2260
|
+
):
|
|
2261
|
+
"""Let SQLite resolve a copied journal without ever opening the source."""
|
|
2262
|
+
connection = None
|
|
2263
|
+
try:
|
|
2264
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2265
|
+
connection = sqlite3.connect(
|
|
2266
|
+
f"file:{_sqlite_uri_path(snapshot_path)}?mode=rw",
|
|
2267
|
+
uri=True,
|
|
2268
|
+
timeout=0,
|
|
2269
|
+
)
|
|
2270
|
+
_install_read_control_progress_handler(
|
|
2271
|
+
connection,
|
|
2272
|
+
cancelled_callback,
|
|
2273
|
+
deadline,
|
|
2274
|
+
)
|
|
2275
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2276
|
+
connection.execute("PRAGMA query_only = ON")
|
|
2277
|
+
connection.execute("SELECT 1 FROM sqlite_schema LIMIT 1").fetchone()
|
|
2278
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2279
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
2280
|
+
raise
|
|
2281
|
+
except sqlite3.Error as error:
|
|
2282
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2283
|
+
raise ReadOnlyDatabaseAccessError(
|
|
2284
|
+
"qPlot could not recover a transaction-consistent private copy of "
|
|
2285
|
+
f"{database_path}. The database is busy or temporarily unavailable; "
|
|
2286
|
+
"finish the current transaction or refresh to retry. qPlot did not "
|
|
2287
|
+
"modify the source database or its SQLite sidecars."
|
|
2288
|
+
) from error
|
|
2289
|
+
finally:
|
|
2290
|
+
if connection is not None:
|
|
2291
|
+
try:
|
|
2292
|
+
connection.close()
|
|
2293
|
+
except sqlite3.Error:
|
|
2294
|
+
pass
|
|
2295
|
+
|
|
2296
|
+
|
|
2297
|
+
def _prepare_read_target(
|
|
2298
|
+
database_path,
|
|
2299
|
+
*,
|
|
2300
|
+
ignore_unpaired_wal=False,
|
|
2301
|
+
expected_database_identity=None,
|
|
2302
|
+
cancelled_callback=None,
|
|
2303
|
+
deadline=None,
|
|
2304
|
+
):
|
|
2305
|
+
"""Capture a stable private SQLite snapshot for every viewer read."""
|
|
2306
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2307
|
+
_require_publication_complete(database_path)
|
|
2308
|
+
if not database_path.is_file():
|
|
2309
|
+
raise FileNotFoundError(database_path)
|
|
2310
|
+
|
|
2311
|
+
_require_expected_database_instance(
|
|
2312
|
+
database_path,
|
|
2313
|
+
expected_database_identity,
|
|
2314
|
+
)
|
|
2315
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2316
|
+
prepared_database_identity = database_file_identity(database_path)
|
|
2317
|
+
if prepared_database_identity is None:
|
|
2318
|
+
raise FileNotFoundError(database_path)
|
|
2319
|
+
|
|
2320
|
+
try:
|
|
2321
|
+
generation_marker = database_has_qplot_generation_marker(database_path)
|
|
2322
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
2323
|
+
raise
|
|
2324
|
+
except OSError as error:
|
|
2325
|
+
raise ReadOnlyDatabaseAccessError(
|
|
2326
|
+
f"Could not inspect the database publication marker: {error}"
|
|
2327
|
+
) from error
|
|
2328
|
+
|
|
2329
|
+
wal_path = _wal_path(database_path)
|
|
2330
|
+
journal_path = _journal_path(database_path)
|
|
2331
|
+
|
|
2332
|
+
for _attempt in range(WAL_SNAPSHOT_ATTEMPTS):
|
|
2333
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2334
|
+
_require_expected_database_instance(
|
|
2335
|
+
database_path,
|
|
2336
|
+
expected_database_identity,
|
|
2337
|
+
)
|
|
2338
|
+
try:
|
|
2339
|
+
before = _source_signature_with_controls(
|
|
2340
|
+
database_path,
|
|
2341
|
+
cancelled_callback,
|
|
2342
|
+
deadline,
|
|
2343
|
+
)
|
|
2344
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
2345
|
+
raise
|
|
2346
|
+
except OSError as error:
|
|
2347
|
+
raise ReadOnlyDatabaseAccessError(
|
|
2348
|
+
f"Could not inspect a read-only view of {database_path}: {error}"
|
|
2349
|
+
) from error
|
|
2350
|
+
if before.database is None:
|
|
2351
|
+
raise FileNotFoundError(database_path)
|
|
2352
|
+
if not before.stable:
|
|
2353
|
+
continue
|
|
2354
|
+
|
|
2355
|
+
policy = _source_read_policy(
|
|
2356
|
+
database_path,
|
|
2357
|
+
before,
|
|
2358
|
+
generation_marker=generation_marker,
|
|
2359
|
+
ignore_unpaired_wal=ignore_unpaired_wal,
|
|
2360
|
+
)
|
|
2361
|
+
journal = policy.journal
|
|
2362
|
+
quarantined = policy.quarantined
|
|
2363
|
+
ignore_wal = policy.ignore_wal
|
|
2364
|
+
include_wal = policy.include_wal
|
|
2365
|
+
|
|
2366
|
+
if include_wal and not generation_marker:
|
|
2367
|
+
# SQLite WAL salts and checksums validate only the WAL's own frame
|
|
2368
|
+
# sequence. They do not commit to the base main file, and SQLite
|
|
2369
|
+
# will replay a structurally compatible unrelated WAL. Revalidate
|
|
2370
|
+
# the observed pair before rejecting it, but do not allocate or
|
|
2371
|
+
# copy a private snapshot whose contents can never be trusted.
|
|
2372
|
+
_require_expected_database_instance(
|
|
2373
|
+
database_path,
|
|
2374
|
+
expected_database_identity,
|
|
2375
|
+
)
|
|
2376
|
+
_require_prepared_database_instance(
|
|
2377
|
+
database_path,
|
|
2378
|
+
prepared_database_identity,
|
|
2379
|
+
)
|
|
2380
|
+
after = _source_signature_for_validation(
|
|
2381
|
+
database_path,
|
|
2382
|
+
cancelled_callback=cancelled_callback,
|
|
2383
|
+
deadline=deadline,
|
|
2384
|
+
)
|
|
2385
|
+
if after != before or not after.stable:
|
|
2386
|
+
continue
|
|
2387
|
+
raise _unverifiable_unmarked_wal_error(database_path)
|
|
2388
|
+
|
|
2389
|
+
snapshot = tempfile.TemporaryDirectory(prefix="qplot-readonly-")
|
|
2390
|
+
snapshot_path = Path(snapshot.name) / "database.db"
|
|
2391
|
+
main_snapshot_path = Path(snapshot.name) / "database-main.db"
|
|
2392
|
+
try:
|
|
2393
|
+
_copy_file_cooperatively(
|
|
2394
|
+
database_path,
|
|
2395
|
+
snapshot_path,
|
|
2396
|
+
cancelled_callback=cancelled_callback,
|
|
2397
|
+
deadline=deadline,
|
|
2398
|
+
)
|
|
2399
|
+
if include_wal and generation_marker:
|
|
2400
|
+
_copy_file_cooperatively(
|
|
2401
|
+
snapshot_path,
|
|
2402
|
+
main_snapshot_path,
|
|
2403
|
+
cancelled_callback=cancelled_callback,
|
|
2404
|
+
deadline=deadline,
|
|
2405
|
+
)
|
|
2406
|
+
if include_wal:
|
|
2407
|
+
_copy_file_cooperatively(
|
|
2408
|
+
wal_path,
|
|
2409
|
+
_wal_path(snapshot_path),
|
|
2410
|
+
cancelled_callback=cancelled_callback,
|
|
2411
|
+
deadline=deadline,
|
|
2412
|
+
)
|
|
2413
|
+
if journal is not None:
|
|
2414
|
+
_copy_file_cooperatively(
|
|
2415
|
+
journal_path,
|
|
2416
|
+
_journal_path(snapshot_path),
|
|
2417
|
+
cancelled_callback=cancelled_callback,
|
|
2418
|
+
deadline=deadline,
|
|
2419
|
+
)
|
|
2420
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
2421
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2422
|
+
raise
|
|
2423
|
+
except FileNotFoundError:
|
|
2424
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2425
|
+
continue
|
|
2426
|
+
except OSError as err:
|
|
2427
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2428
|
+
raise ReadOnlyDatabaseAccessError(
|
|
2429
|
+
f"Could not copy a read-only view of {database_path}: {err}"
|
|
2430
|
+
) from err
|
|
2431
|
+
|
|
2432
|
+
try:
|
|
2433
|
+
after_copy = _source_signature_with_controls(
|
|
2434
|
+
database_path,
|
|
2435
|
+
cancelled_callback,
|
|
2436
|
+
deadline,
|
|
2437
|
+
)
|
|
2438
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
2439
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2440
|
+
raise
|
|
2441
|
+
except OSError as error:
|
|
2442
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2443
|
+
raise ReadOnlyDatabaseAccessError(
|
|
2444
|
+
f"Could not validate a read-only copy of {database_path}: {error}"
|
|
2445
|
+
) from error
|
|
2446
|
+
if after_copy != before or not after_copy.stable:
|
|
2447
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2448
|
+
continue
|
|
2449
|
+
|
|
2450
|
+
try:
|
|
2451
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2452
|
+
_require_expected_database_instance(
|
|
2453
|
+
database_path,
|
|
2454
|
+
expected_database_identity,
|
|
2455
|
+
)
|
|
2456
|
+
_require_prepared_database_instance(
|
|
2457
|
+
database_path,
|
|
2458
|
+
prepared_database_identity,
|
|
2459
|
+
)
|
|
2460
|
+
committed_transactions = None
|
|
2461
|
+
if include_wal and generation_marker:
|
|
2462
|
+
try:
|
|
2463
|
+
committed_transactions = _committed_wal_transaction_pages(
|
|
2464
|
+
_wal_path(snapshot_path),
|
|
2465
|
+
cancelled_callback=cancelled_callback,
|
|
2466
|
+
deadline=deadline,
|
|
2467
|
+
)
|
|
2468
|
+
except (ReadOnlyDatabaseCancelledError, TimeoutError):
|
|
2469
|
+
raise
|
|
2470
|
+
except (OSError, ValueError) as error:
|
|
2471
|
+
if _source_signature_for_validation(
|
|
2472
|
+
database_path,
|
|
2473
|
+
cancelled_callback=cancelled_callback,
|
|
2474
|
+
deadline=deadline,
|
|
2475
|
+
) != before:
|
|
2476
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2477
|
+
continue
|
|
2478
|
+
raise _unverifiable_generated_wal_error(
|
|
2479
|
+
database_path,
|
|
2480
|
+
"the WAL has no verifiable committed lineage events",
|
|
2481
|
+
) from error
|
|
2482
|
+
if journal is not None:
|
|
2483
|
+
try:
|
|
2484
|
+
_recover_private_rollback_journal(
|
|
2485
|
+
database_path,
|
|
2486
|
+
snapshot_path,
|
|
2487
|
+
cancelled_callback=cancelled_callback,
|
|
2488
|
+
deadline=deadline,
|
|
2489
|
+
)
|
|
2490
|
+
except ReadOnlyDatabaseAccessError:
|
|
2491
|
+
if _source_signature_for_validation(
|
|
2492
|
+
database_path,
|
|
2493
|
+
cancelled_callback=cancelled_callback,
|
|
2494
|
+
deadline=deadline,
|
|
2495
|
+
) != before:
|
|
2496
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2497
|
+
continue
|
|
2498
|
+
raise
|
|
2499
|
+
if include_wal and generation_marker:
|
|
2500
|
+
try:
|
|
2501
|
+
_require_matching_generated_wal_provenance(
|
|
2502
|
+
database_path,
|
|
2503
|
+
main_snapshot_path,
|
|
2504
|
+
snapshot_path,
|
|
2505
|
+
committed_transactions,
|
|
2506
|
+
cancelled_callback=cancelled_callback,
|
|
2507
|
+
deadline=deadline,
|
|
2508
|
+
)
|
|
2509
|
+
except UnverifiableDatabaseWalError:
|
|
2510
|
+
if _source_signature_for_validation(
|
|
2511
|
+
database_path,
|
|
2512
|
+
cancelled_callback=cancelled_callback,
|
|
2513
|
+
deadline=deadline,
|
|
2514
|
+
) != before:
|
|
2515
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2516
|
+
continue
|
|
2517
|
+
if not (ignore_unpaired_wal or quarantined):
|
|
2518
|
+
raise
|
|
2519
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2520
|
+
return main_snapshot_path, True, snapshot, True, before
|
|
2521
|
+
|
|
2522
|
+
if _source_signature_for_validation(
|
|
2523
|
+
database_path,
|
|
2524
|
+
cancelled_callback=cancelled_callback,
|
|
2525
|
+
deadline=deadline,
|
|
2526
|
+
) != before:
|
|
2527
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2528
|
+
continue
|
|
2529
|
+
if include_wal and generation_marker:
|
|
2530
|
+
if not _release_wal_quarantine(
|
|
2531
|
+
database_path,
|
|
2532
|
+
prepared_database_identity,
|
|
2533
|
+
):
|
|
2534
|
+
raise DatabaseInstanceChangedError(
|
|
2535
|
+
"The database was replaced while qPlot was validating "
|
|
2536
|
+
"its WAL provenance."
|
|
2537
|
+
)
|
|
2538
|
+
_raise_if_read_interrupted(cancelled_callback, deadline)
|
|
2539
|
+
return snapshot_path, not include_wal, snapshot, ignore_wal, before
|
|
2540
|
+
except Exception:
|
|
2541
|
+
_cleanup_failed_connection_open(None, snapshot)
|
|
2542
|
+
raise
|
|
2543
|
+
|
|
2544
|
+
raise ReadOnlyDatabaseAccessError(
|
|
2545
|
+
"The database main file or SQLite sidecars changed continuously while "
|
|
2546
|
+
"qPlot tried to capture a transaction-consistent snapshot. The database "
|
|
2547
|
+
"is busy or temporarily unavailable; refresh to retry. qPlot did not "
|
|
2548
|
+
"modify the source database or its SQLite sidecars."
|
|
2549
|
+
)
|
|
2550
|
+
|
|
2551
|
+
|
|
2552
|
+
def _attach_snapshot_cleanup(conn, snapshot):
|
|
2553
|
+
"""Make an AtomicConnection clean up its private snapshot on close or GC."""
|
|
2554
|
+
connection_ref = weakref.ref(conn)
|
|
2555
|
+
finalizer = weakref.finalize(conn, snapshot.cleanup)
|
|
2556
|
+
|
|
2557
|
+
def close_snapshot():
|
|
2558
|
+
connection = connection_ref()
|
|
2559
|
+
try:
|
|
2560
|
+
if connection is not None:
|
|
2561
|
+
sqlite3.Connection.close(connection)
|
|
2562
|
+
finally:
|
|
2563
|
+
finalizer()
|
|
2564
|
+
|
|
2565
|
+
conn.close = close_snapshot
|
|
2566
|
+
conn._qplot_snapshot_finalizer = finalizer
|