qplotter 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qplot/__init__.py +46 -0
- qplot/__main__.py +232 -0
- qplot/_auxiliary_paths.py +67 -0
- qplot/_build_info.json +1 -0
- qplot/_metadata.py +75 -0
- qplot/_repair.py +14 -0
- qplot/_shutdown_supervisor.py +4049 -0
- qplot/_version.py +12 -0
- qplot/_windows_shutdown_job.py +1166 -0
- qplot/configuration/__init__.py +3 -0
- qplot/configuration/config.py +578 -0
- qplot/configuration/config_schema.json +357 -0
- qplot/configuration/fonts.py +3 -0
- qplot/configuration/scripts.py +255 -0
- qplot/configuration/themes/__init__.py +13 -0
- qplot/configuration/themes/_base.py +592 -0
- qplot/configuration/themes/blank.py +17 -0
- qplot/configuration/themes/dark.py +54 -0
- qplot/configuration/themes/light.py +54 -0
- qplot/datahandling/LoadFromDB.py +224 -0
- qplot/datahandling/__init__.py +47 -0
- qplot/datahandling/_trusted_live_helper.py +812 -0
- qplot/datahandling/_trusted_live_protocol.py +1745 -0
- qplot/datahandling/database.py +2340 -0
- qplot/datahandling/dimensions.py +43 -0
- qplot/datahandling/file_identity.py +461 -0
- qplot/datahandling/parameter_data.py +149 -0
- qplot/datahandling/plot_progress.py +17 -0
- qplot/datahandling/qcodes_cache.py +277 -0
- qplot/datahandling/qcodes_compat.py +14 -0
- qplot/datahandling/readSQL.py +2442 -0
- qplot/datahandling/readonly.py +2566 -0
- qplot/datahandling/source_activity.py +21 -0
- qplot/datahandling/trusted_array_preview.py +148 -0
- qplot/datahandling/trusted_derived_cache.py +1605 -0
- qplot/datahandling/trusted_derived_rendering.py +876 -0
- qplot/datahandling/trusted_heatmap.py +246 -0
- qplot/datahandling/trusted_live.py +2808 -0
- qplot/datahandling/trusted_live_queries.py +5907 -0
- qplot/datahandling/trusted_live_service.py +1919 -0
- qplot/datahandling/trusted_live_supervisor.py +2161 -0
- qplot/datahandling/trusted_plot.py +283 -0
- qplot/datahandling/trusted_presentation.py +1065 -0
- qplot/datahandling/trusted_snapshot.py +1132 -0
- qplot/datahandling/trusted_work_coordinator.py +2157 -0
- qplot/datahandling/trusted_work_scheduler.py +1204 -0
- qplot/diagnostics.py +241 -0
- qplot/resources/testdata/qplot_test_db_01_10mb.csv +11 -0
- qplot/resources/testdata/qplot_test_db_02_25mb.csv +14 -0
- qplot/resources/testdata/qplot_test_db_03_50mb.csv +17 -0
- qplot/resources/testdata/qplot_test_db_04_100mb.csv +20 -0
- qplot/resources/testdata/qplot_test_db_05_250mb.csv +23 -0
- qplot/resources/testdata/qplot_test_db_06_500mb.csv +26 -0
- qplot/resources/testdata/qplot_test_db_07_1gb.csv +29 -0
- qplot/resources/testdata/qplot_test_db_08_5gb.csv +32 -0
- qplot/resources/testdata/qplot_test_db_09_10gb.csv +35 -0
- qplot/resources/testdata/qplot_test_db_10_30gb.csv +38 -0
- qplot/testdata.py +1804 -0
- qplot/tools/__init__.py +13 -0
- qplot/tools/finite_means.py +123 -0
- qplot/tools/general.py +139 -0
- qplot/tools/heatmap_geometry.py +391 -0
- qplot/tools/operation_registry.py +179 -0
- qplot/tools/plot_tools.py +650 -0
- qplot/tools/sample_statistics.py +111 -0
- qplot/tools/worker.py +2838 -0
- qplot/windows/__init__.py +9 -0
- qplot/windows/_colorbar.py +422 -0
- qplot/windows/_commands.py +523 -0
- qplot/windows/_config_persistence.py +123 -0
- qplot/windows/_database_actions.py +4933 -0
- qplot/windows/_dataset_handle.py +221 -0
- qplot/windows/_display_precision.py +48 -0
- qplot/windows/_dragdrop.py +70 -0
- qplot/windows/_export_paths.py +1167 -0
- qplot/windows/_help.py +210 -0
- qplot/windows/_mixed_appearance.py +303 -0
- qplot/windows/_mixed_plots.py +461 -0
- qplot/windows/_native_averaging.py +323 -0
- qplot/windows/_native_transforms.py +551 -0
- qplot/windows/_plot1d_snap.py +612 -0
- qplot/windows/_plot1d_traces.py +2236 -0
- qplot/windows/_plot2d_colorbar.py +1127 -0
- qplot/windows/_plot2d_colorbar_dialog.py +786 -0
- qplot/windows/_plot2d_layers.py +2291 -0
- qplot/windows/_plot2d_sweeps.py +712 -0
- qplot/windows/_plotWin.py +2082 -0
- qplot/windows/_plot_actions.py +2703 -0
- qplot/windows/_plot_annotation.py +186 -0
- qplot/windows/_plot_appearance.py +283 -0
- qplot/windows/_plot_axis_scaling.py +1521 -0
- qplot/windows/_plot_data_viewbox.py +94 -0
- qplot/windows/_plot_drop_axes.py +224 -0
- qplot/windows/_plot_export.py +1445 -0
- qplot/windows/_plot_feedback.py +112 -0
- qplot/windows/_plot_marquee.py +659 -0
- qplot/windows/_plot_refresh.py +950 -0
- qplot/windows/_plot_source_promotion.py +178 -0
- qplot/windows/_plot_state.py +237 -0
- qplot/windows/_plot_status.py +115 -0
- qplot/windows/_plot_tiling.py +35 -0
- qplot/windows/_plot_transform_labels.py +60 -0
- qplot/windows/_preferences.py +724 -0
- qplot/windows/_refresh_interval.py +172 -0
- qplot/windows/_run_controls.py +789 -0
- qplot/windows/_shortcuts.py +47 -0
- qplot/windows/_subplots/__init__.py +8 -0
- qplot/windows/_subplots/subplot1d.py +532 -0
- qplot/windows/_subplots/subplot2d.py +919 -0
- qplot/windows/_svg_export.py +209 -0
- qplot/windows/_trusted_derived_qt.py +1759 -0
- qplot/windows/_widgets/__init__.py +23 -0
- qplot/windows/_widgets/_run_formatting.py +387 -0
- qplot/windows/_widgets/details_tables.py +1119 -0
- qplot/windows/_widgets/dropbox.py +290 -0
- qplot/windows/_widgets/operations.py +460 -0
- qplot/windows/_widgets/preview.py +2997 -0
- qplot/windows/_widgets/run_list_items.py +488 -0
- qplot/windows/_widgets/toolbar.py +111 -0
- qplot/windows/_widgets/treeWidgets.py +2572 -0
- qplot/windows/_window_controls.py +414 -0
- qplot/windows/main.py +1632 -0
- qplot/windows/plot1d.py +313 -0
- qplot/windows/plot2d.py +1935 -0
- qplotter-1.6.0.dist-info/METADATA +322 -0
- qplotter-1.6.0.dist-info/RECORD +130 -0
- qplotter-1.6.0.dist-info/WHEEL +5 -0
- qplotter-1.6.0.dist-info/entry_points.txt +4 -0
- qplotter-1.6.0.dist-info/licenses/LICENSE +21 -0
- qplotter-1.6.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2161 @@
|
|
|
1
|
+
"""Persistent spawned-process supervision for :mod:`trusted_live`.
|
|
2
|
+
|
|
3
|
+
This is the application-independent Stage 3 boundary. It deliberately has no
|
|
4
|
+
Qt dependencies and is not yet integrated into qPlot's windows or workers.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import atexit
|
|
10
|
+
import math
|
|
11
|
+
import multiprocessing
|
|
12
|
+
import os
|
|
13
|
+
import queue
|
|
14
|
+
import secrets
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import weakref
|
|
18
|
+
from collections.abc import Sequence
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from multiprocessing.connection import Connection
|
|
21
|
+
from multiprocessing.process import BaseProcess
|
|
22
|
+
from types import TracebackType
|
|
23
|
+
from typing import Any, Generic, Literal, TypeVar, cast
|
|
24
|
+
|
|
25
|
+
from qplot.datahandling._trusted_live_helper import trusted_live_helper_main
|
|
26
|
+
from qplot.datahandling._trusted_live_protocol import (
|
|
27
|
+
MAX_GENERATION,
|
|
28
|
+
MAX_OPERATION_TIMEOUT_MS,
|
|
29
|
+
MAX_REPLY_BYTES,
|
|
30
|
+
ProtocolEnvelope,
|
|
31
|
+
TrustedLiveProtocolValidationError,
|
|
32
|
+
decode_reply_frame,
|
|
33
|
+
decode_reply_payload,
|
|
34
|
+
encode_cancel,
|
|
35
|
+
encode_job_request,
|
|
36
|
+
encode_shutdown,
|
|
37
|
+
encode_startup_request,
|
|
38
|
+
error_code_is_terminal,
|
|
39
|
+
normalize_query_batch,
|
|
40
|
+
normalize_query_specification,
|
|
41
|
+
validate_job_success,
|
|
42
|
+
validate_shutdown_success,
|
|
43
|
+
validate_startup_success,
|
|
44
|
+
)
|
|
45
|
+
from qplot.datahandling.file_identity import DatabaseInstance, database_instances_differ
|
|
46
|
+
from qplot.datahandling.trusted_live import (
|
|
47
|
+
DEFAULT_TRUSTED_OPERATION_TIMEOUT_SECONDS,
|
|
48
|
+
SqliteBindings,
|
|
49
|
+
TrustedLiveBusyTimeoutError,
|
|
50
|
+
TrustedLiveCancelledError,
|
|
51
|
+
TrustedLiveCleanupError,
|
|
52
|
+
TrustedLiveDeadlineExceededError,
|
|
53
|
+
TrustedLiveInvalidDatabaseError,
|
|
54
|
+
TrustedLiveQueryError,
|
|
55
|
+
TrustedLiveReaderClosedError,
|
|
56
|
+
TrustedLiveReaderError,
|
|
57
|
+
TrustedLiveReaderThreadError,
|
|
58
|
+
TrustedLiveReaderUnavailableError,
|
|
59
|
+
TrustedLiveResultLimitError,
|
|
60
|
+
TrustedLiveSourceChangedError,
|
|
61
|
+
TrustedLiveSourceIdentity,
|
|
62
|
+
TrustedLiveSourceIOError,
|
|
63
|
+
TrustedLiveSqlRejectedError,
|
|
64
|
+
TrustedLiveTransactionError,
|
|
65
|
+
TrustedLiveUnsupportedSourceError,
|
|
66
|
+
TrustedQuery,
|
|
67
|
+
TrustedQueryResult,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
_T = TypeVar("_T")
|
|
71
|
+
_DEFAULT_STARTUP_TIMEOUT_SECONDS = 10.0
|
|
72
|
+
_DEFAULT_REPLY_TIMEOUT_SECONDS = 10.0
|
|
73
|
+
_DEFAULT_CANCELLATION_GRACE_SECONDS = 0.75
|
|
74
|
+
_DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 2.0
|
|
75
|
+
_DEFAULT_TERMINATE_TIMEOUT_SECONDS = 1.0
|
|
76
|
+
_DEFAULT_KILL_TIMEOUT_SECONDS = 1.0
|
|
77
|
+
_DEFAULT_SEND_TIMEOUT_SECONDS = 1.0
|
|
78
|
+
_DEFAULT_ORPHAN_GRACE_SECONDS = 1.0
|
|
79
|
+
_ATEXIT_LOCK_TIMEOUT_SECONDS = 0.25
|
|
80
|
+
_ATEXIT_PROCESS_TIMEOUT_SECONDS = 0.25
|
|
81
|
+
_REPLY_RECEIVER_JOIN_TIMEOUT_SECONDS = 0.5
|
|
82
|
+
_MAX_PARENT_TIMEOUT_SECONDS = 300.0
|
|
83
|
+
_POLL_QUANTUM_SECONDS = 0.05
|
|
84
|
+
_NO_VALUE = object()
|
|
85
|
+
_PRIVATE_TEST_FAULTS = frozenset(
|
|
86
|
+
{
|
|
87
|
+
"cleanup_quarantine",
|
|
88
|
+
"crash_before_reply",
|
|
89
|
+
"hang_before_operation",
|
|
90
|
+
"hang_close",
|
|
91
|
+
"hang_during_startup",
|
|
92
|
+
"hang_startup",
|
|
93
|
+
"malformed_reply",
|
|
94
|
+
"oversized_reply",
|
|
95
|
+
"partial_job_body",
|
|
96
|
+
"partial_job_header",
|
|
97
|
+
"partial_shutdown_body",
|
|
98
|
+
"partial_shutdown_header",
|
|
99
|
+
"partial_startup_body",
|
|
100
|
+
"partial_startup_header",
|
|
101
|
+
"statement_limit_install",
|
|
102
|
+
"statement_limit_restore",
|
|
103
|
+
"statement_limit_verify",
|
|
104
|
+
"stale_generation_reply",
|
|
105
|
+
"stale_session_reply",
|
|
106
|
+
"wrong_version_reply",
|
|
107
|
+
}
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class TrustedLiveSupervisorError(TrustedLiveReaderError):
|
|
112
|
+
"""Base class for failures introduced by the process boundary."""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class TrustedLiveHelperStartupError(
|
|
116
|
+
TrustedLiveReaderUnavailableError,
|
|
117
|
+
TrustedLiveSupervisorError,
|
|
118
|
+
):
|
|
119
|
+
"""Raised when a helper cannot start and report readiness in time."""
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class TrustedLiveProtocolError(
|
|
123
|
+
TrustedLiveReaderUnavailableError,
|
|
124
|
+
TrustedLiveSupervisorError,
|
|
125
|
+
):
|
|
126
|
+
"""Raised when either endpoint violates the bounded IPC protocol."""
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class TrustedLiveHelperExitedError(
|
|
130
|
+
TrustedLiveReaderUnavailableError,
|
|
131
|
+
TrustedLiveSupervisorError,
|
|
132
|
+
):
|
|
133
|
+
"""Raised when a helper exits unexpectedly before its reply."""
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class TrustedLiveHelperReplyTimeoutError(
|
|
137
|
+
TimeoutError,
|
|
138
|
+
TrustedLiveSupervisorError,
|
|
139
|
+
):
|
|
140
|
+
"""Raised after a parent reply deadline and cooperative retirement."""
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class TrustedLiveHelperForcedTerminationError(
|
|
144
|
+
TimeoutError,
|
|
145
|
+
TrustedLiveSupervisorError,
|
|
146
|
+
):
|
|
147
|
+
"""Raised when cancellation grace expires and the helper is terminated."""
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class TrustedLiveSupervisorClosedError(
|
|
151
|
+
TrustedLiveReaderClosedError,
|
|
152
|
+
TrustedLiveSupervisorError,
|
|
153
|
+
):
|
|
154
|
+
"""Raised when an operation targets a closed supervisor."""
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@dataclass(slots=True)
|
|
158
|
+
class TrustedLiveJob(Generic[_T]):
|
|
159
|
+
"""Typed handle for one generation-bound helper operation."""
|
|
160
|
+
|
|
161
|
+
incarnation: int
|
|
162
|
+
session: str
|
|
163
|
+
generation: int
|
|
164
|
+
operation: str
|
|
165
|
+
_owner_nonce: str = field(repr=False)
|
|
166
|
+
_expected_result_count: int | None = field(default=None, repr=False)
|
|
167
|
+
_done: threading.Event = field(default_factory=threading.Event, repr=False)
|
|
168
|
+
_value: object = field(default=_NO_VALUE, repr=False)
|
|
169
|
+
_error: BaseException | None = field(default=None, repr=False)
|
|
170
|
+
_cancel_requested: bool = field(default=False, repr=False)
|
|
171
|
+
_cancel_deadline: float | None = field(default=None, repr=False)
|
|
172
|
+
_parent_timed_out: bool = field(default=False, repr=False)
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def done(self) -> bool:
|
|
176
|
+
"""Return whether the parent has accepted a terminal outcome."""
|
|
177
|
+
|
|
178
|
+
return self._done.is_set()
|
|
179
|
+
|
|
180
|
+
@property
|
|
181
|
+
def cancellation_requested(self) -> bool:
|
|
182
|
+
"""Return whether exact-generation cancellation was requested."""
|
|
183
|
+
|
|
184
|
+
return self._cancel_requested
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@dataclass(frozen=True, slots=True)
|
|
188
|
+
class TrustedLiveSupervisorLiveness:
|
|
189
|
+
"""Conservative snapshot of every resource owned by one supervisor."""
|
|
190
|
+
|
|
191
|
+
helper_pid: int | None
|
|
192
|
+
process_alive: bool
|
|
193
|
+
receiver_alive: bool
|
|
194
|
+
open_endpoints: int
|
|
195
|
+
active_incarnation: bool
|
|
196
|
+
unreaped_incarnation: bool
|
|
197
|
+
active_job: bool
|
|
198
|
+
closing: bool
|
|
199
|
+
closed: bool
|
|
200
|
+
|
|
201
|
+
@property
|
|
202
|
+
def resources_owned(self) -> bool:
|
|
203
|
+
"""Return whether any helper-side resource remains strongly owned."""
|
|
204
|
+
|
|
205
|
+
return bool(
|
|
206
|
+
self.active_incarnation
|
|
207
|
+
or self.unreaped_incarnation
|
|
208
|
+
or self.active_job
|
|
209
|
+
or self.process_alive
|
|
210
|
+
or self.receiver_alive
|
|
211
|
+
or self.open_endpoints
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
_LOCK_CONTENTION_LIVENESS = TrustedLiveSupervisorLiveness(
|
|
216
|
+
helper_pid=None,
|
|
217
|
+
process_alive=True,
|
|
218
|
+
receiver_alive=True,
|
|
219
|
+
open_endpoints=1,
|
|
220
|
+
active_incarnation=True,
|
|
221
|
+
unreaped_incarnation=True,
|
|
222
|
+
active_job=True,
|
|
223
|
+
closing=True,
|
|
224
|
+
closed=False,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@dataclass(frozen=True, slots=True)
|
|
229
|
+
class _ReplyReceiveFailure:
|
|
230
|
+
"""One terminal raw-pipe failure published by the sole receiver."""
|
|
231
|
+
|
|
232
|
+
kind: Literal["eof", "connection", "unexpected"]
|
|
233
|
+
description: str
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
_ReplyInboxItem = bytes | _ReplyReceiveFailure
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@dataclass(slots=True)
|
|
240
|
+
class _HelperIncarnation:
|
|
241
|
+
number: int
|
|
242
|
+
session: str
|
|
243
|
+
process: BaseProcess
|
|
244
|
+
command_send: Connection
|
|
245
|
+
reply_connection: Connection
|
|
246
|
+
control_send: Connection
|
|
247
|
+
test_notify_receive: Connection | None
|
|
248
|
+
reply_inbox: queue.Queue[_ReplyInboxItem]
|
|
249
|
+
reply_ready: threading.Event
|
|
250
|
+
reply_receiver_stop: threading.Event
|
|
251
|
+
reply_receiver_done: threading.Event
|
|
252
|
+
reply_receiver: threading.Thread
|
|
253
|
+
reply_receiver_started: bool = False
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _publish_reply_item(
|
|
257
|
+
inbox: queue.Queue[_ReplyInboxItem],
|
|
258
|
+
ready: threading.Event,
|
|
259
|
+
stop: threading.Event,
|
|
260
|
+
item: _ReplyInboxItem,
|
|
261
|
+
) -> bool:
|
|
262
|
+
"""Publish into the one-slot inbox without ever stranding its producer."""
|
|
263
|
+
|
|
264
|
+
while not stop.is_set():
|
|
265
|
+
# The event is private test observability only. Set it before the
|
|
266
|
+
# queue publication so a consumer can never clear the event between
|
|
267
|
+
# publication and notification and leave a stale signalled state.
|
|
268
|
+
ready.set()
|
|
269
|
+
try:
|
|
270
|
+
inbox.put(item, timeout=_POLL_QUANTUM_SECONDS)
|
|
271
|
+
except queue.Full:
|
|
272
|
+
continue
|
|
273
|
+
return True
|
|
274
|
+
ready.clear()
|
|
275
|
+
return False
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _reply_receiver_loop(
|
|
279
|
+
connection: Connection,
|
|
280
|
+
inbox: queue.Queue[_ReplyInboxItem],
|
|
281
|
+
ready: threading.Event,
|
|
282
|
+
stop: threading.Event,
|
|
283
|
+
done: threading.Event,
|
|
284
|
+
) -> None:
|
|
285
|
+
"""Sole raw reply receiver for one helper process incarnation.
|
|
286
|
+
|
|
287
|
+
``Connection.poll()`` only establishes that some bytes are readable on
|
|
288
|
+
stream-backed multiprocessing pipes; it does not establish that a complete
|
|
289
|
+
frame is available. This one persistent thread may therefore block in
|
|
290
|
+
``recv_bytes`` while all public callers retain their own finite waits on the
|
|
291
|
+
bounded inbox. Incarnation retirement closes the peer and this endpoint,
|
|
292
|
+
then proves that this thread has stopped before the helper is reusable.
|
|
293
|
+
"""
|
|
294
|
+
|
|
295
|
+
try:
|
|
296
|
+
while not stop.is_set():
|
|
297
|
+
try:
|
|
298
|
+
item: _ReplyInboxItem = connection.recv_bytes(MAX_REPLY_BYTES)
|
|
299
|
+
terminal = False
|
|
300
|
+
except EOFError:
|
|
301
|
+
item = _ReplyReceiveFailure(
|
|
302
|
+
"eof",
|
|
303
|
+
"The helper closed its reply endpoint.",
|
|
304
|
+
)
|
|
305
|
+
terminal = True
|
|
306
|
+
except (OSError, ValueError) as error:
|
|
307
|
+
item = _ReplyReceiveFailure(
|
|
308
|
+
"connection",
|
|
309
|
+
"The helper reply connection failed while receiving "
|
|
310
|
+
f"a frame ({type(error).__name__}).",
|
|
311
|
+
)
|
|
312
|
+
terminal = True
|
|
313
|
+
except BaseException as error:
|
|
314
|
+
# Never let an arbitrary receiver failure escape through a
|
|
315
|
+
# public operation or leave its incarnation reusable.
|
|
316
|
+
item = _ReplyReceiveFailure(
|
|
317
|
+
"unexpected",
|
|
318
|
+
"The helper reply receiver failed unexpectedly "
|
|
319
|
+
f"({type(error).__name__}).",
|
|
320
|
+
)
|
|
321
|
+
terminal = True
|
|
322
|
+
if not _publish_reply_item(inbox, ready, stop, item):
|
|
323
|
+
return
|
|
324
|
+
if terminal:
|
|
325
|
+
return
|
|
326
|
+
finally:
|
|
327
|
+
done.set()
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _duration(
|
|
331
|
+
value: object,
|
|
332
|
+
*,
|
|
333
|
+
description: str,
|
|
334
|
+
allow_zero: bool,
|
|
335
|
+
) -> float:
|
|
336
|
+
if isinstance(value, bool):
|
|
337
|
+
raise ValueError(f"{description} must be a finite number of seconds.")
|
|
338
|
+
try:
|
|
339
|
+
duration = float(cast(Any, value))
|
|
340
|
+
except (TypeError, ValueError) as error:
|
|
341
|
+
raise ValueError(
|
|
342
|
+
f"{description} must be a finite number of seconds."
|
|
343
|
+
) from error
|
|
344
|
+
if (
|
|
345
|
+
not math.isfinite(duration)
|
|
346
|
+
or duration < 0
|
|
347
|
+
or (duration == 0 and not allow_zero)
|
|
348
|
+
or duration > _MAX_PARENT_TIMEOUT_SECONDS
|
|
349
|
+
):
|
|
350
|
+
qualifier = "non-negative" if allow_zero else "positive"
|
|
351
|
+
raise ValueError(
|
|
352
|
+
f"{description} must be a finite {qualifier} duration no greater "
|
|
353
|
+
f"than {_MAX_PARENT_TIMEOUT_SECONDS:g} seconds."
|
|
354
|
+
)
|
|
355
|
+
return duration
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _timeout_milliseconds(value: object, *, description: str) -> int:
|
|
359
|
+
seconds = _duration(value, description=description, allow_zero=True)
|
|
360
|
+
milliseconds = math.ceil(seconds * 1_000.0)
|
|
361
|
+
if milliseconds > MAX_OPERATION_TIMEOUT_MS:
|
|
362
|
+
raise ValueError(f"{description} exceeds the protocol operation-timeout limit.")
|
|
363
|
+
return milliseconds
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _close_connection(connection: Connection | None) -> None:
|
|
367
|
+
if connection is None:
|
|
368
|
+
return
|
|
369
|
+
try:
|
|
370
|
+
connection.close()
|
|
371
|
+
except OSError:
|
|
372
|
+
pass
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _bounded_send_bytes(
|
|
376
|
+
connection: Connection,
|
|
377
|
+
frame: bytes,
|
|
378
|
+
timeout: float,
|
|
379
|
+
) -> None:
|
|
380
|
+
"""Bound a potentially blocking pipe write without serialising objects."""
|
|
381
|
+
|
|
382
|
+
completed = threading.Event()
|
|
383
|
+
errors: list[BaseException] = []
|
|
384
|
+
|
|
385
|
+
def send() -> None:
|
|
386
|
+
try:
|
|
387
|
+
connection.send_bytes(frame)
|
|
388
|
+
except BaseException as error:
|
|
389
|
+
errors.append(error)
|
|
390
|
+
finally:
|
|
391
|
+
completed.set()
|
|
392
|
+
|
|
393
|
+
sender = threading.Thread(
|
|
394
|
+
target=send,
|
|
395
|
+
name="qplot-trusted-reader-ipc-send",
|
|
396
|
+
daemon=True,
|
|
397
|
+
)
|
|
398
|
+
sender.start()
|
|
399
|
+
if not completed.wait(timeout):
|
|
400
|
+
_close_connection(connection)
|
|
401
|
+
completed.wait(min(0.1, timeout))
|
|
402
|
+
raise TimeoutError("A bounded trusted-helper IPC send did not finish.")
|
|
403
|
+
sender.join(timeout=0)
|
|
404
|
+
if errors:
|
|
405
|
+
raise OSError("The trusted-helper IPC endpoint rejected a frame.") from errors[
|
|
406
|
+
0
|
|
407
|
+
]
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _error_from_payload(code: str, message: str) -> BaseException:
|
|
411
|
+
error_types: dict[str, type[BaseException]] = {
|
|
412
|
+
"reader_unavailable": TrustedLiveReaderUnavailableError,
|
|
413
|
+
"unsupported_source": TrustedLiveUnsupportedSourceError,
|
|
414
|
+
"source_changed": TrustedLiveSourceChangedError,
|
|
415
|
+
"source_io": TrustedLiveSourceIOError,
|
|
416
|
+
"sql_rejected": TrustedLiveSqlRejectedError,
|
|
417
|
+
"query_failed": TrustedLiveQueryError,
|
|
418
|
+
"result_limit": TrustedLiveResultLimitError,
|
|
419
|
+
"busy_timeout": TrustedLiveBusyTimeoutError,
|
|
420
|
+
"cancelled": TrustedLiveCancelledError,
|
|
421
|
+
"operation_deadline": TrustedLiveDeadlineExceededError,
|
|
422
|
+
"invalid_database": TrustedLiveInvalidDatabaseError,
|
|
423
|
+
"cleanup_quarantine": TrustedLiveCleanupError,
|
|
424
|
+
"reader_closed": TrustedLiveReaderClosedError,
|
|
425
|
+
"reader_thread": TrustedLiveReaderThreadError,
|
|
426
|
+
"transaction": TrustedLiveTransactionError,
|
|
427
|
+
"reader_error": TrustedLiveReaderError,
|
|
428
|
+
"protocol_error": TrustedLiveProtocolError,
|
|
429
|
+
"internal_error": TrustedLiveHelperExitedError,
|
|
430
|
+
}
|
|
431
|
+
return error_types[code](message)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
class TrustedLiveReaderSupervisor:
|
|
435
|
+
"""Own one persistent spawned helper for one accepted database instance.
|
|
436
|
+
|
|
437
|
+
Only one database job may be active. A failed job is never replayed. If
|
|
438
|
+
its process incarnation is retired, a later explicit submit or
|
|
439
|
+
:meth:`restart` may spawn a fresh helper bound to the originally accepted
|
|
440
|
+
:class:`DatabaseInstance`.
|
|
441
|
+
"""
|
|
442
|
+
|
|
443
|
+
def __init__(
|
|
444
|
+
self,
|
|
445
|
+
database_path: str | os.PathLike[str],
|
|
446
|
+
*,
|
|
447
|
+
expected_database_instance: DatabaseInstance | None = None,
|
|
448
|
+
busy_timeout_ms: int = 5_000,
|
|
449
|
+
operation_timeout_seconds: float = DEFAULT_TRUSTED_OPERATION_TIMEOUT_SECONDS,
|
|
450
|
+
startup_timeout_seconds: float = _DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
|
451
|
+
reply_timeout_seconds: float = _DEFAULT_REPLY_TIMEOUT_SECONDS,
|
|
452
|
+
cancellation_grace_seconds: float = (_DEFAULT_CANCELLATION_GRACE_SECONDS),
|
|
453
|
+
shutdown_timeout_seconds: float = _DEFAULT_SHUTDOWN_TIMEOUT_SECONDS,
|
|
454
|
+
terminate_timeout_seconds: float = _DEFAULT_TERMINATE_TIMEOUT_SECONDS,
|
|
455
|
+
kill_timeout_seconds: float = _DEFAULT_KILL_TIMEOUT_SECONDS,
|
|
456
|
+
_test_fault: str | None = None,
|
|
457
|
+
) -> None:
|
|
458
|
+
try:
|
|
459
|
+
selected_path = os.fspath(database_path)
|
|
460
|
+
except TypeError as error:
|
|
461
|
+
raise TypeError("database_path must be a path-like value.") from error
|
|
462
|
+
if isinstance(selected_path, bytes):
|
|
463
|
+
selected_path = os.fsdecode(selected_path)
|
|
464
|
+
if (
|
|
465
|
+
not isinstance(selected_path, str)
|
|
466
|
+
or not selected_path
|
|
467
|
+
or "\x00" in selected_path
|
|
468
|
+
):
|
|
469
|
+
raise ValueError("database_path must be non-empty text without NUL bytes.")
|
|
470
|
+
try:
|
|
471
|
+
path_bytes = selected_path.encode("utf-8", errors="strict")
|
|
472
|
+
except UnicodeEncodeError as error:
|
|
473
|
+
raise ValueError(
|
|
474
|
+
"database_path must contain valid Unicode text."
|
|
475
|
+
) from error
|
|
476
|
+
if len(path_bytes) > 32 * 1024:
|
|
477
|
+
raise ValueError("database_path exceeds the bounded helper path limit.")
|
|
478
|
+
if expected_database_instance is not None and not isinstance(
|
|
479
|
+
expected_database_instance, DatabaseInstance
|
|
480
|
+
):
|
|
481
|
+
raise TypeError(
|
|
482
|
+
"expected_database_instance must be a DatabaseInstance or None."
|
|
483
|
+
)
|
|
484
|
+
if (
|
|
485
|
+
type(busy_timeout_ms) is not int
|
|
486
|
+
or busy_timeout_ms < 0
|
|
487
|
+
or busy_timeout_ms > 2_147_483_647
|
|
488
|
+
):
|
|
489
|
+
raise ValueError(
|
|
490
|
+
"busy_timeout_ms must be an integer from 0 through 2147483647."
|
|
491
|
+
)
|
|
492
|
+
if _test_fault is not None and (
|
|
493
|
+
type(_test_fault) is not str or _test_fault not in _PRIVATE_TEST_FAULTS
|
|
494
|
+
):
|
|
495
|
+
raise ValueError("The private helper test fault mode is invalid.")
|
|
496
|
+
self._database_path = selected_path
|
|
497
|
+
self._configured_expected_instance = expected_database_instance
|
|
498
|
+
self._accepted_source: TrustedLiveSourceIdentity | None = None
|
|
499
|
+
self._busy_timeout_ms = busy_timeout_ms
|
|
500
|
+
self._operation_timeout_ms = _timeout_milliseconds(
|
|
501
|
+
operation_timeout_seconds,
|
|
502
|
+
description="default child operation timeout",
|
|
503
|
+
)
|
|
504
|
+
if self._operation_timeout_ms == 0:
|
|
505
|
+
raise ValueError("default child operation timeout must be positive.")
|
|
506
|
+
self._startup_timeout = _duration(
|
|
507
|
+
startup_timeout_seconds,
|
|
508
|
+
description="helper startup timeout",
|
|
509
|
+
allow_zero=False,
|
|
510
|
+
)
|
|
511
|
+
self._reply_timeout = _duration(
|
|
512
|
+
reply_timeout_seconds,
|
|
513
|
+
description="parent reply timeout",
|
|
514
|
+
allow_zero=False,
|
|
515
|
+
)
|
|
516
|
+
self._cancellation_grace = _duration(
|
|
517
|
+
cancellation_grace_seconds,
|
|
518
|
+
description="cancellation grace period",
|
|
519
|
+
allow_zero=True,
|
|
520
|
+
)
|
|
521
|
+
self._shutdown_timeout = _duration(
|
|
522
|
+
shutdown_timeout_seconds,
|
|
523
|
+
description="helper shutdown timeout",
|
|
524
|
+
allow_zero=True,
|
|
525
|
+
)
|
|
526
|
+
self._terminate_timeout = _duration(
|
|
527
|
+
terminate_timeout_seconds,
|
|
528
|
+
description="helper termination timeout",
|
|
529
|
+
allow_zero=True,
|
|
530
|
+
)
|
|
531
|
+
self._kill_timeout = _duration(
|
|
532
|
+
kill_timeout_seconds,
|
|
533
|
+
description="helper kill timeout",
|
|
534
|
+
allow_zero=True,
|
|
535
|
+
)
|
|
536
|
+
self._send_timeout = min(_DEFAULT_SEND_TIMEOUT_SECONDS, self._reply_timeout)
|
|
537
|
+
self._orphan_grace = min(
|
|
538
|
+
_DEFAULT_ORPHAN_GRACE_SECONDS,
|
|
539
|
+
max(0.05, self._shutdown_timeout),
|
|
540
|
+
)
|
|
541
|
+
self._context = multiprocessing.get_context("spawn")
|
|
542
|
+
self._owner_nonce = secrets.token_hex(16)
|
|
543
|
+
self._next_generation = 1
|
|
544
|
+
self._incarnation_counter = 0
|
|
545
|
+
self._helper: _HelperIncarnation | None = None
|
|
546
|
+
self._unreaped_helper: _HelperIncarnation | None = None
|
|
547
|
+
self._active_job: TrustedLiveJob[Any] | None = None
|
|
548
|
+
self._closing = False
|
|
549
|
+
self._closed = False
|
|
550
|
+
self._lock = threading.RLock()
|
|
551
|
+
self._reply_lock = threading.Lock()
|
|
552
|
+
self._test_fault = _test_fault
|
|
553
|
+
self._test_fault_consumed = False
|
|
554
|
+
self._atexit_registered = False
|
|
555
|
+
supervisor_reference = weakref.ref(self)
|
|
556
|
+
|
|
557
|
+
def cleanup_at_exit() -> None:
|
|
558
|
+
supervisor = supervisor_reference()
|
|
559
|
+
if supervisor is not None:
|
|
560
|
+
supervisor._cleanup_at_exit()
|
|
561
|
+
|
|
562
|
+
self._atexit_callback = cleanup_at_exit
|
|
563
|
+
atexit.register(self._atexit_callback)
|
|
564
|
+
self._atexit_registered = True
|
|
565
|
+
|
|
566
|
+
@classmethod
|
|
567
|
+
def open(
|
|
568
|
+
cls,
|
|
569
|
+
database_path: str | os.PathLike[str],
|
|
570
|
+
*,
|
|
571
|
+
expected_database_instance: DatabaseInstance | None = None,
|
|
572
|
+
busy_timeout_ms: int = 5_000,
|
|
573
|
+
operation_timeout_seconds: float = DEFAULT_TRUSTED_OPERATION_TIMEOUT_SECONDS,
|
|
574
|
+
startup_timeout_seconds: float = _DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
|
575
|
+
reply_timeout_seconds: float = _DEFAULT_REPLY_TIMEOUT_SECONDS,
|
|
576
|
+
cancellation_grace_seconds: float = (_DEFAULT_CANCELLATION_GRACE_SECONDS),
|
|
577
|
+
shutdown_timeout_seconds: float = _DEFAULT_SHUTDOWN_TIMEOUT_SECONDS,
|
|
578
|
+
terminate_timeout_seconds: float = _DEFAULT_TERMINATE_TIMEOUT_SECONDS,
|
|
579
|
+
kill_timeout_seconds: float = _DEFAULT_KILL_TIMEOUT_SECONDS,
|
|
580
|
+
_test_fault: str | None = None,
|
|
581
|
+
) -> TrustedLiveReaderSupervisor:
|
|
582
|
+
"""Spawn and open one helper, failing within ``startup_timeout_seconds``."""
|
|
583
|
+
|
|
584
|
+
supervisor = cls(
|
|
585
|
+
database_path,
|
|
586
|
+
expected_database_instance=expected_database_instance,
|
|
587
|
+
busy_timeout_ms=busy_timeout_ms,
|
|
588
|
+
operation_timeout_seconds=operation_timeout_seconds,
|
|
589
|
+
startup_timeout_seconds=startup_timeout_seconds,
|
|
590
|
+
reply_timeout_seconds=reply_timeout_seconds,
|
|
591
|
+
cancellation_grace_seconds=cancellation_grace_seconds,
|
|
592
|
+
shutdown_timeout_seconds=shutdown_timeout_seconds,
|
|
593
|
+
terminate_timeout_seconds=terminate_timeout_seconds,
|
|
594
|
+
kill_timeout_seconds=kill_timeout_seconds,
|
|
595
|
+
_test_fault=_test_fault,
|
|
596
|
+
)
|
|
597
|
+
try:
|
|
598
|
+
with supervisor._lock:
|
|
599
|
+
supervisor._spawn_helper_locked()
|
|
600
|
+
except BaseException:
|
|
601
|
+
supervisor._closed = True
|
|
602
|
+
with supervisor._lock:
|
|
603
|
+
supervisor._discard_helper_locked(force=True)
|
|
604
|
+
supervisor._unregister_atexit()
|
|
605
|
+
raise
|
|
606
|
+
return supervisor
|
|
607
|
+
|
|
608
|
+
@property
|
|
609
|
+
def closed(self) -> bool:
|
|
610
|
+
with self._lock:
|
|
611
|
+
return self._closed
|
|
612
|
+
|
|
613
|
+
@property
|
|
614
|
+
def source_identity(self) -> TrustedLiveSourceIdentity:
|
|
615
|
+
with self._lock:
|
|
616
|
+
source = self._accepted_source
|
|
617
|
+
if source is None:
|
|
618
|
+
raise TrustedLiveReaderUnavailableError(
|
|
619
|
+
"The supervisor has not accepted a trusted database source."
|
|
620
|
+
)
|
|
621
|
+
return source
|
|
622
|
+
|
|
623
|
+
@property
|
|
624
|
+
def database_instance(self) -> DatabaseInstance:
|
|
625
|
+
return self.source_identity.database_instance
|
|
626
|
+
|
|
627
|
+
@property
|
|
628
|
+
def incarnation(self) -> int:
|
|
629
|
+
with self._lock:
|
|
630
|
+
return self._incarnation_counter
|
|
631
|
+
|
|
632
|
+
@property
|
|
633
|
+
def helper_pid(self) -> int | None:
|
|
634
|
+
with self._lock:
|
|
635
|
+
helper = self._helper
|
|
636
|
+
return None if helper is None else helper.process.pid
|
|
637
|
+
|
|
638
|
+
@property
|
|
639
|
+
def helper_alive(self) -> bool:
|
|
640
|
+
with self._lock:
|
|
641
|
+
helper = self._helper
|
|
642
|
+
return helper is not None and helper.process.is_alive()
|
|
643
|
+
|
|
644
|
+
def resource_liveness(self) -> TrustedLiveSupervisorLiveness:
|
|
645
|
+
"""Describe active and quarantined resources without waiting for them."""
|
|
646
|
+
|
|
647
|
+
if not self._lock.acquire(blocking=False):
|
|
648
|
+
# ``close()`` intentionally owns this lock while it performs its
|
|
649
|
+
# bounded helper shutdown. GUI-side shutdown polling must not
|
|
650
|
+
# inherit that deadline. Lock contention therefore means only
|
|
651
|
+
# "cleanup is still pending", never "cleanup is complete".
|
|
652
|
+
return _LOCK_CONTENTION_LIVENESS
|
|
653
|
+
try:
|
|
654
|
+
return self._resource_liveness_locked()
|
|
655
|
+
finally:
|
|
656
|
+
self._lock.release()
|
|
657
|
+
|
|
658
|
+
def reap_closed_resources(self) -> TrustedLiveSupervisorLiveness:
|
|
659
|
+
"""Zero-wait reap a closed supervisor's quarantined incarnation.
|
|
660
|
+
|
|
661
|
+
A bounded ``close()`` may have to leave a process handle or reply
|
|
662
|
+
receiver quarantined. Owners can poll this method off the GUI thread;
|
|
663
|
+
it never terminates an active incarnation and never waits for one.
|
|
664
|
+
"""
|
|
665
|
+
|
|
666
|
+
if not self._lock.acquire(blocking=False):
|
|
667
|
+
return _LOCK_CONTENTION_LIVENESS
|
|
668
|
+
try:
|
|
669
|
+
if self._closed and not self._closing:
|
|
670
|
+
try:
|
|
671
|
+
self._reap_quarantined_helper_locked()
|
|
672
|
+
except TrustedLiveHelperForcedTerminationError:
|
|
673
|
+
pass
|
|
674
|
+
snapshot = self._resource_liveness_locked()
|
|
675
|
+
if snapshot.closed and not snapshot.resources_owned:
|
|
676
|
+
self._unregister_atexit_locked()
|
|
677
|
+
return snapshot
|
|
678
|
+
finally:
|
|
679
|
+
self._lock.release()
|
|
680
|
+
|
|
681
|
+
def _resource_liveness_locked(self) -> TrustedLiveSupervisorLiveness:
|
|
682
|
+
helpers = tuple(
|
|
683
|
+
helper
|
|
684
|
+
for helper in (self._helper, self._unreaped_helper)
|
|
685
|
+
if helper is not None
|
|
686
|
+
)
|
|
687
|
+
process_alive = False
|
|
688
|
+
receiver_alive = False
|
|
689
|
+
open_endpoints = 0
|
|
690
|
+
helper_pid: int | None = None
|
|
691
|
+
for helper in helpers:
|
|
692
|
+
if helper_pid is None:
|
|
693
|
+
try:
|
|
694
|
+
helper_pid = helper.process.pid
|
|
695
|
+
except (AssertionError, OSError, ValueError):
|
|
696
|
+
pass
|
|
697
|
+
try:
|
|
698
|
+
process_alive = process_alive or helper.process.is_alive()
|
|
699
|
+
except (AssertionError, OSError, ValueError):
|
|
700
|
+
# An incarnation is still owned, so an uninspectable process
|
|
701
|
+
# handle must conservatively keep shutdown pending.
|
|
702
|
+
process_alive = True
|
|
703
|
+
if helper.reply_receiver_started:
|
|
704
|
+
try:
|
|
705
|
+
receiver_alive = receiver_alive or helper.reply_receiver.is_alive()
|
|
706
|
+
except (AssertionError, RuntimeError):
|
|
707
|
+
receiver_alive = True
|
|
708
|
+
for connection in (
|
|
709
|
+
helper.command_send,
|
|
710
|
+
helper.reply_connection,
|
|
711
|
+
helper.control_send,
|
|
712
|
+
helper.test_notify_receive,
|
|
713
|
+
):
|
|
714
|
+
if connection is None:
|
|
715
|
+
continue
|
|
716
|
+
try:
|
|
717
|
+
is_open = not connection.closed
|
|
718
|
+
except (AttributeError, OSError, ValueError):
|
|
719
|
+
is_open = True
|
|
720
|
+
open_endpoints += int(is_open)
|
|
721
|
+
active_job = self._active_job
|
|
722
|
+
return TrustedLiveSupervisorLiveness(
|
|
723
|
+
helper_pid=helper_pid,
|
|
724
|
+
process_alive=process_alive,
|
|
725
|
+
receiver_alive=receiver_alive,
|
|
726
|
+
open_endpoints=open_endpoints,
|
|
727
|
+
active_incarnation=self._helper is not None,
|
|
728
|
+
unreaped_incarnation=self._unreaped_helper is not None,
|
|
729
|
+
active_job=active_job is not None and not active_job.done,
|
|
730
|
+
closing=self._closing,
|
|
731
|
+
closed=self._closed,
|
|
732
|
+
)
|
|
733
|
+
|
|
734
|
+
@property
|
|
735
|
+
def active_job(self) -> TrustedLiveJob[Any] | None:
|
|
736
|
+
with self._lock:
|
|
737
|
+
return self._active_job
|
|
738
|
+
|
|
739
|
+
def _require_open_locked(self) -> None:
|
|
740
|
+
if self._closed or self._closing:
|
|
741
|
+
raise TrustedLiveSupervisorClosedError(
|
|
742
|
+
"The trusted live-reader supervisor is closing or closed."
|
|
743
|
+
)
|
|
744
|
+
|
|
745
|
+
def _require_not_closed_locked(self) -> None:
|
|
746
|
+
if self._closed:
|
|
747
|
+
raise TrustedLiveSupervisorClosedError(
|
|
748
|
+
"The trusted live-reader supervisor is closed."
|
|
749
|
+
)
|
|
750
|
+
|
|
751
|
+
def _expected_instance_for_spawn(self) -> DatabaseInstance | None:
|
|
752
|
+
accepted_source = self._accepted_source
|
|
753
|
+
if accepted_source is not None:
|
|
754
|
+
return accepted_source.database_instance
|
|
755
|
+
return self._configured_expected_instance
|
|
756
|
+
|
|
757
|
+
def _spawn_helper_locked(self) -> _HelperIncarnation:
|
|
758
|
+
self._require_open_locked()
|
|
759
|
+
self._reap_quarantined_helper_locked()
|
|
760
|
+
if self._helper is not None:
|
|
761
|
+
raise TrustedLiveTransactionError(
|
|
762
|
+
"A trusted helper incarnation is already installed."
|
|
763
|
+
)
|
|
764
|
+
if self._next_generation > MAX_GENERATION:
|
|
765
|
+
raise TrustedLiveProtocolError(
|
|
766
|
+
"The trusted helper job-generation space is exhausted."
|
|
767
|
+
)
|
|
768
|
+
session = secrets.token_hex(16)
|
|
769
|
+
try:
|
|
770
|
+
startup_frame = encode_startup_request(
|
|
771
|
+
session,
|
|
772
|
+
self._next_generation,
|
|
773
|
+
self._database_path,
|
|
774
|
+
self._expected_instance_for_spawn(),
|
|
775
|
+
self._busy_timeout_ms,
|
|
776
|
+
self._operation_timeout_ms,
|
|
777
|
+
math.ceil(self._orphan_grace * 1_000.0),
|
|
778
|
+
)
|
|
779
|
+
except TrustedLiveProtocolValidationError as error:
|
|
780
|
+
raise TrustedLiveProtocolError(
|
|
781
|
+
"The helper startup configuration exceeds the bounded IPC protocol."
|
|
782
|
+
) from error
|
|
783
|
+
self._incarnation_counter += 1
|
|
784
|
+
number = self._incarnation_counter
|
|
785
|
+
command_receive, command_send = self._context.Pipe(duplex=False)
|
|
786
|
+
reply_receive, reply_send = self._context.Pipe(duplex=False)
|
|
787
|
+
control_receive, control_send = self._context.Pipe(duplex=False)
|
|
788
|
+
test_notify_receive: Connection | None = None
|
|
789
|
+
test_notify_send: Connection | None = None
|
|
790
|
+
fault = None
|
|
791
|
+
if self._test_fault is not None and not self._test_fault_consumed:
|
|
792
|
+
fault_aliases = {"hang_during_startup": "hang_startup"}
|
|
793
|
+
fault = fault_aliases.get(self._test_fault, self._test_fault)
|
|
794
|
+
self._test_fault_consumed = True
|
|
795
|
+
test_notify_receive, test_notify_send = self._context.Pipe(duplex=False)
|
|
796
|
+
process = self._context.Process(
|
|
797
|
+
target=trusted_live_helper_main,
|
|
798
|
+
args=(
|
|
799
|
+
command_receive,
|
|
800
|
+
reply_send,
|
|
801
|
+
control_receive,
|
|
802
|
+
startup_frame,
|
|
803
|
+
fault,
|
|
804
|
+
test_notify_send,
|
|
805
|
+
),
|
|
806
|
+
name=f"qplot-trusted-live-reader-{number}",
|
|
807
|
+
)
|
|
808
|
+
# The explicit bounded atexit cleanup below normally owns retirement.
|
|
809
|
+
# Daemonic status is a final interpreter-shutdown fallback so an
|
|
810
|
+
# abandoned supervisor cannot make multiprocessing join forever.
|
|
811
|
+
process.daemon = True
|
|
812
|
+
reply_inbox: queue.Queue[_ReplyInboxItem] = queue.Queue(maxsize=1)
|
|
813
|
+
reply_ready = threading.Event()
|
|
814
|
+
reply_receiver_stop = threading.Event()
|
|
815
|
+
reply_receiver_done = threading.Event()
|
|
816
|
+
reply_receiver = threading.Thread(
|
|
817
|
+
target=_reply_receiver_loop,
|
|
818
|
+
args=(
|
|
819
|
+
reply_receive,
|
|
820
|
+
reply_inbox,
|
|
821
|
+
reply_ready,
|
|
822
|
+
reply_receiver_stop,
|
|
823
|
+
reply_receiver_done,
|
|
824
|
+
),
|
|
825
|
+
name=f"qplot-trusted-reader-reply-receiver-{number}",
|
|
826
|
+
daemon=True,
|
|
827
|
+
)
|
|
828
|
+
helper = _HelperIncarnation(
|
|
829
|
+
number,
|
|
830
|
+
session,
|
|
831
|
+
process,
|
|
832
|
+
command_send,
|
|
833
|
+
reply_receive,
|
|
834
|
+
control_send,
|
|
835
|
+
test_notify_receive,
|
|
836
|
+
reply_inbox,
|
|
837
|
+
reply_ready,
|
|
838
|
+
reply_receiver_stop,
|
|
839
|
+
reply_receiver_done,
|
|
840
|
+
reply_receiver,
|
|
841
|
+
)
|
|
842
|
+
try:
|
|
843
|
+
process.start()
|
|
844
|
+
except BaseException as error:
|
|
845
|
+
for connection in (
|
|
846
|
+
command_receive,
|
|
847
|
+
command_send,
|
|
848
|
+
reply_receive,
|
|
849
|
+
reply_send,
|
|
850
|
+
control_receive,
|
|
851
|
+
control_send,
|
|
852
|
+
test_notify_receive,
|
|
853
|
+
test_notify_send,
|
|
854
|
+
):
|
|
855
|
+
_close_connection(connection)
|
|
856
|
+
try:
|
|
857
|
+
process.close()
|
|
858
|
+
except (OSError, ValueError):
|
|
859
|
+
pass
|
|
860
|
+
raise TrustedLiveHelperStartupError(
|
|
861
|
+
"The trusted live-reader helper could not be spawned."
|
|
862
|
+
) from error
|
|
863
|
+
# Process start may lazily register multiprocessing's own exit hook.
|
|
864
|
+
# Re-register ours afterwards so LIFO interpreter shutdown runs this
|
|
865
|
+
# bounded cleanup before multiprocessing considers joining children.
|
|
866
|
+
if self._atexit_registered:
|
|
867
|
+
atexit.unregister(self._atexit_callback)
|
|
868
|
+
atexit.register(self._atexit_callback)
|
|
869
|
+
for child_endpoint in (
|
|
870
|
+
command_receive,
|
|
871
|
+
reply_send,
|
|
872
|
+
control_receive,
|
|
873
|
+
test_notify_send,
|
|
874
|
+
):
|
|
875
|
+
_close_connection(child_endpoint)
|
|
876
|
+
self._helper = helper
|
|
877
|
+
try:
|
|
878
|
+
try:
|
|
879
|
+
helper.reply_receiver.start()
|
|
880
|
+
helper.reply_receiver_started = True
|
|
881
|
+
except BaseException as error:
|
|
882
|
+
raise TrustedLiveHelperStartupError(
|
|
883
|
+
"The trusted helper reply receiver could not be started."
|
|
884
|
+
) from error
|
|
885
|
+
envelope = self._wait_startup_reply_locked(helper)
|
|
886
|
+
status, payload = decode_reply_payload(envelope)
|
|
887
|
+
if envelope.generation != 0:
|
|
888
|
+
raise TrustedLiveProtocolError(
|
|
889
|
+
"The helper sent an out-of-order startup reply."
|
|
890
|
+
)
|
|
891
|
+
if envelope.operation == "protocol":
|
|
892
|
+
if status != "error" or payload["code"] != "protocol_error":
|
|
893
|
+
raise TrustedLiveProtocolError(
|
|
894
|
+
"The helper sent an invalid startup protocol outcome."
|
|
895
|
+
)
|
|
896
|
+
raise cast(
|
|
897
|
+
BaseException,
|
|
898
|
+
_error_from_payload(payload["code"], payload["message"]),
|
|
899
|
+
)
|
|
900
|
+
if envelope.operation != "startup":
|
|
901
|
+
raise TrustedLiveProtocolError(
|
|
902
|
+
"The helper sent an out-of-order startup reply."
|
|
903
|
+
)
|
|
904
|
+
if status == "error":
|
|
905
|
+
raise _error_from_payload(payload["code"], payload["message"])
|
|
906
|
+
source = validate_startup_success(payload)
|
|
907
|
+
prior = self._expected_instance_for_spawn()
|
|
908
|
+
if prior is not None and database_instances_differ(
|
|
909
|
+
prior,
|
|
910
|
+
source.database_instance,
|
|
911
|
+
):
|
|
912
|
+
raise TrustedLiveSourceChangedError(
|
|
913
|
+
"The spawned helper reported a different database instance."
|
|
914
|
+
)
|
|
915
|
+
if self._accepted_source is None:
|
|
916
|
+
self._accepted_source = source
|
|
917
|
+
elif database_instances_differ(
|
|
918
|
+
self._accepted_source.database_instance,
|
|
919
|
+
source.database_instance,
|
|
920
|
+
):
|
|
921
|
+
raise TrustedLiveSourceChangedError(
|
|
922
|
+
"A replacement helper opened a different database instance."
|
|
923
|
+
)
|
|
924
|
+
# Preserve the accepted main-file binding while accurately
|
|
925
|
+
# reporting the currently installed WAL/SHM identities and mode.
|
|
926
|
+
self._accepted_source = source
|
|
927
|
+
return helper
|
|
928
|
+
except BaseException:
|
|
929
|
+
self._discard_helper_locked(force=False)
|
|
930
|
+
raise
|
|
931
|
+
|
|
932
|
+
@staticmethod
|
|
933
|
+
def _take_reply_item(
|
|
934
|
+
helper: _HelperIncarnation,
|
|
935
|
+
timeout: float,
|
|
936
|
+
) -> _ReplyInboxItem | None:
|
|
937
|
+
"""Wait boundedly for the sole receiver to publish one complete frame."""
|
|
938
|
+
|
|
939
|
+
try:
|
|
940
|
+
if timeout <= 0:
|
|
941
|
+
item = helper.reply_inbox.get_nowait()
|
|
942
|
+
else:
|
|
943
|
+
item = helper.reply_inbox.get(timeout=timeout)
|
|
944
|
+
except queue.Empty:
|
|
945
|
+
return None
|
|
946
|
+
if helper.reply_inbox.empty():
|
|
947
|
+
helper.reply_ready.clear()
|
|
948
|
+
return item
|
|
949
|
+
|
|
950
|
+
def _wait_startup_reply_locked(
|
|
951
|
+
self,
|
|
952
|
+
helper: _HelperIncarnation,
|
|
953
|
+
) -> ProtocolEnvelope:
|
|
954
|
+
deadline = time.monotonic() + self._startup_timeout
|
|
955
|
+
while True:
|
|
956
|
+
remaining = deadline - time.monotonic()
|
|
957
|
+
if remaining <= 0:
|
|
958
|
+
self._discard_helper_locked(force=True)
|
|
959
|
+
raise TrustedLiveHelperStartupError(
|
|
960
|
+
"The trusted helper did not finish startup before its "
|
|
961
|
+
"parent deadline."
|
|
962
|
+
)
|
|
963
|
+
try:
|
|
964
|
+
item = self._take_reply_item(
|
|
965
|
+
helper,
|
|
966
|
+
min(_POLL_QUANTUM_SECONDS, remaining),
|
|
967
|
+
)
|
|
968
|
+
if item is not None:
|
|
969
|
+
if isinstance(item, _ReplyReceiveFailure):
|
|
970
|
+
if item.kind == "eof":
|
|
971
|
+
raise TrustedLiveHelperExitedError(item.description)
|
|
972
|
+
raise TrustedLiveProtocolError(item.description)
|
|
973
|
+
frame = item
|
|
974
|
+
envelope = decode_reply_frame(frame)
|
|
975
|
+
if envelope.session != helper.session:
|
|
976
|
+
raise TrustedLiveProtocolError(
|
|
977
|
+
"A stale startup reply targeted another incarnation."
|
|
978
|
+
)
|
|
979
|
+
return envelope
|
|
980
|
+
except TrustedLiveProtocolValidationError as error:
|
|
981
|
+
raise TrustedLiveProtocolError(
|
|
982
|
+
"The helper sent malformed or oversized startup IPC."
|
|
983
|
+
) from error
|
|
984
|
+
except EOFError as error:
|
|
985
|
+
raise TrustedLiveHelperExitedError(
|
|
986
|
+
"The trusted helper's startup endpoint closed unexpectedly."
|
|
987
|
+
) from error
|
|
988
|
+
except (OSError, ValueError) as error:
|
|
989
|
+
raise TrustedLiveProtocolError(
|
|
990
|
+
"The helper sent malformed or oversized startup IPC."
|
|
991
|
+
) from error
|
|
992
|
+
if not helper.process.is_alive():
|
|
993
|
+
exit_code = helper.process.exitcode
|
|
994
|
+
if helper.reply_receiver_done.is_set() and helper.reply_inbox.empty():
|
|
995
|
+
raise TrustedLiveHelperExitedError(
|
|
996
|
+
"The trusted helper exited during startup "
|
|
997
|
+
f"(exit code {exit_code})."
|
|
998
|
+
)
|
|
999
|
+
|
|
1000
|
+
def _ensure_helper_locked(self) -> _HelperIncarnation:
|
|
1001
|
+
self._require_open_locked()
|
|
1002
|
+
self._reap_quarantined_helper_locked()
|
|
1003
|
+
helper = self._helper
|
|
1004
|
+
if (
|
|
1005
|
+
helper is not None
|
|
1006
|
+
and helper.process.is_alive()
|
|
1007
|
+
and helper.reply_receiver_started
|
|
1008
|
+
and helper.reply_receiver.is_alive()
|
|
1009
|
+
):
|
|
1010
|
+
return helper
|
|
1011
|
+
if helper is not None:
|
|
1012
|
+
# A live process without its sole receiver is unusable and must not
|
|
1013
|
+
# accept another generation.
|
|
1014
|
+
self._discard_helper_locked(force=helper.process.is_alive())
|
|
1015
|
+
return self._spawn_helper_locked()
|
|
1016
|
+
|
|
1017
|
+
def _stop_incarnation(
|
|
1018
|
+
self,
|
|
1019
|
+
helper: _HelperIncarnation,
|
|
1020
|
+
*,
|
|
1021
|
+
force: bool,
|
|
1022
|
+
terminate_timeout: float | None = None,
|
|
1023
|
+
kill_timeout: float | None = None,
|
|
1024
|
+
) -> tuple[bool, bool]:
|
|
1025
|
+
"""Join and close one exact process, escalating without unbounded waits."""
|
|
1026
|
+
|
|
1027
|
+
forced = False
|
|
1028
|
+
terminate_wait = (
|
|
1029
|
+
self._terminate_timeout if terminate_timeout is None else terminate_timeout
|
|
1030
|
+
)
|
|
1031
|
+
kill_wait = self._kill_timeout if kill_timeout is None else kill_timeout
|
|
1032
|
+
|
|
1033
|
+
def process_alive() -> bool:
|
|
1034
|
+
try:
|
|
1035
|
+
return helper.process.is_alive()
|
|
1036
|
+
except (AssertionError, OSError, ValueError):
|
|
1037
|
+
return True
|
|
1038
|
+
|
|
1039
|
+
def bounded_join(timeout: float) -> None:
|
|
1040
|
+
try:
|
|
1041
|
+
helper.process.join(timeout=timeout)
|
|
1042
|
+
except (AssertionError, OSError, ValueError):
|
|
1043
|
+
pass
|
|
1044
|
+
|
|
1045
|
+
try:
|
|
1046
|
+
if not force and process_alive():
|
|
1047
|
+
_close_connection(helper.command_send)
|
|
1048
|
+
_close_connection(helper.control_send)
|
|
1049
|
+
bounded_join(self._shutdown_timeout)
|
|
1050
|
+
if process_alive():
|
|
1051
|
+
forced = True
|
|
1052
|
+
try:
|
|
1053
|
+
helper.process.terminate()
|
|
1054
|
+
except (AttributeError, OSError, ValueError):
|
|
1055
|
+
pass
|
|
1056
|
+
bounded_join(terminate_wait)
|
|
1057
|
+
if process_alive():
|
|
1058
|
+
forced = True
|
|
1059
|
+
try:
|
|
1060
|
+
kill = getattr(helper.process, "kill", None)
|
|
1061
|
+
if callable(kill):
|
|
1062
|
+
kill()
|
|
1063
|
+
else:
|
|
1064
|
+
helper.process.terminate()
|
|
1065
|
+
except (AttributeError, OSError, ValueError):
|
|
1066
|
+
pass
|
|
1067
|
+
bounded_join(kill_wait)
|
|
1068
|
+
finally:
|
|
1069
|
+
for connection in (
|
|
1070
|
+
helper.command_send,
|
|
1071
|
+
helper.control_send,
|
|
1072
|
+
helper.test_notify_receive,
|
|
1073
|
+
):
|
|
1074
|
+
_close_connection(connection)
|
|
1075
|
+
helper.reply_receiver_stop.set()
|
|
1076
|
+
_close_connection(helper.reply_connection)
|
|
1077
|
+
if helper.reply_receiver_started:
|
|
1078
|
+
try:
|
|
1079
|
+
helper.reply_receiver.join(
|
|
1080
|
+
timeout=_REPLY_RECEIVER_JOIN_TIMEOUT_SECONDS
|
|
1081
|
+
)
|
|
1082
|
+
except RuntimeError:
|
|
1083
|
+
pass
|
|
1084
|
+
receiver_alive = (
|
|
1085
|
+
helper.reply_receiver_started and helper.reply_receiver.is_alive()
|
|
1086
|
+
)
|
|
1087
|
+
if process_alive() or receiver_alive:
|
|
1088
|
+
# Keep the Process handle quarantined. A later spawn may proceed
|
|
1089
|
+
# only after both its process and sole receiver are reaped.
|
|
1090
|
+
return True, False
|
|
1091
|
+
bounded_join(0)
|
|
1092
|
+
try:
|
|
1093
|
+
helper.process.close()
|
|
1094
|
+
except (OSError, ValueError):
|
|
1095
|
+
pass
|
|
1096
|
+
return forced, True
|
|
1097
|
+
|
|
1098
|
+
def _reap_quarantined_helper_locked(self) -> None:
|
|
1099
|
+
helper = self._unreaped_helper
|
|
1100
|
+
if helper is None:
|
|
1101
|
+
return
|
|
1102
|
+
helper.reply_receiver_stop.set()
|
|
1103
|
+
_close_connection(helper.reply_connection)
|
|
1104
|
+
if helper.reply_receiver_started:
|
|
1105
|
+
try:
|
|
1106
|
+
helper.reply_receiver.join(timeout=0)
|
|
1107
|
+
except RuntimeError:
|
|
1108
|
+
pass
|
|
1109
|
+
try:
|
|
1110
|
+
helper.process.join(timeout=0)
|
|
1111
|
+
alive = helper.process.is_alive()
|
|
1112
|
+
except (AssertionError, OSError, ValueError):
|
|
1113
|
+
alive = True
|
|
1114
|
+
receiver_alive = (
|
|
1115
|
+
helper.reply_receiver_started and helper.reply_receiver.is_alive()
|
|
1116
|
+
)
|
|
1117
|
+
if alive or receiver_alive:
|
|
1118
|
+
raise TrustedLiveHelperForcedTerminationError(
|
|
1119
|
+
"A previously killed trusted helper or its reply receiver has "
|
|
1120
|
+
"not been reaped; a replacement process will not be started."
|
|
1121
|
+
)
|
|
1122
|
+
try:
|
|
1123
|
+
helper.process.close()
|
|
1124
|
+
except (OSError, ValueError):
|
|
1125
|
+
pass
|
|
1126
|
+
self._unreaped_helper = None
|
|
1127
|
+
|
|
1128
|
+
def _force_quarantined_helper_locked(
|
|
1129
|
+
self,
|
|
1130
|
+
*,
|
|
1131
|
+
terminate_timeout: float | None = None,
|
|
1132
|
+
kill_timeout: float | None = None,
|
|
1133
|
+
) -> bool:
|
|
1134
|
+
helper = self._unreaped_helper
|
|
1135
|
+
if helper is None:
|
|
1136
|
+
return False
|
|
1137
|
+
forced, reaped = self._stop_incarnation(
|
|
1138
|
+
helper,
|
|
1139
|
+
force=True,
|
|
1140
|
+
terminate_timeout=terminate_timeout,
|
|
1141
|
+
kill_timeout=kill_timeout,
|
|
1142
|
+
)
|
|
1143
|
+
if reaped:
|
|
1144
|
+
self._unreaped_helper = None
|
|
1145
|
+
return forced or not reaped
|
|
1146
|
+
|
|
1147
|
+
def _discard_helper_locked(
|
|
1148
|
+
self,
|
|
1149
|
+
*,
|
|
1150
|
+
force: bool,
|
|
1151
|
+
terminate_timeout: float | None = None,
|
|
1152
|
+
kill_timeout: float | None = None,
|
|
1153
|
+
) -> bool:
|
|
1154
|
+
helper, self._helper = self._helper, None
|
|
1155
|
+
if helper is None:
|
|
1156
|
+
return False
|
|
1157
|
+
forced, reaped = self._stop_incarnation(
|
|
1158
|
+
helper,
|
|
1159
|
+
force=force,
|
|
1160
|
+
terminate_timeout=terminate_timeout,
|
|
1161
|
+
kill_timeout=kill_timeout,
|
|
1162
|
+
)
|
|
1163
|
+
if not reaped:
|
|
1164
|
+
self._unreaped_helper = helper
|
|
1165
|
+
return forced
|
|
1166
|
+
|
|
1167
|
+
def _validate_job_locked(self, job: TrustedLiveJob[_T]) -> None:
|
|
1168
|
+
if not isinstance(job, TrustedLiveJob) or job._owner_nonce != self._owner_nonce:
|
|
1169
|
+
raise ValueError("The job does not belong to this supervisor.")
|
|
1170
|
+
|
|
1171
|
+
def _submit(
|
|
1172
|
+
self,
|
|
1173
|
+
operation: str,
|
|
1174
|
+
queries: Sequence[TrustedQuery] | None,
|
|
1175
|
+
timeout: float | None,
|
|
1176
|
+
) -> TrustedLiveJob[Any]:
|
|
1177
|
+
timeout_ms = (
|
|
1178
|
+
self._operation_timeout_ms
|
|
1179
|
+
if timeout is None
|
|
1180
|
+
else _timeout_milliseconds(
|
|
1181
|
+
timeout,
|
|
1182
|
+
description="child operation timeout",
|
|
1183
|
+
)
|
|
1184
|
+
)
|
|
1185
|
+
with self._lock:
|
|
1186
|
+
self._require_open_locked()
|
|
1187
|
+
if self._active_job is not None:
|
|
1188
|
+
raise TrustedLiveTransactionError(
|
|
1189
|
+
"Only one trusted database job may be active per helper."
|
|
1190
|
+
)
|
|
1191
|
+
helper = self._ensure_helper_locked()
|
|
1192
|
+
generation = self._next_generation
|
|
1193
|
+
if generation > MAX_GENERATION:
|
|
1194
|
+
raise TrustedLiveProtocolError(
|
|
1195
|
+
"The trusted helper job-generation space is exhausted."
|
|
1196
|
+
)
|
|
1197
|
+
try:
|
|
1198
|
+
frame = encode_job_request(
|
|
1199
|
+
helper.session,
|
|
1200
|
+
generation,
|
|
1201
|
+
operation,
|
|
1202
|
+
queries,
|
|
1203
|
+
timeout_ms,
|
|
1204
|
+
)
|
|
1205
|
+
except TrustedLiveProtocolValidationError as error:
|
|
1206
|
+
raise TrustedLiveProtocolError(
|
|
1207
|
+
"The database job exceeds the bounded IPC protocol."
|
|
1208
|
+
) from error
|
|
1209
|
+
self._next_generation += 1
|
|
1210
|
+
expected_result_count = (
|
|
1211
|
+
len(queries)
|
|
1212
|
+
if operation in {"query", "query_batch"} and queries is not None
|
|
1213
|
+
else None
|
|
1214
|
+
)
|
|
1215
|
+
job: TrustedLiveJob[Any] = TrustedLiveJob(
|
|
1216
|
+
helper.number,
|
|
1217
|
+
helper.session,
|
|
1218
|
+
generation,
|
|
1219
|
+
operation,
|
|
1220
|
+
self._owner_nonce,
|
|
1221
|
+
expected_result_count,
|
|
1222
|
+
)
|
|
1223
|
+
self._active_job = job
|
|
1224
|
+
try:
|
|
1225
|
+
_bounded_send_bytes(helper.command_send, frame, self._send_timeout)
|
|
1226
|
+
except BaseException as error:
|
|
1227
|
+
self._active_job = None
|
|
1228
|
+
self._discard_helper_locked(force=True)
|
|
1229
|
+
raise TrustedLiveHelperExitedError(
|
|
1230
|
+
"The trusted helper did not accept the bounded job frame."
|
|
1231
|
+
) from error
|
|
1232
|
+
return job
|
|
1233
|
+
|
|
1234
|
+
def submit_query(
|
|
1235
|
+
self,
|
|
1236
|
+
sql: str,
|
|
1237
|
+
bindings: SqliteBindings = None,
|
|
1238
|
+
*,
|
|
1239
|
+
timeout: float | None = None,
|
|
1240
|
+
) -> TrustedLiveJob[TrustedQueryResult]:
|
|
1241
|
+
"""Submit one finite read-only statement without waiting for its reply."""
|
|
1242
|
+
|
|
1243
|
+
try:
|
|
1244
|
+
query = normalize_query_specification(sql, bindings)
|
|
1245
|
+
except TrustedLiveProtocolValidationError as error:
|
|
1246
|
+
raise TrustedLiveProtocolError(
|
|
1247
|
+
"The database job exceeds the bounded IPC protocol."
|
|
1248
|
+
) from error
|
|
1249
|
+
return cast(
|
|
1250
|
+
TrustedLiveJob[TrustedQueryResult],
|
|
1251
|
+
self._submit("query", (query,), timeout),
|
|
1252
|
+
)
|
|
1253
|
+
|
|
1254
|
+
def submit_query_batch(
|
|
1255
|
+
self,
|
|
1256
|
+
queries: Sequence[TrustedQuery],
|
|
1257
|
+
*,
|
|
1258
|
+
timeout: float | None = None,
|
|
1259
|
+
) -> TrustedLiveJob[tuple[TrustedQueryResult, ...]]:
|
|
1260
|
+
"""Submit one finite repeatable-read query batch."""
|
|
1261
|
+
|
|
1262
|
+
try:
|
|
1263
|
+
normalized_queries = normalize_query_batch(queries)
|
|
1264
|
+
except TrustedLiveProtocolValidationError as error:
|
|
1265
|
+
raise TrustedLiveProtocolError(
|
|
1266
|
+
"The database job exceeds the bounded IPC protocol."
|
|
1267
|
+
) from error
|
|
1268
|
+
return cast(
|
|
1269
|
+
TrustedLiveJob[tuple[TrustedQueryResult, ...]],
|
|
1270
|
+
self._submit("query_batch", normalized_queries, timeout),
|
|
1271
|
+
)
|
|
1272
|
+
|
|
1273
|
+
def submit_data_version(
|
|
1274
|
+
self,
|
|
1275
|
+
*,
|
|
1276
|
+
timeout: float | None = None,
|
|
1277
|
+
) -> TrustedLiveJob[int]:
|
|
1278
|
+
"""Submit a finite ``PRAGMA data_version`` operation."""
|
|
1279
|
+
|
|
1280
|
+
return cast(
|
|
1281
|
+
TrustedLiveJob[int],
|
|
1282
|
+
self._submit("data_version", None, timeout),
|
|
1283
|
+
)
|
|
1284
|
+
|
|
1285
|
+
@staticmethod
|
|
1286
|
+
def _complete_job(
|
|
1287
|
+
job: TrustedLiveJob[Any],
|
|
1288
|
+
*,
|
|
1289
|
+
value: object = _NO_VALUE,
|
|
1290
|
+
error: BaseException | None = None,
|
|
1291
|
+
) -> None:
|
|
1292
|
+
if job.done:
|
|
1293
|
+
return
|
|
1294
|
+
job._value = value
|
|
1295
|
+
job._error = error
|
|
1296
|
+
job._done.set()
|
|
1297
|
+
|
|
1298
|
+
def _protocol_failure_locked(
|
|
1299
|
+
self,
|
|
1300
|
+
job: TrustedLiveJob[Any],
|
|
1301
|
+
message: str,
|
|
1302
|
+
cause: BaseException | None = None,
|
|
1303
|
+
) -> None:
|
|
1304
|
+
error = TrustedLiveProtocolError(message)
|
|
1305
|
+
if cause is not None:
|
|
1306
|
+
error.__cause__ = cause
|
|
1307
|
+
if self._active_job is job:
|
|
1308
|
+
self._active_job = None
|
|
1309
|
+
self._discard_helper_locked(force=True)
|
|
1310
|
+
self._complete_job(job, error=error)
|
|
1311
|
+
|
|
1312
|
+
def _unexpected_exit_locked(
|
|
1313
|
+
self,
|
|
1314
|
+
job: TrustedLiveJob[Any],
|
|
1315
|
+
message: str,
|
|
1316
|
+
) -> None:
|
|
1317
|
+
if self._active_job is job:
|
|
1318
|
+
self._active_job = None
|
|
1319
|
+
self._discard_helper_locked(force=False)
|
|
1320
|
+
self._complete_job(job, error=TrustedLiveHelperExitedError(message))
|
|
1321
|
+
|
|
1322
|
+
def _handle_job_reply_locked(
|
|
1323
|
+
self,
|
|
1324
|
+
job: TrustedLiveJob[Any],
|
|
1325
|
+
envelope: ProtocolEnvelope,
|
|
1326
|
+
) -> None:
|
|
1327
|
+
helper = self._helper
|
|
1328
|
+
if helper is None or self._active_job is not job:
|
|
1329
|
+
self._protocol_failure_locked(
|
|
1330
|
+
job,
|
|
1331
|
+
"A reply arrived without its exact active helper job.",
|
|
1332
|
+
)
|
|
1333
|
+
return
|
|
1334
|
+
if envelope.session != job.session or envelope.session != helper.session:
|
|
1335
|
+
self._protocol_failure_locked(
|
|
1336
|
+
job,
|
|
1337
|
+
"A stale reply from another process incarnation was rejected.",
|
|
1338
|
+
)
|
|
1339
|
+
return
|
|
1340
|
+
if envelope.generation != job.generation:
|
|
1341
|
+
self._protocol_failure_locked(
|
|
1342
|
+
job,
|
|
1343
|
+
"A duplicate, stale, or out-of-order job reply was rejected.",
|
|
1344
|
+
)
|
|
1345
|
+
return
|
|
1346
|
+
try:
|
|
1347
|
+
status, payload = decode_reply_payload(envelope)
|
|
1348
|
+
if envelope.operation == "protocol":
|
|
1349
|
+
if status != "error" or payload["code"] != "protocol_error":
|
|
1350
|
+
raise TrustedLiveProtocolValidationError(
|
|
1351
|
+
"A protocol failure reply has invalid fields."
|
|
1352
|
+
)
|
|
1353
|
+
error = _error_from_payload(payload["code"], payload["message"])
|
|
1354
|
+
terminal = True
|
|
1355
|
+
value = _NO_VALUE
|
|
1356
|
+
else:
|
|
1357
|
+
if envelope.operation != job.operation:
|
|
1358
|
+
raise TrustedLiveProtocolValidationError(
|
|
1359
|
+
"The helper reply operation does not match its job."
|
|
1360
|
+
)
|
|
1361
|
+
if status == "error":
|
|
1362
|
+
code = payload["code"]
|
|
1363
|
+
error = _error_from_payload(code, payload["message"])
|
|
1364
|
+
terminal = error_code_is_terminal(code)
|
|
1365
|
+
value = _NO_VALUE
|
|
1366
|
+
else:
|
|
1367
|
+
value = validate_job_success(job.operation, payload)
|
|
1368
|
+
if job.operation == "query_batch" and (
|
|
1369
|
+
not isinstance(value, tuple)
|
|
1370
|
+
or len(value) != job._expected_result_count
|
|
1371
|
+
):
|
|
1372
|
+
raise TrustedLiveProtocolValidationError(
|
|
1373
|
+
"A query-batch reply does not contain one result "
|
|
1374
|
+
"for every submitted statement."
|
|
1375
|
+
)
|
|
1376
|
+
error = None
|
|
1377
|
+
terminal = False
|
|
1378
|
+
except TrustedLiveProtocolValidationError as protocol_error:
|
|
1379
|
+
self._protocol_failure_locked(
|
|
1380
|
+
job,
|
|
1381
|
+
"The helper reply payload failed closed validation.",
|
|
1382
|
+
protocol_error,
|
|
1383
|
+
)
|
|
1384
|
+
return
|
|
1385
|
+
self._active_job = None
|
|
1386
|
+
if job._parent_timed_out:
|
|
1387
|
+
forced = self._discard_helper_locked(force=False)
|
|
1388
|
+
if forced:
|
|
1389
|
+
error = TrustedLiveHelperForcedTerminationError(
|
|
1390
|
+
"The parent reply deadline expired and helper retirement "
|
|
1391
|
+
"required forced termination."
|
|
1392
|
+
)
|
|
1393
|
+
else:
|
|
1394
|
+
error = TrustedLiveHelperReplyTimeoutError(
|
|
1395
|
+
"The parent reply deadline expired; the completed helper "
|
|
1396
|
+
"incarnation was retired without accepting its late reply."
|
|
1397
|
+
)
|
|
1398
|
+
value = _NO_VALUE
|
|
1399
|
+
elif terminal:
|
|
1400
|
+
self._discard_helper_locked(force=False)
|
|
1401
|
+
self._complete_job(job, value=value, error=error)
|
|
1402
|
+
|
|
1403
|
+
def _poll_job_once(self, job: TrustedLiveJob[Any], timeout: float) -> None:
|
|
1404
|
+
with self._lock:
|
|
1405
|
+
if job.done:
|
|
1406
|
+
return
|
|
1407
|
+
helper = self._helper
|
|
1408
|
+
if (
|
|
1409
|
+
helper is None
|
|
1410
|
+
or self._active_job is not job
|
|
1411
|
+
or helper.session != job.session
|
|
1412
|
+
):
|
|
1413
|
+
self._unexpected_exit_locked(
|
|
1414
|
+
job,
|
|
1415
|
+
"The job's helper incarnation is no longer available.",
|
|
1416
|
+
)
|
|
1417
|
+
return
|
|
1418
|
+
process = helper.process
|
|
1419
|
+
try:
|
|
1420
|
+
item = self._take_reply_item(helper, max(0.0, timeout))
|
|
1421
|
+
except Exception as error:
|
|
1422
|
+
with self._lock:
|
|
1423
|
+
if job.done:
|
|
1424
|
+
return
|
|
1425
|
+
if self._helper is not helper or self._active_job is not job:
|
|
1426
|
+
self._complete_job(
|
|
1427
|
+
job,
|
|
1428
|
+
error=TrustedLiveHelperExitedError(
|
|
1429
|
+
"The job's helper was retired while polling its reply."
|
|
1430
|
+
),
|
|
1431
|
+
)
|
|
1432
|
+
return
|
|
1433
|
+
self._protocol_failure_locked(
|
|
1434
|
+
job,
|
|
1435
|
+
"The helper reply endpoint failed closed.",
|
|
1436
|
+
error,
|
|
1437
|
+
)
|
|
1438
|
+
return
|
|
1439
|
+
if item is not None:
|
|
1440
|
+
if isinstance(item, _ReplyReceiveFailure):
|
|
1441
|
+
with self._lock:
|
|
1442
|
+
if job.done:
|
|
1443
|
+
return
|
|
1444
|
+
if self._helper is not helper or self._active_job is not job:
|
|
1445
|
+
self._complete_job(
|
|
1446
|
+
job,
|
|
1447
|
+
error=TrustedLiveHelperExitedError(
|
|
1448
|
+
"The job's helper was retired while its sole "
|
|
1449
|
+
"reply receiver was stopping."
|
|
1450
|
+
),
|
|
1451
|
+
)
|
|
1452
|
+
return
|
|
1453
|
+
if item.kind == "eof":
|
|
1454
|
+
try:
|
|
1455
|
+
exit_code = process.exitcode
|
|
1456
|
+
except (AssertionError, OSError, ValueError):
|
|
1457
|
+
exit_code = None
|
|
1458
|
+
self._unexpected_exit_locked(
|
|
1459
|
+
job,
|
|
1460
|
+
"The trusted helper closed its reply endpoint before "
|
|
1461
|
+
f"replying (exit code {exit_code}).",
|
|
1462
|
+
)
|
|
1463
|
+
else:
|
|
1464
|
+
self._protocol_failure_locked(job, item.description)
|
|
1465
|
+
return
|
|
1466
|
+
try:
|
|
1467
|
+
frame = item
|
|
1468
|
+
envelope = decode_reply_frame(frame)
|
|
1469
|
+
except TrustedLiveProtocolValidationError as error:
|
|
1470
|
+
with self._lock:
|
|
1471
|
+
if job.done:
|
|
1472
|
+
return
|
|
1473
|
+
if self._helper is not helper or self._active_job is not job:
|
|
1474
|
+
self._complete_job(
|
|
1475
|
+
job,
|
|
1476
|
+
error=TrustedLiveHelperExitedError(
|
|
1477
|
+
"The job's helper was retired before its reply "
|
|
1478
|
+
"could be validated."
|
|
1479
|
+
),
|
|
1480
|
+
)
|
|
1481
|
+
return
|
|
1482
|
+
self._protocol_failure_locked(
|
|
1483
|
+
job,
|
|
1484
|
+
"The helper sent malformed or oversized reply IPC.",
|
|
1485
|
+
error,
|
|
1486
|
+
)
|
|
1487
|
+
return
|
|
1488
|
+
except EOFError:
|
|
1489
|
+
with self._lock:
|
|
1490
|
+
if job.done:
|
|
1491
|
+
return
|
|
1492
|
+
if self._helper is not helper or self._active_job is not job:
|
|
1493
|
+
self._complete_job(
|
|
1494
|
+
job,
|
|
1495
|
+
error=TrustedLiveHelperExitedError(
|
|
1496
|
+
"The job's helper was retired while receiving "
|
|
1497
|
+
"its reply."
|
|
1498
|
+
),
|
|
1499
|
+
)
|
|
1500
|
+
return
|
|
1501
|
+
try:
|
|
1502
|
+
exit_code = process.exitcode
|
|
1503
|
+
except (AssertionError, OSError, ValueError):
|
|
1504
|
+
exit_code = None
|
|
1505
|
+
self._unexpected_exit_locked(
|
|
1506
|
+
job,
|
|
1507
|
+
"The trusted helper closed its reply endpoint before "
|
|
1508
|
+
f"replying (exit code {exit_code}).",
|
|
1509
|
+
)
|
|
1510
|
+
return
|
|
1511
|
+
except (OSError, ValueError) as error:
|
|
1512
|
+
with self._lock:
|
|
1513
|
+
if job.done:
|
|
1514
|
+
return
|
|
1515
|
+
if self._helper is not helper or self._active_job is not job:
|
|
1516
|
+
self._complete_job(
|
|
1517
|
+
job,
|
|
1518
|
+
error=TrustedLiveHelperExitedError(
|
|
1519
|
+
"The job's helper was retired while receiving "
|
|
1520
|
+
"its reply frame."
|
|
1521
|
+
),
|
|
1522
|
+
)
|
|
1523
|
+
return
|
|
1524
|
+
self._protocol_failure_locked(
|
|
1525
|
+
job,
|
|
1526
|
+
"The helper reply frame was truncated or oversized.",
|
|
1527
|
+
error,
|
|
1528
|
+
)
|
|
1529
|
+
return
|
|
1530
|
+
with self._lock:
|
|
1531
|
+
if job.done:
|
|
1532
|
+
return
|
|
1533
|
+
if self._helper is not helper or self._active_job is not job:
|
|
1534
|
+
self._complete_job(
|
|
1535
|
+
job,
|
|
1536
|
+
error=TrustedLiveHelperExitedError(
|
|
1537
|
+
"The job's helper was retired before its reply "
|
|
1538
|
+
"could be accepted."
|
|
1539
|
+
),
|
|
1540
|
+
)
|
|
1541
|
+
return
|
|
1542
|
+
self._handle_job_reply_locked(job, envelope)
|
|
1543
|
+
return
|
|
1544
|
+
try:
|
|
1545
|
+
process_alive = process.is_alive()
|
|
1546
|
+
exit_code = None if process_alive else process.exitcode
|
|
1547
|
+
except (AssertionError, OSError, ValueError) as error:
|
|
1548
|
+
with self._lock:
|
|
1549
|
+
if job.done:
|
|
1550
|
+
return
|
|
1551
|
+
if self._helper is not helper or self._active_job is not job:
|
|
1552
|
+
self._complete_job(
|
|
1553
|
+
job,
|
|
1554
|
+
error=TrustedLiveHelperExitedError(
|
|
1555
|
+
"The job's helper was retired while its process "
|
|
1556
|
+
"state was inspected."
|
|
1557
|
+
),
|
|
1558
|
+
)
|
|
1559
|
+
return
|
|
1560
|
+
self._protocol_failure_locked(
|
|
1561
|
+
job,
|
|
1562
|
+
"The trusted helper process state failed closed.",
|
|
1563
|
+
error,
|
|
1564
|
+
)
|
|
1565
|
+
return
|
|
1566
|
+
if not process_alive:
|
|
1567
|
+
with self._lock:
|
|
1568
|
+
if job.done:
|
|
1569
|
+
return
|
|
1570
|
+
if self._helper is not helper or self._active_job is not job:
|
|
1571
|
+
self._complete_job(
|
|
1572
|
+
job,
|
|
1573
|
+
error=TrustedLiveHelperExitedError(
|
|
1574
|
+
"The job's helper was retired before process-exit "
|
|
1575
|
+
"handling completed."
|
|
1576
|
+
),
|
|
1577
|
+
)
|
|
1578
|
+
return
|
|
1579
|
+
# A complete frame may have become readable with the process
|
|
1580
|
+
# sentinel. The receiver publishes before marking itself
|
|
1581
|
+
# done, so classify the exit only after its inbox is drained.
|
|
1582
|
+
if helper.reply_receiver_done.is_set() and helper.reply_inbox.empty():
|
|
1583
|
+
self._unexpected_exit_locked(
|
|
1584
|
+
job,
|
|
1585
|
+
f"The trusted helper exited before replying "
|
|
1586
|
+
f"(exit code {exit_code}).",
|
|
1587
|
+
)
|
|
1588
|
+
|
|
1589
|
+
def _request_cancel_locked(
|
|
1590
|
+
self,
|
|
1591
|
+
job: TrustedLiveJob[Any],
|
|
1592
|
+
*,
|
|
1593
|
+
grace_timeout: float,
|
|
1594
|
+
parent_timeout: bool,
|
|
1595
|
+
) -> bool:
|
|
1596
|
+
if job.done:
|
|
1597
|
+
return False
|
|
1598
|
+
helper = self._helper
|
|
1599
|
+
if self._active_job is not job or helper is None:
|
|
1600
|
+
self._unexpected_exit_locked(
|
|
1601
|
+
job,
|
|
1602
|
+
"The current helper disappeared before cancellation.",
|
|
1603
|
+
)
|
|
1604
|
+
return False
|
|
1605
|
+
if job._cancel_requested:
|
|
1606
|
+
if parent_timeout:
|
|
1607
|
+
job._parent_timed_out = True
|
|
1608
|
+
return True
|
|
1609
|
+
frame = encode_cancel(helper.session, job.generation)
|
|
1610
|
+
job._cancel_requested = True
|
|
1611
|
+
job._parent_timed_out = parent_timeout
|
|
1612
|
+
job._cancel_deadline = time.monotonic() + grace_timeout
|
|
1613
|
+
try:
|
|
1614
|
+
_bounded_send_bytes(
|
|
1615
|
+
helper.control_send,
|
|
1616
|
+
frame,
|
|
1617
|
+
min(self._send_timeout, max(0.05, grace_timeout)),
|
|
1618
|
+
)
|
|
1619
|
+
except BaseException:
|
|
1620
|
+
self._force_job_locked(
|
|
1621
|
+
job,
|
|
1622
|
+
"The cancellation frame could not reach the trusted helper.",
|
|
1623
|
+
)
|
|
1624
|
+
return True
|
|
1625
|
+
|
|
1626
|
+
def _force_job_locked(self, job: TrustedLiveJob[Any], message: str) -> None:
|
|
1627
|
+
if job.done:
|
|
1628
|
+
return
|
|
1629
|
+
if self._active_job is job:
|
|
1630
|
+
self._active_job = None
|
|
1631
|
+
self._discard_helper_locked(force=True)
|
|
1632
|
+
self._complete_job(
|
|
1633
|
+
job,
|
|
1634
|
+
error=TrustedLiveHelperForcedTerminationError(message),
|
|
1635
|
+
)
|
|
1636
|
+
|
|
1637
|
+
def _drive_job(
|
|
1638
|
+
self,
|
|
1639
|
+
job: TrustedLiveJob[Any],
|
|
1640
|
+
parent_deadline: float,
|
|
1641
|
+
) -> None:
|
|
1642
|
+
while not job.done:
|
|
1643
|
+
now = time.monotonic()
|
|
1644
|
+
with self._lock:
|
|
1645
|
+
cancel_deadline = job._cancel_deadline
|
|
1646
|
+
if cancel_deadline is not None and now >= cancel_deadline:
|
|
1647
|
+
self._force_job_locked(
|
|
1648
|
+
job,
|
|
1649
|
+
"Cooperative cancellation did not finish within its "
|
|
1650
|
+
"bounded grace period; the helper was forcibly stopped.",
|
|
1651
|
+
)
|
|
1652
|
+
return
|
|
1653
|
+
if cancel_deadline is None and now >= parent_deadline:
|
|
1654
|
+
self._request_cancel_locked(
|
|
1655
|
+
job,
|
|
1656
|
+
grace_timeout=self._cancellation_grace,
|
|
1657
|
+
parent_timeout=True,
|
|
1658
|
+
)
|
|
1659
|
+
continue
|
|
1660
|
+
effective_deadline = (
|
|
1661
|
+
parent_deadline
|
|
1662
|
+
if cancel_deadline is None
|
|
1663
|
+
else min(parent_deadline, cancel_deadline)
|
|
1664
|
+
)
|
|
1665
|
+
if job._parent_timed_out and cancel_deadline is not None:
|
|
1666
|
+
effective_deadline = cancel_deadline
|
|
1667
|
+
quantum = max(
|
|
1668
|
+
0.0,
|
|
1669
|
+
min(_POLL_QUANTUM_SECONDS, effective_deadline - time.monotonic()),
|
|
1670
|
+
)
|
|
1671
|
+
self._poll_job_once(job, quantum)
|
|
1672
|
+
|
|
1673
|
+
def _job_outcome(self, job: TrustedLiveJob[_T]) -> _T:
|
|
1674
|
+
error = job._error
|
|
1675
|
+
if error is not None:
|
|
1676
|
+
raise error
|
|
1677
|
+
if job._value is _NO_VALUE:
|
|
1678
|
+
raise TrustedLiveProtocolError(
|
|
1679
|
+
"The helper job completed without a result or error."
|
|
1680
|
+
)
|
|
1681
|
+
return cast(_T, job._value)
|
|
1682
|
+
|
|
1683
|
+
def wait(
|
|
1684
|
+
self,
|
|
1685
|
+
job: TrustedLiveJob[_T],
|
|
1686
|
+
*,
|
|
1687
|
+
timeout: float | None = None,
|
|
1688
|
+
) -> _T:
|
|
1689
|
+
"""Wait no longer than the parent deadline, then cancel and retire."""
|
|
1690
|
+
|
|
1691
|
+
wait_timeout = (
|
|
1692
|
+
self._reply_timeout
|
|
1693
|
+
if timeout is None
|
|
1694
|
+
else _duration(
|
|
1695
|
+
timeout,
|
|
1696
|
+
description="parent job wait timeout",
|
|
1697
|
+
allow_zero=True,
|
|
1698
|
+
)
|
|
1699
|
+
)
|
|
1700
|
+
with self._lock:
|
|
1701
|
+
self._validate_job_locked(job)
|
|
1702
|
+
if job.done:
|
|
1703
|
+
return self._job_outcome(job)
|
|
1704
|
+
parent_deadline = time.monotonic() + wait_timeout
|
|
1705
|
+
while not job.done:
|
|
1706
|
+
remaining = max(0.0, parent_deadline - time.monotonic())
|
|
1707
|
+
if job._cancel_deadline is not None:
|
|
1708
|
+
remaining = max(
|
|
1709
|
+
0.0,
|
|
1710
|
+
job._cancel_deadline - time.monotonic(),
|
|
1711
|
+
)
|
|
1712
|
+
acquired = self._reply_lock.acquire(
|
|
1713
|
+
timeout=min(_POLL_QUANTUM_SECONDS, remaining) if remaining > 0 else 0
|
|
1714
|
+
)
|
|
1715
|
+
if acquired:
|
|
1716
|
+
try:
|
|
1717
|
+
self._drive_job(job, parent_deadline)
|
|
1718
|
+
finally:
|
|
1719
|
+
self._reply_lock.release()
|
|
1720
|
+
break
|
|
1721
|
+
if time.monotonic() >= parent_deadline:
|
|
1722
|
+
with self._lock:
|
|
1723
|
+
self._request_cancel_locked(
|
|
1724
|
+
job,
|
|
1725
|
+
grace_timeout=self._cancellation_grace,
|
|
1726
|
+
parent_timeout=True,
|
|
1727
|
+
)
|
|
1728
|
+
if job._done.wait(self._cancellation_grace):
|
|
1729
|
+
break
|
|
1730
|
+
with self._lock:
|
|
1731
|
+
self._force_job_locked(
|
|
1732
|
+
job,
|
|
1733
|
+
"The parent reply deadline and cancellation grace "
|
|
1734
|
+
"expired; the helper was forcibly stopped.",
|
|
1735
|
+
)
|
|
1736
|
+
break
|
|
1737
|
+
return self._job_outcome(job)
|
|
1738
|
+
|
|
1739
|
+
def cancel(
|
|
1740
|
+
self,
|
|
1741
|
+
job: TrustedLiveJob[Any] | None = None,
|
|
1742
|
+
*,
|
|
1743
|
+
grace_timeout: float | None = None,
|
|
1744
|
+
) -> bool:
|
|
1745
|
+
"""Cancel one exact active generation and enforce bounded escalation."""
|
|
1746
|
+
|
|
1747
|
+
grace = (
|
|
1748
|
+
self._cancellation_grace
|
|
1749
|
+
if grace_timeout is None
|
|
1750
|
+
else _duration(
|
|
1751
|
+
grace_timeout,
|
|
1752
|
+
description="cancellation grace period",
|
|
1753
|
+
allow_zero=True,
|
|
1754
|
+
)
|
|
1755
|
+
)
|
|
1756
|
+
with self._lock:
|
|
1757
|
+
self._require_not_closed_locked()
|
|
1758
|
+
selected = self._active_job if job is None else job
|
|
1759
|
+
if selected is None:
|
|
1760
|
+
return False
|
|
1761
|
+
self._validate_job_locked(selected)
|
|
1762
|
+
if selected.done or self._active_job is not selected:
|
|
1763
|
+
return False
|
|
1764
|
+
self._request_cancel_locked(
|
|
1765
|
+
selected,
|
|
1766
|
+
grace_timeout=grace,
|
|
1767
|
+
parent_timeout=False,
|
|
1768
|
+
)
|
|
1769
|
+
if selected.done:
|
|
1770
|
+
return True
|
|
1771
|
+
if self._reply_lock.acquire(blocking=False):
|
|
1772
|
+
try:
|
|
1773
|
+
self._drive_job(
|
|
1774
|
+
selected,
|
|
1775
|
+
selected._cancel_deadline or time.monotonic(),
|
|
1776
|
+
)
|
|
1777
|
+
finally:
|
|
1778
|
+
self._reply_lock.release()
|
|
1779
|
+
elif not selected._done.wait(grace):
|
|
1780
|
+
with self._lock:
|
|
1781
|
+
self._force_job_locked(
|
|
1782
|
+
selected,
|
|
1783
|
+
"Cooperative cancellation did not finish within its "
|
|
1784
|
+
"bounded grace period; the helper was forcibly stopped.",
|
|
1785
|
+
)
|
|
1786
|
+
return True
|
|
1787
|
+
|
|
1788
|
+
def query(
|
|
1789
|
+
self,
|
|
1790
|
+
sql: str,
|
|
1791
|
+
bindings: SqliteBindings = None,
|
|
1792
|
+
*,
|
|
1793
|
+
timeout: float | None = None,
|
|
1794
|
+
wait_timeout: float | None = None,
|
|
1795
|
+
) -> TrustedQueryResult:
|
|
1796
|
+
"""Submit and boundedly wait for one trusted query."""
|
|
1797
|
+
|
|
1798
|
+
return self.wait(
|
|
1799
|
+
self.submit_query(sql, bindings, timeout=timeout),
|
|
1800
|
+
timeout=wait_timeout,
|
|
1801
|
+
)
|
|
1802
|
+
|
|
1803
|
+
def query_batch(
|
|
1804
|
+
self,
|
|
1805
|
+
queries: Sequence[TrustedQuery],
|
|
1806
|
+
*,
|
|
1807
|
+
timeout: float | None = None,
|
|
1808
|
+
wait_timeout: float | None = None,
|
|
1809
|
+
) -> tuple[TrustedQueryResult, ...]:
|
|
1810
|
+
"""Submit and boundedly wait for one repeatable-read query batch."""
|
|
1811
|
+
|
|
1812
|
+
return self.wait(
|
|
1813
|
+
self.submit_query_batch(queries, timeout=timeout),
|
|
1814
|
+
timeout=wait_timeout,
|
|
1815
|
+
)
|
|
1816
|
+
|
|
1817
|
+
def data_version(
|
|
1818
|
+
self,
|
|
1819
|
+
*,
|
|
1820
|
+
timeout: float | None = None,
|
|
1821
|
+
wait_timeout: float | None = None,
|
|
1822
|
+
) -> int:
|
|
1823
|
+
"""Submit and boundedly wait for ``PRAGMA data_version``."""
|
|
1824
|
+
|
|
1825
|
+
return self.wait(
|
|
1826
|
+
self.submit_data_version(timeout=timeout),
|
|
1827
|
+
timeout=wait_timeout,
|
|
1828
|
+
)
|
|
1829
|
+
|
|
1830
|
+
def restart(self) -> None:
|
|
1831
|
+
"""Explicitly retire any idle incarnation and spawn a fresh one."""
|
|
1832
|
+
|
|
1833
|
+
with self._lock:
|
|
1834
|
+
self._require_open_locked()
|
|
1835
|
+
if self._active_job is not None:
|
|
1836
|
+
raise TrustedLiveTransactionError(
|
|
1837
|
+
"Cancel and finish the active job before restarting its helper."
|
|
1838
|
+
)
|
|
1839
|
+
self._discard_helper_locked(force=False)
|
|
1840
|
+
self._spawn_helper_locked()
|
|
1841
|
+
|
|
1842
|
+
def _shutdown_helper_locked(self, timeout: float) -> BaseException | None:
|
|
1843
|
+
helper = self._helper
|
|
1844
|
+
if helper is None:
|
|
1845
|
+
if self._force_quarantined_helper_locked():
|
|
1846
|
+
return TrustedLiveHelperForcedTerminationError(
|
|
1847
|
+
"A quarantined trusted helper required forced retirement "
|
|
1848
|
+
"during shutdown."
|
|
1849
|
+
)
|
|
1850
|
+
return None
|
|
1851
|
+
if not helper.process.is_alive():
|
|
1852
|
+
self._discard_helper_locked(force=False)
|
|
1853
|
+
return None
|
|
1854
|
+
generation = self._next_generation
|
|
1855
|
+
if generation > MAX_GENERATION:
|
|
1856
|
+
forced = self._discard_helper_locked(force=True)
|
|
1857
|
+
if forced:
|
|
1858
|
+
return TrustedLiveHelperForcedTerminationError(
|
|
1859
|
+
"The helper generation space exhausted during shutdown."
|
|
1860
|
+
)
|
|
1861
|
+
return TrustedLiveProtocolError(
|
|
1862
|
+
"The helper generation space exhausted during shutdown."
|
|
1863
|
+
)
|
|
1864
|
+
self._next_generation += 1
|
|
1865
|
+
try:
|
|
1866
|
+
frame = encode_shutdown(helper.session, generation)
|
|
1867
|
+
_bounded_send_bytes(
|
|
1868
|
+
helper.command_send,
|
|
1869
|
+
frame,
|
|
1870
|
+
min(self._send_timeout, max(0.05, timeout)),
|
|
1871
|
+
)
|
|
1872
|
+
except BaseException as error:
|
|
1873
|
+
self._discard_helper_locked(force=True)
|
|
1874
|
+
shutdown_send_error = TrustedLiveHelperForcedTerminationError(
|
|
1875
|
+
"The helper shutdown request could not be delivered."
|
|
1876
|
+
)
|
|
1877
|
+
shutdown_send_error.__cause__ = error
|
|
1878
|
+
return shutdown_send_error
|
|
1879
|
+
deadline = time.monotonic() + timeout
|
|
1880
|
+
shutdown_error: BaseException | None = None
|
|
1881
|
+
acknowledged = False
|
|
1882
|
+
while time.monotonic() < deadline:
|
|
1883
|
+
remaining = deadline - time.monotonic()
|
|
1884
|
+
try:
|
|
1885
|
+
item = self._take_reply_item(
|
|
1886
|
+
helper,
|
|
1887
|
+
min(_POLL_QUANTUM_SECONDS, remaining),
|
|
1888
|
+
)
|
|
1889
|
+
if item is None and not helper.process.is_alive():
|
|
1890
|
+
# Process exit and a final buffered pipe frame can become
|
|
1891
|
+
# observable in either order. The sole receiver publishes
|
|
1892
|
+
# before marking itself done, so wait for that transition.
|
|
1893
|
+
if (
|
|
1894
|
+
helper.reply_receiver_done.is_set()
|
|
1895
|
+
and helper.reply_inbox.empty()
|
|
1896
|
+
):
|
|
1897
|
+
shutdown_error = TrustedLiveHelperExitedError(
|
|
1898
|
+
"The trusted helper exited without acknowledging "
|
|
1899
|
+
"shutdown "
|
|
1900
|
+
f"(exit code {helper.process.exitcode})."
|
|
1901
|
+
)
|
|
1902
|
+
break
|
|
1903
|
+
if item is not None:
|
|
1904
|
+
if isinstance(item, _ReplyReceiveFailure):
|
|
1905
|
+
if item.kind == "eof":
|
|
1906
|
+
shutdown_error = TrustedLiveHelperExitedError(
|
|
1907
|
+
"The helper exited while acknowledging shutdown."
|
|
1908
|
+
)
|
|
1909
|
+
else:
|
|
1910
|
+
shutdown_error = TrustedLiveProtocolError(item.description)
|
|
1911
|
+
break
|
|
1912
|
+
envelope = decode_reply_frame(item)
|
|
1913
|
+
if (
|
|
1914
|
+
envelope.session != helper.session
|
|
1915
|
+
or envelope.generation != generation
|
|
1916
|
+
or envelope.operation not in {"shutdown", "protocol"}
|
|
1917
|
+
):
|
|
1918
|
+
raise TrustedLiveProtocolValidationError(
|
|
1919
|
+
"The shutdown reply was stale or out of order."
|
|
1920
|
+
)
|
|
1921
|
+
status, payload = decode_reply_payload(envelope)
|
|
1922
|
+
if envelope.operation == "protocol":
|
|
1923
|
+
if status != "error" or payload["code"] != "protocol_error":
|
|
1924
|
+
raise TrustedLiveProtocolValidationError(
|
|
1925
|
+
"A protocol shutdown reply has invalid fields."
|
|
1926
|
+
)
|
|
1927
|
+
shutdown_error = _error_from_payload(
|
|
1928
|
+
payload["code"], payload["message"]
|
|
1929
|
+
)
|
|
1930
|
+
else:
|
|
1931
|
+
if status == "error":
|
|
1932
|
+
shutdown_error = _error_from_payload(
|
|
1933
|
+
payload["code"], payload["message"]
|
|
1934
|
+
)
|
|
1935
|
+
else:
|
|
1936
|
+
validate_shutdown_success(payload)
|
|
1937
|
+
acknowledged = True
|
|
1938
|
+
break
|
|
1939
|
+
except TrustedLiveProtocolValidationError as error:
|
|
1940
|
+
shutdown_error = TrustedLiveProtocolError(
|
|
1941
|
+
"The helper sent invalid shutdown IPC."
|
|
1942
|
+
)
|
|
1943
|
+
shutdown_error.__cause__ = error
|
|
1944
|
+
break
|
|
1945
|
+
except (EOFError, OSError, ValueError) as error:
|
|
1946
|
+
shutdown_error = TrustedLiveHelperExitedError(
|
|
1947
|
+
"The helper exited while acknowledging shutdown."
|
|
1948
|
+
)
|
|
1949
|
+
shutdown_error.__cause__ = error
|
|
1950
|
+
break
|
|
1951
|
+
timed_out = not acknowledged and shutdown_error is None
|
|
1952
|
+
forced = self._discard_helper_locked(force=False)
|
|
1953
|
+
if forced or timed_out:
|
|
1954
|
+
return TrustedLiveHelperForcedTerminationError(
|
|
1955
|
+
"The trusted helper did not shut down within its bounded deadline."
|
|
1956
|
+
)
|
|
1957
|
+
return shutdown_error
|
|
1958
|
+
|
|
1959
|
+
def close(self, *, timeout: float | None = None) -> None:
|
|
1960
|
+
"""Cancel active work, then boundedly close, join, and release the helper."""
|
|
1961
|
+
|
|
1962
|
+
shutdown_timeout = (
|
|
1963
|
+
self._shutdown_timeout
|
|
1964
|
+
if timeout is None
|
|
1965
|
+
else _duration(
|
|
1966
|
+
timeout,
|
|
1967
|
+
description="helper shutdown timeout",
|
|
1968
|
+
allow_zero=True,
|
|
1969
|
+
)
|
|
1970
|
+
)
|
|
1971
|
+
with self._lock:
|
|
1972
|
+
if self._closed:
|
|
1973
|
+
return
|
|
1974
|
+
if self._closing:
|
|
1975
|
+
raise TrustedLiveSupervisorClosedError(
|
|
1976
|
+
"The trusted live-reader supervisor is already closing."
|
|
1977
|
+
)
|
|
1978
|
+
self._closing = True
|
|
1979
|
+
active = self._active_job
|
|
1980
|
+
cancel_failure: BaseException | None = None
|
|
1981
|
+
if active is not None and not active.done:
|
|
1982
|
+
try:
|
|
1983
|
+
self.cancel(active)
|
|
1984
|
+
except BaseException as error:
|
|
1985
|
+
cancel_failure = error
|
|
1986
|
+
if active is not None and not active.done:
|
|
1987
|
+
with self._lock:
|
|
1988
|
+
self._force_job_locked(
|
|
1989
|
+
active,
|
|
1990
|
+
"Supervisor close could not obtain a terminal job reply; "
|
|
1991
|
+
"the helper was forcibly stopped before shutdown.",
|
|
1992
|
+
)
|
|
1993
|
+
active_forced_error = (
|
|
1994
|
+
active._error
|
|
1995
|
+
if active is not None
|
|
1996
|
+
and isinstance(active._error, TrustedLiveHelperForcedTerminationError)
|
|
1997
|
+
else None
|
|
1998
|
+
)
|
|
1999
|
+
with self._lock:
|
|
2000
|
+
try:
|
|
2001
|
+
shutdown_error = self._shutdown_helper_locked(shutdown_timeout)
|
|
2002
|
+
except BaseException as unexpected_shutdown_error:
|
|
2003
|
+
self._discard_helper_locked(force=True)
|
|
2004
|
+
shutdown_error = unexpected_shutdown_error
|
|
2005
|
+
finally:
|
|
2006
|
+
self._active_job = None
|
|
2007
|
+
self._closed = True
|
|
2008
|
+
self._closing = False
|
|
2009
|
+
self._unregister_atexit()
|
|
2010
|
+
if active_forced_error is not None:
|
|
2011
|
+
raise active_forced_error
|
|
2012
|
+
if cancel_failure is not None:
|
|
2013
|
+
raise cancel_failure
|
|
2014
|
+
if shutdown_error is not None:
|
|
2015
|
+
raise shutdown_error
|
|
2016
|
+
|
|
2017
|
+
def _unregister_atexit(self) -> None:
|
|
2018
|
+
lock = getattr(self, "_lock", None)
|
|
2019
|
+
if lock is None:
|
|
2020
|
+
return
|
|
2021
|
+
with lock:
|
|
2022
|
+
self._unregister_atexit_locked()
|
|
2023
|
+
|
|
2024
|
+
def _unregister_atexit_locked(self) -> None:
|
|
2025
|
+
if self._helper is not None or self._unreaped_helper is not None:
|
|
2026
|
+
return
|
|
2027
|
+
if not self._atexit_registered:
|
|
2028
|
+
return
|
|
2029
|
+
self._atexit_registered = False
|
|
2030
|
+
atexit.unregister(self._atexit_callback)
|
|
2031
|
+
|
|
2032
|
+
def _cleanup_at_exit(self) -> None:
|
|
2033
|
+
"""Best-effort bounded cleanup before multiprocessing's exit hook."""
|
|
2034
|
+
|
|
2035
|
+
lock = getattr(self, "_lock", None)
|
|
2036
|
+
if lock is None:
|
|
2037
|
+
return
|
|
2038
|
+
acquired = False
|
|
2039
|
+
try:
|
|
2040
|
+
acquired = lock.acquire(timeout=_ATEXIT_LOCK_TIMEOUT_SECONDS)
|
|
2041
|
+
if not acquired:
|
|
2042
|
+
return
|
|
2043
|
+
self._atexit_registered = False
|
|
2044
|
+
active, self._active_job = self._active_job, None
|
|
2045
|
+
if active is not None and not active.done:
|
|
2046
|
+
self._complete_job(
|
|
2047
|
+
active,
|
|
2048
|
+
error=TrustedLiveHelperForcedTerminationError(
|
|
2049
|
+
"The abandoned supervisor forcibly retired its active helper."
|
|
2050
|
+
),
|
|
2051
|
+
)
|
|
2052
|
+
self._closing = False
|
|
2053
|
+
self._closed = True
|
|
2054
|
+
self._discard_helper_locked(
|
|
2055
|
+
force=True,
|
|
2056
|
+
terminate_timeout=_ATEXIT_PROCESS_TIMEOUT_SECONDS,
|
|
2057
|
+
kill_timeout=_ATEXIT_PROCESS_TIMEOUT_SECONDS,
|
|
2058
|
+
)
|
|
2059
|
+
self._force_quarantined_helper_locked(
|
|
2060
|
+
terminate_timeout=_ATEXIT_PROCESS_TIMEOUT_SECONDS,
|
|
2061
|
+
kill_timeout=_ATEXIT_PROCESS_TIMEOUT_SECONDS,
|
|
2062
|
+
)
|
|
2063
|
+
except BaseException:
|
|
2064
|
+
# The daemon-process fallback remains in force. Interpreter exit
|
|
2065
|
+
# must never be delayed by cleanup diagnostics.
|
|
2066
|
+
pass
|
|
2067
|
+
finally:
|
|
2068
|
+
if acquired:
|
|
2069
|
+
try:
|
|
2070
|
+
lock.release()
|
|
2071
|
+
except BaseException:
|
|
2072
|
+
pass
|
|
2073
|
+
|
|
2074
|
+
def _wait_for_test_notification(self, expected: bytes, timeout: float) -> None:
|
|
2075
|
+
"""Private deterministic barrier; not part of the production protocol."""
|
|
2076
|
+
|
|
2077
|
+
wait_timeout = _duration(
|
|
2078
|
+
timeout,
|
|
2079
|
+
description="private test notification timeout",
|
|
2080
|
+
allow_zero=True,
|
|
2081
|
+
)
|
|
2082
|
+
with self._lock:
|
|
2083
|
+
helper = self._helper
|
|
2084
|
+
connection = None if helper is None else helper.test_notify_receive
|
|
2085
|
+
if connection is None or not connection.poll(wait_timeout):
|
|
2086
|
+
raise TimeoutError("The private helper test notification did not arrive.")
|
|
2087
|
+
value = connection.recv_bytes(256)
|
|
2088
|
+
if value != expected:
|
|
2089
|
+
raise AssertionError(
|
|
2090
|
+
f"Expected private helper notification {expected!r}, got {value!r}."
|
|
2091
|
+
)
|
|
2092
|
+
|
|
2093
|
+
def _send_test_command_frame(self, frame: bytes) -> None:
|
|
2094
|
+
"""Private raw-frame injection used only by protocol tests."""
|
|
2095
|
+
|
|
2096
|
+
if not isinstance(frame, bytes):
|
|
2097
|
+
raise TypeError("A private test frame must be bytes.")
|
|
2098
|
+
with self._lock:
|
|
2099
|
+
helper = self._ensure_helper_locked()
|
|
2100
|
+
_bounded_send_bytes(helper.command_send, frame, self._send_timeout)
|
|
2101
|
+
|
|
2102
|
+
def _send_test_control_frame(self, frame: bytes) -> None:
|
|
2103
|
+
"""Private raw control injection used only by protocol tests."""
|
|
2104
|
+
|
|
2105
|
+
if not isinstance(frame, bytes):
|
|
2106
|
+
raise TypeError("A private test frame must be bytes.")
|
|
2107
|
+
with self._lock:
|
|
2108
|
+
helper = self._ensure_helper_locked()
|
|
2109
|
+
_bounded_send_bytes(helper.control_send, frame, self._send_timeout)
|
|
2110
|
+
|
|
2111
|
+
def __enter__(self) -> TrustedLiveReaderSupervisor:
|
|
2112
|
+
with self._lock:
|
|
2113
|
+
self._require_open_locked()
|
|
2114
|
+
return self
|
|
2115
|
+
|
|
2116
|
+
def __exit__(
|
|
2117
|
+
self,
|
|
2118
|
+
exception_type: type[BaseException] | None,
|
|
2119
|
+
exception: BaseException | None,
|
|
2120
|
+
traceback: TracebackType | None,
|
|
2121
|
+
) -> None:
|
|
2122
|
+
del exception_type, traceback
|
|
2123
|
+
if exception is None:
|
|
2124
|
+
self.close()
|
|
2125
|
+
return
|
|
2126
|
+
try:
|
|
2127
|
+
self.close()
|
|
2128
|
+
except BaseException as close_error:
|
|
2129
|
+
exception.add_note(
|
|
2130
|
+
"The trusted live-reader supervisor also failed during "
|
|
2131
|
+
f"context cleanup: {type(close_error).__name__}: {close_error}"
|
|
2132
|
+
)
|
|
2133
|
+
|
|
2134
|
+
def __del__(self) -> None:
|
|
2135
|
+
try:
|
|
2136
|
+
if (
|
|
2137
|
+
getattr(self, "_closed", True)
|
|
2138
|
+
and getattr(self, "_helper", None) is None
|
|
2139
|
+
and getattr(self, "_unreaped_helper", None) is None
|
|
2140
|
+
):
|
|
2141
|
+
return
|
|
2142
|
+
self._cleanup_at_exit()
|
|
2143
|
+
callback = getattr(self, "_atexit_callback", None)
|
|
2144
|
+
if callback is not None:
|
|
2145
|
+
atexit.unregister(callback)
|
|
2146
|
+
except BaseException:
|
|
2147
|
+
pass
|
|
2148
|
+
|
|
2149
|
+
|
|
2150
|
+
__all__ = [
|
|
2151
|
+
"TrustedLiveHelperExitedError",
|
|
2152
|
+
"TrustedLiveHelperForcedTerminationError",
|
|
2153
|
+
"TrustedLiveHelperReplyTimeoutError",
|
|
2154
|
+
"TrustedLiveHelperStartupError",
|
|
2155
|
+
"TrustedLiveJob",
|
|
2156
|
+
"TrustedLiveProtocolError",
|
|
2157
|
+
"TrustedLiveReaderSupervisor",
|
|
2158
|
+
"TrustedLiveSupervisorClosedError",
|
|
2159
|
+
"TrustedLiveSupervisorError",
|
|
2160
|
+
"TrustedLiveSupervisorLiveness",
|
|
2161
|
+
]
|