qplotter 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. qplot/__init__.py +46 -0
  2. qplot/__main__.py +232 -0
  3. qplot/_auxiliary_paths.py +67 -0
  4. qplot/_build_info.json +1 -0
  5. qplot/_metadata.py +75 -0
  6. qplot/_repair.py +14 -0
  7. qplot/_shutdown_supervisor.py +4049 -0
  8. qplot/_version.py +12 -0
  9. qplot/_windows_shutdown_job.py +1166 -0
  10. qplot/configuration/__init__.py +3 -0
  11. qplot/configuration/config.py +578 -0
  12. qplot/configuration/config_schema.json +357 -0
  13. qplot/configuration/fonts.py +3 -0
  14. qplot/configuration/scripts.py +255 -0
  15. qplot/configuration/themes/__init__.py +13 -0
  16. qplot/configuration/themes/_base.py +592 -0
  17. qplot/configuration/themes/blank.py +17 -0
  18. qplot/configuration/themes/dark.py +54 -0
  19. qplot/configuration/themes/light.py +54 -0
  20. qplot/datahandling/LoadFromDB.py +224 -0
  21. qplot/datahandling/__init__.py +47 -0
  22. qplot/datahandling/_trusted_live_helper.py +812 -0
  23. qplot/datahandling/_trusted_live_protocol.py +1745 -0
  24. qplot/datahandling/database.py +2340 -0
  25. qplot/datahandling/dimensions.py +43 -0
  26. qplot/datahandling/file_identity.py +461 -0
  27. qplot/datahandling/parameter_data.py +149 -0
  28. qplot/datahandling/plot_progress.py +17 -0
  29. qplot/datahandling/qcodes_cache.py +277 -0
  30. qplot/datahandling/qcodes_compat.py +14 -0
  31. qplot/datahandling/readSQL.py +2442 -0
  32. qplot/datahandling/readonly.py +2566 -0
  33. qplot/datahandling/source_activity.py +21 -0
  34. qplot/datahandling/trusted_array_preview.py +148 -0
  35. qplot/datahandling/trusted_derived_cache.py +1605 -0
  36. qplot/datahandling/trusted_derived_rendering.py +876 -0
  37. qplot/datahandling/trusted_heatmap.py +246 -0
  38. qplot/datahandling/trusted_live.py +2808 -0
  39. qplot/datahandling/trusted_live_queries.py +5907 -0
  40. qplot/datahandling/trusted_live_service.py +1919 -0
  41. qplot/datahandling/trusted_live_supervisor.py +2161 -0
  42. qplot/datahandling/trusted_plot.py +283 -0
  43. qplot/datahandling/trusted_presentation.py +1065 -0
  44. qplot/datahandling/trusted_snapshot.py +1132 -0
  45. qplot/datahandling/trusted_work_coordinator.py +2157 -0
  46. qplot/datahandling/trusted_work_scheduler.py +1204 -0
  47. qplot/diagnostics.py +241 -0
  48. qplot/resources/testdata/qplot_test_db_01_10mb.csv +11 -0
  49. qplot/resources/testdata/qplot_test_db_02_25mb.csv +14 -0
  50. qplot/resources/testdata/qplot_test_db_03_50mb.csv +17 -0
  51. qplot/resources/testdata/qplot_test_db_04_100mb.csv +20 -0
  52. qplot/resources/testdata/qplot_test_db_05_250mb.csv +23 -0
  53. qplot/resources/testdata/qplot_test_db_06_500mb.csv +26 -0
  54. qplot/resources/testdata/qplot_test_db_07_1gb.csv +29 -0
  55. qplot/resources/testdata/qplot_test_db_08_5gb.csv +32 -0
  56. qplot/resources/testdata/qplot_test_db_09_10gb.csv +35 -0
  57. qplot/resources/testdata/qplot_test_db_10_30gb.csv +38 -0
  58. qplot/testdata.py +1804 -0
  59. qplot/tools/__init__.py +13 -0
  60. qplot/tools/finite_means.py +123 -0
  61. qplot/tools/general.py +139 -0
  62. qplot/tools/heatmap_geometry.py +391 -0
  63. qplot/tools/operation_registry.py +179 -0
  64. qplot/tools/plot_tools.py +650 -0
  65. qplot/tools/sample_statistics.py +111 -0
  66. qplot/tools/worker.py +2838 -0
  67. qplot/windows/__init__.py +9 -0
  68. qplot/windows/_colorbar.py +422 -0
  69. qplot/windows/_commands.py +523 -0
  70. qplot/windows/_config_persistence.py +123 -0
  71. qplot/windows/_database_actions.py +4933 -0
  72. qplot/windows/_dataset_handle.py +221 -0
  73. qplot/windows/_display_precision.py +48 -0
  74. qplot/windows/_dragdrop.py +70 -0
  75. qplot/windows/_export_paths.py +1167 -0
  76. qplot/windows/_help.py +210 -0
  77. qplot/windows/_mixed_appearance.py +303 -0
  78. qplot/windows/_mixed_plots.py +461 -0
  79. qplot/windows/_native_averaging.py +323 -0
  80. qplot/windows/_native_transforms.py +551 -0
  81. qplot/windows/_plot1d_snap.py +612 -0
  82. qplot/windows/_plot1d_traces.py +2236 -0
  83. qplot/windows/_plot2d_colorbar.py +1127 -0
  84. qplot/windows/_plot2d_colorbar_dialog.py +786 -0
  85. qplot/windows/_plot2d_layers.py +2291 -0
  86. qplot/windows/_plot2d_sweeps.py +712 -0
  87. qplot/windows/_plotWin.py +2082 -0
  88. qplot/windows/_plot_actions.py +2703 -0
  89. qplot/windows/_plot_annotation.py +186 -0
  90. qplot/windows/_plot_appearance.py +283 -0
  91. qplot/windows/_plot_axis_scaling.py +1521 -0
  92. qplot/windows/_plot_data_viewbox.py +94 -0
  93. qplot/windows/_plot_drop_axes.py +224 -0
  94. qplot/windows/_plot_export.py +1445 -0
  95. qplot/windows/_plot_feedback.py +112 -0
  96. qplot/windows/_plot_marquee.py +659 -0
  97. qplot/windows/_plot_refresh.py +950 -0
  98. qplot/windows/_plot_source_promotion.py +178 -0
  99. qplot/windows/_plot_state.py +237 -0
  100. qplot/windows/_plot_status.py +115 -0
  101. qplot/windows/_plot_tiling.py +35 -0
  102. qplot/windows/_plot_transform_labels.py +60 -0
  103. qplot/windows/_preferences.py +724 -0
  104. qplot/windows/_refresh_interval.py +172 -0
  105. qplot/windows/_run_controls.py +789 -0
  106. qplot/windows/_shortcuts.py +47 -0
  107. qplot/windows/_subplots/__init__.py +8 -0
  108. qplot/windows/_subplots/subplot1d.py +532 -0
  109. qplot/windows/_subplots/subplot2d.py +919 -0
  110. qplot/windows/_svg_export.py +209 -0
  111. qplot/windows/_trusted_derived_qt.py +1759 -0
  112. qplot/windows/_widgets/__init__.py +23 -0
  113. qplot/windows/_widgets/_run_formatting.py +387 -0
  114. qplot/windows/_widgets/details_tables.py +1119 -0
  115. qplot/windows/_widgets/dropbox.py +290 -0
  116. qplot/windows/_widgets/operations.py +460 -0
  117. qplot/windows/_widgets/preview.py +2997 -0
  118. qplot/windows/_widgets/run_list_items.py +488 -0
  119. qplot/windows/_widgets/toolbar.py +111 -0
  120. qplot/windows/_widgets/treeWidgets.py +2572 -0
  121. qplot/windows/_window_controls.py +414 -0
  122. qplot/windows/main.py +1632 -0
  123. qplot/windows/plot1d.py +313 -0
  124. qplot/windows/plot2d.py +1935 -0
  125. qplotter-1.6.0.dist-info/METADATA +322 -0
  126. qplotter-1.6.0.dist-info/RECORD +130 -0
  127. qplotter-1.6.0.dist-info/WHEEL +5 -0
  128. qplotter-1.6.0.dist-info/entry_points.txt +4 -0
  129. qplotter-1.6.0.dist-info/licenses/LICENSE +21 -0
  130. qplotter-1.6.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2161 @@
1
+ """Persistent spawned-process supervision for :mod:`trusted_live`.
2
+
3
+ This is the application-independent Stage 3 boundary. It deliberately has no
4
+ Qt dependencies and is not yet integrated into qPlot's windows or workers.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import atexit
10
+ import math
11
+ import multiprocessing
12
+ import os
13
+ import queue
14
+ import secrets
15
+ import threading
16
+ import time
17
+ import weakref
18
+ from collections.abc import Sequence
19
+ from dataclasses import dataclass, field
20
+ from multiprocessing.connection import Connection
21
+ from multiprocessing.process import BaseProcess
22
+ from types import TracebackType
23
+ from typing import Any, Generic, Literal, TypeVar, cast
24
+
25
+ from qplot.datahandling._trusted_live_helper import trusted_live_helper_main
26
+ from qplot.datahandling._trusted_live_protocol import (
27
+ MAX_GENERATION,
28
+ MAX_OPERATION_TIMEOUT_MS,
29
+ MAX_REPLY_BYTES,
30
+ ProtocolEnvelope,
31
+ TrustedLiveProtocolValidationError,
32
+ decode_reply_frame,
33
+ decode_reply_payload,
34
+ encode_cancel,
35
+ encode_job_request,
36
+ encode_shutdown,
37
+ encode_startup_request,
38
+ error_code_is_terminal,
39
+ normalize_query_batch,
40
+ normalize_query_specification,
41
+ validate_job_success,
42
+ validate_shutdown_success,
43
+ validate_startup_success,
44
+ )
45
+ from qplot.datahandling.file_identity import DatabaseInstance, database_instances_differ
46
+ from qplot.datahandling.trusted_live import (
47
+ DEFAULT_TRUSTED_OPERATION_TIMEOUT_SECONDS,
48
+ SqliteBindings,
49
+ TrustedLiveBusyTimeoutError,
50
+ TrustedLiveCancelledError,
51
+ TrustedLiveCleanupError,
52
+ TrustedLiveDeadlineExceededError,
53
+ TrustedLiveInvalidDatabaseError,
54
+ TrustedLiveQueryError,
55
+ TrustedLiveReaderClosedError,
56
+ TrustedLiveReaderError,
57
+ TrustedLiveReaderThreadError,
58
+ TrustedLiveReaderUnavailableError,
59
+ TrustedLiveResultLimitError,
60
+ TrustedLiveSourceChangedError,
61
+ TrustedLiveSourceIdentity,
62
+ TrustedLiveSourceIOError,
63
+ TrustedLiveSqlRejectedError,
64
+ TrustedLiveTransactionError,
65
+ TrustedLiveUnsupportedSourceError,
66
+ TrustedQuery,
67
+ TrustedQueryResult,
68
+ )
69
+
70
+ _T = TypeVar("_T")
71
+ _DEFAULT_STARTUP_TIMEOUT_SECONDS = 10.0
72
+ _DEFAULT_REPLY_TIMEOUT_SECONDS = 10.0
73
+ _DEFAULT_CANCELLATION_GRACE_SECONDS = 0.75
74
+ _DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 2.0
75
+ _DEFAULT_TERMINATE_TIMEOUT_SECONDS = 1.0
76
+ _DEFAULT_KILL_TIMEOUT_SECONDS = 1.0
77
+ _DEFAULT_SEND_TIMEOUT_SECONDS = 1.0
78
+ _DEFAULT_ORPHAN_GRACE_SECONDS = 1.0
79
+ _ATEXIT_LOCK_TIMEOUT_SECONDS = 0.25
80
+ _ATEXIT_PROCESS_TIMEOUT_SECONDS = 0.25
81
+ _REPLY_RECEIVER_JOIN_TIMEOUT_SECONDS = 0.5
82
+ _MAX_PARENT_TIMEOUT_SECONDS = 300.0
83
+ _POLL_QUANTUM_SECONDS = 0.05
84
+ _NO_VALUE = object()
85
+ _PRIVATE_TEST_FAULTS = frozenset(
86
+ {
87
+ "cleanup_quarantine",
88
+ "crash_before_reply",
89
+ "hang_before_operation",
90
+ "hang_close",
91
+ "hang_during_startup",
92
+ "hang_startup",
93
+ "malformed_reply",
94
+ "oversized_reply",
95
+ "partial_job_body",
96
+ "partial_job_header",
97
+ "partial_shutdown_body",
98
+ "partial_shutdown_header",
99
+ "partial_startup_body",
100
+ "partial_startup_header",
101
+ "statement_limit_install",
102
+ "statement_limit_restore",
103
+ "statement_limit_verify",
104
+ "stale_generation_reply",
105
+ "stale_session_reply",
106
+ "wrong_version_reply",
107
+ }
108
+ )
109
+
110
+
111
+ class TrustedLiveSupervisorError(TrustedLiveReaderError):
112
+ """Base class for failures introduced by the process boundary."""
113
+
114
+
115
+ class TrustedLiveHelperStartupError(
116
+ TrustedLiveReaderUnavailableError,
117
+ TrustedLiveSupervisorError,
118
+ ):
119
+ """Raised when a helper cannot start and report readiness in time."""
120
+
121
+
122
+ class TrustedLiveProtocolError(
123
+ TrustedLiveReaderUnavailableError,
124
+ TrustedLiveSupervisorError,
125
+ ):
126
+ """Raised when either endpoint violates the bounded IPC protocol."""
127
+
128
+
129
+ class TrustedLiveHelperExitedError(
130
+ TrustedLiveReaderUnavailableError,
131
+ TrustedLiveSupervisorError,
132
+ ):
133
+ """Raised when a helper exits unexpectedly before its reply."""
134
+
135
+
136
+ class TrustedLiveHelperReplyTimeoutError(
137
+ TimeoutError,
138
+ TrustedLiveSupervisorError,
139
+ ):
140
+ """Raised after a parent reply deadline and cooperative retirement."""
141
+
142
+
143
+ class TrustedLiveHelperForcedTerminationError(
144
+ TimeoutError,
145
+ TrustedLiveSupervisorError,
146
+ ):
147
+ """Raised when cancellation grace expires and the helper is terminated."""
148
+
149
+
150
+ class TrustedLiveSupervisorClosedError(
151
+ TrustedLiveReaderClosedError,
152
+ TrustedLiveSupervisorError,
153
+ ):
154
+ """Raised when an operation targets a closed supervisor."""
155
+
156
+
157
+ @dataclass(slots=True)
158
+ class TrustedLiveJob(Generic[_T]):
159
+ """Typed handle for one generation-bound helper operation."""
160
+
161
+ incarnation: int
162
+ session: str
163
+ generation: int
164
+ operation: str
165
+ _owner_nonce: str = field(repr=False)
166
+ _expected_result_count: int | None = field(default=None, repr=False)
167
+ _done: threading.Event = field(default_factory=threading.Event, repr=False)
168
+ _value: object = field(default=_NO_VALUE, repr=False)
169
+ _error: BaseException | None = field(default=None, repr=False)
170
+ _cancel_requested: bool = field(default=False, repr=False)
171
+ _cancel_deadline: float | None = field(default=None, repr=False)
172
+ _parent_timed_out: bool = field(default=False, repr=False)
173
+
174
+ @property
175
+ def done(self) -> bool:
176
+ """Return whether the parent has accepted a terminal outcome."""
177
+
178
+ return self._done.is_set()
179
+
180
+ @property
181
+ def cancellation_requested(self) -> bool:
182
+ """Return whether exact-generation cancellation was requested."""
183
+
184
+ return self._cancel_requested
185
+
186
+
187
+ @dataclass(frozen=True, slots=True)
188
+ class TrustedLiveSupervisorLiveness:
189
+ """Conservative snapshot of every resource owned by one supervisor."""
190
+
191
+ helper_pid: int | None
192
+ process_alive: bool
193
+ receiver_alive: bool
194
+ open_endpoints: int
195
+ active_incarnation: bool
196
+ unreaped_incarnation: bool
197
+ active_job: bool
198
+ closing: bool
199
+ closed: bool
200
+
201
+ @property
202
+ def resources_owned(self) -> bool:
203
+ """Return whether any helper-side resource remains strongly owned."""
204
+
205
+ return bool(
206
+ self.active_incarnation
207
+ or self.unreaped_incarnation
208
+ or self.active_job
209
+ or self.process_alive
210
+ or self.receiver_alive
211
+ or self.open_endpoints
212
+ )
213
+
214
+
215
+ _LOCK_CONTENTION_LIVENESS = TrustedLiveSupervisorLiveness(
216
+ helper_pid=None,
217
+ process_alive=True,
218
+ receiver_alive=True,
219
+ open_endpoints=1,
220
+ active_incarnation=True,
221
+ unreaped_incarnation=True,
222
+ active_job=True,
223
+ closing=True,
224
+ closed=False,
225
+ )
226
+
227
+
228
+ @dataclass(frozen=True, slots=True)
229
+ class _ReplyReceiveFailure:
230
+ """One terminal raw-pipe failure published by the sole receiver."""
231
+
232
+ kind: Literal["eof", "connection", "unexpected"]
233
+ description: str
234
+
235
+
236
+ _ReplyInboxItem = bytes | _ReplyReceiveFailure
237
+
238
+
239
+ @dataclass(slots=True)
240
+ class _HelperIncarnation:
241
+ number: int
242
+ session: str
243
+ process: BaseProcess
244
+ command_send: Connection
245
+ reply_connection: Connection
246
+ control_send: Connection
247
+ test_notify_receive: Connection | None
248
+ reply_inbox: queue.Queue[_ReplyInboxItem]
249
+ reply_ready: threading.Event
250
+ reply_receiver_stop: threading.Event
251
+ reply_receiver_done: threading.Event
252
+ reply_receiver: threading.Thread
253
+ reply_receiver_started: bool = False
254
+
255
+
256
+ def _publish_reply_item(
257
+ inbox: queue.Queue[_ReplyInboxItem],
258
+ ready: threading.Event,
259
+ stop: threading.Event,
260
+ item: _ReplyInboxItem,
261
+ ) -> bool:
262
+ """Publish into the one-slot inbox without ever stranding its producer."""
263
+
264
+ while not stop.is_set():
265
+ # The event is private test observability only. Set it before the
266
+ # queue publication so a consumer can never clear the event between
267
+ # publication and notification and leave a stale signalled state.
268
+ ready.set()
269
+ try:
270
+ inbox.put(item, timeout=_POLL_QUANTUM_SECONDS)
271
+ except queue.Full:
272
+ continue
273
+ return True
274
+ ready.clear()
275
+ return False
276
+
277
+
278
+ def _reply_receiver_loop(
279
+ connection: Connection,
280
+ inbox: queue.Queue[_ReplyInboxItem],
281
+ ready: threading.Event,
282
+ stop: threading.Event,
283
+ done: threading.Event,
284
+ ) -> None:
285
+ """Sole raw reply receiver for one helper process incarnation.
286
+
287
+ ``Connection.poll()`` only establishes that some bytes are readable on
288
+ stream-backed multiprocessing pipes; it does not establish that a complete
289
+ frame is available. This one persistent thread may therefore block in
290
+ ``recv_bytes`` while all public callers retain their own finite waits on the
291
+ bounded inbox. Incarnation retirement closes the peer and this endpoint,
292
+ then proves that this thread has stopped before the helper is reusable.
293
+ """
294
+
295
+ try:
296
+ while not stop.is_set():
297
+ try:
298
+ item: _ReplyInboxItem = connection.recv_bytes(MAX_REPLY_BYTES)
299
+ terminal = False
300
+ except EOFError:
301
+ item = _ReplyReceiveFailure(
302
+ "eof",
303
+ "The helper closed its reply endpoint.",
304
+ )
305
+ terminal = True
306
+ except (OSError, ValueError) as error:
307
+ item = _ReplyReceiveFailure(
308
+ "connection",
309
+ "The helper reply connection failed while receiving "
310
+ f"a frame ({type(error).__name__}).",
311
+ )
312
+ terminal = True
313
+ except BaseException as error:
314
+ # Never let an arbitrary receiver failure escape through a
315
+ # public operation or leave its incarnation reusable.
316
+ item = _ReplyReceiveFailure(
317
+ "unexpected",
318
+ "The helper reply receiver failed unexpectedly "
319
+ f"({type(error).__name__}).",
320
+ )
321
+ terminal = True
322
+ if not _publish_reply_item(inbox, ready, stop, item):
323
+ return
324
+ if terminal:
325
+ return
326
+ finally:
327
+ done.set()
328
+
329
+
330
+ def _duration(
331
+ value: object,
332
+ *,
333
+ description: str,
334
+ allow_zero: bool,
335
+ ) -> float:
336
+ if isinstance(value, bool):
337
+ raise ValueError(f"{description} must be a finite number of seconds.")
338
+ try:
339
+ duration = float(cast(Any, value))
340
+ except (TypeError, ValueError) as error:
341
+ raise ValueError(
342
+ f"{description} must be a finite number of seconds."
343
+ ) from error
344
+ if (
345
+ not math.isfinite(duration)
346
+ or duration < 0
347
+ or (duration == 0 and not allow_zero)
348
+ or duration > _MAX_PARENT_TIMEOUT_SECONDS
349
+ ):
350
+ qualifier = "non-negative" if allow_zero else "positive"
351
+ raise ValueError(
352
+ f"{description} must be a finite {qualifier} duration no greater "
353
+ f"than {_MAX_PARENT_TIMEOUT_SECONDS:g} seconds."
354
+ )
355
+ return duration
356
+
357
+
358
+ def _timeout_milliseconds(value: object, *, description: str) -> int:
359
+ seconds = _duration(value, description=description, allow_zero=True)
360
+ milliseconds = math.ceil(seconds * 1_000.0)
361
+ if milliseconds > MAX_OPERATION_TIMEOUT_MS:
362
+ raise ValueError(f"{description} exceeds the protocol operation-timeout limit.")
363
+ return milliseconds
364
+
365
+
366
+ def _close_connection(connection: Connection | None) -> None:
367
+ if connection is None:
368
+ return
369
+ try:
370
+ connection.close()
371
+ except OSError:
372
+ pass
373
+
374
+
375
+ def _bounded_send_bytes(
376
+ connection: Connection,
377
+ frame: bytes,
378
+ timeout: float,
379
+ ) -> None:
380
+ """Bound a potentially blocking pipe write without serialising objects."""
381
+
382
+ completed = threading.Event()
383
+ errors: list[BaseException] = []
384
+
385
+ def send() -> None:
386
+ try:
387
+ connection.send_bytes(frame)
388
+ except BaseException as error:
389
+ errors.append(error)
390
+ finally:
391
+ completed.set()
392
+
393
+ sender = threading.Thread(
394
+ target=send,
395
+ name="qplot-trusted-reader-ipc-send",
396
+ daemon=True,
397
+ )
398
+ sender.start()
399
+ if not completed.wait(timeout):
400
+ _close_connection(connection)
401
+ completed.wait(min(0.1, timeout))
402
+ raise TimeoutError("A bounded trusted-helper IPC send did not finish.")
403
+ sender.join(timeout=0)
404
+ if errors:
405
+ raise OSError("The trusted-helper IPC endpoint rejected a frame.") from errors[
406
+ 0
407
+ ]
408
+
409
+
410
+ def _error_from_payload(code: str, message: str) -> BaseException:
411
+ error_types: dict[str, type[BaseException]] = {
412
+ "reader_unavailable": TrustedLiveReaderUnavailableError,
413
+ "unsupported_source": TrustedLiveUnsupportedSourceError,
414
+ "source_changed": TrustedLiveSourceChangedError,
415
+ "source_io": TrustedLiveSourceIOError,
416
+ "sql_rejected": TrustedLiveSqlRejectedError,
417
+ "query_failed": TrustedLiveQueryError,
418
+ "result_limit": TrustedLiveResultLimitError,
419
+ "busy_timeout": TrustedLiveBusyTimeoutError,
420
+ "cancelled": TrustedLiveCancelledError,
421
+ "operation_deadline": TrustedLiveDeadlineExceededError,
422
+ "invalid_database": TrustedLiveInvalidDatabaseError,
423
+ "cleanup_quarantine": TrustedLiveCleanupError,
424
+ "reader_closed": TrustedLiveReaderClosedError,
425
+ "reader_thread": TrustedLiveReaderThreadError,
426
+ "transaction": TrustedLiveTransactionError,
427
+ "reader_error": TrustedLiveReaderError,
428
+ "protocol_error": TrustedLiveProtocolError,
429
+ "internal_error": TrustedLiveHelperExitedError,
430
+ }
431
+ return error_types[code](message)
432
+
433
+
434
+ class TrustedLiveReaderSupervisor:
435
+ """Own one persistent spawned helper for one accepted database instance.
436
+
437
+ Only one database job may be active. A failed job is never replayed. If
438
+ its process incarnation is retired, a later explicit submit or
439
+ :meth:`restart` may spawn a fresh helper bound to the originally accepted
440
+ :class:`DatabaseInstance`.
441
+ """
442
+
443
+ def __init__(
444
+ self,
445
+ database_path: str | os.PathLike[str],
446
+ *,
447
+ expected_database_instance: DatabaseInstance | None = None,
448
+ busy_timeout_ms: int = 5_000,
449
+ operation_timeout_seconds: float = DEFAULT_TRUSTED_OPERATION_TIMEOUT_SECONDS,
450
+ startup_timeout_seconds: float = _DEFAULT_STARTUP_TIMEOUT_SECONDS,
451
+ reply_timeout_seconds: float = _DEFAULT_REPLY_TIMEOUT_SECONDS,
452
+ cancellation_grace_seconds: float = (_DEFAULT_CANCELLATION_GRACE_SECONDS),
453
+ shutdown_timeout_seconds: float = _DEFAULT_SHUTDOWN_TIMEOUT_SECONDS,
454
+ terminate_timeout_seconds: float = _DEFAULT_TERMINATE_TIMEOUT_SECONDS,
455
+ kill_timeout_seconds: float = _DEFAULT_KILL_TIMEOUT_SECONDS,
456
+ _test_fault: str | None = None,
457
+ ) -> None:
458
+ try:
459
+ selected_path = os.fspath(database_path)
460
+ except TypeError as error:
461
+ raise TypeError("database_path must be a path-like value.") from error
462
+ if isinstance(selected_path, bytes):
463
+ selected_path = os.fsdecode(selected_path)
464
+ if (
465
+ not isinstance(selected_path, str)
466
+ or not selected_path
467
+ or "\x00" in selected_path
468
+ ):
469
+ raise ValueError("database_path must be non-empty text without NUL bytes.")
470
+ try:
471
+ path_bytes = selected_path.encode("utf-8", errors="strict")
472
+ except UnicodeEncodeError as error:
473
+ raise ValueError(
474
+ "database_path must contain valid Unicode text."
475
+ ) from error
476
+ if len(path_bytes) > 32 * 1024:
477
+ raise ValueError("database_path exceeds the bounded helper path limit.")
478
+ if expected_database_instance is not None and not isinstance(
479
+ expected_database_instance, DatabaseInstance
480
+ ):
481
+ raise TypeError(
482
+ "expected_database_instance must be a DatabaseInstance or None."
483
+ )
484
+ if (
485
+ type(busy_timeout_ms) is not int
486
+ or busy_timeout_ms < 0
487
+ or busy_timeout_ms > 2_147_483_647
488
+ ):
489
+ raise ValueError(
490
+ "busy_timeout_ms must be an integer from 0 through 2147483647."
491
+ )
492
+ if _test_fault is not None and (
493
+ type(_test_fault) is not str or _test_fault not in _PRIVATE_TEST_FAULTS
494
+ ):
495
+ raise ValueError("The private helper test fault mode is invalid.")
496
+ self._database_path = selected_path
497
+ self._configured_expected_instance = expected_database_instance
498
+ self._accepted_source: TrustedLiveSourceIdentity | None = None
499
+ self._busy_timeout_ms = busy_timeout_ms
500
+ self._operation_timeout_ms = _timeout_milliseconds(
501
+ operation_timeout_seconds,
502
+ description="default child operation timeout",
503
+ )
504
+ if self._operation_timeout_ms == 0:
505
+ raise ValueError("default child operation timeout must be positive.")
506
+ self._startup_timeout = _duration(
507
+ startup_timeout_seconds,
508
+ description="helper startup timeout",
509
+ allow_zero=False,
510
+ )
511
+ self._reply_timeout = _duration(
512
+ reply_timeout_seconds,
513
+ description="parent reply timeout",
514
+ allow_zero=False,
515
+ )
516
+ self._cancellation_grace = _duration(
517
+ cancellation_grace_seconds,
518
+ description="cancellation grace period",
519
+ allow_zero=True,
520
+ )
521
+ self._shutdown_timeout = _duration(
522
+ shutdown_timeout_seconds,
523
+ description="helper shutdown timeout",
524
+ allow_zero=True,
525
+ )
526
+ self._terminate_timeout = _duration(
527
+ terminate_timeout_seconds,
528
+ description="helper termination timeout",
529
+ allow_zero=True,
530
+ )
531
+ self._kill_timeout = _duration(
532
+ kill_timeout_seconds,
533
+ description="helper kill timeout",
534
+ allow_zero=True,
535
+ )
536
+ self._send_timeout = min(_DEFAULT_SEND_TIMEOUT_SECONDS, self._reply_timeout)
537
+ self._orphan_grace = min(
538
+ _DEFAULT_ORPHAN_GRACE_SECONDS,
539
+ max(0.05, self._shutdown_timeout),
540
+ )
541
+ self._context = multiprocessing.get_context("spawn")
542
+ self._owner_nonce = secrets.token_hex(16)
543
+ self._next_generation = 1
544
+ self._incarnation_counter = 0
545
+ self._helper: _HelperIncarnation | None = None
546
+ self._unreaped_helper: _HelperIncarnation | None = None
547
+ self._active_job: TrustedLiveJob[Any] | None = None
548
+ self._closing = False
549
+ self._closed = False
550
+ self._lock = threading.RLock()
551
+ self._reply_lock = threading.Lock()
552
+ self._test_fault = _test_fault
553
+ self._test_fault_consumed = False
554
+ self._atexit_registered = False
555
+ supervisor_reference = weakref.ref(self)
556
+
557
+ def cleanup_at_exit() -> None:
558
+ supervisor = supervisor_reference()
559
+ if supervisor is not None:
560
+ supervisor._cleanup_at_exit()
561
+
562
+ self._atexit_callback = cleanup_at_exit
563
+ atexit.register(self._atexit_callback)
564
+ self._atexit_registered = True
565
+
566
+ @classmethod
567
+ def open(
568
+ cls,
569
+ database_path: str | os.PathLike[str],
570
+ *,
571
+ expected_database_instance: DatabaseInstance | None = None,
572
+ busy_timeout_ms: int = 5_000,
573
+ operation_timeout_seconds: float = DEFAULT_TRUSTED_OPERATION_TIMEOUT_SECONDS,
574
+ startup_timeout_seconds: float = _DEFAULT_STARTUP_TIMEOUT_SECONDS,
575
+ reply_timeout_seconds: float = _DEFAULT_REPLY_TIMEOUT_SECONDS,
576
+ cancellation_grace_seconds: float = (_DEFAULT_CANCELLATION_GRACE_SECONDS),
577
+ shutdown_timeout_seconds: float = _DEFAULT_SHUTDOWN_TIMEOUT_SECONDS,
578
+ terminate_timeout_seconds: float = _DEFAULT_TERMINATE_TIMEOUT_SECONDS,
579
+ kill_timeout_seconds: float = _DEFAULT_KILL_TIMEOUT_SECONDS,
580
+ _test_fault: str | None = None,
581
+ ) -> TrustedLiveReaderSupervisor:
582
+ """Spawn and open one helper, failing within ``startup_timeout_seconds``."""
583
+
584
+ supervisor = cls(
585
+ database_path,
586
+ expected_database_instance=expected_database_instance,
587
+ busy_timeout_ms=busy_timeout_ms,
588
+ operation_timeout_seconds=operation_timeout_seconds,
589
+ startup_timeout_seconds=startup_timeout_seconds,
590
+ reply_timeout_seconds=reply_timeout_seconds,
591
+ cancellation_grace_seconds=cancellation_grace_seconds,
592
+ shutdown_timeout_seconds=shutdown_timeout_seconds,
593
+ terminate_timeout_seconds=terminate_timeout_seconds,
594
+ kill_timeout_seconds=kill_timeout_seconds,
595
+ _test_fault=_test_fault,
596
+ )
597
+ try:
598
+ with supervisor._lock:
599
+ supervisor._spawn_helper_locked()
600
+ except BaseException:
601
+ supervisor._closed = True
602
+ with supervisor._lock:
603
+ supervisor._discard_helper_locked(force=True)
604
+ supervisor._unregister_atexit()
605
+ raise
606
+ return supervisor
607
+
608
+ @property
609
+ def closed(self) -> bool:
610
+ with self._lock:
611
+ return self._closed
612
+
613
+ @property
614
+ def source_identity(self) -> TrustedLiveSourceIdentity:
615
+ with self._lock:
616
+ source = self._accepted_source
617
+ if source is None:
618
+ raise TrustedLiveReaderUnavailableError(
619
+ "The supervisor has not accepted a trusted database source."
620
+ )
621
+ return source
622
+
623
+ @property
624
+ def database_instance(self) -> DatabaseInstance:
625
+ return self.source_identity.database_instance
626
+
627
+ @property
628
+ def incarnation(self) -> int:
629
+ with self._lock:
630
+ return self._incarnation_counter
631
+
632
+ @property
633
+ def helper_pid(self) -> int | None:
634
+ with self._lock:
635
+ helper = self._helper
636
+ return None if helper is None else helper.process.pid
637
+
638
+ @property
639
+ def helper_alive(self) -> bool:
640
+ with self._lock:
641
+ helper = self._helper
642
+ return helper is not None and helper.process.is_alive()
643
+
644
+ def resource_liveness(self) -> TrustedLiveSupervisorLiveness:
645
+ """Describe active and quarantined resources without waiting for them."""
646
+
647
+ if not self._lock.acquire(blocking=False):
648
+ # ``close()`` intentionally owns this lock while it performs its
649
+ # bounded helper shutdown. GUI-side shutdown polling must not
650
+ # inherit that deadline. Lock contention therefore means only
651
+ # "cleanup is still pending", never "cleanup is complete".
652
+ return _LOCK_CONTENTION_LIVENESS
653
+ try:
654
+ return self._resource_liveness_locked()
655
+ finally:
656
+ self._lock.release()
657
+
658
+ def reap_closed_resources(self) -> TrustedLiveSupervisorLiveness:
659
+ """Zero-wait reap a closed supervisor's quarantined incarnation.
660
+
661
+ A bounded ``close()`` may have to leave a process handle or reply
662
+ receiver quarantined. Owners can poll this method off the GUI thread;
663
+ it never terminates an active incarnation and never waits for one.
664
+ """
665
+
666
+ if not self._lock.acquire(blocking=False):
667
+ return _LOCK_CONTENTION_LIVENESS
668
+ try:
669
+ if self._closed and not self._closing:
670
+ try:
671
+ self._reap_quarantined_helper_locked()
672
+ except TrustedLiveHelperForcedTerminationError:
673
+ pass
674
+ snapshot = self._resource_liveness_locked()
675
+ if snapshot.closed and not snapshot.resources_owned:
676
+ self._unregister_atexit_locked()
677
+ return snapshot
678
+ finally:
679
+ self._lock.release()
680
+
681
+ def _resource_liveness_locked(self) -> TrustedLiveSupervisorLiveness:
682
+ helpers = tuple(
683
+ helper
684
+ for helper in (self._helper, self._unreaped_helper)
685
+ if helper is not None
686
+ )
687
+ process_alive = False
688
+ receiver_alive = False
689
+ open_endpoints = 0
690
+ helper_pid: int | None = None
691
+ for helper in helpers:
692
+ if helper_pid is None:
693
+ try:
694
+ helper_pid = helper.process.pid
695
+ except (AssertionError, OSError, ValueError):
696
+ pass
697
+ try:
698
+ process_alive = process_alive or helper.process.is_alive()
699
+ except (AssertionError, OSError, ValueError):
700
+ # An incarnation is still owned, so an uninspectable process
701
+ # handle must conservatively keep shutdown pending.
702
+ process_alive = True
703
+ if helper.reply_receiver_started:
704
+ try:
705
+ receiver_alive = receiver_alive or helper.reply_receiver.is_alive()
706
+ except (AssertionError, RuntimeError):
707
+ receiver_alive = True
708
+ for connection in (
709
+ helper.command_send,
710
+ helper.reply_connection,
711
+ helper.control_send,
712
+ helper.test_notify_receive,
713
+ ):
714
+ if connection is None:
715
+ continue
716
+ try:
717
+ is_open = not connection.closed
718
+ except (AttributeError, OSError, ValueError):
719
+ is_open = True
720
+ open_endpoints += int(is_open)
721
+ active_job = self._active_job
722
+ return TrustedLiveSupervisorLiveness(
723
+ helper_pid=helper_pid,
724
+ process_alive=process_alive,
725
+ receiver_alive=receiver_alive,
726
+ open_endpoints=open_endpoints,
727
+ active_incarnation=self._helper is not None,
728
+ unreaped_incarnation=self._unreaped_helper is not None,
729
+ active_job=active_job is not None and not active_job.done,
730
+ closing=self._closing,
731
+ closed=self._closed,
732
+ )
733
+
734
+ @property
735
+ def active_job(self) -> TrustedLiveJob[Any] | None:
736
+ with self._lock:
737
+ return self._active_job
738
+
739
+ def _require_open_locked(self) -> None:
740
+ if self._closed or self._closing:
741
+ raise TrustedLiveSupervisorClosedError(
742
+ "The trusted live-reader supervisor is closing or closed."
743
+ )
744
+
745
+ def _require_not_closed_locked(self) -> None:
746
+ if self._closed:
747
+ raise TrustedLiveSupervisorClosedError(
748
+ "The trusted live-reader supervisor is closed."
749
+ )
750
+
751
+ def _expected_instance_for_spawn(self) -> DatabaseInstance | None:
752
+ accepted_source = self._accepted_source
753
+ if accepted_source is not None:
754
+ return accepted_source.database_instance
755
+ return self._configured_expected_instance
756
+
757
+ def _spawn_helper_locked(self) -> _HelperIncarnation:
758
+ self._require_open_locked()
759
+ self._reap_quarantined_helper_locked()
760
+ if self._helper is not None:
761
+ raise TrustedLiveTransactionError(
762
+ "A trusted helper incarnation is already installed."
763
+ )
764
+ if self._next_generation > MAX_GENERATION:
765
+ raise TrustedLiveProtocolError(
766
+ "The trusted helper job-generation space is exhausted."
767
+ )
768
+ session = secrets.token_hex(16)
769
+ try:
770
+ startup_frame = encode_startup_request(
771
+ session,
772
+ self._next_generation,
773
+ self._database_path,
774
+ self._expected_instance_for_spawn(),
775
+ self._busy_timeout_ms,
776
+ self._operation_timeout_ms,
777
+ math.ceil(self._orphan_grace * 1_000.0),
778
+ )
779
+ except TrustedLiveProtocolValidationError as error:
780
+ raise TrustedLiveProtocolError(
781
+ "The helper startup configuration exceeds the bounded IPC protocol."
782
+ ) from error
783
+ self._incarnation_counter += 1
784
+ number = self._incarnation_counter
785
+ command_receive, command_send = self._context.Pipe(duplex=False)
786
+ reply_receive, reply_send = self._context.Pipe(duplex=False)
787
+ control_receive, control_send = self._context.Pipe(duplex=False)
788
+ test_notify_receive: Connection | None = None
789
+ test_notify_send: Connection | None = None
790
+ fault = None
791
+ if self._test_fault is not None and not self._test_fault_consumed:
792
+ fault_aliases = {"hang_during_startup": "hang_startup"}
793
+ fault = fault_aliases.get(self._test_fault, self._test_fault)
794
+ self._test_fault_consumed = True
795
+ test_notify_receive, test_notify_send = self._context.Pipe(duplex=False)
796
+ process = self._context.Process(
797
+ target=trusted_live_helper_main,
798
+ args=(
799
+ command_receive,
800
+ reply_send,
801
+ control_receive,
802
+ startup_frame,
803
+ fault,
804
+ test_notify_send,
805
+ ),
806
+ name=f"qplot-trusted-live-reader-{number}",
807
+ )
808
+ # The explicit bounded atexit cleanup below normally owns retirement.
809
+ # Daemonic status is a final interpreter-shutdown fallback so an
810
+ # abandoned supervisor cannot make multiprocessing join forever.
811
+ process.daemon = True
812
+ reply_inbox: queue.Queue[_ReplyInboxItem] = queue.Queue(maxsize=1)
813
+ reply_ready = threading.Event()
814
+ reply_receiver_stop = threading.Event()
815
+ reply_receiver_done = threading.Event()
816
+ reply_receiver = threading.Thread(
817
+ target=_reply_receiver_loop,
818
+ args=(
819
+ reply_receive,
820
+ reply_inbox,
821
+ reply_ready,
822
+ reply_receiver_stop,
823
+ reply_receiver_done,
824
+ ),
825
+ name=f"qplot-trusted-reader-reply-receiver-{number}",
826
+ daemon=True,
827
+ )
828
+ helper = _HelperIncarnation(
829
+ number,
830
+ session,
831
+ process,
832
+ command_send,
833
+ reply_receive,
834
+ control_send,
835
+ test_notify_receive,
836
+ reply_inbox,
837
+ reply_ready,
838
+ reply_receiver_stop,
839
+ reply_receiver_done,
840
+ reply_receiver,
841
+ )
842
+ try:
843
+ process.start()
844
+ except BaseException as error:
845
+ for connection in (
846
+ command_receive,
847
+ command_send,
848
+ reply_receive,
849
+ reply_send,
850
+ control_receive,
851
+ control_send,
852
+ test_notify_receive,
853
+ test_notify_send,
854
+ ):
855
+ _close_connection(connection)
856
+ try:
857
+ process.close()
858
+ except (OSError, ValueError):
859
+ pass
860
+ raise TrustedLiveHelperStartupError(
861
+ "The trusted live-reader helper could not be spawned."
862
+ ) from error
863
+ # Process start may lazily register multiprocessing's own exit hook.
864
+ # Re-register ours afterwards so LIFO interpreter shutdown runs this
865
+ # bounded cleanup before multiprocessing considers joining children.
866
+ if self._atexit_registered:
867
+ atexit.unregister(self._atexit_callback)
868
+ atexit.register(self._atexit_callback)
869
+ for child_endpoint in (
870
+ command_receive,
871
+ reply_send,
872
+ control_receive,
873
+ test_notify_send,
874
+ ):
875
+ _close_connection(child_endpoint)
876
+ self._helper = helper
877
+ try:
878
+ try:
879
+ helper.reply_receiver.start()
880
+ helper.reply_receiver_started = True
881
+ except BaseException as error:
882
+ raise TrustedLiveHelperStartupError(
883
+ "The trusted helper reply receiver could not be started."
884
+ ) from error
885
+ envelope = self._wait_startup_reply_locked(helper)
886
+ status, payload = decode_reply_payload(envelope)
887
+ if envelope.generation != 0:
888
+ raise TrustedLiveProtocolError(
889
+ "The helper sent an out-of-order startup reply."
890
+ )
891
+ if envelope.operation == "protocol":
892
+ if status != "error" or payload["code"] != "protocol_error":
893
+ raise TrustedLiveProtocolError(
894
+ "The helper sent an invalid startup protocol outcome."
895
+ )
896
+ raise cast(
897
+ BaseException,
898
+ _error_from_payload(payload["code"], payload["message"]),
899
+ )
900
+ if envelope.operation != "startup":
901
+ raise TrustedLiveProtocolError(
902
+ "The helper sent an out-of-order startup reply."
903
+ )
904
+ if status == "error":
905
+ raise _error_from_payload(payload["code"], payload["message"])
906
+ source = validate_startup_success(payload)
907
+ prior = self._expected_instance_for_spawn()
908
+ if prior is not None and database_instances_differ(
909
+ prior,
910
+ source.database_instance,
911
+ ):
912
+ raise TrustedLiveSourceChangedError(
913
+ "The spawned helper reported a different database instance."
914
+ )
915
+ if self._accepted_source is None:
916
+ self._accepted_source = source
917
+ elif database_instances_differ(
918
+ self._accepted_source.database_instance,
919
+ source.database_instance,
920
+ ):
921
+ raise TrustedLiveSourceChangedError(
922
+ "A replacement helper opened a different database instance."
923
+ )
924
+ # Preserve the accepted main-file binding while accurately
925
+ # reporting the currently installed WAL/SHM identities and mode.
926
+ self._accepted_source = source
927
+ return helper
928
+ except BaseException:
929
+ self._discard_helper_locked(force=False)
930
+ raise
931
+
932
+ @staticmethod
933
+ def _take_reply_item(
934
+ helper: _HelperIncarnation,
935
+ timeout: float,
936
+ ) -> _ReplyInboxItem | None:
937
+ """Wait boundedly for the sole receiver to publish one complete frame."""
938
+
939
+ try:
940
+ if timeout <= 0:
941
+ item = helper.reply_inbox.get_nowait()
942
+ else:
943
+ item = helper.reply_inbox.get(timeout=timeout)
944
+ except queue.Empty:
945
+ return None
946
+ if helper.reply_inbox.empty():
947
+ helper.reply_ready.clear()
948
+ return item
949
+
950
+ def _wait_startup_reply_locked(
951
+ self,
952
+ helper: _HelperIncarnation,
953
+ ) -> ProtocolEnvelope:
954
+ deadline = time.monotonic() + self._startup_timeout
955
+ while True:
956
+ remaining = deadline - time.monotonic()
957
+ if remaining <= 0:
958
+ self._discard_helper_locked(force=True)
959
+ raise TrustedLiveHelperStartupError(
960
+ "The trusted helper did not finish startup before its "
961
+ "parent deadline."
962
+ )
963
+ try:
964
+ item = self._take_reply_item(
965
+ helper,
966
+ min(_POLL_QUANTUM_SECONDS, remaining),
967
+ )
968
+ if item is not None:
969
+ if isinstance(item, _ReplyReceiveFailure):
970
+ if item.kind == "eof":
971
+ raise TrustedLiveHelperExitedError(item.description)
972
+ raise TrustedLiveProtocolError(item.description)
973
+ frame = item
974
+ envelope = decode_reply_frame(frame)
975
+ if envelope.session != helper.session:
976
+ raise TrustedLiveProtocolError(
977
+ "A stale startup reply targeted another incarnation."
978
+ )
979
+ return envelope
980
+ except TrustedLiveProtocolValidationError as error:
981
+ raise TrustedLiveProtocolError(
982
+ "The helper sent malformed or oversized startup IPC."
983
+ ) from error
984
+ except EOFError as error:
985
+ raise TrustedLiveHelperExitedError(
986
+ "The trusted helper's startup endpoint closed unexpectedly."
987
+ ) from error
988
+ except (OSError, ValueError) as error:
989
+ raise TrustedLiveProtocolError(
990
+ "The helper sent malformed or oversized startup IPC."
991
+ ) from error
992
+ if not helper.process.is_alive():
993
+ exit_code = helper.process.exitcode
994
+ if helper.reply_receiver_done.is_set() and helper.reply_inbox.empty():
995
+ raise TrustedLiveHelperExitedError(
996
+ "The trusted helper exited during startup "
997
+ f"(exit code {exit_code})."
998
+ )
999
+
1000
+ def _ensure_helper_locked(self) -> _HelperIncarnation:
1001
+ self._require_open_locked()
1002
+ self._reap_quarantined_helper_locked()
1003
+ helper = self._helper
1004
+ if (
1005
+ helper is not None
1006
+ and helper.process.is_alive()
1007
+ and helper.reply_receiver_started
1008
+ and helper.reply_receiver.is_alive()
1009
+ ):
1010
+ return helper
1011
+ if helper is not None:
1012
+ # A live process without its sole receiver is unusable and must not
1013
+ # accept another generation.
1014
+ self._discard_helper_locked(force=helper.process.is_alive())
1015
+ return self._spawn_helper_locked()
1016
+
1017
+ def _stop_incarnation(
1018
+ self,
1019
+ helper: _HelperIncarnation,
1020
+ *,
1021
+ force: bool,
1022
+ terminate_timeout: float | None = None,
1023
+ kill_timeout: float | None = None,
1024
+ ) -> tuple[bool, bool]:
1025
+ """Join and close one exact process, escalating without unbounded waits."""
1026
+
1027
+ forced = False
1028
+ terminate_wait = (
1029
+ self._terminate_timeout if terminate_timeout is None else terminate_timeout
1030
+ )
1031
+ kill_wait = self._kill_timeout if kill_timeout is None else kill_timeout
1032
+
1033
+ def process_alive() -> bool:
1034
+ try:
1035
+ return helper.process.is_alive()
1036
+ except (AssertionError, OSError, ValueError):
1037
+ return True
1038
+
1039
+ def bounded_join(timeout: float) -> None:
1040
+ try:
1041
+ helper.process.join(timeout=timeout)
1042
+ except (AssertionError, OSError, ValueError):
1043
+ pass
1044
+
1045
+ try:
1046
+ if not force and process_alive():
1047
+ _close_connection(helper.command_send)
1048
+ _close_connection(helper.control_send)
1049
+ bounded_join(self._shutdown_timeout)
1050
+ if process_alive():
1051
+ forced = True
1052
+ try:
1053
+ helper.process.terminate()
1054
+ except (AttributeError, OSError, ValueError):
1055
+ pass
1056
+ bounded_join(terminate_wait)
1057
+ if process_alive():
1058
+ forced = True
1059
+ try:
1060
+ kill = getattr(helper.process, "kill", None)
1061
+ if callable(kill):
1062
+ kill()
1063
+ else:
1064
+ helper.process.terminate()
1065
+ except (AttributeError, OSError, ValueError):
1066
+ pass
1067
+ bounded_join(kill_wait)
1068
+ finally:
1069
+ for connection in (
1070
+ helper.command_send,
1071
+ helper.control_send,
1072
+ helper.test_notify_receive,
1073
+ ):
1074
+ _close_connection(connection)
1075
+ helper.reply_receiver_stop.set()
1076
+ _close_connection(helper.reply_connection)
1077
+ if helper.reply_receiver_started:
1078
+ try:
1079
+ helper.reply_receiver.join(
1080
+ timeout=_REPLY_RECEIVER_JOIN_TIMEOUT_SECONDS
1081
+ )
1082
+ except RuntimeError:
1083
+ pass
1084
+ receiver_alive = (
1085
+ helper.reply_receiver_started and helper.reply_receiver.is_alive()
1086
+ )
1087
+ if process_alive() or receiver_alive:
1088
+ # Keep the Process handle quarantined. A later spawn may proceed
1089
+ # only after both its process and sole receiver are reaped.
1090
+ return True, False
1091
+ bounded_join(0)
1092
+ try:
1093
+ helper.process.close()
1094
+ except (OSError, ValueError):
1095
+ pass
1096
+ return forced, True
1097
+
1098
+ def _reap_quarantined_helper_locked(self) -> None:
1099
+ helper = self._unreaped_helper
1100
+ if helper is None:
1101
+ return
1102
+ helper.reply_receiver_stop.set()
1103
+ _close_connection(helper.reply_connection)
1104
+ if helper.reply_receiver_started:
1105
+ try:
1106
+ helper.reply_receiver.join(timeout=0)
1107
+ except RuntimeError:
1108
+ pass
1109
+ try:
1110
+ helper.process.join(timeout=0)
1111
+ alive = helper.process.is_alive()
1112
+ except (AssertionError, OSError, ValueError):
1113
+ alive = True
1114
+ receiver_alive = (
1115
+ helper.reply_receiver_started and helper.reply_receiver.is_alive()
1116
+ )
1117
+ if alive or receiver_alive:
1118
+ raise TrustedLiveHelperForcedTerminationError(
1119
+ "A previously killed trusted helper or its reply receiver has "
1120
+ "not been reaped; a replacement process will not be started."
1121
+ )
1122
+ try:
1123
+ helper.process.close()
1124
+ except (OSError, ValueError):
1125
+ pass
1126
+ self._unreaped_helper = None
1127
+
1128
+ def _force_quarantined_helper_locked(
1129
+ self,
1130
+ *,
1131
+ terminate_timeout: float | None = None,
1132
+ kill_timeout: float | None = None,
1133
+ ) -> bool:
1134
+ helper = self._unreaped_helper
1135
+ if helper is None:
1136
+ return False
1137
+ forced, reaped = self._stop_incarnation(
1138
+ helper,
1139
+ force=True,
1140
+ terminate_timeout=terminate_timeout,
1141
+ kill_timeout=kill_timeout,
1142
+ )
1143
+ if reaped:
1144
+ self._unreaped_helper = None
1145
+ return forced or not reaped
1146
+
1147
+ def _discard_helper_locked(
1148
+ self,
1149
+ *,
1150
+ force: bool,
1151
+ terminate_timeout: float | None = None,
1152
+ kill_timeout: float | None = None,
1153
+ ) -> bool:
1154
+ helper, self._helper = self._helper, None
1155
+ if helper is None:
1156
+ return False
1157
+ forced, reaped = self._stop_incarnation(
1158
+ helper,
1159
+ force=force,
1160
+ terminate_timeout=terminate_timeout,
1161
+ kill_timeout=kill_timeout,
1162
+ )
1163
+ if not reaped:
1164
+ self._unreaped_helper = helper
1165
+ return forced
1166
+
1167
+ def _validate_job_locked(self, job: TrustedLiveJob[_T]) -> None:
1168
+ if not isinstance(job, TrustedLiveJob) or job._owner_nonce != self._owner_nonce:
1169
+ raise ValueError("The job does not belong to this supervisor.")
1170
+
1171
+ def _submit(
1172
+ self,
1173
+ operation: str,
1174
+ queries: Sequence[TrustedQuery] | None,
1175
+ timeout: float | None,
1176
+ ) -> TrustedLiveJob[Any]:
1177
+ timeout_ms = (
1178
+ self._operation_timeout_ms
1179
+ if timeout is None
1180
+ else _timeout_milliseconds(
1181
+ timeout,
1182
+ description="child operation timeout",
1183
+ )
1184
+ )
1185
+ with self._lock:
1186
+ self._require_open_locked()
1187
+ if self._active_job is not None:
1188
+ raise TrustedLiveTransactionError(
1189
+ "Only one trusted database job may be active per helper."
1190
+ )
1191
+ helper = self._ensure_helper_locked()
1192
+ generation = self._next_generation
1193
+ if generation > MAX_GENERATION:
1194
+ raise TrustedLiveProtocolError(
1195
+ "The trusted helper job-generation space is exhausted."
1196
+ )
1197
+ try:
1198
+ frame = encode_job_request(
1199
+ helper.session,
1200
+ generation,
1201
+ operation,
1202
+ queries,
1203
+ timeout_ms,
1204
+ )
1205
+ except TrustedLiveProtocolValidationError as error:
1206
+ raise TrustedLiveProtocolError(
1207
+ "The database job exceeds the bounded IPC protocol."
1208
+ ) from error
1209
+ self._next_generation += 1
1210
+ expected_result_count = (
1211
+ len(queries)
1212
+ if operation in {"query", "query_batch"} and queries is not None
1213
+ else None
1214
+ )
1215
+ job: TrustedLiveJob[Any] = TrustedLiveJob(
1216
+ helper.number,
1217
+ helper.session,
1218
+ generation,
1219
+ operation,
1220
+ self._owner_nonce,
1221
+ expected_result_count,
1222
+ )
1223
+ self._active_job = job
1224
+ try:
1225
+ _bounded_send_bytes(helper.command_send, frame, self._send_timeout)
1226
+ except BaseException as error:
1227
+ self._active_job = None
1228
+ self._discard_helper_locked(force=True)
1229
+ raise TrustedLiveHelperExitedError(
1230
+ "The trusted helper did not accept the bounded job frame."
1231
+ ) from error
1232
+ return job
1233
+
1234
+ def submit_query(
1235
+ self,
1236
+ sql: str,
1237
+ bindings: SqliteBindings = None,
1238
+ *,
1239
+ timeout: float | None = None,
1240
+ ) -> TrustedLiveJob[TrustedQueryResult]:
1241
+ """Submit one finite read-only statement without waiting for its reply."""
1242
+
1243
+ try:
1244
+ query = normalize_query_specification(sql, bindings)
1245
+ except TrustedLiveProtocolValidationError as error:
1246
+ raise TrustedLiveProtocolError(
1247
+ "The database job exceeds the bounded IPC protocol."
1248
+ ) from error
1249
+ return cast(
1250
+ TrustedLiveJob[TrustedQueryResult],
1251
+ self._submit("query", (query,), timeout),
1252
+ )
1253
+
1254
+ def submit_query_batch(
1255
+ self,
1256
+ queries: Sequence[TrustedQuery],
1257
+ *,
1258
+ timeout: float | None = None,
1259
+ ) -> TrustedLiveJob[tuple[TrustedQueryResult, ...]]:
1260
+ """Submit one finite repeatable-read query batch."""
1261
+
1262
+ try:
1263
+ normalized_queries = normalize_query_batch(queries)
1264
+ except TrustedLiveProtocolValidationError as error:
1265
+ raise TrustedLiveProtocolError(
1266
+ "The database job exceeds the bounded IPC protocol."
1267
+ ) from error
1268
+ return cast(
1269
+ TrustedLiveJob[tuple[TrustedQueryResult, ...]],
1270
+ self._submit("query_batch", normalized_queries, timeout),
1271
+ )
1272
+
1273
+ def submit_data_version(
1274
+ self,
1275
+ *,
1276
+ timeout: float | None = None,
1277
+ ) -> TrustedLiveJob[int]:
1278
+ """Submit a finite ``PRAGMA data_version`` operation."""
1279
+
1280
+ return cast(
1281
+ TrustedLiveJob[int],
1282
+ self._submit("data_version", None, timeout),
1283
+ )
1284
+
1285
+ @staticmethod
1286
+ def _complete_job(
1287
+ job: TrustedLiveJob[Any],
1288
+ *,
1289
+ value: object = _NO_VALUE,
1290
+ error: BaseException | None = None,
1291
+ ) -> None:
1292
+ if job.done:
1293
+ return
1294
+ job._value = value
1295
+ job._error = error
1296
+ job._done.set()
1297
+
1298
+ def _protocol_failure_locked(
1299
+ self,
1300
+ job: TrustedLiveJob[Any],
1301
+ message: str,
1302
+ cause: BaseException | None = None,
1303
+ ) -> None:
1304
+ error = TrustedLiveProtocolError(message)
1305
+ if cause is not None:
1306
+ error.__cause__ = cause
1307
+ if self._active_job is job:
1308
+ self._active_job = None
1309
+ self._discard_helper_locked(force=True)
1310
+ self._complete_job(job, error=error)
1311
+
1312
+ def _unexpected_exit_locked(
1313
+ self,
1314
+ job: TrustedLiveJob[Any],
1315
+ message: str,
1316
+ ) -> None:
1317
+ if self._active_job is job:
1318
+ self._active_job = None
1319
+ self._discard_helper_locked(force=False)
1320
+ self._complete_job(job, error=TrustedLiveHelperExitedError(message))
1321
+
1322
+ def _handle_job_reply_locked(
1323
+ self,
1324
+ job: TrustedLiveJob[Any],
1325
+ envelope: ProtocolEnvelope,
1326
+ ) -> None:
1327
+ helper = self._helper
1328
+ if helper is None or self._active_job is not job:
1329
+ self._protocol_failure_locked(
1330
+ job,
1331
+ "A reply arrived without its exact active helper job.",
1332
+ )
1333
+ return
1334
+ if envelope.session != job.session or envelope.session != helper.session:
1335
+ self._protocol_failure_locked(
1336
+ job,
1337
+ "A stale reply from another process incarnation was rejected.",
1338
+ )
1339
+ return
1340
+ if envelope.generation != job.generation:
1341
+ self._protocol_failure_locked(
1342
+ job,
1343
+ "A duplicate, stale, or out-of-order job reply was rejected.",
1344
+ )
1345
+ return
1346
+ try:
1347
+ status, payload = decode_reply_payload(envelope)
1348
+ if envelope.operation == "protocol":
1349
+ if status != "error" or payload["code"] != "protocol_error":
1350
+ raise TrustedLiveProtocolValidationError(
1351
+ "A protocol failure reply has invalid fields."
1352
+ )
1353
+ error = _error_from_payload(payload["code"], payload["message"])
1354
+ terminal = True
1355
+ value = _NO_VALUE
1356
+ else:
1357
+ if envelope.operation != job.operation:
1358
+ raise TrustedLiveProtocolValidationError(
1359
+ "The helper reply operation does not match its job."
1360
+ )
1361
+ if status == "error":
1362
+ code = payload["code"]
1363
+ error = _error_from_payload(code, payload["message"])
1364
+ terminal = error_code_is_terminal(code)
1365
+ value = _NO_VALUE
1366
+ else:
1367
+ value = validate_job_success(job.operation, payload)
1368
+ if job.operation == "query_batch" and (
1369
+ not isinstance(value, tuple)
1370
+ or len(value) != job._expected_result_count
1371
+ ):
1372
+ raise TrustedLiveProtocolValidationError(
1373
+ "A query-batch reply does not contain one result "
1374
+ "for every submitted statement."
1375
+ )
1376
+ error = None
1377
+ terminal = False
1378
+ except TrustedLiveProtocolValidationError as protocol_error:
1379
+ self._protocol_failure_locked(
1380
+ job,
1381
+ "The helper reply payload failed closed validation.",
1382
+ protocol_error,
1383
+ )
1384
+ return
1385
+ self._active_job = None
1386
+ if job._parent_timed_out:
1387
+ forced = self._discard_helper_locked(force=False)
1388
+ if forced:
1389
+ error = TrustedLiveHelperForcedTerminationError(
1390
+ "The parent reply deadline expired and helper retirement "
1391
+ "required forced termination."
1392
+ )
1393
+ else:
1394
+ error = TrustedLiveHelperReplyTimeoutError(
1395
+ "The parent reply deadline expired; the completed helper "
1396
+ "incarnation was retired without accepting its late reply."
1397
+ )
1398
+ value = _NO_VALUE
1399
+ elif terminal:
1400
+ self._discard_helper_locked(force=False)
1401
+ self._complete_job(job, value=value, error=error)
1402
+
1403
+ def _poll_job_once(self, job: TrustedLiveJob[Any], timeout: float) -> None:
1404
+ with self._lock:
1405
+ if job.done:
1406
+ return
1407
+ helper = self._helper
1408
+ if (
1409
+ helper is None
1410
+ or self._active_job is not job
1411
+ or helper.session != job.session
1412
+ ):
1413
+ self._unexpected_exit_locked(
1414
+ job,
1415
+ "The job's helper incarnation is no longer available.",
1416
+ )
1417
+ return
1418
+ process = helper.process
1419
+ try:
1420
+ item = self._take_reply_item(helper, max(0.0, timeout))
1421
+ except Exception as error:
1422
+ with self._lock:
1423
+ if job.done:
1424
+ return
1425
+ if self._helper is not helper or self._active_job is not job:
1426
+ self._complete_job(
1427
+ job,
1428
+ error=TrustedLiveHelperExitedError(
1429
+ "The job's helper was retired while polling its reply."
1430
+ ),
1431
+ )
1432
+ return
1433
+ self._protocol_failure_locked(
1434
+ job,
1435
+ "The helper reply endpoint failed closed.",
1436
+ error,
1437
+ )
1438
+ return
1439
+ if item is not None:
1440
+ if isinstance(item, _ReplyReceiveFailure):
1441
+ with self._lock:
1442
+ if job.done:
1443
+ return
1444
+ if self._helper is not helper or self._active_job is not job:
1445
+ self._complete_job(
1446
+ job,
1447
+ error=TrustedLiveHelperExitedError(
1448
+ "The job's helper was retired while its sole "
1449
+ "reply receiver was stopping."
1450
+ ),
1451
+ )
1452
+ return
1453
+ if item.kind == "eof":
1454
+ try:
1455
+ exit_code = process.exitcode
1456
+ except (AssertionError, OSError, ValueError):
1457
+ exit_code = None
1458
+ self._unexpected_exit_locked(
1459
+ job,
1460
+ "The trusted helper closed its reply endpoint before "
1461
+ f"replying (exit code {exit_code}).",
1462
+ )
1463
+ else:
1464
+ self._protocol_failure_locked(job, item.description)
1465
+ return
1466
+ try:
1467
+ frame = item
1468
+ envelope = decode_reply_frame(frame)
1469
+ except TrustedLiveProtocolValidationError as error:
1470
+ with self._lock:
1471
+ if job.done:
1472
+ return
1473
+ if self._helper is not helper or self._active_job is not job:
1474
+ self._complete_job(
1475
+ job,
1476
+ error=TrustedLiveHelperExitedError(
1477
+ "The job's helper was retired before its reply "
1478
+ "could be validated."
1479
+ ),
1480
+ )
1481
+ return
1482
+ self._protocol_failure_locked(
1483
+ job,
1484
+ "The helper sent malformed or oversized reply IPC.",
1485
+ error,
1486
+ )
1487
+ return
1488
+ except EOFError:
1489
+ with self._lock:
1490
+ if job.done:
1491
+ return
1492
+ if self._helper is not helper or self._active_job is not job:
1493
+ self._complete_job(
1494
+ job,
1495
+ error=TrustedLiveHelperExitedError(
1496
+ "The job's helper was retired while receiving "
1497
+ "its reply."
1498
+ ),
1499
+ )
1500
+ return
1501
+ try:
1502
+ exit_code = process.exitcode
1503
+ except (AssertionError, OSError, ValueError):
1504
+ exit_code = None
1505
+ self._unexpected_exit_locked(
1506
+ job,
1507
+ "The trusted helper closed its reply endpoint before "
1508
+ f"replying (exit code {exit_code}).",
1509
+ )
1510
+ return
1511
+ except (OSError, ValueError) as error:
1512
+ with self._lock:
1513
+ if job.done:
1514
+ return
1515
+ if self._helper is not helper or self._active_job is not job:
1516
+ self._complete_job(
1517
+ job,
1518
+ error=TrustedLiveHelperExitedError(
1519
+ "The job's helper was retired while receiving "
1520
+ "its reply frame."
1521
+ ),
1522
+ )
1523
+ return
1524
+ self._protocol_failure_locked(
1525
+ job,
1526
+ "The helper reply frame was truncated or oversized.",
1527
+ error,
1528
+ )
1529
+ return
1530
+ with self._lock:
1531
+ if job.done:
1532
+ return
1533
+ if self._helper is not helper or self._active_job is not job:
1534
+ self._complete_job(
1535
+ job,
1536
+ error=TrustedLiveHelperExitedError(
1537
+ "The job's helper was retired before its reply "
1538
+ "could be accepted."
1539
+ ),
1540
+ )
1541
+ return
1542
+ self._handle_job_reply_locked(job, envelope)
1543
+ return
1544
+ try:
1545
+ process_alive = process.is_alive()
1546
+ exit_code = None if process_alive else process.exitcode
1547
+ except (AssertionError, OSError, ValueError) as error:
1548
+ with self._lock:
1549
+ if job.done:
1550
+ return
1551
+ if self._helper is not helper or self._active_job is not job:
1552
+ self._complete_job(
1553
+ job,
1554
+ error=TrustedLiveHelperExitedError(
1555
+ "The job's helper was retired while its process "
1556
+ "state was inspected."
1557
+ ),
1558
+ )
1559
+ return
1560
+ self._protocol_failure_locked(
1561
+ job,
1562
+ "The trusted helper process state failed closed.",
1563
+ error,
1564
+ )
1565
+ return
1566
+ if not process_alive:
1567
+ with self._lock:
1568
+ if job.done:
1569
+ return
1570
+ if self._helper is not helper or self._active_job is not job:
1571
+ self._complete_job(
1572
+ job,
1573
+ error=TrustedLiveHelperExitedError(
1574
+ "The job's helper was retired before process-exit "
1575
+ "handling completed."
1576
+ ),
1577
+ )
1578
+ return
1579
+ # A complete frame may have become readable with the process
1580
+ # sentinel. The receiver publishes before marking itself
1581
+ # done, so classify the exit only after its inbox is drained.
1582
+ if helper.reply_receiver_done.is_set() and helper.reply_inbox.empty():
1583
+ self._unexpected_exit_locked(
1584
+ job,
1585
+ f"The trusted helper exited before replying "
1586
+ f"(exit code {exit_code}).",
1587
+ )
1588
+
1589
+ def _request_cancel_locked(
1590
+ self,
1591
+ job: TrustedLiveJob[Any],
1592
+ *,
1593
+ grace_timeout: float,
1594
+ parent_timeout: bool,
1595
+ ) -> bool:
1596
+ if job.done:
1597
+ return False
1598
+ helper = self._helper
1599
+ if self._active_job is not job or helper is None:
1600
+ self._unexpected_exit_locked(
1601
+ job,
1602
+ "The current helper disappeared before cancellation.",
1603
+ )
1604
+ return False
1605
+ if job._cancel_requested:
1606
+ if parent_timeout:
1607
+ job._parent_timed_out = True
1608
+ return True
1609
+ frame = encode_cancel(helper.session, job.generation)
1610
+ job._cancel_requested = True
1611
+ job._parent_timed_out = parent_timeout
1612
+ job._cancel_deadline = time.monotonic() + grace_timeout
1613
+ try:
1614
+ _bounded_send_bytes(
1615
+ helper.control_send,
1616
+ frame,
1617
+ min(self._send_timeout, max(0.05, grace_timeout)),
1618
+ )
1619
+ except BaseException:
1620
+ self._force_job_locked(
1621
+ job,
1622
+ "The cancellation frame could not reach the trusted helper.",
1623
+ )
1624
+ return True
1625
+
1626
+ def _force_job_locked(self, job: TrustedLiveJob[Any], message: str) -> None:
1627
+ if job.done:
1628
+ return
1629
+ if self._active_job is job:
1630
+ self._active_job = None
1631
+ self._discard_helper_locked(force=True)
1632
+ self._complete_job(
1633
+ job,
1634
+ error=TrustedLiveHelperForcedTerminationError(message),
1635
+ )
1636
+
1637
+ def _drive_job(
1638
+ self,
1639
+ job: TrustedLiveJob[Any],
1640
+ parent_deadline: float,
1641
+ ) -> None:
1642
+ while not job.done:
1643
+ now = time.monotonic()
1644
+ with self._lock:
1645
+ cancel_deadline = job._cancel_deadline
1646
+ if cancel_deadline is not None and now >= cancel_deadline:
1647
+ self._force_job_locked(
1648
+ job,
1649
+ "Cooperative cancellation did not finish within its "
1650
+ "bounded grace period; the helper was forcibly stopped.",
1651
+ )
1652
+ return
1653
+ if cancel_deadline is None and now >= parent_deadline:
1654
+ self._request_cancel_locked(
1655
+ job,
1656
+ grace_timeout=self._cancellation_grace,
1657
+ parent_timeout=True,
1658
+ )
1659
+ continue
1660
+ effective_deadline = (
1661
+ parent_deadline
1662
+ if cancel_deadline is None
1663
+ else min(parent_deadline, cancel_deadline)
1664
+ )
1665
+ if job._parent_timed_out and cancel_deadline is not None:
1666
+ effective_deadline = cancel_deadline
1667
+ quantum = max(
1668
+ 0.0,
1669
+ min(_POLL_QUANTUM_SECONDS, effective_deadline - time.monotonic()),
1670
+ )
1671
+ self._poll_job_once(job, quantum)
1672
+
1673
+ def _job_outcome(self, job: TrustedLiveJob[_T]) -> _T:
1674
+ error = job._error
1675
+ if error is not None:
1676
+ raise error
1677
+ if job._value is _NO_VALUE:
1678
+ raise TrustedLiveProtocolError(
1679
+ "The helper job completed without a result or error."
1680
+ )
1681
+ return cast(_T, job._value)
1682
+
1683
+ def wait(
1684
+ self,
1685
+ job: TrustedLiveJob[_T],
1686
+ *,
1687
+ timeout: float | None = None,
1688
+ ) -> _T:
1689
+ """Wait no longer than the parent deadline, then cancel and retire."""
1690
+
1691
+ wait_timeout = (
1692
+ self._reply_timeout
1693
+ if timeout is None
1694
+ else _duration(
1695
+ timeout,
1696
+ description="parent job wait timeout",
1697
+ allow_zero=True,
1698
+ )
1699
+ )
1700
+ with self._lock:
1701
+ self._validate_job_locked(job)
1702
+ if job.done:
1703
+ return self._job_outcome(job)
1704
+ parent_deadline = time.monotonic() + wait_timeout
1705
+ while not job.done:
1706
+ remaining = max(0.0, parent_deadline - time.monotonic())
1707
+ if job._cancel_deadline is not None:
1708
+ remaining = max(
1709
+ 0.0,
1710
+ job._cancel_deadline - time.monotonic(),
1711
+ )
1712
+ acquired = self._reply_lock.acquire(
1713
+ timeout=min(_POLL_QUANTUM_SECONDS, remaining) if remaining > 0 else 0
1714
+ )
1715
+ if acquired:
1716
+ try:
1717
+ self._drive_job(job, parent_deadline)
1718
+ finally:
1719
+ self._reply_lock.release()
1720
+ break
1721
+ if time.monotonic() >= parent_deadline:
1722
+ with self._lock:
1723
+ self._request_cancel_locked(
1724
+ job,
1725
+ grace_timeout=self._cancellation_grace,
1726
+ parent_timeout=True,
1727
+ )
1728
+ if job._done.wait(self._cancellation_grace):
1729
+ break
1730
+ with self._lock:
1731
+ self._force_job_locked(
1732
+ job,
1733
+ "The parent reply deadline and cancellation grace "
1734
+ "expired; the helper was forcibly stopped.",
1735
+ )
1736
+ break
1737
+ return self._job_outcome(job)
1738
+
1739
+ def cancel(
1740
+ self,
1741
+ job: TrustedLiveJob[Any] | None = None,
1742
+ *,
1743
+ grace_timeout: float | None = None,
1744
+ ) -> bool:
1745
+ """Cancel one exact active generation and enforce bounded escalation."""
1746
+
1747
+ grace = (
1748
+ self._cancellation_grace
1749
+ if grace_timeout is None
1750
+ else _duration(
1751
+ grace_timeout,
1752
+ description="cancellation grace period",
1753
+ allow_zero=True,
1754
+ )
1755
+ )
1756
+ with self._lock:
1757
+ self._require_not_closed_locked()
1758
+ selected = self._active_job if job is None else job
1759
+ if selected is None:
1760
+ return False
1761
+ self._validate_job_locked(selected)
1762
+ if selected.done or self._active_job is not selected:
1763
+ return False
1764
+ self._request_cancel_locked(
1765
+ selected,
1766
+ grace_timeout=grace,
1767
+ parent_timeout=False,
1768
+ )
1769
+ if selected.done:
1770
+ return True
1771
+ if self._reply_lock.acquire(blocking=False):
1772
+ try:
1773
+ self._drive_job(
1774
+ selected,
1775
+ selected._cancel_deadline or time.monotonic(),
1776
+ )
1777
+ finally:
1778
+ self._reply_lock.release()
1779
+ elif not selected._done.wait(grace):
1780
+ with self._lock:
1781
+ self._force_job_locked(
1782
+ selected,
1783
+ "Cooperative cancellation did not finish within its "
1784
+ "bounded grace period; the helper was forcibly stopped.",
1785
+ )
1786
+ return True
1787
+
1788
+ def query(
1789
+ self,
1790
+ sql: str,
1791
+ bindings: SqliteBindings = None,
1792
+ *,
1793
+ timeout: float | None = None,
1794
+ wait_timeout: float | None = None,
1795
+ ) -> TrustedQueryResult:
1796
+ """Submit and boundedly wait for one trusted query."""
1797
+
1798
+ return self.wait(
1799
+ self.submit_query(sql, bindings, timeout=timeout),
1800
+ timeout=wait_timeout,
1801
+ )
1802
+
1803
+ def query_batch(
1804
+ self,
1805
+ queries: Sequence[TrustedQuery],
1806
+ *,
1807
+ timeout: float | None = None,
1808
+ wait_timeout: float | None = None,
1809
+ ) -> tuple[TrustedQueryResult, ...]:
1810
+ """Submit and boundedly wait for one repeatable-read query batch."""
1811
+
1812
+ return self.wait(
1813
+ self.submit_query_batch(queries, timeout=timeout),
1814
+ timeout=wait_timeout,
1815
+ )
1816
+
1817
+ def data_version(
1818
+ self,
1819
+ *,
1820
+ timeout: float | None = None,
1821
+ wait_timeout: float | None = None,
1822
+ ) -> int:
1823
+ """Submit and boundedly wait for ``PRAGMA data_version``."""
1824
+
1825
+ return self.wait(
1826
+ self.submit_data_version(timeout=timeout),
1827
+ timeout=wait_timeout,
1828
+ )
1829
+
1830
+ def restart(self) -> None:
1831
+ """Explicitly retire any idle incarnation and spawn a fresh one."""
1832
+
1833
+ with self._lock:
1834
+ self._require_open_locked()
1835
+ if self._active_job is not None:
1836
+ raise TrustedLiveTransactionError(
1837
+ "Cancel and finish the active job before restarting its helper."
1838
+ )
1839
+ self._discard_helper_locked(force=False)
1840
+ self._spawn_helper_locked()
1841
+
1842
+ def _shutdown_helper_locked(self, timeout: float) -> BaseException | None:
1843
+ helper = self._helper
1844
+ if helper is None:
1845
+ if self._force_quarantined_helper_locked():
1846
+ return TrustedLiveHelperForcedTerminationError(
1847
+ "A quarantined trusted helper required forced retirement "
1848
+ "during shutdown."
1849
+ )
1850
+ return None
1851
+ if not helper.process.is_alive():
1852
+ self._discard_helper_locked(force=False)
1853
+ return None
1854
+ generation = self._next_generation
1855
+ if generation > MAX_GENERATION:
1856
+ forced = self._discard_helper_locked(force=True)
1857
+ if forced:
1858
+ return TrustedLiveHelperForcedTerminationError(
1859
+ "The helper generation space exhausted during shutdown."
1860
+ )
1861
+ return TrustedLiveProtocolError(
1862
+ "The helper generation space exhausted during shutdown."
1863
+ )
1864
+ self._next_generation += 1
1865
+ try:
1866
+ frame = encode_shutdown(helper.session, generation)
1867
+ _bounded_send_bytes(
1868
+ helper.command_send,
1869
+ frame,
1870
+ min(self._send_timeout, max(0.05, timeout)),
1871
+ )
1872
+ except BaseException as error:
1873
+ self._discard_helper_locked(force=True)
1874
+ shutdown_send_error = TrustedLiveHelperForcedTerminationError(
1875
+ "The helper shutdown request could not be delivered."
1876
+ )
1877
+ shutdown_send_error.__cause__ = error
1878
+ return shutdown_send_error
1879
+ deadline = time.monotonic() + timeout
1880
+ shutdown_error: BaseException | None = None
1881
+ acknowledged = False
1882
+ while time.monotonic() < deadline:
1883
+ remaining = deadline - time.monotonic()
1884
+ try:
1885
+ item = self._take_reply_item(
1886
+ helper,
1887
+ min(_POLL_QUANTUM_SECONDS, remaining),
1888
+ )
1889
+ if item is None and not helper.process.is_alive():
1890
+ # Process exit and a final buffered pipe frame can become
1891
+ # observable in either order. The sole receiver publishes
1892
+ # before marking itself done, so wait for that transition.
1893
+ if (
1894
+ helper.reply_receiver_done.is_set()
1895
+ and helper.reply_inbox.empty()
1896
+ ):
1897
+ shutdown_error = TrustedLiveHelperExitedError(
1898
+ "The trusted helper exited without acknowledging "
1899
+ "shutdown "
1900
+ f"(exit code {helper.process.exitcode})."
1901
+ )
1902
+ break
1903
+ if item is not None:
1904
+ if isinstance(item, _ReplyReceiveFailure):
1905
+ if item.kind == "eof":
1906
+ shutdown_error = TrustedLiveHelperExitedError(
1907
+ "The helper exited while acknowledging shutdown."
1908
+ )
1909
+ else:
1910
+ shutdown_error = TrustedLiveProtocolError(item.description)
1911
+ break
1912
+ envelope = decode_reply_frame(item)
1913
+ if (
1914
+ envelope.session != helper.session
1915
+ or envelope.generation != generation
1916
+ or envelope.operation not in {"shutdown", "protocol"}
1917
+ ):
1918
+ raise TrustedLiveProtocolValidationError(
1919
+ "The shutdown reply was stale or out of order."
1920
+ )
1921
+ status, payload = decode_reply_payload(envelope)
1922
+ if envelope.operation == "protocol":
1923
+ if status != "error" or payload["code"] != "protocol_error":
1924
+ raise TrustedLiveProtocolValidationError(
1925
+ "A protocol shutdown reply has invalid fields."
1926
+ )
1927
+ shutdown_error = _error_from_payload(
1928
+ payload["code"], payload["message"]
1929
+ )
1930
+ else:
1931
+ if status == "error":
1932
+ shutdown_error = _error_from_payload(
1933
+ payload["code"], payload["message"]
1934
+ )
1935
+ else:
1936
+ validate_shutdown_success(payload)
1937
+ acknowledged = True
1938
+ break
1939
+ except TrustedLiveProtocolValidationError as error:
1940
+ shutdown_error = TrustedLiveProtocolError(
1941
+ "The helper sent invalid shutdown IPC."
1942
+ )
1943
+ shutdown_error.__cause__ = error
1944
+ break
1945
+ except (EOFError, OSError, ValueError) as error:
1946
+ shutdown_error = TrustedLiveHelperExitedError(
1947
+ "The helper exited while acknowledging shutdown."
1948
+ )
1949
+ shutdown_error.__cause__ = error
1950
+ break
1951
+ timed_out = not acknowledged and shutdown_error is None
1952
+ forced = self._discard_helper_locked(force=False)
1953
+ if forced or timed_out:
1954
+ return TrustedLiveHelperForcedTerminationError(
1955
+ "The trusted helper did not shut down within its bounded deadline."
1956
+ )
1957
+ return shutdown_error
1958
+
1959
+ def close(self, *, timeout: float | None = None) -> None:
1960
+ """Cancel active work, then boundedly close, join, and release the helper."""
1961
+
1962
+ shutdown_timeout = (
1963
+ self._shutdown_timeout
1964
+ if timeout is None
1965
+ else _duration(
1966
+ timeout,
1967
+ description="helper shutdown timeout",
1968
+ allow_zero=True,
1969
+ )
1970
+ )
1971
+ with self._lock:
1972
+ if self._closed:
1973
+ return
1974
+ if self._closing:
1975
+ raise TrustedLiveSupervisorClosedError(
1976
+ "The trusted live-reader supervisor is already closing."
1977
+ )
1978
+ self._closing = True
1979
+ active = self._active_job
1980
+ cancel_failure: BaseException | None = None
1981
+ if active is not None and not active.done:
1982
+ try:
1983
+ self.cancel(active)
1984
+ except BaseException as error:
1985
+ cancel_failure = error
1986
+ if active is not None and not active.done:
1987
+ with self._lock:
1988
+ self._force_job_locked(
1989
+ active,
1990
+ "Supervisor close could not obtain a terminal job reply; "
1991
+ "the helper was forcibly stopped before shutdown.",
1992
+ )
1993
+ active_forced_error = (
1994
+ active._error
1995
+ if active is not None
1996
+ and isinstance(active._error, TrustedLiveHelperForcedTerminationError)
1997
+ else None
1998
+ )
1999
+ with self._lock:
2000
+ try:
2001
+ shutdown_error = self._shutdown_helper_locked(shutdown_timeout)
2002
+ except BaseException as unexpected_shutdown_error:
2003
+ self._discard_helper_locked(force=True)
2004
+ shutdown_error = unexpected_shutdown_error
2005
+ finally:
2006
+ self._active_job = None
2007
+ self._closed = True
2008
+ self._closing = False
2009
+ self._unregister_atexit()
2010
+ if active_forced_error is not None:
2011
+ raise active_forced_error
2012
+ if cancel_failure is not None:
2013
+ raise cancel_failure
2014
+ if shutdown_error is not None:
2015
+ raise shutdown_error
2016
+
2017
+ def _unregister_atexit(self) -> None:
2018
+ lock = getattr(self, "_lock", None)
2019
+ if lock is None:
2020
+ return
2021
+ with lock:
2022
+ self._unregister_atexit_locked()
2023
+
2024
+ def _unregister_atexit_locked(self) -> None:
2025
+ if self._helper is not None or self._unreaped_helper is not None:
2026
+ return
2027
+ if not self._atexit_registered:
2028
+ return
2029
+ self._atexit_registered = False
2030
+ atexit.unregister(self._atexit_callback)
2031
+
2032
+ def _cleanup_at_exit(self) -> None:
2033
+ """Best-effort bounded cleanup before multiprocessing's exit hook."""
2034
+
2035
+ lock = getattr(self, "_lock", None)
2036
+ if lock is None:
2037
+ return
2038
+ acquired = False
2039
+ try:
2040
+ acquired = lock.acquire(timeout=_ATEXIT_LOCK_TIMEOUT_SECONDS)
2041
+ if not acquired:
2042
+ return
2043
+ self._atexit_registered = False
2044
+ active, self._active_job = self._active_job, None
2045
+ if active is not None and not active.done:
2046
+ self._complete_job(
2047
+ active,
2048
+ error=TrustedLiveHelperForcedTerminationError(
2049
+ "The abandoned supervisor forcibly retired its active helper."
2050
+ ),
2051
+ )
2052
+ self._closing = False
2053
+ self._closed = True
2054
+ self._discard_helper_locked(
2055
+ force=True,
2056
+ terminate_timeout=_ATEXIT_PROCESS_TIMEOUT_SECONDS,
2057
+ kill_timeout=_ATEXIT_PROCESS_TIMEOUT_SECONDS,
2058
+ )
2059
+ self._force_quarantined_helper_locked(
2060
+ terminate_timeout=_ATEXIT_PROCESS_TIMEOUT_SECONDS,
2061
+ kill_timeout=_ATEXIT_PROCESS_TIMEOUT_SECONDS,
2062
+ )
2063
+ except BaseException:
2064
+ # The daemon-process fallback remains in force. Interpreter exit
2065
+ # must never be delayed by cleanup diagnostics.
2066
+ pass
2067
+ finally:
2068
+ if acquired:
2069
+ try:
2070
+ lock.release()
2071
+ except BaseException:
2072
+ pass
2073
+
2074
+ def _wait_for_test_notification(self, expected: bytes, timeout: float) -> None:
2075
+ """Private deterministic barrier; not part of the production protocol."""
2076
+
2077
+ wait_timeout = _duration(
2078
+ timeout,
2079
+ description="private test notification timeout",
2080
+ allow_zero=True,
2081
+ )
2082
+ with self._lock:
2083
+ helper = self._helper
2084
+ connection = None if helper is None else helper.test_notify_receive
2085
+ if connection is None or not connection.poll(wait_timeout):
2086
+ raise TimeoutError("The private helper test notification did not arrive.")
2087
+ value = connection.recv_bytes(256)
2088
+ if value != expected:
2089
+ raise AssertionError(
2090
+ f"Expected private helper notification {expected!r}, got {value!r}."
2091
+ )
2092
+
2093
+ def _send_test_command_frame(self, frame: bytes) -> None:
2094
+ """Private raw-frame injection used only by protocol tests."""
2095
+
2096
+ if not isinstance(frame, bytes):
2097
+ raise TypeError("A private test frame must be bytes.")
2098
+ with self._lock:
2099
+ helper = self._ensure_helper_locked()
2100
+ _bounded_send_bytes(helper.command_send, frame, self._send_timeout)
2101
+
2102
+ def _send_test_control_frame(self, frame: bytes) -> None:
2103
+ """Private raw control injection used only by protocol tests."""
2104
+
2105
+ if not isinstance(frame, bytes):
2106
+ raise TypeError("A private test frame must be bytes.")
2107
+ with self._lock:
2108
+ helper = self._ensure_helper_locked()
2109
+ _bounded_send_bytes(helper.control_send, frame, self._send_timeout)
2110
+
2111
+ def __enter__(self) -> TrustedLiveReaderSupervisor:
2112
+ with self._lock:
2113
+ self._require_open_locked()
2114
+ return self
2115
+
2116
+ def __exit__(
2117
+ self,
2118
+ exception_type: type[BaseException] | None,
2119
+ exception: BaseException | None,
2120
+ traceback: TracebackType | None,
2121
+ ) -> None:
2122
+ del exception_type, traceback
2123
+ if exception is None:
2124
+ self.close()
2125
+ return
2126
+ try:
2127
+ self.close()
2128
+ except BaseException as close_error:
2129
+ exception.add_note(
2130
+ "The trusted live-reader supervisor also failed during "
2131
+ f"context cleanup: {type(close_error).__name__}: {close_error}"
2132
+ )
2133
+
2134
+ def __del__(self) -> None:
2135
+ try:
2136
+ if (
2137
+ getattr(self, "_closed", True)
2138
+ and getattr(self, "_helper", None) is None
2139
+ and getattr(self, "_unreaped_helper", None) is None
2140
+ ):
2141
+ return
2142
+ self._cleanup_at_exit()
2143
+ callback = getattr(self, "_atexit_callback", None)
2144
+ if callback is not None:
2145
+ atexit.unregister(callback)
2146
+ except BaseException:
2147
+ pass
2148
+
2149
+
2150
+ __all__ = [
2151
+ "TrustedLiveHelperExitedError",
2152
+ "TrustedLiveHelperForcedTerminationError",
2153
+ "TrustedLiveHelperReplyTimeoutError",
2154
+ "TrustedLiveHelperStartupError",
2155
+ "TrustedLiveJob",
2156
+ "TrustedLiveProtocolError",
2157
+ "TrustedLiveReaderSupervisor",
2158
+ "TrustedLiveSupervisorClosedError",
2159
+ "TrustedLiveSupervisorError",
2160
+ "TrustedLiveSupervisorLiveness",
2161
+ ]