execsql2 2.35.0__py3-none-any.whl → 2.38.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +111 -53
- execsql/cli/commands/run.py +2 -2
- execsql/cli/lint.py +60 -7
- execsql/cli/run.py +10 -10
- execsql/config.py +7 -0
- execsql/data/execsql.conf.template +31 -7
- execsql/db/base.py +64 -26
- execsql/db/duckdb.py +51 -5
- execsql/db/firebird.py +8 -0
- execsql/db/mysql.py +8 -3
- execsql/db/oracle.py +5 -0
- execsql/db/postgres.py +2 -2
- execsql/db/sqlite.py +117 -10
- execsql/db/sqlserver.py +11 -7
- execsql/db/target_types.py +151 -4
- execsql/debug/repl.py +4 -3
- execsql/exceptions.py +14 -0
- execsql/exporters/html.py +26 -42
- execsql/exporters/ods.py +93 -21
- execsql/exporters/sqlite.py +82 -47
- execsql/format.py +12 -4
- execsql/gui/desktop.py +2 -1
- execsql/gui/tui.py +5 -2
- execsql/importers/base.py +9 -7
- execsql/importers/json.py +37 -13
- execsql/importers/ods.py +22 -2
- execsql/lsp/document.py +6 -1
- execsql/lsp/hover.py +2 -1
- execsql/metacommands/conditions.py +3 -1
- execsql/metacommands/connect.py +28 -12
- execsql/metacommands/dispatch.py +10 -0
- execsql/metacommands/io.py +1 -0
- execsql/metacommands/io_export.py +1 -2
- execsql/metacommands/io_fileops.py +32 -93
- execsql/metacommands/io_import.py +26 -8
- execsql/metacommands/io_write.py +3 -1
- execsql/metacommands/reference.py +7 -0
- execsql/metacommands/script_ext.py +13 -10
- execsql/metacommands/system.py +13 -6
- execsql/parser.py +14 -37
- execsql/script/ast.py +21 -0
- execsql/script/control.py +73 -3
- execsql/script/executor.py +503 -227
- execsql/script/parser.py +102 -52
- execsql/script/preparation.py +42 -0
- execsql/state.py +23 -0
- execsql/types.py +6 -5
- execsql/utils/errors.py +24 -6
- execsql/utils/fileio.py +342 -134
- execsql/utils/sigterm.py +154 -0
- execsql/utils/spreadsheet.py +74 -0
- execsql/utils/strings.py +13 -7
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/README.md +5 -5
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/example_config_prompt.sql +2 -2
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/METADATA +32 -119
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/RECORD +71 -68
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/WHEEL +1 -1
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/licenses/NOTICE +0 -0
execsql/utils/fileio.py
CHANGED
|
@@ -8,7 +8,7 @@ Provides:
|
|
|
8
8
|
- :class:`EncodedFile` — opens files with a specific encoding and error
|
|
9
9
|
handler; supports the script-, output-, and import-encoding settings.
|
|
10
10
|
- :class:`FileWriter` — asynchronous text-file writer thread, one per run,
|
|
11
|
-
used for
|
|
11
|
+
used for WRITE/TEE output to avoid blocking the main execution loop.
|
|
12
12
|
- :class:`Logger` — wraps a log file with timestamped write methods;
|
|
13
13
|
mirrors output to stderr when ``tee_write_log`` is set.
|
|
14
14
|
- :class:`TempFileMgr` — registry of temporary files that are cleaned up
|
|
@@ -23,6 +23,7 @@ import atexit
|
|
|
23
23
|
import codecs
|
|
24
24
|
import collections
|
|
25
25
|
import errno
|
|
26
|
+
from functools import partial
|
|
26
27
|
import io
|
|
27
28
|
import os
|
|
28
29
|
import queue
|
|
@@ -92,6 +93,8 @@ def check_zip_decompression_ratio(
|
|
|
92
93
|
*,
|
|
93
94
|
max_uncompressed_mb: int = 500,
|
|
94
95
|
max_ratio: int = 100,
|
|
96
|
+
xml_only: bool = False,
|
|
97
|
+
limit_names: tuple[str, str] | None = None,
|
|
95
98
|
) -> None:
|
|
96
99
|
"""Reject a zip-based workbook that looks like a decompression bomb.
|
|
97
100
|
|
|
@@ -107,6 +110,9 @@ def check_zip_decompression_ratio(
|
|
|
107
110
|
* ``max_ratio`` — per-member uncompressed:compressed ratio
|
|
108
111
|
(default 100:1; legitimate XML-heavy XLSX rarely exceeds 30:1).
|
|
109
112
|
|
|
113
|
+
Zero disables the corresponding bound. With *xml_only*, the aggregate
|
|
114
|
+
counts XML members only (ODS); compression ratios still cover every member.
|
|
115
|
+
*limit_names* adds configuration setting names to rejection messages.
|
|
110
116
|
No-op when *path* is not a zipfile (e.g. legacy ``.xls`` OLE-CDF).
|
|
111
117
|
"""
|
|
112
118
|
import zipfile
|
|
@@ -115,26 +121,41 @@ def check_zip_decompression_ratio(
|
|
|
115
121
|
with zipfile.ZipFile(path) as zf:
|
|
116
122
|
total_uncompressed = 0
|
|
117
123
|
for info in zf.infolist():
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
124
|
+
if not xml_only or info.filename.lower().endswith(".xml"):
|
|
125
|
+
total_uncompressed += info.file_size
|
|
126
|
+
if max_ratio and info.file_size:
|
|
127
|
+
ratio = info.file_size / info.compress_size if info.compress_size else float("inf")
|
|
121
128
|
if ratio > max_ratio:
|
|
129
|
+
sizes = (
|
|
130
|
+
f" ({info.file_size:,} uncompressed / {info.compress_size:,} compressed bytes)"
|
|
131
|
+
if limit_names
|
|
132
|
+
else ""
|
|
133
|
+
)
|
|
122
134
|
raise ErrInfo(
|
|
123
135
|
type="error",
|
|
124
136
|
other_msg=(
|
|
125
137
|
f"Refusing to open '{path}': member '{info.filename}' has "
|
|
126
|
-
f"compression ratio {ratio:.1f}:1
|
|
138
|
+
f"compression ratio {ratio:.1f}:1"
|
|
139
|
+
f"{sizes} "
|
|
140
|
+
f"(limit {max_ratio}:1"
|
|
141
|
+
f"{(' set by ' + limit_names[1]) if limit_names else ''}) - "
|
|
127
142
|
"possible zip-bomb."
|
|
128
143
|
),
|
|
129
144
|
)
|
|
130
145
|
limit_bytes = max_uncompressed_mb * 1024 * 1024
|
|
131
|
-
if total_uncompressed > limit_bytes:
|
|
146
|
+
if max_uncompressed_mb and total_uncompressed > limit_bytes:
|
|
147
|
+
observed = (
|
|
148
|
+
f"{total_uncompressed:,} XML bytes" if xml_only else f"{total_uncompressed / 1024 / 1024:.0f} MB"
|
|
149
|
+
)
|
|
150
|
+
maximum = (
|
|
151
|
+
f"{limit_bytes:,} bytes ({max_uncompressed_mb} MiB)" if limit_names else f"{max_uncompressed_mb} MB"
|
|
152
|
+
)
|
|
132
153
|
raise ErrInfo(
|
|
133
154
|
type="error",
|
|
134
155
|
other_msg=(
|
|
135
156
|
f"Refusing to open '{path}': total uncompressed size "
|
|
136
|
-
f"{
|
|
137
|
-
f"{
|
|
157
|
+
f"{observed} exceeds limit {maximum}"
|
|
158
|
+
f"{(' set by ' + limit_names[0]) if limit_names else ''} - possible zip-bomb."
|
|
138
159
|
),
|
|
139
160
|
)
|
|
140
161
|
except zipfile.BadZipFile:
|
|
@@ -219,24 +240,27 @@ class FileWriter(threading.Thread):
|
|
|
219
240
|
STATUS_OPEN, STATUS_WAITING, STATUS_UNOPENED, STATUS_CLOSED, STATUS_OPENFAILURE, STATUS_WRITEFAILURE = range(6)
|
|
220
241
|
FAILED = (STATUS_OPENFAILURE, STATUS_WRITEFAILURE)
|
|
221
242
|
|
|
222
|
-
def __init__(
|
|
243
|
+
def __init__(
|
|
244
|
+
self,
|
|
245
|
+
fn: str,
|
|
246
|
+
open_timeout: int,
|
|
247
|
+
encoding: str = "utf-8",
|
|
248
|
+
*,
|
|
249
|
+
writer: FileWriter | None = None,
|
|
250
|
+
) -> None:
|
|
223
251
|
self.filename = fn
|
|
224
252
|
self.open_timeout = open_timeout
|
|
225
253
|
self.encoding = encoding
|
|
254
|
+
self._writer_ref = weakref.ref(writer) if writer is not None else None
|
|
255
|
+
self._discarded = False
|
|
226
256
|
self.openmode = "a"
|
|
227
257
|
self.handle: io.TextIOWrapper | None = None
|
|
228
258
|
self.status = self.STATUS_UNOPENED
|
|
229
259
|
self.open_start_time: float | None = None
|
|
230
260
|
self.output_queue: collections.deque[str] = collections.deque()
|
|
231
261
|
self.close_after_write = False
|
|
232
|
-
# Lines discarded because the file could not be opened within
|
|
233
|
-
# open_timeout, or could not be written; reported back to the
|
|
234
|
-
# caller, never silently lost.
|
|
235
262
|
self.dropped = 0
|
|
236
|
-
# The OS error that failed the file; None for an open that timed out.
|
|
237
263
|
self.error: str | None = None
|
|
238
|
-
# Lines discarded because the file was closed while still locked
|
|
239
|
-
# (a halt, or an exporter about to write the same path).
|
|
240
264
|
self.abandoned = 0
|
|
241
265
|
|
|
242
266
|
def __del__(self) -> None:
|
|
@@ -245,31 +269,97 @@ class FileWriter(threading.Thread):
|
|
|
245
269
|
except Exception:
|
|
246
270
|
pass # Best-effort cleanup at interpreter shutdown.
|
|
247
271
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
self.
|
|
251
|
-
|
|
272
|
+
@property
|
|
273
|
+
def writer(self) -> FileWriter | None:
|
|
274
|
+
return self._writer_ref() if self._writer_ref is not None else None
|
|
275
|
+
|
|
276
|
+
def _stopped(self) -> bool:
|
|
277
|
+
writer = self.writer
|
|
278
|
+
return self._discarded or (writer is not None and writer.abandoned.is_set())
|
|
279
|
+
|
|
280
|
+
def _retire(self, count: int) -> None:
|
|
281
|
+
writer = self.writer
|
|
282
|
+
if writer is not None:
|
|
283
|
+
writer.completed(self.filename, count)
|
|
284
|
+
|
|
285
|
+
def _io(self, operation: str, action: Callable[[], Any], *, cleanup: bool = False) -> Any:
|
|
286
|
+
if not cleanup and self._stopped():
|
|
287
|
+
return None
|
|
288
|
+
writer = self.writer
|
|
289
|
+
if writer is not None:
|
|
290
|
+
writer.current_op = (operation, self.filename, time.monotonic())
|
|
291
|
+
try:
|
|
292
|
+
# Publish the operation before the final abandonment check: a
|
|
293
|
+
# call reserved here is the one that may finish after timeout.
|
|
294
|
+
if not cleanup and self._stopped():
|
|
295
|
+
return None
|
|
296
|
+
return action()
|
|
297
|
+
finally:
|
|
298
|
+
if writer is not None:
|
|
299
|
+
writer.current_op = None
|
|
300
|
+
|
|
301
|
+
def _close_handle(self, *, discard: bool = False) -> None:
|
|
302
|
+
handle = self.handle
|
|
303
|
+
if handle is None:
|
|
304
|
+
return
|
|
305
|
+
if discard or self._stopped():
|
|
306
|
+
# TextIOWrapper.close() flushes. Close the raw file first when
|
|
307
|
+
# abandoning a write/flush so its buffered bytes cannot escape
|
|
308
|
+
# later; keep normal text encoding/newline/BOM behavior intact.
|
|
309
|
+
raw = getattr(getattr(handle, "buffer", None), "raw", None)
|
|
310
|
+
if raw is not None:
|
|
311
|
+
try:
|
|
312
|
+
self._io("close", raw.close, cleanup=True)
|
|
313
|
+
except Exception:
|
|
314
|
+
pass
|
|
315
|
+
try:
|
|
316
|
+
self._io("close", handle.close, cleanup=True)
|
|
317
|
+
except Exception:
|
|
318
|
+
pass
|
|
319
|
+
else:
|
|
320
|
+
self._io("close", handle.close)
|
|
321
|
+
if self._stopped():
|
|
322
|
+
self._close_handle(discard=True)
|
|
323
|
+
self.handle = None
|
|
324
|
+
|
|
325
|
+
def discard(self) -> None:
|
|
326
|
+
"""Drop queued text and close without flushing abandoned buffers."""
|
|
327
|
+
self._discarded = True
|
|
252
328
|
self.dropped += len(self.output_queue)
|
|
329
|
+
self._retire(len(self.output_queue))
|
|
253
330
|
self.output_queue.clear()
|
|
254
331
|
self.close_after_write = False
|
|
255
|
-
|
|
256
|
-
if
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
332
|
+
self._close_handle(discard=True)
|
|
333
|
+
if self.status not in self.FAILED:
|
|
334
|
+
self.status = self.STATUS_CLOSED
|
|
335
|
+
|
|
336
|
+
def fail(self, status: int, exc: BaseException) -> None:
|
|
337
|
+
"""Fail this file alone, discarding any text it cannot safely write."""
|
|
338
|
+
self.status = status
|
|
339
|
+
self.error = str(exc)
|
|
340
|
+
self.discard()
|
|
261
341
|
|
|
262
342
|
def write_queue(self) -> None:
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
343
|
+
if self._stopped():
|
|
344
|
+
return
|
|
345
|
+
handle = self.handle
|
|
346
|
+
assert handle is not None
|
|
347
|
+
while self.output_queue and not self._stopped():
|
|
348
|
+
message = self.output_queue.pop()
|
|
266
349
|
try:
|
|
267
|
-
self.handle.write
|
|
268
|
-
|
|
350
|
+
self._io("write", partial(handle.write, message))
|
|
351
|
+
if self._stopped():
|
|
352
|
+
self.dropped += 1
|
|
353
|
+
self._retire(1)
|
|
354
|
+
return
|
|
355
|
+
self._io("write", handle.flush)
|
|
356
|
+
except Exception as exc:
|
|
269
357
|
self.dropped += 1
|
|
358
|
+
self._retire(1)
|
|
270
359
|
self.fail(self.STATUS_WRITEFAILURE, exc)
|
|
271
360
|
return
|
|
272
|
-
|
|
361
|
+
self._retire(1)
|
|
362
|
+
if self.close_after_write and not self._stopped():
|
|
273
363
|
self.close_after_write = False
|
|
274
364
|
self.close()
|
|
275
365
|
|
|
@@ -278,14 +368,16 @@ class FileWriter(threading.Thread):
|
|
|
278
368
|
self.openmode = "w"
|
|
279
369
|
|
|
280
370
|
def try_open(self) -> None:
|
|
371
|
+
if self._stopped():
|
|
372
|
+
return
|
|
281
373
|
if self.status != self.STATUS_OPEN or self.handle is None:
|
|
282
374
|
if self.open_start_time is None:
|
|
283
375
|
self.open_start_time = time.time()
|
|
284
376
|
if time.time() - self.open_start_time < self.open_timeout:
|
|
285
377
|
try:
|
|
286
|
-
self.handle =
|
|
287
|
-
|
|
288
|
-
open( # noqa: SIM115
|
|
378
|
+
self.handle = self._io(
|
|
379
|
+
"open",
|
|
380
|
+
lambda: open( # noqa: SIM115
|
|
289
381
|
file=self.filename,
|
|
290
382
|
mode=self.openmode,
|
|
291
383
|
encoding=self.encoding,
|
|
@@ -293,25 +385,29 @@ class FileWriter(threading.Thread):
|
|
|
293
385
|
),
|
|
294
386
|
)
|
|
295
387
|
except Exception as exc:
|
|
296
|
-
if _never_opens
|
|
388
|
+
if self._io("open", partial(_never_opens, self.filename, exc)):
|
|
297
389
|
self.fail(self.STATUS_OPENFAILURE, exc)
|
|
298
390
|
else:
|
|
299
391
|
self.status = self.STATUS_WAITING
|
|
300
392
|
else:
|
|
301
|
-
self.
|
|
302
|
-
|
|
303
|
-
|
|
393
|
+
if self.handle is not None:
|
|
394
|
+
self.status = self.STATUS_OPEN
|
|
395
|
+
self.openmode = "a"
|
|
396
|
+
self.open_start_time = None
|
|
304
397
|
else:
|
|
305
|
-
# Give up: discard what was buffered (counting it so the
|
|
306
|
-
# caller can report the loss) and stop buffering.
|
|
307
398
|
self.status = self.STATUS_OPENFAILURE
|
|
308
399
|
self.dropped += len(self.output_queue)
|
|
400
|
+
self._retire(len(self.output_queue))
|
|
309
401
|
self.output_queue.clear()
|
|
310
402
|
self.close_after_write = False
|
|
311
403
|
|
|
312
404
|
def close(self) -> None:
|
|
405
|
+
if self._stopped():
|
|
406
|
+
self.discard()
|
|
407
|
+
return
|
|
313
408
|
if self.status == self.STATUS_WAITING:
|
|
314
409
|
self.abandoned += len(self.output_queue)
|
|
410
|
+
self._retire(len(self.output_queue))
|
|
315
411
|
self.output_queue.clear()
|
|
316
412
|
self.close_after_write = False
|
|
317
413
|
if self.status == self.STATUS_OPEN and self.handle is not None:
|
|
@@ -319,16 +415,14 @@ class FileWriter(threading.Thread):
|
|
|
319
415
|
if self.status == self.STATUS_WRITEFAILURE:
|
|
320
416
|
return
|
|
321
417
|
try:
|
|
322
|
-
self.
|
|
323
|
-
except OSError as exc:
|
|
324
|
-
self.handle = None
|
|
418
|
+
self._close_handle()
|
|
419
|
+
except OSError as exc:
|
|
325
420
|
self.fail(self.STATUS_WRITEFAILURE, exc)
|
|
326
421
|
return
|
|
327
|
-
self.handle = None
|
|
328
422
|
if self.status in self.FAILED:
|
|
329
|
-
return
|
|
423
|
+
return
|
|
330
424
|
self.status = self.STATUS_CLOSED
|
|
331
|
-
self.openmode = "a"
|
|
425
|
+
self.openmode = "a"
|
|
332
426
|
|
|
333
427
|
def clean_close(self) -> None:
|
|
334
428
|
if self.status == self.STATUS_WAITING:
|
|
@@ -337,8 +431,9 @@ class FileWriter(threading.Thread):
|
|
|
337
431
|
self.close()
|
|
338
432
|
|
|
339
433
|
def write(self, content: str) -> None:
|
|
340
|
-
if self.status in self.FAILED:
|
|
434
|
+
if self._stopped() or self.status in self.FAILED:
|
|
341
435
|
self.dropped += 1
|
|
436
|
+
self._retire(1)
|
|
342
437
|
return
|
|
343
438
|
self.output_queue.appendleft(content)
|
|
344
439
|
self.try_open()
|
|
@@ -348,14 +443,19 @@ class FileWriter(threading.Thread):
|
|
|
348
443
|
def __init__(self, file_encoding: str = "utf-8", open_timeout: int = 600) -> None:
|
|
349
444
|
# open_timeout is the maximum time, in seconds, that opening will be retried
|
|
350
445
|
# if a file cannot be initially opened for writing.
|
|
351
|
-
#
|
|
352
|
-
#
|
|
446
|
+
# A daemon cannot hold interpreter shutdown on an abandoned I/O call.
|
|
447
|
+
# Normal callers flush first, then request shutdown.
|
|
353
448
|
super().__init__(name="execsql-filewriter", daemon=True)
|
|
354
449
|
self.input_queue: queue.Queue[tuple[int, tuple[Any, ...]]] = queue.Queue()
|
|
355
450
|
self.return_msg_queue: queue.Queue[Any] = queue.Queue()
|
|
356
451
|
self.file_encoding = file_encoding
|
|
357
452
|
self.open_timeout = open_timeout
|
|
358
453
|
self.files: dict[str, FileWriter.FileControl] = {}
|
|
454
|
+
self.abandoned = threading.Event()
|
|
455
|
+
self.current_op: tuple[str, str, float] | None = None
|
|
456
|
+
self.output_error: ErrInfo | None = None
|
|
457
|
+
self._pending: dict[str, int] = {}
|
|
458
|
+
self._pending_lock = threading.Lock()
|
|
359
459
|
# Commands that raised something other than a file's own OSError,
|
|
360
460
|
# reported with the failed files.
|
|
361
461
|
self.errors: list[LostOutput] = []
|
|
@@ -373,6 +473,39 @@ class FileWriter(threading.Thread):
|
|
|
373
473
|
self.open_failures,
|
|
374
474
|
)
|
|
375
475
|
|
|
476
|
+
def submit(self, command: int, arguments: tuple[Any, ...]) -> None:
|
|
477
|
+
"""Queue a command, recording pending destinations without holding locks during I/O."""
|
|
478
|
+
with self._pending_lock:
|
|
479
|
+
if self.abandoned.is_set():
|
|
480
|
+
raise _abandoned_writer_error(self)
|
|
481
|
+
if command == self.CMD_WRITE:
|
|
482
|
+
filename = arguments[0]
|
|
483
|
+
self._pending[filename] = self._pending.get(filename, 0) + 1
|
|
484
|
+
self.input_queue.put((command, arguments))
|
|
485
|
+
|
|
486
|
+
def completed(self, filename: str, count: int) -> None:
|
|
487
|
+
"""Retire writes only after flushing them or recording their loss."""
|
|
488
|
+
with self._pending_lock:
|
|
489
|
+
remaining = self._pending.get(filename, 0) - count
|
|
490
|
+
if remaining > 0:
|
|
491
|
+
self._pending[filename] = remaining
|
|
492
|
+
else:
|
|
493
|
+
self._pending.pop(filename, None)
|
|
494
|
+
|
|
495
|
+
def _resolved_write(self, requested: str, filename: str) -> None:
|
|
496
|
+
"""Move this write's accounting to the worker-resolved destination."""
|
|
497
|
+
if requested == filename:
|
|
498
|
+
return
|
|
499
|
+
with self._pending_lock:
|
|
500
|
+
count = self._pending.get(requested, 0)
|
|
501
|
+
if not count:
|
|
502
|
+
return
|
|
503
|
+
if count == 1:
|
|
504
|
+
del self._pending[requested]
|
|
505
|
+
else:
|
|
506
|
+
self._pending[requested] = count - 1
|
|
507
|
+
self._pending[filename] = self._pending.get(filename, 0) + 1
|
|
508
|
+
|
|
376
509
|
def __del__(self) -> None:
|
|
377
510
|
try:
|
|
378
511
|
self.close_all()
|
|
@@ -387,12 +520,21 @@ class FileWriter(threading.Thread):
|
|
|
387
520
|
except Exception as exc:
|
|
388
521
|
fc.fail(fc.STATUS_WRITEFAILURE, exc)
|
|
389
522
|
|
|
523
|
+
def _resolve_filename(self, filename: str) -> str:
|
|
524
|
+
self.current_op = ("open", filename, time.monotonic())
|
|
525
|
+
try:
|
|
526
|
+
if self.abandoned.is_set():
|
|
527
|
+
raise _abandoned_writer_error(self)
|
|
528
|
+
return str(Path(filename).resolve())
|
|
529
|
+
finally:
|
|
530
|
+
self.current_op = None
|
|
531
|
+
|
|
390
532
|
def close_all(self) -> None:
|
|
391
533
|
for fc in getattr(self, "files", {}).values():
|
|
392
534
|
self._on_file(fc, fc.close)
|
|
393
535
|
|
|
394
536
|
def close_if_open(self, fn: str) -> None:
|
|
395
|
-
filename =
|
|
537
|
+
filename = self._resolve_filename(fn)
|
|
396
538
|
if filename in self.files:
|
|
397
539
|
fc = self.files[filename]
|
|
398
540
|
self._on_file(fc, fc.close)
|
|
@@ -452,33 +594,32 @@ class FileWriter(threading.Thread):
|
|
|
452
594
|
self.errors = []
|
|
453
595
|
self.return_msg_queue.put(failed)
|
|
454
596
|
|
|
455
|
-
def
|
|
456
|
-
|
|
597
|
+
def _file_control(self, filename: str) -> FileWriter.FileControl:
|
|
598
|
+
"""Get or create the control for a worker-resolved filename."""
|
|
457
599
|
if filename not in self.files:
|
|
458
600
|
self.files[filename] = self.FileControl(
|
|
459
601
|
filename,
|
|
460
602
|
open_timeout=self.open_timeout,
|
|
461
603
|
encoding=self.file_encoding,
|
|
604
|
+
writer=self,
|
|
462
605
|
)
|
|
463
|
-
|
|
606
|
+
return self.files[filename]
|
|
607
|
+
|
|
608
|
+
def open_as_new(self, fn: str) -> None:
|
|
609
|
+
fc = self._file_control(self._resolve_filename(fn))
|
|
464
610
|
self._on_file(fc, fc.open_as_new)
|
|
465
611
|
|
|
466
612
|
def status(self, fn: str) -> None:
|
|
467
|
-
filename =
|
|
613
|
+
filename = self._resolve_filename(fn)
|
|
468
614
|
if filename in self.files:
|
|
469
615
|
self.return_msg_queue.put(self.files[filename].status)
|
|
470
616
|
else:
|
|
471
617
|
self.return_msg_queue.put(self.FileControl.STATUS_UNOPENED)
|
|
472
618
|
|
|
473
619
|
def write(self, fn: str, content: str) -> None:
|
|
474
|
-
filename =
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
filename,
|
|
478
|
-
open_timeout=self.open_timeout,
|
|
479
|
-
encoding=self.file_encoding,
|
|
480
|
-
)
|
|
481
|
-
fc = self.files[filename]
|
|
620
|
+
filename = self._resolve_filename(fn)
|
|
621
|
+
fc = self._file_control(filename)
|
|
622
|
+
self._resolved_write(fn, filename)
|
|
482
623
|
self._on_file(fc, lambda: fc.write(content))
|
|
483
624
|
|
|
484
625
|
def shutdown(self) -> None:
|
|
@@ -492,22 +633,39 @@ class FileWriter(threading.Thread):
|
|
|
492
633
|
# Waiting files are retried on a timer rather than only when the queue
|
|
493
634
|
# is idle: a caller polling CMD_CLOSED_STATUS keeps the queue busy.
|
|
494
635
|
last_retry = time.monotonic()
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
command, argtuple = self.input_queue.get(timeout=0.1)
|
|
498
|
-
except queue.Empty:
|
|
499
|
-
pass
|
|
500
|
-
else:
|
|
501
|
-
try:
|
|
502
|
-
self.execvec[command](*argtuple)
|
|
503
|
-
except Exception as exc:
|
|
504
|
-
self._command_failed(command, argtuple, exc)
|
|
505
|
-
if time.monotonic() - last_retry >= 0.1:
|
|
636
|
+
try:
|
|
637
|
+
while self.active and not self.abandoned.is_set():
|
|
506
638
|
try:
|
|
507
|
-
self.
|
|
508
|
-
except
|
|
509
|
-
|
|
510
|
-
|
|
639
|
+
command, argtuple = self.input_queue.get(timeout=0.1)
|
|
640
|
+
except queue.Empty:
|
|
641
|
+
pass
|
|
642
|
+
else:
|
|
643
|
+
if self.abandoned.is_set():
|
|
644
|
+
break
|
|
645
|
+
try:
|
|
646
|
+
self.execvec[command](*argtuple)
|
|
647
|
+
except Exception as exc:
|
|
648
|
+
self._command_failed(command, argtuple, exc)
|
|
649
|
+
if self.abandoned.is_set():
|
|
650
|
+
break
|
|
651
|
+
if time.monotonic() - last_retry >= 0.1:
|
|
652
|
+
try:
|
|
653
|
+
self.retry_waiting()
|
|
654
|
+
except Exception as exc:
|
|
655
|
+
self.errors.append(LostOutput("", 0, str(exc), "writer"))
|
|
656
|
+
last_retry = time.monotonic()
|
|
657
|
+
finally:
|
|
658
|
+
if self.abandoned.is_set():
|
|
659
|
+
self.active = False
|
|
660
|
+
while True:
|
|
661
|
+
try:
|
|
662
|
+
self.input_queue.get_nowait()
|
|
663
|
+
except queue.Empty:
|
|
664
|
+
break
|
|
665
|
+
for fc in self.files.values():
|
|
666
|
+
fc.discard()
|
|
667
|
+
with self._pending_lock:
|
|
668
|
+
self._pending.clear()
|
|
511
669
|
|
|
512
670
|
def _command_failed(self, command: int, argtuple: tuple[Any, ...], exc: Exception) -> None:
|
|
513
671
|
"""Record a command that raised, and still answer it if the caller waits for an answer.
|
|
@@ -517,7 +675,10 @@ class FileWriter(threading.Thread):
|
|
|
517
675
|
every later write, to any file).
|
|
518
676
|
"""
|
|
519
677
|
filename = argtuple[0] if argtuple and isinstance(argtuple[0], str) else ""
|
|
520
|
-
|
|
678
|
+
dropped = 1 if command == self.CMD_WRITE else 0
|
|
679
|
+
if dropped:
|
|
680
|
+
self.completed(filename, dropped)
|
|
681
|
+
lost = LostOutput(filename, dropped, str(exc), "writer")
|
|
521
682
|
if command == self.CMD_OPEN_FAILURES:
|
|
522
683
|
self.return_msg_queue.put([lost])
|
|
523
684
|
return
|
|
@@ -572,18 +733,23 @@ def _active_writer() -> FileWriter | None:
|
|
|
572
733
|
import execsql.state as _state
|
|
573
734
|
|
|
574
735
|
fw = _state.filewriter
|
|
575
|
-
if fw is
|
|
736
|
+
if fw is None or fw.ident is None:
|
|
737
|
+
return None
|
|
738
|
+
if fw.abandoned.is_set():
|
|
739
|
+
raise _abandoned_writer_error(fw)
|
|
740
|
+
if _writer_stopped(fw):
|
|
576
741
|
raise _writer_stopped_error()
|
|
577
|
-
return fw if fw
|
|
742
|
+
return fw if fw.is_alive() else None
|
|
578
743
|
|
|
579
744
|
|
|
580
745
|
def _abspath(filename: str) -> str:
|
|
581
746
|
"""*filename* made absolute against the working directory as it is now.
|
|
582
747
|
|
|
583
|
-
|
|
584
|
-
|
|
748
|
+
Capture the current directory here so a later ``CD`` cannot redirect
|
|
749
|
+
queued output. Physical resolution belongs to the writer: metadata I/O
|
|
750
|
+
must not block the caller. Preserve ``..`` until symlinks are resolved.
|
|
585
751
|
"""
|
|
586
|
-
return str(Path(filename).
|
|
752
|
+
return str(Path(filename).absolute())
|
|
587
753
|
|
|
588
754
|
|
|
589
755
|
#: Sentinel returned by :func:`_writer_reply` when the writer died before
|
|
@@ -591,30 +757,75 @@ def _abspath(filename: str) -> str:
|
|
|
591
757
|
_NO_REPLY = object()
|
|
592
758
|
|
|
593
759
|
|
|
594
|
-
def
|
|
595
|
-
|
|
760
|
+
def _abandoned_writer_error(fw: FileWriter) -> ErrInfo:
|
|
761
|
+
return fw.output_error or ErrInfo(
|
|
762
|
+
"error",
|
|
763
|
+
other_msg="File output lost: the file writer was abandoned; queued output was discarded.",
|
|
764
|
+
)
|
|
596
765
|
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
766
|
+
|
|
767
|
+
def _abandon_writer(fw: FileWriter, wait: float) -> ErrInfo:
|
|
768
|
+
"""Stop admission immediately; the daemon may still be inside its last I/O call."""
|
|
769
|
+
with fw._pending_lock:
|
|
770
|
+
if fw.output_error is not None:
|
|
771
|
+
return fw.output_error
|
|
772
|
+
operation = fw.current_op
|
|
773
|
+
fw.abandoned.set()
|
|
774
|
+
pending = tuple(fw._pending)
|
|
775
|
+
if operation is not None:
|
|
776
|
+
phase, filename, started = operation
|
|
777
|
+
elapsed = time.monotonic() - started
|
|
778
|
+
where = f"in {phase} of {filename}"
|
|
779
|
+
else:
|
|
780
|
+
elapsed = wait
|
|
781
|
+
where = "while waiting for a file writer reply"
|
|
782
|
+
queued = "; queued output files: " + ", ".join(pending) if pending else ""
|
|
783
|
+
fw.output_error = ErrInfo(
|
|
784
|
+
"error",
|
|
785
|
+
other_msg=(
|
|
786
|
+
f"File output lost: the file writer has been blocked for {elapsed:.3f} seconds {where}; "
|
|
787
|
+
f"output queued after it was discarded{queued}. The in-progress I/O may complete later."
|
|
788
|
+
),
|
|
789
|
+
)
|
|
790
|
+
return fw.output_error
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
def _writer_deadline(fw: FileWriter, max_wait: float | None = None) -> tuple[float, float]:
|
|
794
|
+
wait = fw.open_timeout + 5.0
|
|
795
|
+
if max_wait is not None:
|
|
796
|
+
wait = min(wait, max_wait)
|
|
797
|
+
wait = max(0.0, wait)
|
|
798
|
+
return time.monotonic() + wait, wait
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
def _writer_reply(fw: FileWriter, deadline: float, wait: float) -> Any:
|
|
802
|
+
"""Wait only until the caller's absolute deadline, checking worker liveness."""
|
|
601
803
|
while True:
|
|
804
|
+
if fw.abandoned.is_set():
|
|
805
|
+
raise _abandoned_writer_error(fw)
|
|
806
|
+
if not fw.is_alive():
|
|
807
|
+
return _NO_REPLY
|
|
808
|
+
remaining = deadline - time.monotonic()
|
|
809
|
+
if remaining <= 0:
|
|
810
|
+
raise _abandon_writer(fw, wait)
|
|
602
811
|
try:
|
|
603
|
-
return fw.return_msg_queue.get(timeout=
|
|
812
|
+
return fw.return_msg_queue.get(timeout=min(0.05, remaining))
|
|
604
813
|
except queue.Empty:
|
|
605
|
-
|
|
606
|
-
|
|
814
|
+
pass
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
def _filewriter_status(fw: FileWriter, filename: str, deadline: float, wait: float) -> int:
|
|
818
|
+
fw.submit(FileWriter.CMD_GET_STATUS, (filename,))
|
|
819
|
+
reply = _writer_reply(fw, deadline, wait)
|
|
820
|
+
return FileWriter.FileControl.STATUS_CLOSED if reply is _NO_REPLY else cast(int, reply)
|
|
607
821
|
|
|
608
822
|
|
|
609
823
|
def filewriter_filestatus(filename: str) -> int:
|
|
610
824
|
fw = _active_writer()
|
|
611
825
|
if fw is None:
|
|
612
826
|
return FileWriter.FileControl.STATUS_CLOSED
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
if reply is _NO_REPLY:
|
|
616
|
-
return FileWriter.FileControl.STATUS_CLOSED
|
|
617
|
-
return cast(int, reply)
|
|
827
|
+
deadline, wait = _writer_deadline(fw)
|
|
828
|
+
return _filewriter_status(fw, _abspath(filename), deadline, wait)
|
|
618
829
|
|
|
619
830
|
|
|
620
831
|
#: Set once the first dropped write has been reported, so a script writing
|
|
@@ -651,7 +862,7 @@ def filewriter_write(filename: str, message: str) -> None:
|
|
|
651
862
|
if fw is None:
|
|
652
863
|
_warn_write_dropped(filename)
|
|
653
864
|
return
|
|
654
|
-
fw.
|
|
865
|
+
fw.submit(FileWriter.CMD_WRITE, (_abspath(filename), message))
|
|
655
866
|
|
|
656
867
|
|
|
657
868
|
def filewriter_open_as_new(filename: str) -> None:
|
|
@@ -661,59 +872,52 @@ def filewriter_open_as_new(filename: str) -> None:
|
|
|
661
872
|
if fw is None:
|
|
662
873
|
_warn_write_dropped(filename)
|
|
663
874
|
return
|
|
664
|
-
fw.
|
|
875
|
+
fw.submit(FileWriter.CMD_OPEN_AS_NEW, (_abspath(filename),))
|
|
665
876
|
|
|
666
877
|
|
|
667
878
|
def filewriter_close(filename: str) -> None:
|
|
668
|
-
|
|
669
|
-
# file is closed before it writes to that file itself.
|
|
879
|
+
"""Release a writer-held path before synchronous output, using one bounded deadline."""
|
|
670
880
|
fw = _active_writer()
|
|
671
881
|
if fw is None:
|
|
672
882
|
return
|
|
673
883
|
filename = _abspath(filename)
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
884
|
+
deadline, wait = _writer_deadline(fw)
|
|
885
|
+
fw.submit(FileWriter.CMD_CLOSE_IF_OPEN, (filename,))
|
|
886
|
+
while _filewriter_status(fw, filename, deadline, wait) == FileWriter.FileControl.STATUS_OPEN:
|
|
887
|
+
remaining = deadline - time.monotonic()
|
|
888
|
+
if remaining <= 0:
|
|
889
|
+
raise _abandon_writer(fw, wait)
|
|
890
|
+
time.sleep(min(0.05, remaining))
|
|
677
891
|
|
|
678
892
|
|
|
679
893
|
def filewriter_close_all_after_write(max_wait: float | None = None) -> None:
|
|
680
|
-
"""Flush and
|
|
894
|
+
"""Flush and report output loss within one absolute deadline, including reply waits.
|
|
681
895
|
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
and this raises :class:`ErrInfo` naming each file whose output was lost
|
|
686
|
-
and how much — a lost write is an error, not a silent success.
|
|
896
|
+
The deadline is open_timeout + 5 seconds, capped by max_wait when supplied.
|
|
897
|
+
On expiry the daemon is abandoned, not killed: its current I/O may finish
|
|
898
|
+
later, but it will discard queued writes and close without flushing them.
|
|
687
899
|
"""
|
|
688
900
|
fw = _active_writer()
|
|
689
901
|
if fw is None:
|
|
690
902
|
return
|
|
691
|
-
wait =
|
|
692
|
-
|
|
693
|
-
fw.input_queue.put((FileWriter.CMD_CLOSE_ALL_AFTER_WRITE, ()))
|
|
694
|
-
timed_out = False
|
|
903
|
+
deadline, wait = _writer_deadline(fw, max_wait)
|
|
904
|
+
fw.submit(FileWriter.CMD_CLOSE_ALL_AFTER_WRITE, ())
|
|
695
905
|
while True:
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
fw.input_queue.put((FileWriter.CMD_CLOSED_STATUS, ()))
|
|
699
|
-
close_status = _writer_reply(fw)
|
|
906
|
+
fw.submit(FileWriter.CMD_CLOSED_STATUS, ())
|
|
907
|
+
close_status = _writer_reply(fw, deadline, wait)
|
|
700
908
|
if close_status is _NO_REPLY:
|
|
701
909
|
raise _writer_stopped_error()
|
|
702
910
|
if close_status == FileWriter.FileControl.STATUS_CLOSED:
|
|
703
911
|
break
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
fw.input_queue.put((FileWriter.CMD_OPEN_FAILURES, ()))
|
|
711
|
-
failures = _writer_reply(fw)
|
|
912
|
+
remaining = deadline - time.monotonic()
|
|
913
|
+
if remaining <= 0:
|
|
914
|
+
raise _abandon_writer(fw, wait)
|
|
915
|
+
time.sleep(min(0.05, remaining))
|
|
916
|
+
fw.submit(FileWriter.CMD_OPEN_FAILURES, ())
|
|
917
|
+
failures = _writer_reply(fw, deadline, wait)
|
|
712
918
|
if failures is _NO_REPLY:
|
|
713
919
|
raise _writer_stopped_error()
|
|
714
920
|
messages = [_lost_output_message(lost, fw.open_timeout) for lost in failures]
|
|
715
|
-
if timed_out and not messages:
|
|
716
|
-
messages.append(f"one or more output files were still locked after {wait:.0f} seconds")
|
|
717
921
|
if messages:
|
|
718
922
|
raise ErrInfo("error", other_msg="File output lost: " + "; ".join(messages) + ".")
|
|
719
923
|
|
|
@@ -735,11 +939,13 @@ def _lost_output_message(lost: LostOutput, open_timeout: int) -> str:
|
|
|
735
939
|
|
|
736
940
|
|
|
737
941
|
def filewriter_end(writer: FileWriter | None = None) -> None:
|
|
738
|
-
"""
|
|
942
|
+
"""Request this writer's shutdown, or detach it immediately if already abandoned.
|
|
739
943
|
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
944
|
+
A healthy worker closes its files and stops; a stalled one is abandoned
|
|
945
|
+
after the capped join and may remain alive. Other runs' writers are left
|
|
946
|
+
alone. The join is capped so an ``atexit`` handler cannot wedge
|
|
947
|
+
interpreter shutdown on a writer stuck in I/O. An abandoned writer is
|
|
948
|
+
detached immediately, without a second wait for its in-progress call.
|
|
743
949
|
"""
|
|
744
950
|
import execsql.state as _state
|
|
745
951
|
|
|
@@ -747,9 +953,11 @@ def filewriter_end(writer: FileWriter | None = None) -> None:
|
|
|
747
953
|
if fw is None:
|
|
748
954
|
return
|
|
749
955
|
try:
|
|
750
|
-
if fw.is_alive():
|
|
751
|
-
fw.
|
|
956
|
+
if fw.is_alive() and not fw.abandoned.is_set():
|
|
957
|
+
fw.submit(FileWriter.CMD_SHUTDOWN, ())
|
|
752
958
|
fw.join(timeout=5.0)
|
|
959
|
+
if fw.is_alive():
|
|
960
|
+
_abandon_writer(fw, 5.0)
|
|
753
961
|
except Exception:
|
|
754
962
|
pass # Best-effort cleanup at interpreter shutdown.
|
|
755
963
|
if _state.filewriter is fw:
|