execsql2 2.37.0__py3-none-any.whl → 2.38.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. execsql/api.py +95 -45
  2. execsql/cli/commands/run.py +2 -2
  3. execsql/cli/lint.py +6 -1
  4. execsql/cli/run.py +5 -0
  5. execsql/config.py +7 -0
  6. execsql/data/execsql.conf.template +21 -0
  7. execsql/db/base.py +52 -24
  8. execsql/db/duckdb.py +43 -4
  9. execsql/db/firebird.py +8 -0
  10. execsql/db/mysql.py +8 -3
  11. execsql/db/oracle.py +5 -0
  12. execsql/db/postgres.py +2 -2
  13. execsql/db/sqlite.py +117 -10
  14. execsql/db/sqlserver.py +11 -7
  15. execsql/db/target_types.py +151 -4
  16. execsql/debug/repl.py +2 -1
  17. execsql/exceptions.py +14 -0
  18. execsql/exporters/html.py +26 -42
  19. execsql/exporters/ods.py +93 -21
  20. execsql/exporters/sqlite.py +82 -47
  21. execsql/format.py +12 -4
  22. execsql/gui/desktop.py +2 -1
  23. execsql/gui/tui.py +5 -2
  24. execsql/importers/base.py +9 -7
  25. execsql/importers/json.py +37 -13
  26. execsql/importers/ods.py +22 -2
  27. execsql/lsp/document.py +6 -1
  28. execsql/lsp/hover.py +2 -1
  29. execsql/metacommands/conditions.py +3 -1
  30. execsql/metacommands/connect.py +18 -3
  31. execsql/metacommands/dispatch.py +10 -0
  32. execsql/metacommands/io.py +1 -0
  33. execsql/metacommands/io_export.py +1 -2
  34. execsql/metacommands/io_fileops.py +26 -90
  35. execsql/metacommands/io_import.py +26 -8
  36. execsql/metacommands/reference.py +7 -0
  37. execsql/parser.py +14 -37
  38. execsql/script/control.py +17 -7
  39. execsql/script/executor.py +160 -34
  40. execsql/script/preparation.py +42 -0
  41. execsql/types.py +6 -5
  42. execsql/utils/errors.py +2 -2
  43. execsql/utils/fileio.py +342 -134
  44. execsql/utils/spreadsheet.py +74 -0
  45. execsql/utils/strings.py +13 -7
  46. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/README.md +5 -5
  47. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/example_config_prompt.sql +2 -2
  48. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/METADATA +32 -119
  49. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/RECORD +64 -62
  50. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/WHEEL +1 -1
  51. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
  52. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
  53. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_compare.sql +0 -0
  54. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
  55. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
  56. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
  57. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
  58. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/script_template.sql +0 -0
  59. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
  60. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
  61. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
  62. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/entry_points.txt +0 -0
  63. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/licenses/LICENSE.txt +0 -0
  64. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/licenses/NOTICE +0 -0
execsql/utils/fileio.py CHANGED
@@ -8,7 +8,7 @@ Provides:
8
8
  - :class:`EncodedFile` — opens files with a specific encoding and error
9
9
  handler; supports the script-, output-, and import-encoding settings.
10
10
  - :class:`FileWriter` — asynchronous text-file writer thread, one per run,
11
- used for export output to avoid blocking the main execution loop.
11
+ used for WRITE/TEE output to avoid blocking the main execution loop.
12
12
  - :class:`Logger` — wraps a log file with timestamped write methods;
13
13
  mirrors output to stderr when ``tee_write_log`` is set.
14
14
  - :class:`TempFileMgr` — registry of temporary files that are cleaned up
@@ -23,6 +23,7 @@ import atexit
23
23
  import codecs
24
24
  import collections
25
25
  import errno
26
+ from functools import partial
26
27
  import io
27
28
  import os
28
29
  import queue
@@ -92,6 +93,8 @@ def check_zip_decompression_ratio(
92
93
  *,
93
94
  max_uncompressed_mb: int = 500,
94
95
  max_ratio: int = 100,
96
+ xml_only: bool = False,
97
+ limit_names: tuple[str, str] | None = None,
95
98
  ) -> None:
96
99
  """Reject a zip-based workbook that looks like a decompression bomb.
97
100
 
@@ -107,6 +110,9 @@ def check_zip_decompression_ratio(
107
110
  * ``max_ratio`` — per-member uncompressed:compressed ratio
108
111
  (default 100:1; legitimate XML-heavy XLSX rarely exceeds 30:1).
109
112
 
113
+ Zero disables the corresponding bound. With *xml_only*, the aggregate
114
+ counts XML members only (ODS); compression ratios still cover every member.
115
+ *limit_names* adds configuration setting names to rejection messages.
110
116
  No-op when *path* is not a zipfile (e.g. legacy ``.xls`` OLE-CDF).
111
117
  """
112
118
  import zipfile
@@ -115,26 +121,41 @@ def check_zip_decompression_ratio(
115
121
  with zipfile.ZipFile(path) as zf:
116
122
  total_uncompressed = 0
117
123
  for info in zf.infolist():
118
- total_uncompressed += info.file_size
119
- if info.compress_size > 0:
120
- ratio = info.file_size / info.compress_size
124
+ if not xml_only or info.filename.lower().endswith(".xml"):
125
+ total_uncompressed += info.file_size
126
+ if max_ratio and info.file_size:
127
+ ratio = info.file_size / info.compress_size if info.compress_size else float("inf")
121
128
  if ratio > max_ratio:
129
+ sizes = (
130
+ f" ({info.file_size:,} uncompressed / {info.compress_size:,} compressed bytes)"
131
+ if limit_names
132
+ else ""
133
+ )
122
134
  raise ErrInfo(
123
135
  type="error",
124
136
  other_msg=(
125
137
  f"Refusing to open '{path}': member '{info.filename}' has "
126
- f"compression ratio {ratio:.1f}:1 (limit {max_ratio}:1) - "
138
+ f"compression ratio {ratio:.1f}:1"
139
+ f"{sizes} "
140
+ f"(limit {max_ratio}:1"
141
+ f"{(' set by ' + limit_names[1]) if limit_names else ''}) - "
127
142
  "possible zip-bomb."
128
143
  ),
129
144
  )
130
145
  limit_bytes = max_uncompressed_mb * 1024 * 1024
131
- if total_uncompressed > limit_bytes:
146
+ if max_uncompressed_mb and total_uncompressed > limit_bytes:
147
+ observed = (
148
+ f"{total_uncompressed:,} XML bytes" if xml_only else f"{total_uncompressed / 1024 / 1024:.0f} MB"
149
+ )
150
+ maximum = (
151
+ f"{limit_bytes:,} bytes ({max_uncompressed_mb} MiB)" if limit_names else f"{max_uncompressed_mb} MB"
152
+ )
132
153
  raise ErrInfo(
133
154
  type="error",
134
155
  other_msg=(
135
156
  f"Refusing to open '{path}': total uncompressed size "
136
- f"{total_uncompressed / 1024 / 1024:.0f} MB exceeds limit "
137
- f"{max_uncompressed_mb} MB - possible zip-bomb."
157
+ f"{observed} exceeds limit {maximum}"
158
+ f"{(' set by ' + limit_names[0]) if limit_names else ''} - possible zip-bomb."
138
159
  ),
139
160
  )
140
161
  except zipfile.BadZipFile:
@@ -219,24 +240,27 @@ class FileWriter(threading.Thread):
219
240
  STATUS_OPEN, STATUS_WAITING, STATUS_UNOPENED, STATUS_CLOSED, STATUS_OPENFAILURE, STATUS_WRITEFAILURE = range(6)
220
241
  FAILED = (STATUS_OPENFAILURE, STATUS_WRITEFAILURE)
221
242
 
222
- def __init__(self, fn: str, open_timeout: int, encoding: str = "utf-8") -> None:
243
+ def __init__(
244
+ self,
245
+ fn: str,
246
+ open_timeout: int,
247
+ encoding: str = "utf-8",
248
+ *,
249
+ writer: FileWriter | None = None,
250
+ ) -> None:
223
251
  self.filename = fn
224
252
  self.open_timeout = open_timeout
225
253
  self.encoding = encoding
254
+ self._writer_ref = weakref.ref(writer) if writer is not None else None
255
+ self._discarded = False
226
256
  self.openmode = "a"
227
257
  self.handle: io.TextIOWrapper | None = None
228
258
  self.status = self.STATUS_UNOPENED
229
259
  self.open_start_time: float | None = None
230
260
  self.output_queue: collections.deque[str] = collections.deque()
231
261
  self.close_after_write = False
232
- # Lines discarded because the file could not be opened within
233
- # open_timeout, or could not be written; reported back to the
234
- # caller, never silently lost.
235
262
  self.dropped = 0
236
- # The OS error that failed the file; None for an open that timed out.
237
263
  self.error: str | None = None
238
- # Lines discarded because the file was closed while still locked
239
- # (a halt, or an exporter about to write the same path).
240
264
  self.abandoned = 0
241
265
 
242
266
  def __del__(self) -> None:
@@ -245,31 +269,97 @@ class FileWriter(threading.Thread):
245
269
  except Exception:
246
270
  pass # Best-effort cleanup at interpreter shutdown.
247
271
 
248
- def fail(self, status: int, exc: BaseException) -> None:
249
- """Give up on the file after *exc*: discard (and count) what is buffered, and buffer nothing more."""
250
- self.status = status
251
- self.error = str(exc)
272
+ @property
273
+ def writer(self) -> FileWriter | None:
274
+ return self._writer_ref() if self._writer_ref is not None else None
275
+
276
+ def _stopped(self) -> bool:
277
+ writer = self.writer
278
+ return self._discarded or (writer is not None and writer.abandoned.is_set())
279
+
280
+ def _retire(self, count: int) -> None:
281
+ writer = self.writer
282
+ if writer is not None:
283
+ writer.completed(self.filename, count)
284
+
285
+ def _io(self, operation: str, action: Callable[[], Any], *, cleanup: bool = False) -> Any:
286
+ if not cleanup and self._stopped():
287
+ return None
288
+ writer = self.writer
289
+ if writer is not None:
290
+ writer.current_op = (operation, self.filename, time.monotonic())
291
+ try:
292
+ # Publish the operation before the final abandonment check: a
293
+ # call reserved here is the one that may finish after timeout.
294
+ if not cleanup and self._stopped():
295
+ return None
296
+ return action()
297
+ finally:
298
+ if writer is not None:
299
+ writer.current_op = None
300
+
301
+ def _close_handle(self, *, discard: bool = False) -> None:
302
+ handle = self.handle
303
+ if handle is None:
304
+ return
305
+ if discard or self._stopped():
306
+ # TextIOWrapper.close() flushes. Close the raw file first when
307
+ # abandoning a write/flush so its buffered bytes cannot escape
308
+ # later; keep normal text encoding/newline/BOM behavior intact.
309
+ raw = getattr(getattr(handle, "buffer", None), "raw", None)
310
+ if raw is not None:
311
+ try:
312
+ self._io("close", raw.close, cleanup=True)
313
+ except Exception:
314
+ pass
315
+ try:
316
+ self._io("close", handle.close, cleanup=True)
317
+ except Exception:
318
+ pass
319
+ else:
320
+ self._io("close", handle.close)
321
+ if self._stopped():
322
+ self._close_handle(discard=True)
323
+ self.handle = None
324
+
325
+ def discard(self) -> None:
326
+ """Drop queued text and close without flushing abandoned buffers."""
327
+ self._discarded = True
252
328
  self.dropped += len(self.output_queue)
329
+ self._retire(len(self.output_queue))
253
330
  self.output_queue.clear()
254
331
  self.close_after_write = False
255
- handle, self.handle = self.handle, None
256
- if handle is not None:
257
- try:
258
- handle.close()
259
- except OSError:
260
- pass # The file has already failed; its error is the one to report.
332
+ self._close_handle(discard=True)
333
+ if self.status not in self.FAILED:
334
+ self.status = self.STATUS_CLOSED
335
+
336
+ def fail(self, status: int, exc: BaseException) -> None:
337
+ """Fail this file alone, discarding any text it cannot safely write."""
338
+ self.status = status
339
+ self.error = str(exc)
340
+ self.discard()
261
341
 
262
342
  def write_queue(self) -> None:
263
- assert self.handle is not None
264
- while len(self.output_queue) > 0:
265
- m = self.output_queue.pop()
343
+ if self._stopped():
344
+ return
345
+ handle = self.handle
346
+ assert handle is not None
347
+ while self.output_queue and not self._stopped():
348
+ message = self.output_queue.pop()
266
349
  try:
267
- self.handle.write(m)
268
- except OSError as exc: # disk full, a share that dropped, ...
350
+ self._io("write", partial(handle.write, message))
351
+ if self._stopped():
352
+ self.dropped += 1
353
+ self._retire(1)
354
+ return
355
+ self._io("write", handle.flush)
356
+ except Exception as exc:
269
357
  self.dropped += 1
358
+ self._retire(1)
270
359
  self.fail(self.STATUS_WRITEFAILURE, exc)
271
360
  return
272
- if self.close_after_write:
361
+ self._retire(1)
362
+ if self.close_after_write and not self._stopped():
273
363
  self.close_after_write = False
274
364
  self.close()
275
365
 
@@ -278,14 +368,16 @@ class FileWriter(threading.Thread):
278
368
  self.openmode = "w"
279
369
 
280
370
  def try_open(self) -> None:
371
+ if self._stopped():
372
+ return
281
373
  if self.status != self.STATUS_OPEN or self.handle is None:
282
374
  if self.open_start_time is None:
283
375
  self.open_start_time = time.time()
284
376
  if time.time() - self.open_start_time < self.open_timeout:
285
377
  try:
286
- self.handle = cast(
287
- io.TextIOWrapper,
288
- open( # noqa: SIM115
378
+ self.handle = self._io(
379
+ "open",
380
+ lambda: open( # noqa: SIM115
289
381
  file=self.filename,
290
382
  mode=self.openmode,
291
383
  encoding=self.encoding,
@@ -293,25 +385,29 @@ class FileWriter(threading.Thread):
293
385
  ),
294
386
  )
295
387
  except Exception as exc:
296
- if _never_opens(self.filename, exc):
388
+ if self._io("open", partial(_never_opens, self.filename, exc)):
297
389
  self.fail(self.STATUS_OPENFAILURE, exc)
298
390
  else:
299
391
  self.status = self.STATUS_WAITING
300
392
  else:
301
- self.status = self.STATUS_OPEN
302
- self.openmode = "a" # Return to default for next open command.
303
- self.open_start_time = None
393
+ if self.handle is not None:
394
+ self.status = self.STATUS_OPEN
395
+ self.openmode = "a"
396
+ self.open_start_time = None
304
397
  else:
305
- # Give up: discard what was buffered (counting it so the
306
- # caller can report the loss) and stop buffering.
307
398
  self.status = self.STATUS_OPENFAILURE
308
399
  self.dropped += len(self.output_queue)
400
+ self._retire(len(self.output_queue))
309
401
  self.output_queue.clear()
310
402
  self.close_after_write = False
311
403
 
312
404
  def close(self) -> None:
405
+ if self._stopped():
406
+ self.discard()
407
+ return
313
408
  if self.status == self.STATUS_WAITING:
314
409
  self.abandoned += len(self.output_queue)
410
+ self._retire(len(self.output_queue))
315
411
  self.output_queue.clear()
316
412
  self.close_after_write = False
317
413
  if self.status == self.STATUS_OPEN and self.handle is not None:
@@ -319,16 +415,14 @@ class FileWriter(threading.Thread):
319
415
  if self.status == self.STATUS_WRITEFAILURE:
320
416
  return
321
417
  try:
322
- self.handle.close()
323
- except OSError as exc: # the final flush failed
324
- self.handle = None
418
+ self._close_handle()
419
+ except OSError as exc:
325
420
  self.fail(self.STATUS_WRITEFAILURE, exc)
326
421
  return
327
- self.handle = None
328
422
  if self.status in self.FAILED:
329
- return # Kept until the caller has been told about the loss.
423
+ return
330
424
  self.status = self.STATUS_CLOSED
331
- self.openmode = "a" # Return this to the default in case it had been changed for the last open.
425
+ self.openmode = "a"
332
426
 
333
427
  def clean_close(self) -> None:
334
428
  if self.status == self.STATUS_WAITING:
@@ -337,8 +431,9 @@ class FileWriter(threading.Thread):
337
431
  self.close()
338
432
 
339
433
  def write(self, content: str) -> None:
340
- if self.status in self.FAILED:
434
+ if self._stopped() or self.status in self.FAILED:
341
435
  self.dropped += 1
436
+ self._retire(1)
342
437
  return
343
438
  self.output_queue.appendleft(content)
344
439
  self.try_open()
@@ -348,14 +443,19 @@ class FileWriter(threading.Thread):
348
443
  def __init__(self, file_encoding: str = "utf-8", open_timeout: int = 600) -> None:
349
444
  # open_timeout is the maximum time, in seconds, that opening will be retried
350
445
  # if a file cannot be initially opened for writing.
351
- # Daemon, so a run that never reaches filewriter_end() cannot hold the
352
- # interpreter open; the atexit handlers flush before daemons stop.
446
+ # A daemon cannot hold interpreter shutdown on an abandoned I/O call.
447
+ # Normal callers flush first, then request shutdown.
353
448
  super().__init__(name="execsql-filewriter", daemon=True)
354
449
  self.input_queue: queue.Queue[tuple[int, tuple[Any, ...]]] = queue.Queue()
355
450
  self.return_msg_queue: queue.Queue[Any] = queue.Queue()
356
451
  self.file_encoding = file_encoding
357
452
  self.open_timeout = open_timeout
358
453
  self.files: dict[str, FileWriter.FileControl] = {}
454
+ self.abandoned = threading.Event()
455
+ self.current_op: tuple[str, str, float] | None = None
456
+ self.output_error: ErrInfo | None = None
457
+ self._pending: dict[str, int] = {}
458
+ self._pending_lock = threading.Lock()
359
459
  # Commands that raised something other than a file's own OSError,
360
460
  # reported with the failed files.
361
461
  self.errors: list[LostOutput] = []
@@ -373,6 +473,39 @@ class FileWriter(threading.Thread):
373
473
  self.open_failures,
374
474
  )
375
475
 
476
+ def submit(self, command: int, arguments: tuple[Any, ...]) -> None:
477
+ """Queue a command, recording pending destinations without holding locks during I/O."""
478
+ with self._pending_lock:
479
+ if self.abandoned.is_set():
480
+ raise _abandoned_writer_error(self)
481
+ if command == self.CMD_WRITE:
482
+ filename = arguments[0]
483
+ self._pending[filename] = self._pending.get(filename, 0) + 1
484
+ self.input_queue.put((command, arguments))
485
+
486
+ def completed(self, filename: str, count: int) -> None:
487
+ """Retire writes only after flushing them or recording their loss."""
488
+ with self._pending_lock:
489
+ remaining = self._pending.get(filename, 0) - count
490
+ if remaining > 0:
491
+ self._pending[filename] = remaining
492
+ else:
493
+ self._pending.pop(filename, None)
494
+
495
+ def _resolved_write(self, requested: str, filename: str) -> None:
496
+ """Move this write's accounting to the worker-resolved destination."""
497
+ if requested == filename:
498
+ return
499
+ with self._pending_lock:
500
+ count = self._pending.get(requested, 0)
501
+ if not count:
502
+ return
503
+ if count == 1:
504
+ del self._pending[requested]
505
+ else:
506
+ self._pending[requested] = count - 1
507
+ self._pending[filename] = self._pending.get(filename, 0) + 1
508
+
376
509
  def __del__(self) -> None:
377
510
  try:
378
511
  self.close_all()
@@ -387,12 +520,21 @@ class FileWriter(threading.Thread):
387
520
  except Exception as exc:
388
521
  fc.fail(fc.STATUS_WRITEFAILURE, exc)
389
522
 
523
+ def _resolve_filename(self, filename: str) -> str:
524
+ self.current_op = ("open", filename, time.monotonic())
525
+ try:
526
+ if self.abandoned.is_set():
527
+ raise _abandoned_writer_error(self)
528
+ return str(Path(filename).resolve())
529
+ finally:
530
+ self.current_op = None
531
+
390
532
  def close_all(self) -> None:
391
533
  for fc in getattr(self, "files", {}).values():
392
534
  self._on_file(fc, fc.close)
393
535
 
394
536
  def close_if_open(self, fn: str) -> None:
395
- filename = str(Path(fn).resolve())
537
+ filename = self._resolve_filename(fn)
396
538
  if filename in self.files:
397
539
  fc = self.files[filename]
398
540
  self._on_file(fc, fc.close)
@@ -452,33 +594,32 @@ class FileWriter(threading.Thread):
452
594
  self.errors = []
453
595
  self.return_msg_queue.put(failed)
454
596
 
455
- def open_as_new(self, fn: str) -> None:
456
- filename = str(Path(fn).resolve())
597
+ def _file_control(self, filename: str) -> FileWriter.FileControl:
598
+ """Get or create the control for a worker-resolved filename."""
457
599
  if filename not in self.files:
458
600
  self.files[filename] = self.FileControl(
459
601
  filename,
460
602
  open_timeout=self.open_timeout,
461
603
  encoding=self.file_encoding,
604
+ writer=self,
462
605
  )
463
- fc = self.files[filename]
606
+ return self.files[filename]
607
+
608
+ def open_as_new(self, fn: str) -> None:
609
+ fc = self._file_control(self._resolve_filename(fn))
464
610
  self._on_file(fc, fc.open_as_new)
465
611
 
466
612
  def status(self, fn: str) -> None:
467
- filename = str(Path(fn).resolve())
613
+ filename = self._resolve_filename(fn)
468
614
  if filename in self.files:
469
615
  self.return_msg_queue.put(self.files[filename].status)
470
616
  else:
471
617
  self.return_msg_queue.put(self.FileControl.STATUS_UNOPENED)
472
618
 
473
619
  def write(self, fn: str, content: str) -> None:
474
- filename = str(Path(fn).resolve())
475
- if filename not in self.files:
476
- self.files[filename] = self.FileControl(
477
- filename,
478
- open_timeout=self.open_timeout,
479
- encoding=self.file_encoding,
480
- )
481
- fc = self.files[filename]
620
+ filename = self._resolve_filename(fn)
621
+ fc = self._file_control(filename)
622
+ self._resolved_write(fn, filename)
482
623
  self._on_file(fc, lambda: fc.write(content))
483
624
 
484
625
  def shutdown(self) -> None:
@@ -492,22 +633,39 @@ class FileWriter(threading.Thread):
492
633
  # Waiting files are retried on a timer rather than only when the queue
493
634
  # is idle: a caller polling CMD_CLOSED_STATUS keeps the queue busy.
494
635
  last_retry = time.monotonic()
495
- while self.active:
496
- try:
497
- command, argtuple = self.input_queue.get(timeout=0.1)
498
- except queue.Empty:
499
- pass
500
- else:
501
- try:
502
- self.execvec[command](*argtuple)
503
- except Exception as exc:
504
- self._command_failed(command, argtuple, exc)
505
- if time.monotonic() - last_retry >= 0.1:
636
+ try:
637
+ while self.active and not self.abandoned.is_set():
506
638
  try:
507
- self.retry_waiting()
508
- except Exception as exc:
509
- self.errors.append(LostOutput("", 0, str(exc), "writer"))
510
- last_retry = time.monotonic()
639
+ command, argtuple = self.input_queue.get(timeout=0.1)
640
+ except queue.Empty:
641
+ pass
642
+ else:
643
+ if self.abandoned.is_set():
644
+ break
645
+ try:
646
+ self.execvec[command](*argtuple)
647
+ except Exception as exc:
648
+ self._command_failed(command, argtuple, exc)
649
+ if self.abandoned.is_set():
650
+ break
651
+ if time.monotonic() - last_retry >= 0.1:
652
+ try:
653
+ self.retry_waiting()
654
+ except Exception as exc:
655
+ self.errors.append(LostOutput("", 0, str(exc), "writer"))
656
+ last_retry = time.monotonic()
657
+ finally:
658
+ if self.abandoned.is_set():
659
+ self.active = False
660
+ while True:
661
+ try:
662
+ self.input_queue.get_nowait()
663
+ except queue.Empty:
664
+ break
665
+ for fc in self.files.values():
666
+ fc.discard()
667
+ with self._pending_lock:
668
+ self._pending.clear()
511
669
 
512
670
  def _command_failed(self, command: int, argtuple: tuple[Any, ...], exc: Exception) -> None:
513
671
  """Record a command that raised, and still answer it if the caller waits for an answer.
@@ -517,7 +675,10 @@ class FileWriter(threading.Thread):
517
675
  every later write, to any file).
518
676
  """
519
677
  filename = argtuple[0] if argtuple and isinstance(argtuple[0], str) else ""
520
- lost = LostOutput(filename, 0, str(exc), "writer")
678
+ dropped = 1 if command == self.CMD_WRITE else 0
679
+ if dropped:
680
+ self.completed(filename, dropped)
681
+ lost = LostOutput(filename, dropped, str(exc), "writer")
521
682
  if command == self.CMD_OPEN_FAILURES:
522
683
  self.return_msg_queue.put([lost])
523
684
  return
@@ -572,18 +733,23 @@ def _active_writer() -> FileWriter | None:
572
733
  import execsql.state as _state
573
734
 
574
735
  fw = _state.filewriter
575
- if fw is not None and _writer_stopped(fw):
736
+ if fw is None or fw.ident is None:
737
+ return None
738
+ if fw.abandoned.is_set():
739
+ raise _abandoned_writer_error(fw)
740
+ if _writer_stopped(fw):
576
741
  raise _writer_stopped_error()
577
- return fw if fw is not None and fw.is_alive() else None
742
+ return fw if fw.is_alive() else None
578
743
 
579
744
 
580
745
  def _abspath(filename: str) -> str:
581
746
  """*filename* made absolute against the working directory as it is now.
582
747
 
583
- Resolved here, on the run's thread, rather than in the writer: a ``CD``
584
- later in the script must not redirect output already queued.
748
+ Capture the current directory here so a later ``CD`` cannot redirect
749
+ queued output. Physical resolution belongs to the writer: metadata I/O
750
+ must not block the caller. Preserve ``..`` until symlinks are resolved.
585
751
  """
586
- return str(Path(filename).resolve())
752
+ return str(Path(filename).absolute())
587
753
 
588
754
 
589
755
  #: Sentinel returned by :func:`_writer_reply` when the writer died before
@@ -591,30 +757,75 @@ def _abspath(filename: str) -> str:
591
757
  _NO_REPLY = object()
592
758
 
593
759
 
594
- def _writer_reply(fw: FileWriter) -> Any:
595
- """Wait for *fw*'s answer to the command just sent.
760
+ def _abandoned_writer_error(fw: FileWriter) -> ErrInfo:
761
+ return fw.output_error or ErrInfo(
762
+ "error",
763
+ other_msg="File output lost: the file writer was abandoned; queued output was discarded.",
764
+ )
596
765
 
597
- Polls with a short timeout and re-checks liveness between polls, so a
598
- writer that dies mid-request cannot block the caller forever. Returns
599
- :data:`_NO_REPLY` if it died.
600
- """
766
+
767
+ def _abandon_writer(fw: FileWriter, wait: float) -> ErrInfo:
768
+ """Stop admission immediately; the daemon may still be inside its last I/O call."""
769
+ with fw._pending_lock:
770
+ if fw.output_error is not None:
771
+ return fw.output_error
772
+ operation = fw.current_op
773
+ fw.abandoned.set()
774
+ pending = tuple(fw._pending)
775
+ if operation is not None:
776
+ phase, filename, started = operation
777
+ elapsed = time.monotonic() - started
778
+ where = f"in {phase} of {filename}"
779
+ else:
780
+ elapsed = wait
781
+ where = "while waiting for a file writer reply"
782
+ queued = "; queued output files: " + ", ".join(pending) if pending else ""
783
+ fw.output_error = ErrInfo(
784
+ "error",
785
+ other_msg=(
786
+ f"File output lost: the file writer has been blocked for {elapsed:.3f} seconds {where}; "
787
+ f"output queued after it was discarded{queued}. The in-progress I/O may complete later."
788
+ ),
789
+ )
790
+ return fw.output_error
791
+
792
+
793
+ def _writer_deadline(fw: FileWriter, max_wait: float | None = None) -> tuple[float, float]:
794
+ wait = fw.open_timeout + 5.0
795
+ if max_wait is not None:
796
+ wait = min(wait, max_wait)
797
+ wait = max(0.0, wait)
798
+ return time.monotonic() + wait, wait
799
+
800
+
801
+ def _writer_reply(fw: FileWriter, deadline: float, wait: float) -> Any:
802
+ """Wait only until the caller's absolute deadline, checking worker liveness."""
601
803
  while True:
804
+ if fw.abandoned.is_set():
805
+ raise _abandoned_writer_error(fw)
806
+ if not fw.is_alive():
807
+ return _NO_REPLY
808
+ remaining = deadline - time.monotonic()
809
+ if remaining <= 0:
810
+ raise _abandon_writer(fw, wait)
602
811
  try:
603
- return fw.return_msg_queue.get(timeout=2.0)
812
+ return fw.return_msg_queue.get(timeout=min(0.05, remaining))
604
813
  except queue.Empty:
605
- if not fw.is_alive():
606
- return _NO_REPLY
814
+ pass
815
+
816
+
817
+ def _filewriter_status(fw: FileWriter, filename: str, deadline: float, wait: float) -> int:
818
+ fw.submit(FileWriter.CMD_GET_STATUS, (filename,))
819
+ reply = _writer_reply(fw, deadline, wait)
820
+ return FileWriter.FileControl.STATUS_CLOSED if reply is _NO_REPLY else cast(int, reply)
607
821
 
608
822
 
609
823
  def filewriter_filestatus(filename: str) -> int:
610
824
  fw = _active_writer()
611
825
  if fw is None:
612
826
  return FileWriter.FileControl.STATUS_CLOSED
613
- fw.input_queue.put((FileWriter.CMD_GET_STATUS, (_abspath(filename),)))
614
- reply = _writer_reply(fw)
615
- if reply is _NO_REPLY:
616
- return FileWriter.FileControl.STATUS_CLOSED
617
- return cast(int, reply)
827
+ deadline, wait = _writer_deadline(fw)
828
+ return _filewriter_status(fw, _abspath(filename), deadline, wait)
618
829
 
619
830
 
620
831
  #: Set once the first dropped write has been reported, so a script writing
@@ -651,7 +862,7 @@ def filewriter_write(filename: str, message: str) -> None:
651
862
  if fw is None:
652
863
  _warn_write_dropped(filename)
653
864
  return
654
- fw.input_queue.put((FileWriter.CMD_WRITE, (_abspath(filename), message)))
865
+ fw.submit(FileWriter.CMD_WRITE, (_abspath(filename), message))
655
866
 
656
867
 
657
868
  def filewriter_open_as_new(filename: str) -> None:
@@ -661,59 +872,52 @@ def filewriter_open_as_new(filename: str) -> None:
661
872
  if fw is None:
662
873
  _warn_write_dropped(filename)
663
874
  return
664
- fw.input_queue.put((FileWriter.CMD_OPEN_AS_NEW, (_abspath(filename),)))
875
+ fw.submit(FileWriter.CMD_OPEN_AS_NEW, (_abspath(filename),))
665
876
 
666
877
 
667
878
  def filewriter_close(filename: str) -> None:
668
- # This is intended to be used by the executing thread to ensure that a
669
- # file is closed before it writes to that file itself.
879
+ """Release a writer-held path before synchronous output, using one bounded deadline."""
670
880
  fw = _active_writer()
671
881
  if fw is None:
672
882
  return
673
883
  filename = _abspath(filename)
674
- fw.input_queue.put((FileWriter.CMD_CLOSE_IF_OPEN, (filename,)))
675
- while filewriter_filestatus(filename) == FileWriter.FileControl.STATUS_OPEN:
676
- time.sleep(0.05)
884
+ deadline, wait = _writer_deadline(fw)
885
+ fw.submit(FileWriter.CMD_CLOSE_IF_OPEN, (filename,))
886
+ while _filewriter_status(fw, filename, deadline, wait) == FileWriter.FileControl.STATUS_OPEN:
887
+ remaining = deadline - time.monotonic()
888
+ if remaining <= 0:
889
+ raise _abandon_writer(fw, wait)
890
+ time.sleep(min(0.05, remaining))
677
891
 
678
892
 
679
893
  def filewriter_close_all_after_write(max_wait: float | None = None) -> None:
680
- """Flush and close every file the active run's writer holds, then report lost output.
894
+ """Flush and report output loss within one absolute deadline, including reply waits.
681
895
 
682
- Waits for files still locked by another process for up to the writer's
683
- ``open_timeout`` (plus a small margin), or *max_wait* seconds if that is
684
- shorter. A file still locked then is closed and its output discarded,
685
- and this raises :class:`ErrInfo` naming each file whose output was lost
686
- and how much — a lost write is an error, not a silent success.
896
+ The deadline is open_timeout + 5 seconds, capped by max_wait when supplied.
897
+ On expiry the daemon is abandoned, not killed: its current I/O may finish
898
+ later, but it will discard queued writes and close without flushing them.
687
899
  """
688
900
  fw = _active_writer()
689
901
  if fw is None:
690
902
  return
691
- wait = fw.open_timeout + 5.0 if max_wait is None else min(fw.open_timeout + 5.0, max_wait)
692
- deadline = time.monotonic() + wait
693
- fw.input_queue.put((FileWriter.CMD_CLOSE_ALL_AFTER_WRITE, ()))
694
- timed_out = False
903
+ deadline, wait = _writer_deadline(fw, max_wait)
904
+ fw.submit(FileWriter.CMD_CLOSE_ALL_AFTER_WRITE, ())
695
905
  while True:
696
- if not fw.is_alive():
697
- raise _writer_stopped_error()
698
- fw.input_queue.put((FileWriter.CMD_CLOSED_STATUS, ()))
699
- close_status = _writer_reply(fw)
906
+ fw.submit(FileWriter.CMD_CLOSED_STATUS, ())
907
+ close_status = _writer_reply(fw, deadline, wait)
700
908
  if close_status is _NO_REPLY:
701
909
  raise _writer_stopped_error()
702
910
  if close_status == FileWriter.FileControl.STATUS_CLOSED:
703
911
  break
704
- if time.monotonic() > deadline:
705
- timed_out = True
706
- # Close what is still locked, so its discarded output is counted.
707
- fw.input_queue.put((FileWriter.CMD_CLOSE_ALL, ()))
708
- break
709
- time.sleep(0.05)
710
- fw.input_queue.put((FileWriter.CMD_OPEN_FAILURES, ()))
711
- failures = _writer_reply(fw)
912
+ remaining = deadline - time.monotonic()
913
+ if remaining <= 0:
914
+ raise _abandon_writer(fw, wait)
915
+ time.sleep(min(0.05, remaining))
916
+ fw.submit(FileWriter.CMD_OPEN_FAILURES, ())
917
+ failures = _writer_reply(fw, deadline, wait)
712
918
  if failures is _NO_REPLY:
713
919
  raise _writer_stopped_error()
714
920
  messages = [_lost_output_message(lost, fw.open_timeout) for lost in failures]
715
- if timed_out and not messages:
716
- messages.append(f"one or more output files were still locked after {wait:.0f} seconds")
717
921
  if messages:
718
922
  raise ErrInfo("error", other_msg="File output lost: " + "; ".join(messages) + ".")
719
923
 
@@ -735,11 +939,13 @@ def _lost_output_message(lost: LostOutput, open_timeout: int) -> str:
735
939
 
736
940
 
737
941
  def filewriter_end(writer: FileWriter | None = None) -> None:
738
- """Close every file and stop *writer*, or the active run's writer if omitted.
942
+ """Request this writer's shutdown, or detach it immediately if already abandoned.
739
943
 
740
- Only that one writer is stopped: a run that halts leaves other runs'
741
- output alone. The join is capped so an ``atexit`` handler cannot wedge
742
- interpreter shutdown on a writer stuck opening a file.
944
+ A healthy worker closes its files and stops; a stalled one is abandoned
945
+ after the capped join and may remain alive. Other runs' writers are left
946
+ alone. The join is capped so an ``atexit`` handler cannot wedge
947
+ interpreter shutdown on a writer stuck in I/O. An abandoned writer is
948
+ detached immediately, without a second wait for its in-progress call.
743
949
  """
744
950
  import execsql.state as _state
745
951
 
@@ -747,9 +953,11 @@ def filewriter_end(writer: FileWriter | None = None) -> None:
747
953
  if fw is None:
748
954
  return
749
955
  try:
750
- if fw.is_alive():
751
- fw.input_queue.put((FileWriter.CMD_SHUTDOWN, ()))
956
+ if fw.is_alive() and not fw.abandoned.is_set():
957
+ fw.submit(FileWriter.CMD_SHUTDOWN, ())
752
958
  fw.join(timeout=5.0)
959
+ if fw.is_alive():
960
+ _abandon_writer(fw, 5.0)
753
961
  except Exception:
754
962
  pass # Best-effort cleanup at interpreter shutdown.
755
963
  if _state.filewriter is fw: