execsql2 2.37.0__py3-none-any.whl → 2.38.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +95 -45
- execsql/cli/commands/run.py +2 -2
- execsql/cli/lint.py +6 -1
- execsql/cli/run.py +5 -0
- execsql/config.py +7 -0
- execsql/data/execsql.conf.template +21 -0
- execsql/db/base.py +52 -24
- execsql/db/duckdb.py +43 -4
- execsql/db/firebird.py +8 -0
- execsql/db/mysql.py +8 -3
- execsql/db/oracle.py +5 -0
- execsql/db/postgres.py +2 -2
- execsql/db/sqlite.py +117 -10
- execsql/db/sqlserver.py +11 -7
- execsql/db/target_types.py +151 -4
- execsql/debug/repl.py +2 -1
- execsql/exceptions.py +14 -0
- execsql/exporters/html.py +26 -42
- execsql/exporters/ods.py +93 -21
- execsql/exporters/sqlite.py +82 -47
- execsql/format.py +12 -4
- execsql/gui/desktop.py +2 -1
- execsql/gui/tui.py +5 -2
- execsql/importers/base.py +9 -7
- execsql/importers/json.py +37 -13
- execsql/importers/ods.py +22 -2
- execsql/lsp/document.py +6 -1
- execsql/lsp/hover.py +2 -1
- execsql/metacommands/conditions.py +3 -1
- execsql/metacommands/connect.py +18 -3
- execsql/metacommands/dispatch.py +10 -0
- execsql/metacommands/io.py +1 -0
- execsql/metacommands/io_export.py +1 -2
- execsql/metacommands/io_fileops.py +26 -90
- execsql/metacommands/io_import.py +26 -8
- execsql/metacommands/reference.py +7 -0
- execsql/parser.py +14 -37
- execsql/script/control.py +17 -7
- execsql/script/executor.py +160 -34
- execsql/script/preparation.py +42 -0
- execsql/types.py +6 -5
- execsql/utils/errors.py +2 -2
- execsql/utils/fileio.py +342 -134
- execsql/utils/spreadsheet.py +74 -0
- execsql/utils/strings.py +13 -7
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/README.md +5 -5
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/example_config_prompt.sql +2 -2
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/METADATA +32 -119
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/RECORD +64 -62
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/WHEEL +1 -1
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/licenses/NOTICE +0 -0
execsql/api.py
CHANGED
|
@@ -39,7 +39,7 @@ from typing import Any, cast
|
|
|
39
39
|
|
|
40
40
|
from execsql.cli.dsn import _parse_connection_string
|
|
41
41
|
from execsql.config import ConfigData, StatObj, WriteHooks
|
|
42
|
-
from execsql.exceptions import ErrInfo
|
|
42
|
+
from execsql.exceptions import ErrInfo, ScriptExit
|
|
43
43
|
from execsql.state import RuntimeContext, active_context
|
|
44
44
|
|
|
45
45
|
__all__ = ["run", "ScriptResult", "ScriptError"]
|
|
@@ -224,6 +224,11 @@ class _LibraryConfig:
|
|
|
224
224
|
self.email_format = "plain"
|
|
225
225
|
self.email_css: str | None = None
|
|
226
226
|
|
|
227
|
+
# Default capabilities when no configuration file is supplied.
|
|
228
|
+
self.allow_system_cmd = True
|
|
229
|
+
self.allow_rm_file = True
|
|
230
|
+
self.allow_serve = True
|
|
231
|
+
|
|
227
232
|
# Includes
|
|
228
233
|
self.include_req: list = []
|
|
229
234
|
self.include_opt: list = []
|
|
@@ -340,9 +345,9 @@ def run(
|
|
|
340
345
|
encoding: str = "utf-8",
|
|
341
346
|
halt_on_error: bool = True,
|
|
342
347
|
new_db: bool = False,
|
|
343
|
-
allow_system_cmd: bool =
|
|
344
|
-
allow_rm_file: bool =
|
|
345
|
-
allow_serve: bool =
|
|
348
|
+
allow_system_cmd: bool | None = None,
|
|
349
|
+
allow_rm_file: bool | None = None,
|
|
350
|
+
allow_serve: bool | None = None,
|
|
346
351
|
) -> ScriptResult:
|
|
347
352
|
"""Execute a SQL script and return the result.
|
|
348
353
|
|
|
@@ -356,10 +361,26 @@ def run(
|
|
|
356
361
|
``"postgresql://user:pass@host/db"``).
|
|
357
362
|
connection: A pre-existing :class:`~execsql.db.base.Database`
|
|
358
363
|
instance. ``run()`` will NOT close this connection on exit.
|
|
364
|
+
Connections opened by the script with CONNECT are run-owned:
|
|
365
|
+
their pending work is rolled back and they are closed on exit,
|
|
366
|
+
including under AUTOCOMMIT OFF. Commit their work in the script
|
|
367
|
+
before it ends if it must be kept.
|
|
368
|
+
What an interrupted SQL statement, metacommand or ``END BATCH``
|
|
369
|
+
did is rolled back before the interrupt is re-raised: an SQL
|
|
370
|
+
statement rolls back its connection's open transaction, and a
|
|
371
|
+
metacommand everything not yet committed on each connection
|
|
372
|
+
with autocommit on (under AUTOCOMMIT OFF, a metacommand's writes
|
|
373
|
+
stay with what the script holds). An interrupt that ends the run
|
|
374
|
+
also rolls back everything else not yet committed on each
|
|
375
|
+
connection with autocommit on. Work the database had already
|
|
376
|
+
committed when the interrupt arrived stays committed. A
|
|
377
|
+
``SystemExit`` from a signal handler is rolled back the same way,
|
|
378
|
+
and the run returns a failed result.
|
|
359
379
|
variables: Substitution variables as ``{"NAME": "value"}``.
|
|
360
380
|
Keys without a ``$`` prefix get one added automatically, so
|
|
361
381
|
``{"SCHEMA": "public"}`` is referenced as ``!!$SCHEMA!!``.
|
|
362
|
-
config_file: Optional execsql configuration file to load.
|
|
382
|
+
config_file: Optional execsql configuration file to load. Configured
|
|
383
|
+
variables and startup includes apply before the main script runs.
|
|
363
384
|
encoding: Script file encoding (default ``"utf-8"``).
|
|
364
385
|
halt_on_error: If ``True`` (default), stop on the first SQL
|
|
365
386
|
error. If ``False``, capture errors and continue.
|
|
@@ -368,20 +389,26 @@ def run(
|
|
|
368
389
|
allow_system_cmd: If ``False``, the SYSTEM_CMD metacommand is
|
|
369
390
|
disabled and will raise an error if encountered. Matches the
|
|
370
391
|
``--no-system-cmd`` CLI flag and the ``allow_system_cmd``
|
|
371
|
-
config option.
|
|
392
|
+
config option. None (default) inherits the configured value,
|
|
393
|
+
or True without a config file; explicit True overrides a restriction.
|
|
372
394
|
allow_rm_file: If ``False``, the RM_FILE metacommand is disabled
|
|
373
395
|
and will raise an error if encountered. Matches the
|
|
374
396
|
``--no-rm-file`` CLI flag and the ``allow_rm_file`` config
|
|
375
|
-
option.
|
|
397
|
+
option. None (default) inherits the configured value;
|
|
398
|
+
an explicit boolean overrides it.
|
|
376
399
|
allow_serve: If ``False``, the SERVE metacommand is disabled and
|
|
377
400
|
will raise an error if encountered. Matches the ``--no-serve``
|
|
378
|
-
CLI flag and the ``allow_serve`` config option.
|
|
401
|
+
CLI flag and the ``allow_serve`` config option. None (default)
|
|
402
|
+
inherits the configured value; an explicit boolean overrides it.
|
|
379
403
|
|
|
380
404
|
Returns:
|
|
381
405
|
A :class:`ScriptResult` with execution outcome, timing, errors,
|
|
382
406
|
and final variable state. A script that cannot be read, a bad
|
|
383
407
|
*config_file*, and a *dsn* that cannot be parsed or opened all
|
|
384
|
-
return ``success=False`` rather than raising.
|
|
408
|
+
return ``success=False`` rather than raising. A blocked background
|
|
409
|
+
output writer adds an error with source ``<output>`` and is abandoned
|
|
410
|
+
at its flush deadline. Its current I/O may finish later; queued
|
|
411
|
+
output is discarded and no further opens or writes are started.
|
|
385
412
|
|
|
386
413
|
Raises:
|
|
387
414
|
ValueError: If the argument combination is invalid (e.g. both
|
|
@@ -420,27 +447,6 @@ def run(
|
|
|
420
447
|
# ------------------------------------------------------------------
|
|
421
448
|
ctx = RuntimeContext()
|
|
422
449
|
|
|
423
|
-
# Configuration
|
|
424
|
-
conf_overrides: dict[str, Any] = {"script_encoding": encoding}
|
|
425
|
-
conf: Any
|
|
426
|
-
if config_file is not None:
|
|
427
|
-
# Load a real ConfigData with the explicit config file
|
|
428
|
-
from execsql.script.variables import SubVarSet
|
|
429
|
-
|
|
430
|
-
if not Path(config_file).is_file():
|
|
431
|
-
return _failed(f"Config file {str(config_file)!r} does not exist.", "<config>")
|
|
432
|
-
temp_subvars = SubVarSet()
|
|
433
|
-
script_dir = str(Path(script).resolve().parent) if script else os.getcwd()
|
|
434
|
-
try:
|
|
435
|
-
conf = ConfigData(script_dir, temp_subvars, config_file=str(config_file))
|
|
436
|
-
except Exception as exc:
|
|
437
|
-
return _failed(_error_text(exc), "<config>")
|
|
438
|
-
# Apply any overrides
|
|
439
|
-
for k, v in conf_overrides.items():
|
|
440
|
-
setattr(conf, k, v)
|
|
441
|
-
else:
|
|
442
|
-
conf = _LibraryConfig(**conf_overrides)
|
|
443
|
-
|
|
444
450
|
# Substitution variables
|
|
445
451
|
from execsql.script.variables import SubVarSet
|
|
446
452
|
|
|
@@ -461,6 +467,21 @@ def run(
|
|
|
461
467
|
|
|
462
468
|
subvars.add_substitution("$HOSTNAME", platform.node())
|
|
463
469
|
|
|
470
|
+
# Configuration
|
|
471
|
+
conf: Any
|
|
472
|
+
if config_file is not None:
|
|
473
|
+
# Load a real ConfigData with the explicit config file
|
|
474
|
+
if not Path(config_file).is_file():
|
|
475
|
+
return _failed(f"Config file {str(config_file)!r} does not exist.", "<config>")
|
|
476
|
+
script_dir = str(Path(script).resolve().parent) if script else os.getcwd()
|
|
477
|
+
try:
|
|
478
|
+
conf = ConfigData(script_dir, subvars, config_file=str(config_file))
|
|
479
|
+
except Exception as exc:
|
|
480
|
+
return _failed(_error_text(exc), "<config>")
|
|
481
|
+
conf.script_encoding = encoding
|
|
482
|
+
else:
|
|
483
|
+
conf = _LibraryConfig(script_encoding=encoding)
|
|
484
|
+
|
|
464
485
|
# User-supplied variables
|
|
465
486
|
if variables:
|
|
466
487
|
for name, value in variables.items():
|
|
@@ -476,11 +497,18 @@ def run(
|
|
|
476
497
|
ctx.subvars = subvars
|
|
477
498
|
ctx.status = StatObj()
|
|
478
499
|
ctx.status.halt_on_err = halt_on_error
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
500
|
+
if allow_system_cmd is not None:
|
|
501
|
+
conf.allow_system_cmd = allow_system_cmd
|
|
502
|
+
if allow_rm_file is not None:
|
|
503
|
+
conf.allow_rm_file = allow_rm_file
|
|
504
|
+
if allow_serve is not None:
|
|
505
|
+
conf.allow_serve = allow_serve
|
|
482
506
|
ctx.conf = conf
|
|
483
507
|
|
|
508
|
+
from execsql.script.preparation import prepare_script
|
|
509
|
+
|
|
510
|
+
tree = prepare_script(tree, conf.include_req, conf.include_opt)
|
|
511
|
+
|
|
484
512
|
# Capture output to a buffer (suppress stdout/stderr)
|
|
485
513
|
stdout_buf = io.StringIO()
|
|
486
514
|
stderr_buf = io.StringIO()
|
|
@@ -521,7 +549,7 @@ def run(
|
|
|
521
549
|
# ------------------------------------------------------------------
|
|
522
550
|
# Execute
|
|
523
551
|
# ------------------------------------------------------------------
|
|
524
|
-
from execsql.script.executor import execute
|
|
552
|
+
from execsql.script.executor import _roll_back_after_interrupt, execute
|
|
525
553
|
|
|
526
554
|
errors: list[ScriptError] = []
|
|
527
555
|
t0 = time.perf_counter()
|
|
@@ -534,8 +562,11 @@ def run(
|
|
|
534
562
|
try:
|
|
535
563
|
execute(tree, ctx=ctx)
|
|
536
564
|
except SystemExit as exc:
|
|
537
|
-
# exit_now()
|
|
538
|
-
|
|
565
|
+
# exit_now() raises ScriptExit; any other SystemExit (a signal
|
|
566
|
+
# handler's) interrupted the run. Either is reported as an error.
|
|
567
|
+
if not isinstance(exc, ScriptExit):
|
|
568
|
+
_roll_back_after_interrupt(ctx)
|
|
569
|
+
_capture_errors(ctx, errors, exc.code, interrupted=not isinstance(exc, ScriptExit))
|
|
539
570
|
except ErrInfo as exc:
|
|
540
571
|
errors.append(
|
|
541
572
|
ScriptError(
|
|
@@ -547,6 +578,9 @@ def run(
|
|
|
547
578
|
)
|
|
548
579
|
except Exception as exc:
|
|
549
580
|
errors.append(ScriptError(message=str(exc), source="<runtime>"))
|
|
581
|
+
except BaseException:
|
|
582
|
+
_roll_back_after_interrupt(ctx)
|
|
583
|
+
raise
|
|
550
584
|
|
|
551
585
|
# Non-halting SQL errors (halt_on_error=False) are recorded in
|
|
552
586
|
# status.error_history as execution continues; a halting error never
|
|
@@ -562,13 +596,15 @@ def run(
|
|
|
562
596
|
|
|
563
597
|
# Flush and close every file the script wrote, so the caller can read
|
|
564
598
|
# them the moment run() returns. Output to a file that could not be
|
|
565
|
-
# opened is reported as an error.
|
|
599
|
+
# opened is reported as an error. A blocked writer is abandoned;
|
|
600
|
+
# healthy writers are stopped below.
|
|
566
601
|
from execsql.utils.fileio import filewriter_close_all_after_write
|
|
567
602
|
|
|
568
603
|
try:
|
|
569
604
|
filewriter_close_all_after_write()
|
|
570
605
|
except ErrInfo as exc:
|
|
571
|
-
|
|
606
|
+
if ctx.halt_output_error is not exc:
|
|
607
|
+
errors.append(ScriptError(message=exc.errmsg(), source="<output>"))
|
|
572
608
|
except Exception:
|
|
573
609
|
pass # Best-effort: a failed flush must not mask a script error.
|
|
574
610
|
|
|
@@ -589,9 +625,12 @@ def run(
|
|
|
589
625
|
from execsql.utils.fileio import filewriter_end
|
|
590
626
|
|
|
591
627
|
filewriter_end(ctx.filewriter)
|
|
592
|
-
if
|
|
628
|
+
if ctx.dbs is not None:
|
|
593
629
|
try:
|
|
594
|
-
|
|
630
|
+
if owns_connection:
|
|
631
|
+
ctx.dbs.closeall()
|
|
632
|
+
else:
|
|
633
|
+
ctx.dbs.closeall(except_db=connection)
|
|
595
634
|
except Exception:
|
|
596
635
|
pass
|
|
597
636
|
if ctx.tempfiles is not None:
|
|
@@ -621,8 +660,8 @@ def _start_filewriter(ctx: RuntimeContext, conf: Any) -> None:
|
|
|
621
660
|
writer, and every entry point in :mod:`execsql.utils.fileio` drops the
|
|
622
661
|
write when none is running. A writer per run keeps concurrent runs from
|
|
623
662
|
sharing open files or reply queues, and a halt in one run (which stops
|
|
624
|
-
its writer) from discarding another run's output.
|
|
625
|
-
|
|
663
|
+
its writer) from discarding another run's output. Healthy writers stop
|
|
664
|
+
before run() returns; a blocked daemon is abandoned with an output error.
|
|
626
665
|
"""
|
|
627
666
|
from execsql.utils.fileio import FileWriter
|
|
628
667
|
|
|
@@ -633,7 +672,13 @@ def _start_filewriter(ctx: RuntimeContext, conf: Any) -> None:
|
|
|
633
672
|
ctx.filewriter.start()
|
|
634
673
|
|
|
635
674
|
|
|
636
|
-
def _capture_errors(
|
|
675
|
+
def _capture_errors(
|
|
676
|
+
ctx: RuntimeContext,
|
|
677
|
+
errors: list[ScriptError],
|
|
678
|
+
exit_code: Any,
|
|
679
|
+
*,
|
|
680
|
+
interrupted: bool = False,
|
|
681
|
+
) -> None:
|
|
637
682
|
"""Record the error that halted the run, after ``exit_now()`` raised ``SystemExit``.
|
|
638
683
|
|
|
639
684
|
``exit_now`` leaves the halting :class:`ErrInfo` on the context, and any
|
|
@@ -641,10 +686,15 @@ def _capture_errors(ctx: RuntimeContext, errors: list[ScriptError], exit_code: A
|
|
|
641
686
|
SQL statement's text is in ``$LAST_ERROR``, which the executor sets just
|
|
642
687
|
before halting; it is reported only when the halt came from SQL, so a
|
|
643
688
|
metacommand error is never paired with an earlier statement. A ``HALT``
|
|
644
|
-
carries no error, only its exit status
|
|
689
|
+
carries no error, only its exit status; nor does a run *interrupted* by a
|
|
690
|
+
``SystemExit`` that execsql did not raise (a signal handler's).
|
|
645
691
|
"""
|
|
646
692
|
err = ctx.halt_error
|
|
647
|
-
if err is None:
|
|
693
|
+
if err is None and interrupted:
|
|
694
|
+
msg = "Script interrupted by SystemExit" + (
|
|
695
|
+
f" with exit status {exit_code}." if type(exit_code) is int else "."
|
|
696
|
+
)
|
|
697
|
+
elif err is None:
|
|
648
698
|
msg = f"Script halted by HALT with exit status {exit_code}."
|
|
649
699
|
else:
|
|
650
700
|
msg = err.errmsg()
|
execsql/cli/commands/run.py
CHANGED
|
@@ -114,9 +114,9 @@ def main(
|
|
|
114
114
|
"--output-dir",
|
|
115
115
|
metavar="DIR",
|
|
116
116
|
help=(
|
|
117
|
-
"
|
|
117
|
+
"Containment root for EXPORT output files. "
|
|
118
118
|
"Relative paths in EXPORT metacommands are joined to this directory. "
|
|
119
|
-
"Absolute paths
|
|
119
|
+
"Absolute paths must also stay inside it; stdout is exempt. WRITE is not constrained."
|
|
120
120
|
),
|
|
121
121
|
),
|
|
122
122
|
progress: bool = typer.Option(
|
execsql/cli/lint.py
CHANGED
|
@@ -1079,7 +1079,12 @@ def lint(
|
|
|
1079
1079
|
issues.append(_issue("S001", script_path or script.source, 0, "script is empty"))
|
|
1080
1080
|
return issues
|
|
1081
1081
|
|
|
1082
|
-
|
|
1082
|
+
if script_path:
|
|
1083
|
+
script_dir = Path(script_path).resolve().parent
|
|
1084
|
+
elif base_dir:
|
|
1085
|
+
script_dir = Path(base_dir)
|
|
1086
|
+
else:
|
|
1087
|
+
script_dir = None
|
|
1083
1088
|
script_blocks = _collect_script_blocks(script)
|
|
1084
1089
|
|
|
1085
1090
|
# Pass 1: collect all variable definitions
|
execsql/cli/run.py
CHANGED
|
@@ -23,6 +23,7 @@ from execsql.cli.help import _console, _err_console
|
|
|
23
23
|
from execsql.config import ConfigData, StatObj
|
|
24
24
|
from execsql.exceptions import ConfigError, ErrInfo
|
|
25
25
|
from execsql.script import SubVarSet, current_script_line, substitute_vars
|
|
26
|
+
from execsql.script.preparation import prepare_script
|
|
26
27
|
from execsql.utils.fileio import FileWriter, Logger, filewriter_end
|
|
27
28
|
from execsql.utils.gui import gui_connect, gui_console_isrunning, gui_console_off, gui_console_on, gui_console_wait_user
|
|
28
29
|
|
|
@@ -735,6 +736,8 @@ def _run(
|
|
|
735
736
|
for name, value in named_vars or ():
|
|
736
737
|
_state.subvars.add_substitution(name, value)
|
|
737
738
|
_ast_tree = _load_script(command, script_name, conf.script_encoding)
|
|
739
|
+
if _ast_tree is not None:
|
|
740
|
+
_ast_tree = prepare_script(_ast_tree, conf.include_req, conf.include_opt)
|
|
738
741
|
_print_dry_run(_ast_tree)
|
|
739
742
|
raise SystemExit(0)
|
|
740
743
|
|
|
@@ -795,6 +798,8 @@ def _run(
|
|
|
795
798
|
# Load the SQL script (--dry-run / --ping already exited above)
|
|
796
799
|
# ------------------------------------------------------------------
|
|
797
800
|
_ast_tree = _load_script(command, script_name, conf.script_encoding)
|
|
801
|
+
if _ast_tree is not None:
|
|
802
|
+
_ast_tree = prepare_script(_ast_tree, conf.include_req, conf.include_opt)
|
|
798
803
|
|
|
799
804
|
# ------------------------------------------------------------------
|
|
800
805
|
# Start GUI console if requested
|
execsql/config.py
CHANGED
|
@@ -23,6 +23,7 @@ from configparser import ConfigParser
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
from typing import Protocol
|
|
25
25
|
|
|
26
|
+
from execsql.utils.spreadsheet import INPUT_LIMIT_DEFAULTS
|
|
26
27
|
from execsql.utils.paths import is_network_path
|
|
27
28
|
from execsql.exceptions import ConfigError, ConfigFileError
|
|
28
29
|
from execsql.utils.crypto import Encrypt
|
|
@@ -296,6 +297,10 @@ class ConfigData:
|
|
|
296
297
|
("ignore", "replace", "xmlcharrefreplace", "backslashreplace"),
|
|
297
298
|
)
|
|
298
299
|
# --- [input] ---
|
|
300
|
+
for name in INPUT_LIMIT_DEFAULTS:
|
|
301
|
+
self._get_int(cp, self._INPUT_SECTION, name, name)
|
|
302
|
+
if cp.has_option(self._INPUT_SECTION, name) and getattr(self, name) < 0:
|
|
303
|
+
raise ConfigError(f"Invalid {name}: {getattr(self, name)}; must be >= 0.")
|
|
299
304
|
self._get_int(cp, self._INPUT_SECTION, "max_int", "max_int")
|
|
300
305
|
self._get_bool(cp, self._INPUT_SECTION, "boolean_int", "boolean_int")
|
|
301
306
|
self._get_bool(cp, self._INPUT_SECTION, "boolean_words", "boolean_words")
|
|
@@ -500,6 +505,8 @@ class ConfigData:
|
|
|
500
505
|
self.import_encoding = "utf8"
|
|
501
506
|
self.enc_err_disposition: str | None = None
|
|
502
507
|
self.import_common_cols_only = False
|
|
508
|
+
for name, default in INPUT_LIMIT_DEFAULTS.items():
|
|
509
|
+
setattr(self, name, default)
|
|
503
510
|
self.max_int = 2147483647
|
|
504
511
|
self.boolean_int = True
|
|
505
512
|
self.boolean_words = False
|
|
@@ -77,6 +77,21 @@
|
|
|
77
77
|
[input]
|
|
78
78
|
# Settings for handling imported data.
|
|
79
79
|
|
|
80
|
+
# ODS archive limits, checked before parsing. XML total is in MiB; the
|
|
81
|
+
# compression ratio applies per archive member. 0 disables a limit.
|
|
82
|
+
#ods_max_xml_mb=100
|
|
83
|
+
#ods_max_compression_ratio=100
|
|
84
|
+
|
|
85
|
+
# Materialized spreadsheet limits (currently enforced for ODS imports).
|
|
86
|
+
# Rows include the header after SKIP; cells include rectangular padding.
|
|
87
|
+
# Interior blanks count; trailing empty filler does not. Text counts include
|
|
88
|
+
# repeated/spanned copies. 0 disables an individual limit; values are never truncated.
|
|
89
|
+
#import_max_rows=1048576
|
|
90
|
+
#import_max_columns=16384
|
|
91
|
+
#import_max_cells=10000000
|
|
92
|
+
#import_max_cell_chars=1048576
|
|
93
|
+
#import_max_text_chars=268435456
|
|
94
|
+
|
|
80
95
|
# MS Access: create columns for decimal data as NUMERIC(precision,scale)
|
|
81
96
|
# (Yes) or as DOUBLE (No) when IMPORT or COPY creates a table.
|
|
82
97
|
# Values: Yes or No. Default: No.
|
|
@@ -199,6 +214,9 @@
|
|
|
199
214
|
#css_styles=
|
|
200
215
|
|
|
201
216
|
# Seconds to keep retrying when an output file has an access conflict.
|
|
217
|
+
# Background-writer waits, including blocked writes/closes, end after this
|
|
218
|
+
# interval plus 5 seconds (at most 10 seconds on a halt). On timeout, queued
|
|
219
|
+
# output is discarded; the one I/O call already in progress may finish later.
|
|
202
220
|
# Value: A positive non-zero number. Default: 600.
|
|
203
221
|
#outfile_open_timeout=600
|
|
204
222
|
|
|
@@ -305,6 +323,7 @@
|
|
|
305
323
|
# Whether to allow the SYSTEM_CMD (SHELL) metacommand.
|
|
306
324
|
# Set to No to prevent scripts from executing OS commands.
|
|
307
325
|
# Also controllable via --no-system-cmd CLI flag.
|
|
326
|
+
# Library API: omitted/None inherits this value; an explicit boolean overrides it.
|
|
308
327
|
# Values: Yes or No. Default: Yes.
|
|
309
328
|
#allow_system_cmd=Yes
|
|
310
329
|
|
|
@@ -317,12 +336,14 @@
|
|
|
317
336
|
# Whether to allow the RM_FILE metacommand (which deletes a file).
|
|
318
337
|
# Set to No to prevent scripts from deleting files.
|
|
319
338
|
# Also controllable via --no-rm-file CLI flag.
|
|
339
|
+
# Library API: omitted/None inherits this value; an explicit boolean overrides it.
|
|
320
340
|
# Values: Yes or No. Default: Yes.
|
|
321
341
|
#allow_rm_file=Yes
|
|
322
342
|
|
|
323
343
|
# Whether to allow the SERVE metacommand (which copies a file to stdout
|
|
324
344
|
# with HTTP headers, for CGI use). Set to No to disable.
|
|
325
345
|
# Also controllable via --no-serve CLI flag.
|
|
346
|
+
# Library API: omitted/None inherits this value; an explicit boolean overrides it.
|
|
326
347
|
# Values: Yes or No. Default: Yes.
|
|
327
348
|
#allow_serve=Yes
|
|
328
349
|
|
execsql/db/base.py
CHANGED
|
@@ -24,7 +24,8 @@ from collections.abc import Callable, Generator, Iterator
|
|
|
24
24
|
|
|
25
25
|
from execsql.exceptions import ErrInfo
|
|
26
26
|
from execsql.utils.errors import exception_desc
|
|
27
|
-
from execsql.db.target_types import ColumnKinds, existing_table_converter, kind_for_python_type
|
|
27
|
+
from execsql.db.target_types import ColumnKinds, ColumnMetadata, existing_table_converter, kind_for_python_type
|
|
28
|
+
from execsql.types import DT_Decimal
|
|
28
29
|
from execsql.db.tiers import SupportTier, announce_tier
|
|
29
30
|
from execsql.types import DataType
|
|
30
31
|
import execsql.state as _state
|
|
@@ -285,6 +286,9 @@ class Database(ABC):
|
|
|
285
286
|
#: (an ODBC DSN without transactions): nothing can be held back.
|
|
286
287
|
driver_autocommits: bool = False
|
|
287
288
|
|
|
289
|
+
#: Native Decimal parameter capacity for exact binding; None leaves it unchanged.
|
|
290
|
+
decimal_binding_precision: int | None = None
|
|
291
|
+
|
|
288
292
|
def holding(self) -> bool:
|
|
289
293
|
"""Whether statements on this connection wait for a commit.
|
|
290
294
|
|
|
@@ -372,13 +376,15 @@ class Database(ABC):
|
|
|
372
376
|
try:
|
|
373
377
|
self.conn.rollback()
|
|
374
378
|
except Exception as e:
|
|
375
|
-
# Best-effort; connection may already be closed.
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
379
|
+
# Best-effort; connection may already be closed.
|
|
380
|
+
self._log_failed_rollback(e)
|
|
381
|
+
|
|
382
|
+
def _log_failed_rollback(self, e: Exception) -> None:
|
|
383
|
+
"""Log a rollback that failed, so a cascading failure isn't invisible in a CI / cron log."""
|
|
384
|
+
if _state.exec_log is not None:
|
|
385
|
+
_state.exec_log.log_status_info(
|
|
386
|
+
f"Rollback failed on {self.__class__.__name__}: {e!r}",
|
|
387
|
+
)
|
|
382
388
|
|
|
383
389
|
def needs_explicit_commit_after_ddl(self) -> bool:
|
|
384
390
|
"""Return True if this adapter's driver does NOT auto-commit DDL.
|
|
@@ -622,16 +628,24 @@ class Database(ABC):
|
|
|
622
628
|
*sq_name* is schema-qualified and quoted, as for an ``INSERT``, and
|
|
623
629
|
is looked up the way the ``INSERT`` finds it. Keys are lower-cased
|
|
624
630
|
column names; a column with no fitting execsql type maps to ``None``.
|
|
625
|
-
|
|
626
|
-
|
|
631
|
+
The dict-compatible result also carries known decimal scales in
|
|
632
|
+
``ColumnMetadata.decimal_scales``. Legacy plain-dict providers remain
|
|
633
|
+
supported with unknown scales. This default reads the driver's empty
|
|
634
|
+
query description; adapters with exact catalog types override it.
|
|
627
635
|
"""
|
|
628
636
|
description, _rows = self._column_type_query(sq_name, f"select * from {sq_name} where 1=0;")
|
|
629
|
-
return {d[0].lower(): self.
|
|
637
|
+
return ColumnMetadata({d[0].lower(): self._described_column(d) for d in description})
|
|
630
638
|
|
|
631
639
|
def _described_kind(self, description: Any) -> type[DataType] | None:
|
|
632
640
|
"""The execsql type for one column of a cursor description (a DB-API 7-item sequence)."""
|
|
633
641
|
return kind_for_python_type(description[1], description[5])
|
|
634
642
|
|
|
643
|
+
def _described_column(self, description: Any) -> tuple[type[DataType] | None, int | None]:
|
|
644
|
+
"""Read a kind and decimal scale from a DB-API description."""
|
|
645
|
+
kind = self._described_kind(description)
|
|
646
|
+
scale = description[5]
|
|
647
|
+
return kind, scale if kind is DT_Decimal and type(scale) is int else None
|
|
648
|
+
|
|
635
649
|
def _column_type_query(self, sq_name: str, sql: str, params: list | None = None) -> tuple[Any, list]:
|
|
636
650
|
"""Run *sql* for :meth:`column_kinds` of *sq_name*; return the cursor description and the rows."""
|
|
637
651
|
with self._cursor() as curs:
|
|
@@ -744,7 +758,7 @@ class Database(ABC):
|
|
|
744
758
|
|
|
745
759
|
*rowsource* must be a generator yielding lists of values in column order.
|
|
746
760
|
*tablespec_src* is a zero-argument callable that returns the table's type specification.
|
|
747
|
-
|
|
761
|
+
Only selected columns are converted to the type scanned for each column; with
|
|
748
762
|
*existing_table* (the table was there before the IMPORT or COPY), the
|
|
749
763
|
type of the column it goes into decides instead (see
|
|
750
764
|
``db.target_types.existing_table_converter``).
|
|
@@ -770,9 +784,18 @@ class Database(ABC):
|
|
|
770
784
|
converters = [self._scanned_type_converter(col.column_type()[1]) for col in tablespec.cols]
|
|
771
785
|
if existing_table:
|
|
772
786
|
kinds = self.column_kinds(sq_name)
|
|
787
|
+
scales = kinds.decimal_scales if isinstance(kinds, ColumnMetadata) else {}
|
|
773
788
|
converters = [
|
|
774
|
-
existing_table_converter(
|
|
775
|
-
|
|
789
|
+
existing_table_converter(
|
|
790
|
+
kinds.get(col.name.lower()),
|
|
791
|
+
col.column_type()[1],
|
|
792
|
+
convert,
|
|
793
|
+
col.name,
|
|
794
|
+
sq_name,
|
|
795
|
+
decimal_binding_precision=self.decimal_binding_precision,
|
|
796
|
+
decimal_scale=scales.get(col.name.lower()) if included else None,
|
|
797
|
+
)
|
|
798
|
+
for col, convert, included in zip(tablespec.cols, converters, incl_col)
|
|
776
799
|
]
|
|
777
800
|
# Construct INSERT statement.
|
|
778
801
|
columns = [self.type.quoted(col) for col in sel_cols]
|
|
@@ -871,11 +894,7 @@ class Database(ABC):
|
|
|
871
894
|
line[i] = None
|
|
872
895
|
# Trailing values left off the line are NULL.
|
|
873
896
|
line = list(line) + [None] * (len(ts_colnames) - len(line))
|
|
874
|
-
|
|
875
|
-
row = []
|
|
876
|
-
for i, v in enumerate(lt):
|
|
877
|
-
if incl_col[i]:
|
|
878
|
-
row.append(v)
|
|
897
|
+
row = [converters[i](val) for i, val in enumerate(line) if incl_col[i]]
|
|
879
898
|
add_line = True
|
|
880
899
|
if not _state.conf.empty_rows:
|
|
881
900
|
add_line = not all(c is None for c in row)
|
|
@@ -1076,8 +1095,8 @@ class DatabasePool:
|
|
|
1076
1095
|
return self.pool[self.initial_db]
|
|
1077
1096
|
|
|
1078
1097
|
def aliased_as(self, db_alias: str) -> Database:
|
|
1079
|
-
"""Return the ``Database`` registered under *db_alias
|
|
1080
|
-
return self.pool[db_alias]
|
|
1098
|
+
"""Return the ``Database`` registered under *db_alias* (any letter case)."""
|
|
1099
|
+
return self.pool[db_alias.lower()]
|
|
1081
1100
|
|
|
1082
1101
|
def make_current(self, db_alias: str) -> None:
|
|
1083
1102
|
"""Set the active database to *db_alias*; raises ``ErrInfo`` if the alias is unknown."""
|
|
@@ -1090,7 +1109,8 @@ class DatabasePool:
|
|
|
1090
1109
|
self.current_db = db_alias
|
|
1091
1110
|
|
|
1092
1111
|
def disconnect(self, alias: str) -> None:
|
|
1093
|
-
"""Close and remove the connection registered under *alias* from the pool."""
|
|
1112
|
+
"""Close and remove the connection registered under *alias* (any letter case) from the pool."""
|
|
1113
|
+
alias = alias.lower()
|
|
1094
1114
|
if alias == self.current_db or (alias == "initial" and "initial" in self.pool):
|
|
1095
1115
|
raise ErrInfo(
|
|
1096
1116
|
type="error",
|
|
@@ -1100,9 +1120,17 @@ class DatabasePool:
|
|
|
1100
1120
|
self.pool[alias].close()
|
|
1101
1121
|
del self.pool[alias]
|
|
1102
1122
|
|
|
1103
|
-
def closeall(self) -> None:
|
|
1104
|
-
"""Roll back and close
|
|
1123
|
+
def closeall(self, *, except_db: Database | None = None) -> None:
|
|
1124
|
+
"""Roll back and close the pool's connections except *except_db*.
|
|
1125
|
+
|
|
1126
|
+
The exception is matched by object identity under every alias and is
|
|
1127
|
+
neither rolled back nor closed. All aliases are removed, including
|
|
1128
|
+
aliases of the excluded caller-owned connection.
|
|
1129
|
+
Rollback follows the pool's ``do_rollback`` setting.
|
|
1130
|
+
"""
|
|
1105
1131
|
for alias, db in self.pool.items():
|
|
1132
|
+
if db is except_db:
|
|
1133
|
+
continue
|
|
1106
1134
|
nm = db.name()
|
|
1107
1135
|
try:
|
|
1108
1136
|
if self.do_rollback:
|
execsql/db/duckdb.py
CHANGED
|
@@ -9,6 +9,7 @@ on the CLI.
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import importlib.util
|
|
12
|
+
from decimal import Decimal
|
|
12
13
|
import re
|
|
13
14
|
import sys
|
|
14
15
|
from collections import deque
|
|
@@ -18,8 +19,8 @@ from pathlib import Path
|
|
|
18
19
|
from typing import Any
|
|
19
20
|
|
|
20
21
|
from execsql.db.base import Database, statement_start
|
|
21
|
-
from execsql.db.target_types import kind_for_type_name
|
|
22
|
-
from execsql.types import DataType
|
|
22
|
+
from execsql.db.target_types import kind_for_type_name, kind_and_scale_for_type_name, decimal_parameter
|
|
23
|
+
from execsql.types import DataType, DT_Decimal
|
|
23
24
|
from execsql.exceptions import ErrInfo
|
|
24
25
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
25
26
|
import execsql.state as _state
|
|
@@ -172,6 +173,9 @@ class DuckDBDatabase(Database):
|
|
|
172
173
|
and sees the run's temporary tables.
|
|
173
174
|
"""
|
|
174
175
|
|
|
176
|
+
#: Keep Decimal parameters exact across exponent/width conversion in drivers.
|
|
177
|
+
decimal_binding_precision = 38
|
|
178
|
+
|
|
175
179
|
#: A transaction this adapter or a script's own BEGIN opened, not yet ended.
|
|
176
180
|
in_transaction: bool = False
|
|
177
181
|
#: The cursor whose result the connection holds, if any.
|
|
@@ -266,13 +270,29 @@ class DuckDBDatabase(Database):
|
|
|
266
270
|
self.in_transaction = False
|
|
267
271
|
|
|
268
272
|
def rollback(self) -> None:
|
|
269
|
-
"""Roll back the open transaction.
|
|
273
|
+
"""Roll back the open transaction, if one is open.
|
|
274
|
+
|
|
275
|
+
The driver decides whether one is: ``in_transaction`` misses a
|
|
276
|
+
transaction begun partway through a statement. Outside a transaction
|
|
277
|
+
DuckDB has committed each statement as it ran, and its "no transaction
|
|
278
|
+
is active" error is not logged as a failed rollback.
|
|
279
|
+
"""
|
|
280
|
+
import duckdb
|
|
281
|
+
|
|
270
282
|
try:
|
|
271
283
|
self._take_connection()
|
|
272
284
|
except Exception: # best effort, as the rollback is: a failed transaction may not give up its rows
|
|
273
285
|
self._holder = None
|
|
274
|
-
super().rollback()
|
|
275
286
|
self.in_transaction = False
|
|
287
|
+
if self.conn is None:
|
|
288
|
+
return
|
|
289
|
+
try:
|
|
290
|
+
self.conn.rollback()
|
|
291
|
+
except duckdb.TransactionException as e:
|
|
292
|
+
if "no transaction is active" not in str(e):
|
|
293
|
+
self._log_failed_rollback(e)
|
|
294
|
+
except Exception as e:
|
|
295
|
+
self._log_failed_rollback(e)
|
|
276
296
|
|
|
277
297
|
def close(self) -> None:
|
|
278
298
|
"""Close the connection; an open transaction is rolled back."""
|
|
@@ -333,6 +353,25 @@ class DuckDBDatabase(Database):
|
|
|
333
353
|
"""DuckDB describes a column by its type's name (``DECIMAL(18,3)``, ``TIMESTAMP WITH TIME ZONE``)."""
|
|
334
354
|
return kind_for_type_name(str(description[1]))
|
|
335
355
|
|
|
356
|
+
def _scanned_type_converter(self, data_type: type[DataType]) -> Callable[[Any], Any]:
|
|
357
|
+
convert = super()._scanned_type_converter(data_type)
|
|
358
|
+
if data_type is not DT_Decimal:
|
|
359
|
+
return convert
|
|
360
|
+
|
|
361
|
+
def bind(value: Any) -> Any:
|
|
362
|
+
result = convert(value)
|
|
363
|
+
if isinstance(result, Decimal) and result.is_finite():
|
|
364
|
+
exponent = result.as_tuple().exponent
|
|
365
|
+
if isinstance(exponent, int) and exponent > 0:
|
|
366
|
+
return decimal_parameter(result, self.decimal_binding_precision, "value", "target table")
|
|
367
|
+
return result
|
|
368
|
+
|
|
369
|
+
return bind
|
|
370
|
+
|
|
371
|
+
def _described_column(self, description: Any) -> tuple[type[DataType] | None, int | None]:
|
|
372
|
+
_kind, scale = kind_and_scale_for_type_name(str(description[1]))
|
|
373
|
+
return self._described_kind(description), scale
|
|
374
|
+
|
|
336
375
|
def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
|
|
337
376
|
"""Return True if the named view exists in the DuckDB database."""
|
|
338
377
|
# DuckDB information_schema has no 'views' table; views are listed in 'tables'
|
execsql/db/firebird.py
CHANGED
|
@@ -8,7 +8,10 @@ via the ``firebird-driver`` package. Corresponds to ``-t f`` on the CLI.
|
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
10
|
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
11
13
|
from execsql.db.base import Database
|
|
14
|
+
from execsql.types import DataType, DT_Decimal
|
|
12
15
|
from execsql.db.tiers import SupportTier
|
|
13
16
|
from execsql.exceptions import ErrInfo
|
|
14
17
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
@@ -32,6 +35,11 @@ class FirebirdDatabase(Database):
|
|
|
32
35
|
support_tier = SupportTier.BEST_EFFORT
|
|
33
36
|
support_tier_name = "Firebird"
|
|
34
37
|
|
|
38
|
+
def _described_column(self, description: Any) -> tuple[type[DataType] | None, int | None]:
|
|
39
|
+
kind, scale = super()._described_column(description)
|
|
40
|
+
# firebird-driver exposes the engine's exponent, the negative SQL scale.
|
|
41
|
+
return kind, -scale if kind is DT_Decimal and scale is not None else None
|
|
42
|
+
|
|
35
43
|
def __init__(
|
|
36
44
|
self,
|
|
37
45
|
server_name: str,
|