execsql2 2.35.0__py3-none-any.whl → 2.38.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +111 -53
- execsql/cli/commands/run.py +2 -2
- execsql/cli/lint.py +60 -7
- execsql/cli/run.py +10 -10
- execsql/config.py +7 -0
- execsql/data/execsql.conf.template +31 -7
- execsql/db/base.py +64 -26
- execsql/db/duckdb.py +51 -5
- execsql/db/firebird.py +8 -0
- execsql/db/mysql.py +8 -3
- execsql/db/oracle.py +5 -0
- execsql/db/postgres.py +2 -2
- execsql/db/sqlite.py +117 -10
- execsql/db/sqlserver.py +11 -7
- execsql/db/target_types.py +151 -4
- execsql/debug/repl.py +4 -3
- execsql/exceptions.py +14 -0
- execsql/exporters/html.py +26 -42
- execsql/exporters/ods.py +93 -21
- execsql/exporters/sqlite.py +82 -47
- execsql/format.py +12 -4
- execsql/gui/desktop.py +2 -1
- execsql/gui/tui.py +5 -2
- execsql/importers/base.py +9 -7
- execsql/importers/json.py +37 -13
- execsql/importers/ods.py +22 -2
- execsql/lsp/document.py +6 -1
- execsql/lsp/hover.py +2 -1
- execsql/metacommands/conditions.py +3 -1
- execsql/metacommands/connect.py +28 -12
- execsql/metacommands/dispatch.py +10 -0
- execsql/metacommands/io.py +1 -0
- execsql/metacommands/io_export.py +1 -2
- execsql/metacommands/io_fileops.py +32 -93
- execsql/metacommands/io_import.py +26 -8
- execsql/metacommands/io_write.py +3 -1
- execsql/metacommands/reference.py +7 -0
- execsql/metacommands/script_ext.py +13 -10
- execsql/metacommands/system.py +13 -6
- execsql/parser.py +14 -37
- execsql/script/ast.py +21 -0
- execsql/script/control.py +73 -3
- execsql/script/executor.py +503 -227
- execsql/script/parser.py +102 -52
- execsql/script/preparation.py +42 -0
- execsql/state.py +23 -0
- execsql/types.py +6 -5
- execsql/utils/errors.py +24 -6
- execsql/utils/fileio.py +342 -134
- execsql/utils/sigterm.py +154 -0
- execsql/utils/spreadsheet.py +74 -0
- execsql/utils/strings.py +13 -7
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/README.md +5 -5
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/example_config_prompt.sql +2 -2
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/METADATA +32 -119
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/RECORD +71 -68
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/WHEEL +1 -1
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/licenses/NOTICE +0 -0
execsql/api.py
CHANGED
|
@@ -39,7 +39,7 @@ from typing import Any, cast
|
|
|
39
39
|
|
|
40
40
|
from execsql.cli.dsn import _parse_connection_string
|
|
41
41
|
from execsql.config import ConfigData, StatObj, WriteHooks
|
|
42
|
-
from execsql.exceptions import ErrInfo
|
|
42
|
+
from execsql.exceptions import ErrInfo, ScriptExit
|
|
43
43
|
from execsql.state import RuntimeContext, active_context
|
|
44
44
|
|
|
45
45
|
__all__ = ["run", "ScriptResult", "ScriptError"]
|
|
@@ -224,6 +224,11 @@ class _LibraryConfig:
|
|
|
224
224
|
self.email_format = "plain"
|
|
225
225
|
self.email_css: str | None = None
|
|
226
226
|
|
|
227
|
+
# Default capabilities when no configuration file is supplied.
|
|
228
|
+
self.allow_system_cmd = True
|
|
229
|
+
self.allow_rm_file = True
|
|
230
|
+
self.allow_serve = True
|
|
231
|
+
|
|
227
232
|
# Includes
|
|
228
233
|
self.include_req: list = []
|
|
229
234
|
self.include_opt: list = []
|
|
@@ -340,9 +345,9 @@ def run(
|
|
|
340
345
|
encoding: str = "utf-8",
|
|
341
346
|
halt_on_error: bool = True,
|
|
342
347
|
new_db: bool = False,
|
|
343
|
-
allow_system_cmd: bool =
|
|
344
|
-
allow_rm_file: bool =
|
|
345
|
-
allow_serve: bool =
|
|
348
|
+
allow_system_cmd: bool | None = None,
|
|
349
|
+
allow_rm_file: bool | None = None,
|
|
350
|
+
allow_serve: bool | None = None,
|
|
346
351
|
) -> ScriptResult:
|
|
347
352
|
"""Execute a SQL script and return the result.
|
|
348
353
|
|
|
@@ -356,10 +361,26 @@ def run(
|
|
|
356
361
|
``"postgresql://user:pass@host/db"``).
|
|
357
362
|
connection: A pre-existing :class:`~execsql.db.base.Database`
|
|
358
363
|
instance. ``run()`` will NOT close this connection on exit.
|
|
364
|
+
Connections opened by the script with CONNECT are run-owned:
|
|
365
|
+
their pending work is rolled back and they are closed on exit,
|
|
366
|
+
including under AUTOCOMMIT OFF. Commit their work in the script
|
|
367
|
+
before it ends if it must be kept.
|
|
368
|
+
What an interrupted SQL statement, metacommand or ``END BATCH``
|
|
369
|
+
did is rolled back before the interrupt is re-raised: an SQL
|
|
370
|
+
statement rolls back its connection's open transaction, and a
|
|
371
|
+
metacommand everything not yet committed on each connection
|
|
372
|
+
with autocommit on (under AUTOCOMMIT OFF, a metacommand's writes
|
|
373
|
+
stay with what the script holds). An interrupt that ends the run
|
|
374
|
+
also rolls back everything else not yet committed on each
|
|
375
|
+
connection with autocommit on. Work the database had already
|
|
376
|
+
committed when the interrupt arrived stays committed. A
|
|
377
|
+
``SystemExit`` from a signal handler is rolled back the same way,
|
|
378
|
+
and the run returns a failed result.
|
|
359
379
|
variables: Substitution variables as ``{"NAME": "value"}``.
|
|
360
380
|
Keys without a ``$`` prefix get one added automatically, so
|
|
361
381
|
``{"SCHEMA": "public"}`` is referenced as ``!!$SCHEMA!!``.
|
|
362
|
-
config_file: Optional execsql configuration file to load.
|
|
382
|
+
config_file: Optional execsql configuration file to load. Configured
|
|
383
|
+
variables and startup includes apply before the main script runs.
|
|
363
384
|
encoding: Script file encoding (default ``"utf-8"``).
|
|
364
385
|
halt_on_error: If ``True`` (default), stop on the first SQL
|
|
365
386
|
error. If ``False``, capture errors and continue.
|
|
@@ -368,20 +389,26 @@ def run(
|
|
|
368
389
|
allow_system_cmd: If ``False``, the SYSTEM_CMD metacommand is
|
|
369
390
|
disabled and will raise an error if encountered. Matches the
|
|
370
391
|
``--no-system-cmd`` CLI flag and the ``allow_system_cmd``
|
|
371
|
-
config option.
|
|
392
|
+
config option. None (default) inherits the configured value,
|
|
393
|
+
or True without a config file; explicit True overrides a restriction.
|
|
372
394
|
allow_rm_file: If ``False``, the RM_FILE metacommand is disabled
|
|
373
395
|
and will raise an error if encountered. Matches the
|
|
374
396
|
``--no-rm-file`` CLI flag and the ``allow_rm_file`` config
|
|
375
|
-
option.
|
|
397
|
+
option. None (default) inherits the configured value;
|
|
398
|
+
an explicit boolean overrides it.
|
|
376
399
|
allow_serve: If ``False``, the SERVE metacommand is disabled and
|
|
377
400
|
will raise an error if encountered. Matches the ``--no-serve``
|
|
378
|
-
CLI flag and the ``allow_serve`` config option.
|
|
401
|
+
CLI flag and the ``allow_serve`` config option. None (default)
|
|
402
|
+
inherits the configured value; an explicit boolean overrides it.
|
|
379
403
|
|
|
380
404
|
Returns:
|
|
381
405
|
A :class:`ScriptResult` with execution outcome, timing, errors,
|
|
382
406
|
and final variable state. A script that cannot be read, a bad
|
|
383
407
|
*config_file*, and a *dsn* that cannot be parsed or opened all
|
|
384
|
-
return ``success=False`` rather than raising.
|
|
408
|
+
return ``success=False`` rather than raising. A blocked background
|
|
409
|
+
output writer adds an error with source ``<output>`` and is abandoned
|
|
410
|
+
at its flush deadline. Its current I/O may finish later; queued
|
|
411
|
+
output is discarded and no further opens or writes are started.
|
|
385
412
|
|
|
386
413
|
Raises:
|
|
387
414
|
ValueError: If the argument combination is invalid (e.g. both
|
|
@@ -420,27 +447,6 @@ def run(
|
|
|
420
447
|
# ------------------------------------------------------------------
|
|
421
448
|
ctx = RuntimeContext()
|
|
422
449
|
|
|
423
|
-
# Configuration
|
|
424
|
-
conf_overrides: dict[str, Any] = {"script_encoding": encoding}
|
|
425
|
-
conf: Any
|
|
426
|
-
if config_file is not None:
|
|
427
|
-
# Load a real ConfigData with the explicit config file
|
|
428
|
-
from execsql.script.variables import SubVarSet
|
|
429
|
-
|
|
430
|
-
if not Path(config_file).is_file():
|
|
431
|
-
return _failed(f"Config file {str(config_file)!r} does not exist.", "<config>")
|
|
432
|
-
temp_subvars = SubVarSet()
|
|
433
|
-
script_dir = str(Path(script).resolve().parent) if script else os.getcwd()
|
|
434
|
-
try:
|
|
435
|
-
conf = ConfigData(script_dir, temp_subvars, config_file=str(config_file))
|
|
436
|
-
except Exception as exc:
|
|
437
|
-
return _failed(_error_text(exc), "<config>")
|
|
438
|
-
# Apply any overrides
|
|
439
|
-
for k, v in conf_overrides.items():
|
|
440
|
-
setattr(conf, k, v)
|
|
441
|
-
else:
|
|
442
|
-
conf = _LibraryConfig(**conf_overrides)
|
|
443
|
-
|
|
444
450
|
# Substitution variables
|
|
445
451
|
from execsql.script.variables import SubVarSet
|
|
446
452
|
|
|
@@ -461,6 +467,21 @@ def run(
|
|
|
461
467
|
|
|
462
468
|
subvars.add_substitution("$HOSTNAME", platform.node())
|
|
463
469
|
|
|
470
|
+
# Configuration
|
|
471
|
+
conf: Any
|
|
472
|
+
if config_file is not None:
|
|
473
|
+
# Load a real ConfigData with the explicit config file
|
|
474
|
+
if not Path(config_file).is_file():
|
|
475
|
+
return _failed(f"Config file {str(config_file)!r} does not exist.", "<config>")
|
|
476
|
+
script_dir = str(Path(script).resolve().parent) if script else os.getcwd()
|
|
477
|
+
try:
|
|
478
|
+
conf = ConfigData(script_dir, subvars, config_file=str(config_file))
|
|
479
|
+
except Exception as exc:
|
|
480
|
+
return _failed(_error_text(exc), "<config>")
|
|
481
|
+
conf.script_encoding = encoding
|
|
482
|
+
else:
|
|
483
|
+
conf = _LibraryConfig(script_encoding=encoding)
|
|
484
|
+
|
|
464
485
|
# User-supplied variables
|
|
465
486
|
if variables:
|
|
466
487
|
for name, value in variables.items():
|
|
@@ -476,11 +497,18 @@ def run(
|
|
|
476
497
|
ctx.subvars = subvars
|
|
477
498
|
ctx.status = StatObj()
|
|
478
499
|
ctx.status.halt_on_err = halt_on_error
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
500
|
+
if allow_system_cmd is not None:
|
|
501
|
+
conf.allow_system_cmd = allow_system_cmd
|
|
502
|
+
if allow_rm_file is not None:
|
|
503
|
+
conf.allow_rm_file = allow_rm_file
|
|
504
|
+
if allow_serve is not None:
|
|
505
|
+
conf.allow_serve = allow_serve
|
|
482
506
|
ctx.conf = conf
|
|
483
507
|
|
|
508
|
+
from execsql.script.preparation import prepare_script
|
|
509
|
+
|
|
510
|
+
tree = prepare_script(tree, conf.include_req, conf.include_opt)
|
|
511
|
+
|
|
484
512
|
# Capture output to a buffer (suppress stdout/stderr)
|
|
485
513
|
stdout_buf = io.StringIO()
|
|
486
514
|
stderr_buf = io.StringIO()
|
|
@@ -521,7 +549,7 @@ def run(
|
|
|
521
549
|
# ------------------------------------------------------------------
|
|
522
550
|
# Execute
|
|
523
551
|
# ------------------------------------------------------------------
|
|
524
|
-
from execsql.script.executor import execute
|
|
552
|
+
from execsql.script.executor import _roll_back_after_interrupt, execute
|
|
525
553
|
|
|
526
554
|
errors: list[ScriptError] = []
|
|
527
555
|
t0 = time.perf_counter()
|
|
@@ -534,19 +562,25 @@ def run(
|
|
|
534
562
|
try:
|
|
535
563
|
execute(tree, ctx=ctx)
|
|
536
564
|
except SystemExit as exc:
|
|
537
|
-
# exit_now()
|
|
538
|
-
|
|
565
|
+
# exit_now() raises ScriptExit; any other SystemExit (a signal
|
|
566
|
+
# handler's) interrupted the run. Either is reported as an error.
|
|
567
|
+
if not isinstance(exc, ScriptExit):
|
|
568
|
+
_roll_back_after_interrupt(ctx)
|
|
569
|
+
_capture_errors(ctx, errors, exc.code, interrupted=not isinstance(exc, ScriptExit))
|
|
539
570
|
except ErrInfo as exc:
|
|
540
571
|
errors.append(
|
|
541
572
|
ScriptError(
|
|
542
573
|
message=exc.errmsg(),
|
|
543
|
-
source=_last_source(ctx),
|
|
544
|
-
line=_last_line(ctx),
|
|
574
|
+
source=_last_source(ctx, exc),
|
|
575
|
+
line=_last_line(ctx, exc),
|
|
545
576
|
sql=getattr(exc, "command_text", None),
|
|
546
577
|
),
|
|
547
578
|
)
|
|
548
579
|
except Exception as exc:
|
|
549
580
|
errors.append(ScriptError(message=str(exc), source="<runtime>"))
|
|
581
|
+
except BaseException:
|
|
582
|
+
_roll_back_after_interrupt(ctx)
|
|
583
|
+
raise
|
|
550
584
|
|
|
551
585
|
# Non-halting SQL errors (halt_on_error=False) are recorded in
|
|
552
586
|
# status.error_history as execution continues; a halting error never
|
|
@@ -562,13 +596,15 @@ def run(
|
|
|
562
596
|
|
|
563
597
|
# Flush and close every file the script wrote, so the caller can read
|
|
564
598
|
# them the moment run() returns. Output to a file that could not be
|
|
565
|
-
# opened is reported as an error.
|
|
599
|
+
# opened is reported as an error. A blocked writer is abandoned;
|
|
600
|
+
# healthy writers are stopped below.
|
|
566
601
|
from execsql.utils.fileio import filewriter_close_all_after_write
|
|
567
602
|
|
|
568
603
|
try:
|
|
569
604
|
filewriter_close_all_after_write()
|
|
570
605
|
except ErrInfo as exc:
|
|
571
|
-
|
|
606
|
+
if ctx.halt_output_error is not exc:
|
|
607
|
+
errors.append(ScriptError(message=exc.errmsg(), source="<output>"))
|
|
572
608
|
except Exception:
|
|
573
609
|
pass # Best-effort: a failed flush must not mask a script error.
|
|
574
610
|
|
|
@@ -589,9 +625,12 @@ def run(
|
|
|
589
625
|
from execsql.utils.fileio import filewriter_end
|
|
590
626
|
|
|
591
627
|
filewriter_end(ctx.filewriter)
|
|
592
|
-
if
|
|
628
|
+
if ctx.dbs is not None:
|
|
593
629
|
try:
|
|
594
|
-
|
|
630
|
+
if owns_connection:
|
|
631
|
+
ctx.dbs.closeall()
|
|
632
|
+
else:
|
|
633
|
+
ctx.dbs.closeall(except_db=connection)
|
|
595
634
|
except Exception:
|
|
596
635
|
pass
|
|
597
636
|
if ctx.tempfiles is not None:
|
|
@@ -621,8 +660,8 @@ def _start_filewriter(ctx: RuntimeContext, conf: Any) -> None:
|
|
|
621
660
|
writer, and every entry point in :mod:`execsql.utils.fileio` drops the
|
|
622
661
|
write when none is running. A writer per run keeps concurrent runs from
|
|
623
662
|
sharing open files or reply queues, and a halt in one run (which stops
|
|
624
|
-
its writer) from discarding another run's output.
|
|
625
|
-
|
|
663
|
+
its writer) from discarding another run's output. Healthy writers stop
|
|
664
|
+
before run() returns; a blocked daemon is abandoned with an output error.
|
|
626
665
|
"""
|
|
627
666
|
from execsql.utils.fileio import FileWriter
|
|
628
667
|
|
|
@@ -633,7 +672,13 @@ def _start_filewriter(ctx: RuntimeContext, conf: Any) -> None:
|
|
|
633
672
|
ctx.filewriter.start()
|
|
634
673
|
|
|
635
674
|
|
|
636
|
-
def _capture_errors(
|
|
675
|
+
def _capture_errors(
|
|
676
|
+
ctx: RuntimeContext,
|
|
677
|
+
errors: list[ScriptError],
|
|
678
|
+
exit_code: Any,
|
|
679
|
+
*,
|
|
680
|
+
interrupted: bool = False,
|
|
681
|
+
) -> None:
|
|
637
682
|
"""Record the error that halted the run, after ``exit_now()`` raised ``SystemExit``.
|
|
638
683
|
|
|
639
684
|
``exit_now`` leaves the halting :class:`ErrInfo` on the context, and any
|
|
@@ -641,10 +686,15 @@ def _capture_errors(ctx: RuntimeContext, errors: list[ScriptError], exit_code: A
|
|
|
641
686
|
SQL statement's text is in ``$LAST_ERROR``, which the executor sets just
|
|
642
687
|
before halting; it is reported only when the halt came from SQL, so a
|
|
643
688
|
metacommand error is never paired with an earlier statement. A ``HALT``
|
|
644
|
-
carries no error, only its exit status
|
|
689
|
+
carries no error, only its exit status; nor does a run *interrupted* by a
|
|
690
|
+
``SystemExit`` that execsql did not raise (a signal handler's).
|
|
645
691
|
"""
|
|
646
692
|
err = ctx.halt_error
|
|
647
|
-
if err is None:
|
|
693
|
+
if err is None and interrupted:
|
|
694
|
+
msg = "Script interrupted by SystemExit" + (
|
|
695
|
+
f" with exit status {exit_code}." if type(exit_code) is int else "."
|
|
696
|
+
)
|
|
697
|
+
elif err is None:
|
|
648
698
|
msg = f"Script halted by HALT with exit status {exit_code}."
|
|
649
699
|
else:
|
|
650
700
|
msg = err.errmsg()
|
|
@@ -655,8 +705,8 @@ def _capture_errors(ctx: RuntimeContext, errors: list[ScriptError], exit_code: A
|
|
|
655
705
|
errors.append(
|
|
656
706
|
ScriptError(
|
|
657
707
|
message=msg,
|
|
658
|
-
source=_last_source(ctx),
|
|
659
|
-
line=_last_line(ctx),
|
|
708
|
+
source=_last_source(ctx, err),
|
|
709
|
+
line=_last_line(ctx, err),
|
|
660
710
|
sql=sql,
|
|
661
711
|
),
|
|
662
712
|
)
|
|
@@ -680,16 +730,24 @@ def _error_text(exc: BaseException) -> str:
|
|
|
680
730
|
return exc.errmsg() if isinstance(exc, ErrInfo) else str(exc)
|
|
681
731
|
|
|
682
732
|
|
|
683
|
-
def _last_source(ctx: RuntimeContext) -> str:
|
|
684
|
-
"""
|
|
733
|
+
def _last_source(ctx: RuntimeContext, err: ErrInfo | None = None) -> str:
|
|
734
|
+
"""The source file of *err*, as its report gives it, else of the last executed command.
|
|
735
|
+
|
|
736
|
+
An error can name a line other than the statement on record: an ELSEIF,
|
|
737
|
+
ANDIF or ORIF condition, or a loop condition evaluated after its body.
|
|
738
|
+
"""
|
|
739
|
+
if err is not None and err.script_file and err.script_line_no:
|
|
740
|
+
return err.script_file
|
|
685
741
|
lc = ctx.last_command
|
|
686
742
|
if lc is not None and hasattr(lc, "source"):
|
|
687
743
|
return cast(str, lc.source)
|
|
688
744
|
return "<unknown>"
|
|
689
745
|
|
|
690
746
|
|
|
691
|
-
def _last_line(ctx: RuntimeContext) -> int | None:
|
|
692
|
-
"""
|
|
747
|
+
def _last_line(ctx: RuntimeContext, err: ErrInfo | None = None) -> int | None:
|
|
748
|
+
"""The line of *err*, as its report gives it, else of the last executed command (see :func:`_last_source`)."""
|
|
749
|
+
if err is not None and err.script_file and err.script_line_no:
|
|
750
|
+
return err.script_line_no
|
|
693
751
|
lc = ctx.last_command
|
|
694
752
|
if lc is not None and hasattr(lc, "line_no"):
|
|
695
753
|
return cast(int | None, lc.line_no)
|
execsql/cli/commands/run.py
CHANGED
|
@@ -114,9 +114,9 @@ def main(
|
|
|
114
114
|
"--output-dir",
|
|
115
115
|
metavar="DIR",
|
|
116
116
|
help=(
|
|
117
|
-
"
|
|
117
|
+
"Containment root for EXPORT output files. "
|
|
118
118
|
"Relative paths in EXPORT metacommands are joined to this directory. "
|
|
119
|
-
"Absolute paths
|
|
119
|
+
"Absolute paths must also stay inside it; stdout is exempt. WRITE is not constrained."
|
|
120
120
|
),
|
|
121
121
|
),
|
|
122
122
|
progress: bool = typer.Option(
|
execsql/cli/lint.py
CHANGED
|
@@ -58,6 +58,7 @@ from typing import TYPE_CHECKING, NamedTuple
|
|
|
58
58
|
|
|
59
59
|
from execsql.utils.paths import is_network_path
|
|
60
60
|
from execsql.exceptions import ErrInfo
|
|
61
|
+
from execsql.script.parser import strip_quotes
|
|
61
62
|
from execsql.script.ast import (
|
|
62
63
|
BatchBlock,
|
|
63
64
|
Comment,
|
|
@@ -857,10 +858,13 @@ def _referencing_text(node: Node) -> list[str]:
|
|
|
857
858
|
texts.append(node.command)
|
|
858
859
|
elif isinstance(node, IncludeDirective):
|
|
859
860
|
texts.append(node.target)
|
|
861
|
+
texts.extend(t for t in (node.arguments, node.loop_condition) if t)
|
|
860
862
|
if isinstance(node, IfBlock):
|
|
861
863
|
texts.append(node.condition)
|
|
862
864
|
texts.extend(m.condition for m in node.condition_modifiers)
|
|
863
|
-
|
|
865
|
+
for clause in node.elseif_clauses:
|
|
866
|
+
texts.append(clause.condition)
|
|
867
|
+
texts.extend(m.condition for m in clause.condition_modifiers)
|
|
864
868
|
elif isinstance(node, LoopBlock):
|
|
865
869
|
texts.append(node.condition)
|
|
866
870
|
return texts
|
|
@@ -913,6 +917,24 @@ def _definition_lines(nodes: list[Node], script_dir: Path | None, lines: dict[st
|
|
|
913
917
|
# ---------------------------------------------------------------------------
|
|
914
918
|
|
|
915
919
|
|
|
920
|
+
_RX_FILE_EXISTS = re.compile(r"^\s*FILE_EXISTS\s*\(\s*(?P<path>.+?)\s*\)\s*$", re.I)
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
def _file_exists_guards(condition: str, modifiers: list[ConditionModifier]) -> frozenset[str]:
|
|
924
|
+
"""The files an IF branch runs only if they exist: ``IF (FILE_EXISTS(f))``, with any ``ANDIF (FILE_EXISTS(g))``.
|
|
925
|
+
|
|
926
|
+
An ``ORIF`` lets the branch run without the file, so it guards nothing.
|
|
927
|
+
"""
|
|
928
|
+
if any(m.kind == "OR" for m in modifiers):
|
|
929
|
+
return frozenset()
|
|
930
|
+
paths = set()
|
|
931
|
+
for cond in [condition, *(m.condition for m in modifiers)]:
|
|
932
|
+
m = _RX_FILE_EXISTS.match(cond)
|
|
933
|
+
if m:
|
|
934
|
+
paths.add(strip_quotes(m.group("path")))
|
|
935
|
+
return frozenset(paths)
|
|
936
|
+
|
|
937
|
+
|
|
916
938
|
def _lint_nodes(
|
|
917
939
|
nodes: list[Node],
|
|
918
940
|
script_dir: Path | None,
|
|
@@ -921,8 +943,13 @@ def _lint_nodes(
|
|
|
921
943
|
issues: list[_Issue],
|
|
922
944
|
*,
|
|
923
945
|
visited_scripts: set[str] | None = None,
|
|
946
|
+
guarded: frozenset[str] = frozenset(),
|
|
924
947
|
) -> None:
|
|
925
|
-
"""Walk a list of AST nodes and collect lint issues.
|
|
948
|
+
"""Walk a list of AST nodes and collect lint issues.
|
|
949
|
+
|
|
950
|
+
*guarded* holds the files an enclosing ``IF (FILE_EXISTS(…))`` has
|
|
951
|
+
checked, so an ``INCLUDE`` of one is not reported as missing.
|
|
952
|
+
"""
|
|
926
953
|
if visited_scripts is None:
|
|
927
954
|
visited_scripts = set()
|
|
928
955
|
|
|
@@ -947,6 +974,9 @@ def _lint_nodes(
|
|
|
947
974
|
|
|
948
975
|
# -- IncludeDirective checks --
|
|
949
976
|
elif isinstance(node, IncludeDirective):
|
|
977
|
+
for text in (node.target, node.arguments, node.loop_condition):
|
|
978
|
+
for name in _unresolved_var_refs(text or "", defined_vars):
|
|
979
|
+
_check_var_ref(name, src, lno, defined_vars, issues)
|
|
950
980
|
if node.is_execute_script:
|
|
951
981
|
target = node.target.lower()
|
|
952
982
|
if target not in script_blocks:
|
|
@@ -967,14 +997,22 @@ def _lint_nodes(
|
|
|
967
997
|
else:
|
|
968
998
|
# INCLUDE file existence check
|
|
969
999
|
if not node.if_exists:
|
|
970
|
-
raw_path = node.target.strip()
|
|
971
|
-
if not _RX_VAR_REF.search(raw_path):
|
|
1000
|
+
raw_path = strip_quotes(node.target.strip())
|
|
1001
|
+
if not _RX_VAR_REF.search(raw_path) and raw_path not in guarded:
|
|
972
1002
|
_check_include_path(raw_path, script_dir, src, lno, issues)
|
|
973
1003
|
|
|
974
1004
|
# -- Recurse into block children --
|
|
975
1005
|
if isinstance(node, IfBlock):
|
|
976
1006
|
_check_constant_condition(node, issues)
|
|
977
|
-
_lint_nodes(
|
|
1007
|
+
_lint_nodes(
|
|
1008
|
+
node.body,
|
|
1009
|
+
script_dir,
|
|
1010
|
+
defined_vars,
|
|
1011
|
+
script_blocks,
|
|
1012
|
+
issues,
|
|
1013
|
+
visited_scripts=visited_scripts,
|
|
1014
|
+
guarded=guarded | _file_exists_guards(node.condition, node.condition_modifiers),
|
|
1015
|
+
)
|
|
978
1016
|
for clause in node.elseif_clauses:
|
|
979
1017
|
_lint_nodes(
|
|
980
1018
|
clause.body,
|
|
@@ -983,6 +1021,7 @@ def _lint_nodes(
|
|
|
983
1021
|
script_blocks,
|
|
984
1022
|
issues,
|
|
985
1023
|
visited_scripts=visited_scripts,
|
|
1024
|
+
guarded=guarded | _file_exists_guards(clause.condition, clause.condition_modifiers),
|
|
986
1025
|
)
|
|
987
1026
|
_lint_nodes(
|
|
988
1027
|
node.else_body,
|
|
@@ -991,9 +1030,18 @@ def _lint_nodes(
|
|
|
991
1030
|
script_blocks,
|
|
992
1031
|
issues,
|
|
993
1032
|
visited_scripts=visited_scripts,
|
|
1033
|
+
guarded=guarded,
|
|
994
1034
|
)
|
|
995
1035
|
elif isinstance(node, (LoopBlock, BatchBlock, SqlBlock)):
|
|
996
|
-
_lint_nodes(
|
|
1036
|
+
_lint_nodes(
|
|
1037
|
+
node.body,
|
|
1038
|
+
script_dir,
|
|
1039
|
+
defined_vars,
|
|
1040
|
+
script_blocks,
|
|
1041
|
+
issues,
|
|
1042
|
+
visited_scripts=visited_scripts,
|
|
1043
|
+
guarded=guarded,
|
|
1044
|
+
)
|
|
997
1045
|
elif isinstance(node, ScriptBlock):
|
|
998
1046
|
# Lint the block body once (structural errors are caught by the
|
|
999
1047
|
# parser). Issues carry their own line, which already places them
|
|
@@ -1031,7 +1079,12 @@ def lint(
|
|
|
1031
1079
|
issues.append(_issue("S001", script_path or script.source, 0, "script is empty"))
|
|
1032
1080
|
return issues
|
|
1033
1081
|
|
|
1034
|
-
|
|
1082
|
+
if script_path:
|
|
1083
|
+
script_dir = Path(script_path).resolve().parent
|
|
1084
|
+
elif base_dir:
|
|
1085
|
+
script_dir = Path(base_dir)
|
|
1086
|
+
else:
|
|
1087
|
+
script_dir = None
|
|
1035
1088
|
script_blocks = _collect_script_blocks(script)
|
|
1036
1089
|
|
|
1037
1090
|
# Pass 1: collect all variable definitions
|
execsql/cli/run.py
CHANGED
|
@@ -23,6 +23,7 @@ from execsql.cli.help import _console, _err_console
|
|
|
23
23
|
from execsql.config import ConfigData, StatObj
|
|
24
24
|
from execsql.exceptions import ConfigError, ErrInfo
|
|
25
25
|
from execsql.script import SubVarSet, current_script_line, substitute_vars
|
|
26
|
+
from execsql.script.preparation import prepare_script
|
|
26
27
|
from execsql.utils.fileio import FileWriter, Logger, filewriter_end
|
|
27
28
|
from execsql.utils.gui import gui_connect, gui_console_isrunning, gui_console_off, gui_console_on, gui_console_wait_user
|
|
28
29
|
|
|
@@ -47,14 +48,14 @@ def _print_dry_run(tree: Script | None) -> None:
|
|
|
47
48
|
source location. Variables already populated at parse time are expanded;
|
|
48
49
|
execution-time variables remain unexpanded.
|
|
49
50
|
"""
|
|
50
|
-
from execsql.script.ast import MetaCommandStatement, SqlStatement
|
|
51
|
+
from execsql.script.ast import IncludeDirective, MetaCommandStatement, SqlStatement
|
|
51
52
|
|
|
52
53
|
if tree is None:
|
|
53
54
|
_console.print("[yellow]No commands found in script.[/yellow]")
|
|
54
55
|
return
|
|
55
56
|
nodes = list(tree.walk())
|
|
56
|
-
# Count only executable nodes (SQL and
|
|
57
|
-
executable = [n for n in nodes if isinstance(n, (SqlStatement, MetaCommandStatement))]
|
|
57
|
+
# Count only executable nodes (SQL, metacommands, INCLUDE and EXECUTE SCRIPT)
|
|
58
|
+
executable = [n for n in nodes if isinstance(n, (SqlStatement, MetaCommandStatement, IncludeDirective))]
|
|
58
59
|
if not executable:
|
|
59
60
|
_console.print("[yellow]No commands found in script.[/yellow]")
|
|
60
61
|
return
|
|
@@ -735,6 +736,8 @@ def _run(
|
|
|
735
736
|
for name, value in named_vars or ():
|
|
736
737
|
_state.subvars.add_substitution(name, value)
|
|
737
738
|
_ast_tree = _load_script(command, script_name, conf.script_encoding)
|
|
739
|
+
if _ast_tree is not None:
|
|
740
|
+
_ast_tree = prepare_script(_ast_tree, conf.include_req, conf.include_opt)
|
|
738
741
|
_print_dry_run(_ast_tree)
|
|
739
742
|
raise SystemExit(0)
|
|
740
743
|
|
|
@@ -795,6 +798,8 @@ def _run(
|
|
|
795
798
|
# Load the SQL script (--dry-run / --ping already exited above)
|
|
796
799
|
# ------------------------------------------------------------------
|
|
797
800
|
_ast_tree = _load_script(command, script_name, conf.script_encoding)
|
|
801
|
+
if _ast_tree is not None:
|
|
802
|
+
_ast_tree = prepare_script(_ast_tree, conf.include_req, conf.include_opt)
|
|
798
803
|
|
|
799
804
|
# ------------------------------------------------------------------
|
|
800
805
|
# Start GUI console if requested
|
|
@@ -871,14 +876,9 @@ def _end_run_on_sigterm() -> None:
|
|
|
871
876
|
"""
|
|
872
877
|
import signal
|
|
873
878
|
|
|
874
|
-
|
|
875
|
-
import execsql.state as _state
|
|
876
|
-
|
|
877
|
-
if _state.exec_log is not None:
|
|
878
|
-
_state.exec_log.log_exit_terminated()
|
|
879
|
-
raise SystemExit(128 + signum)
|
|
879
|
+
from execsql.utils.sigterm import terminate
|
|
880
880
|
|
|
881
|
-
signal.signal(signal.SIGTERM,
|
|
881
|
+
signal.signal(signal.SIGTERM, terminate)
|
|
882
882
|
|
|
883
883
|
|
|
884
884
|
def _execute_script_ast(
|
execsql/config.py
CHANGED
|
@@ -23,6 +23,7 @@ from configparser import ConfigParser
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
from typing import Protocol
|
|
25
25
|
|
|
26
|
+
from execsql.utils.spreadsheet import INPUT_LIMIT_DEFAULTS
|
|
26
27
|
from execsql.utils.paths import is_network_path
|
|
27
28
|
from execsql.exceptions import ConfigError, ConfigFileError
|
|
28
29
|
from execsql.utils.crypto import Encrypt
|
|
@@ -296,6 +297,10 @@ class ConfigData:
|
|
|
296
297
|
("ignore", "replace", "xmlcharrefreplace", "backslashreplace"),
|
|
297
298
|
)
|
|
298
299
|
# --- [input] ---
|
|
300
|
+
for name in INPUT_LIMIT_DEFAULTS:
|
|
301
|
+
self._get_int(cp, self._INPUT_SECTION, name, name)
|
|
302
|
+
if cp.has_option(self._INPUT_SECTION, name) and getattr(self, name) < 0:
|
|
303
|
+
raise ConfigError(f"Invalid {name}: {getattr(self, name)}; must be >= 0.")
|
|
299
304
|
self._get_int(cp, self._INPUT_SECTION, "max_int", "max_int")
|
|
300
305
|
self._get_bool(cp, self._INPUT_SECTION, "boolean_int", "boolean_int")
|
|
301
306
|
self._get_bool(cp, self._INPUT_SECTION, "boolean_words", "boolean_words")
|
|
@@ -500,6 +505,8 @@ class ConfigData:
|
|
|
500
505
|
self.import_encoding = "utf8"
|
|
501
506
|
self.enc_err_disposition: str | None = None
|
|
502
507
|
self.import_common_cols_only = False
|
|
508
|
+
for name, default in INPUT_LIMIT_DEFAULTS.items():
|
|
509
|
+
setattr(self, name, default)
|
|
503
510
|
self.max_int = 2147483647
|
|
504
511
|
self.boolean_int = True
|
|
505
512
|
self.boolean_words = False
|
|
@@ -77,6 +77,21 @@
|
|
|
77
77
|
[input]
|
|
78
78
|
# Settings for handling imported data.
|
|
79
79
|
|
|
80
|
+
# ODS archive limits, checked before parsing. XML total is in MiB; the
|
|
81
|
+
# compression ratio applies per archive member. 0 disables a limit.
|
|
82
|
+
#ods_max_xml_mb=100
|
|
83
|
+
#ods_max_compression_ratio=100
|
|
84
|
+
|
|
85
|
+
# Materialized spreadsheet limits (currently enforced for ODS imports).
|
|
86
|
+
# Rows include the header after SKIP; cells include rectangular padding.
|
|
87
|
+
# Interior blanks count; trailing empty filler does not. Text counts include
|
|
88
|
+
# repeated/spanned copies. 0 disables an individual limit; values are never truncated.
|
|
89
|
+
#import_max_rows=1048576
|
|
90
|
+
#import_max_columns=16384
|
|
91
|
+
#import_max_cells=10000000
|
|
92
|
+
#import_max_cell_chars=1048576
|
|
93
|
+
#import_max_text_chars=268435456
|
|
94
|
+
|
|
80
95
|
# MS Access: create columns for decimal data as NUMERIC(precision,scale)
|
|
81
96
|
# (Yes) or as DOUBLE (No) when IMPORT or COPY creates a table.
|
|
82
97
|
# Values: Yes or No. Default: No.
|
|
@@ -199,6 +214,9 @@
|
|
|
199
214
|
#css_styles=
|
|
200
215
|
|
|
201
216
|
# Seconds to keep retrying when an output file has an access conflict.
|
|
217
|
+
# Background-writer waits, including blocked writes/closes, end after this
|
|
218
|
+
# interval plus 5 seconds (at most 10 seconds on a halt). On timeout, queued
|
|
219
|
+
# output is discarded; the one I/O call already in progress may finish later.
|
|
202
220
|
# Value: A positive non-zero number. Default: 600.
|
|
203
221
|
#outfile_open_timeout=600
|
|
204
222
|
|
|
@@ -305,6 +323,7 @@
|
|
|
305
323
|
# Whether to allow the SYSTEM_CMD (SHELL) metacommand.
|
|
306
324
|
# Set to No to prevent scripts from executing OS commands.
|
|
307
325
|
# Also controllable via --no-system-cmd CLI flag.
|
|
326
|
+
# Library API: omitted/None inherits this value; an explicit boolean overrides it.
|
|
308
327
|
# Values: Yes or No. Default: Yes.
|
|
309
328
|
#allow_system_cmd=Yes
|
|
310
329
|
|
|
@@ -317,28 +336,33 @@
|
|
|
317
336
|
# Whether to allow the RM_FILE metacommand (which deletes a file).
|
|
318
337
|
# Set to No to prevent scripts from deleting files.
|
|
319
338
|
# Also controllable via --no-rm-file CLI flag.
|
|
339
|
+
# Library API: omitted/None inherits this value; an explicit boolean overrides it.
|
|
320
340
|
# Values: Yes or No. Default: Yes.
|
|
321
341
|
#allow_rm_file=Yes
|
|
322
342
|
|
|
323
343
|
# Whether to allow the SERVE metacommand (which copies a file to stdout
|
|
324
344
|
# with HTTP headers, for CGI use). Set to No to disable.
|
|
325
345
|
# Also controllable via --no-serve CLI flag.
|
|
346
|
+
# Library API: omitted/None inherits this value; an explicit boolean overrides it.
|
|
326
347
|
# Values: Yes or No. Default: Yes.
|
|
327
348
|
#allow_serve=Yes
|
|
328
349
|
|
|
329
|
-
# Root directory under which
|
|
330
|
-
#
|
|
331
|
-
#
|
|
332
|
-
#
|
|
350
|
+
# Root directory under which the files INCLUDE reads must resolve.
|
|
351
|
+
# When set, attempts to include files outside this root via ../,
|
|
352
|
+
# absolute paths, drive letters, or UNC paths are rejected with an
|
|
353
|
+
# error that stops the run, even with METACOMMAND_ERROR_HALT OFF.
|
|
354
|
+
# Default: no containment (any readable path is permitted).
|
|
333
355
|
#include_root=
|
|
334
356
|
|
|
335
357
|
# Root directory under which SERVE targets must resolve. Same
|
|
336
|
-
# containment
|
|
358
|
+
# containment checks as include_root, but a refused path is an
|
|
359
|
+
# ordinary metacommand error. Default: no containment.
|
|
337
360
|
#serve_root=
|
|
338
361
|
|
|
339
362
|
# Root directory under which Jinja2 / string.Template loader paths
|
|
340
|
-
# must resolve. Same containment
|
|
341
|
-
# Default: no
|
|
363
|
+
# must resolve. Same containment checks as include_root, but a
|
|
364
|
+
# refused path is an ordinary metacommand error. Default: no
|
|
365
|
+
# containment.
|
|
342
366
|
#template_root=
|
|
343
367
|
|
|
344
368
|
# Maximum byte size of any single substitution-variable expansion,
|