execsql2 2.17.3__py3-none-any.whl → 2.18.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/cli/__init__.py +15 -5
- execsql/cli/lint.py +296 -430
- execsql/cli/run.py +29 -2
- execsql/config.py +20 -0
- execsql/db/access.py +6 -0
- execsql/db/base.py +57 -1
- execsql/db/dsn.py +19 -9
- execsql/db/firebird.py +6 -0
- execsql/db/mysql.py +81 -0
- execsql/db/oracle.py +6 -0
- execsql/db/sqlite.py +37 -18
- execsql/db/sqlserver.py +31 -6
- execsql/exporters/base.py +1 -1
- execsql/exporters/duckdb.py +8 -4
- execsql/exporters/ods.py +11 -0
- execsql/exporters/sqlite.py +10 -3
- execsql/exporters/templates.py +10 -0
- execsql/exporters/xls.py +4 -0
- execsql/exporters/xlsx.py +9 -0
- execsql/importers/json.py +49 -32
- execsql/metacommands/conditions.py +7 -2
- execsql/metacommands/dispatch.py +5 -10
- execsql/metacommands/io_export.py +21 -26
- execsql/metacommands/io_fileops.py +21 -3
- execsql/metacommands/io_import.py +23 -3
- execsql/metacommands/script_ext.py +8 -7
- execsql/script/ast.py +8 -0
- execsql/script/engine.py +33 -12
- execsql/script/executor.py +12 -0
- execsql/script/variables.py +41 -15
- execsql/utils/auth.py +49 -1
- execsql/utils/fileio.py +120 -0
- execsql/utils/gui.py +11 -1
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/md_compare.sql +12 -12
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/md_glossary.sql +5 -5
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/md_upsert.sql +13 -13
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/pg_compare.sql +24 -24
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/pg_glossary.sql +5 -5
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/pg_upsert.sql +29 -29
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/script_template.sql +2 -2
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/ss_compare.sql +24 -24
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/ss_glossary.sql +6 -6
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/ss_upsert.sql +2917 -2917
- {execsql2-2.17.3.dist-info → execsql2-2.18.1.dist-info}/METADATA +47 -40
- {execsql2-2.17.3.dist-info → execsql2-2.18.1.dist-info}/RECORD +54 -55
- execsql/cli/lint_ast.py +0 -439
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/README.md +0 -0
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/execsql.conf +0 -0
- {execsql2-2.17.3.data → execsql2-2.18.1.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.17.3.dist-info → execsql2-2.18.1.dist-info}/WHEEL +0 -0
- {execsql2-2.17.3.dist-info → execsql2-2.18.1.dist-info}/entry_points.txt +0 -0
- {execsql2-2.17.3.dist-info → execsql2-2.18.1.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.17.3.dist-info → execsql2-2.18.1.dist-info}/licenses/NOTICE +0 -0
execsql/importers/json.py
CHANGED
|
@@ -4,10 +4,11 @@ from __future__ import annotations
|
|
|
4
4
|
JSON import for execsql.
|
|
5
5
|
|
|
6
6
|
Provides :func:`import_json`, used by ``IMPORT … FORMAT json``.
|
|
7
|
-
Supports JSON arrays of objects (``[{…}, …]``) and
|
|
8
|
-
|
|
9
|
-
dot-separated keys; nested arrays and
|
|
10
|
-
as JSON strings so every column
|
|
7
|
+
Supports JSON arrays of objects (``[{…}, …]``) and `JSON Lines
|
|
8
|
+
<https://jsonlines.org/>`_ (JSONL, one object per line). Nested
|
|
9
|
+
objects are flattened with dot-separated keys; nested arrays and
|
|
10
|
+
non-object values are serialized as JSON strings so every column
|
|
11
|
+
maps to a scalar database value.
|
|
11
12
|
"""
|
|
12
13
|
|
|
13
14
|
import json
|
|
@@ -45,42 +46,58 @@ def _flatten(obj: Any, prefix: str = "", sep: str = ".") -> dict[str, Any]:
|
|
|
45
46
|
def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
|
|
46
47
|
"""Read a JSON file and return a list of flat dicts.
|
|
47
48
|
|
|
48
|
-
Accepts either a JSON array of objects or
|
|
49
|
-
(
|
|
49
|
+
Accepts either a JSON array of objects or `JSON Lines
|
|
50
|
+
<https://jsonlines.org/>`_ (JSONL). The JSONL path streams the
|
|
51
|
+
file line-by-line so the raw text isn't buffered alongside the
|
|
52
|
+
parsed records (B19/F043). The array path still buffers the
|
|
53
|
+
whole file — switching to a streaming parser would require
|
|
54
|
+
``ijson`` as a dependency.
|
|
50
55
|
"""
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
56
|
+
# Peek at the first non-whitespace character to decide which
|
|
57
|
+
# parsing strategy to use, without slurping the entire file.
|
|
58
|
+
with open(filename, encoding=encoding) as fh:
|
|
59
|
+
first_char: str | None = None
|
|
60
|
+
while True:
|
|
61
|
+
ch = fh.read(1)
|
|
62
|
+
if not ch:
|
|
63
|
+
break
|
|
64
|
+
if not ch.isspace():
|
|
65
|
+
first_char = ch
|
|
66
|
+
break
|
|
67
|
+
|
|
68
|
+
if first_char == "[":
|
|
69
|
+
# Standard JSON array — must buffer the whole file (json.loads
|
|
70
|
+
# has no streaming mode; ijson would be needed for that).
|
|
71
|
+
text = Path(filename).read_text(encoding=encoding)
|
|
72
|
+
raw = json.loads(text)
|
|
57
73
|
if not isinstance(raw, list):
|
|
58
74
|
raise ErrInfo(type="error", other_msg="JSON file root is not an array of objects.")
|
|
59
75
|
records = raw
|
|
60
|
-
elif
|
|
61
|
-
#
|
|
76
|
+
elif first_char == "{":
|
|
77
|
+
# JSONL (JSON Lines) — stream the file line-by-line.
|
|
62
78
|
records = []
|
|
63
|
-
|
|
64
|
-
line
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
79
|
+
with open(filename, encoding=encoding) as fh:
|
|
80
|
+
for lineno, line in enumerate(fh, 1):
|
|
81
|
+
line = line.strip()
|
|
82
|
+
if not line:
|
|
83
|
+
continue
|
|
84
|
+
try:
|
|
85
|
+
obj = json.loads(line)
|
|
86
|
+
except json.JSONDecodeError as exc:
|
|
87
|
+
raise ErrInfo(
|
|
88
|
+
type="error",
|
|
89
|
+
other_msg=f"Invalid JSON on line {lineno}: {exc}",
|
|
90
|
+
) from exc
|
|
91
|
+
if not isinstance(obj, dict):
|
|
92
|
+
raise ErrInfo(
|
|
93
|
+
type="error",
|
|
94
|
+
other_msg=f"Line {lineno} is not a JSON object.",
|
|
95
|
+
)
|
|
96
|
+
records.append(obj)
|
|
80
97
|
else:
|
|
81
98
|
raise ErrInfo(
|
|
82
99
|
type="error",
|
|
83
|
-
other_msg="JSON import expects a file starting with '[' (array) or '{' (object/
|
|
100
|
+
other_msg="JSON import expects a file starting with '[' (array) or '{' (object/JSONL).",
|
|
84
101
|
)
|
|
85
102
|
|
|
86
103
|
if not records:
|
|
@@ -75,7 +75,7 @@ def xf_startswith(**kwargs: Any) -> bool:
|
|
|
75
75
|
if kwargs["ignorecase"] and kwargs["ignorecase"].lower() == "i":
|
|
76
76
|
s1 = s1.lower()
|
|
77
77
|
s2 = s2.lower()
|
|
78
|
-
return s1
|
|
78
|
+
return s1.startswith(s2)
|
|
79
79
|
|
|
80
80
|
|
|
81
81
|
def xf_endswith(**kwargs: Any) -> bool:
|
|
@@ -84,7 +84,7 @@ def xf_endswith(**kwargs: Any) -> bool:
|
|
|
84
84
|
if kwargs["ignorecase"] and kwargs["ignorecase"].lower() == "i":
|
|
85
85
|
s1 = s1.lower()
|
|
86
86
|
s2 = s2.lower()
|
|
87
|
-
return s1
|
|
87
|
+
return s1.endswith(s2)
|
|
88
88
|
|
|
89
89
|
|
|
90
90
|
def xf_hasrows(**kwargs: Any) -> bool:
|
|
@@ -374,6 +374,10 @@ def xf_istrue(**kwargs: Any) -> bool:
|
|
|
374
374
|
return unquoted(kwargs["value"].strip()).lower() in ("yes", "y", "true", "t", "1")
|
|
375
375
|
|
|
376
376
|
|
|
377
|
+
def xf_isfalse(**kwargs: Any) -> bool:
|
|
378
|
+
return unquoted(kwargs["value"].strip()).lower() in ("no", "n", "false", "f", "0")
|
|
379
|
+
|
|
380
|
+
|
|
377
381
|
def xf_dbms(**kwargs: Any) -> bool:
|
|
378
382
|
dbms = kwargs["dbms"]
|
|
379
383
|
return _state.dbs.current().type.dbms_id.lower() == dbms.strip().lower()
|
|
@@ -797,6 +801,7 @@ def build_conditional_table() -> Any:
|
|
|
797
801
|
category="condition",
|
|
798
802
|
)
|
|
799
803
|
mcl.add(r"^\s*IS_TRUE\(\s*(?P<value>[^)]*)\s*\)", xf_istrue, description="IS_TRUE", category="condition")
|
|
804
|
+
mcl.add(r"^\s*IS_FALSE\(\s*(?P<value>[^)]*)\s*\)", xf_isfalse, description="IS_FALSE", category="condition")
|
|
800
805
|
|
|
801
806
|
# Boolean literals
|
|
802
807
|
mcl.add(
|
execsql/metacommands/dispatch.py
CHANGED
|
@@ -1377,8 +1377,8 @@ def build_dispatch_table() -> MetaCommandList:
|
|
|
1377
1377
|
# BEGIN / END BATCH / ROLLBACK
|
|
1378
1378
|
# ------------------------------------------------------------------
|
|
1379
1379
|
mcl.add(r"^\s*BEGIN\s+BATCH\s*$", x_begin_batch, description="BEGIN BATCH", category="block")
|
|
1380
|
-
mcl.add(r"^\s*END\s+BATCH\s*$", x_end_batch, "END BATCH",
|
|
1381
|
-
mcl.add(r"^\s*ROLLBACK(:?\s+BATCH)?\s*$", x_rollback, "ROLLBACK BATCH",
|
|
1380
|
+
mcl.add(r"^\s*END\s+BATCH\s*$", x_end_batch, "END BATCH", category="block")
|
|
1381
|
+
mcl.add(r"^\s*ROLLBACK(:?\s+BATCH)?\s*$", x_rollback, "ROLLBACK BATCH", category="block")
|
|
1382
1382
|
|
|
1383
1383
|
# ------------------------------------------------------------------
|
|
1384
1384
|
# ERROR_HALT / METACOMMAND_ERROR_HALT / CANCEL_HALT
|
|
@@ -1727,14 +1727,12 @@ def build_dispatch_table() -> MetaCommandList:
|
|
|
1727
1727
|
x_assert,
|
|
1728
1728
|
description="ASSERT",
|
|
1729
1729
|
category="action",
|
|
1730
|
-
run_when_false=False,
|
|
1731
1730
|
)
|
|
1732
1731
|
mcl.add(
|
|
1733
1732
|
r"^\s*ASSERT\s+(?P<condtest>.+?)\s+(?P<message>(?:\"[^\"]*\"|'[^']*'))\s*$",
|
|
1734
1733
|
x_assert,
|
|
1735
1734
|
description="ASSERT",
|
|
1736
1735
|
category="action",
|
|
1737
|
-
run_when_false=False,
|
|
1738
1736
|
)
|
|
1739
1737
|
|
|
1740
1738
|
# ------------------------------------------------------------------
|
|
@@ -1745,7 +1743,6 @@ def build_dispatch_table() -> MetaCommandList:
|
|
|
1745
1743
|
x_breakpoint,
|
|
1746
1744
|
description="BREAKPOINT",
|
|
1747
1745
|
category="action",
|
|
1748
|
-
run_when_false=False,
|
|
1749
1746
|
)
|
|
1750
1747
|
|
|
1751
1748
|
# ------------------------------------------------------------------
|
|
@@ -1765,26 +1762,24 @@ def build_dispatch_table() -> MetaCommandList:
|
|
|
1765
1762
|
r"^\s*ORIF\s*\(\s*(?P<condtest>.+)\s*\)\s*$",
|
|
1766
1763
|
x_if_orif,
|
|
1767
1764
|
description="ORIF",
|
|
1768
|
-
run_when_false=True,
|
|
1769
1765
|
category="control",
|
|
1770
1766
|
)
|
|
1771
1767
|
mcl.add(
|
|
1772
1768
|
r"^\s*ELSEIF\s*\(\s*(?P<condtest>.+)\s*\)\s*$",
|
|
1773
1769
|
x_if_elseif,
|
|
1774
1770
|
description="ELSEIF",
|
|
1775
|
-
run_when_false=True,
|
|
1776
1771
|
category="control",
|
|
1777
1772
|
)
|
|
1778
1773
|
mcl.add(r"^\s*ANDIF\s*\(\s*(?P<condtest>.+)\s*\)\s*$", x_if_andif, description="ANDIF", category="control")
|
|
1779
|
-
mcl.add(r"^\s*ELSE\s*$", x_if_else, description="ELSE",
|
|
1774
|
+
mcl.add(r"^\s*ELSE\s*$", x_if_else, description="ELSE", category="control")
|
|
1780
1775
|
mcl.add(
|
|
1781
1776
|
r"^\s*IF\s*\(\s*(?P<condtest>.+)\s*\)\s*{\s*(?P<condcmd>.+)\s*}\s*$",
|
|
1782
1777
|
x_if,
|
|
1783
1778
|
description="IF",
|
|
1784
1779
|
category="control",
|
|
1785
1780
|
)
|
|
1786
|
-
mcl.add(r"^\s*IF\s*\(\s*(?P<condtest>.+)\s*\)\s*$", x_if_block,
|
|
1787
|
-
mcl.add(r"^\s*ENDIF\s*$", x_if_end, description="ENDIF",
|
|
1781
|
+
mcl.add(r"^\s*IF\s*\(\s*(?P<condtest>.+)\s*\)\s*$", x_if_block, category="control")
|
|
1782
|
+
mcl.add(r"^\s*ENDIF\s*$", x_if_end, description="ENDIF", category="control")
|
|
1788
1783
|
|
|
1789
1784
|
# ------------------------------------------------------------------
|
|
1790
1785
|
# CONNECT — SQL Server
|
|
@@ -18,7 +18,6 @@ appropriate writer in :mod:`execsql.exporters`.
|
|
|
18
18
|
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
|
-
from pathlib import Path
|
|
22
21
|
from typing import Any
|
|
23
22
|
|
|
24
23
|
import execsql.state as _state
|
|
@@ -44,28 +43,24 @@ from execsql.exporters.yaml import write_query_to_yaml
|
|
|
44
43
|
from execsql.importers.base import import_data_table
|
|
45
44
|
from execsql.script import current_script_line
|
|
46
45
|
from execsql.utils.errors import exception_desc
|
|
47
|
-
from execsql.utils.fileio import check_dir
|
|
46
|
+
from execsql.utils.fileio import check_dir, safe_output_path
|
|
48
47
|
|
|
49
48
|
|
|
50
49
|
def _apply_output_dir(path: str) -> str:
|
|
51
|
-
"""
|
|
50
|
+
"""Resolve *path* against the configured ``--output-dir`` root.
|
|
52
51
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
52
|
+
When ``conf.export_output_dir`` is set, ``--output-dir`` is treated as a
|
|
53
|
+
containment boundary: relative paths are joined to it, absolute paths
|
|
54
|
+
must already fall inside it, and ``..`` segments that escape the root
|
|
55
|
+
are rejected. ``stdout`` is passed through untouched.
|
|
56
|
+
|
|
57
|
+
When ``conf.export_output_dir`` is unset, *path* is returned unchanged
|
|
58
|
+
(no behavior change for users not opting in to ``--output-dir``).
|
|
57
59
|
"""
|
|
58
60
|
output_dir = getattr(_state.conf, "export_output_dir", None)
|
|
59
|
-
if not output_dir:
|
|
60
|
-
return path
|
|
61
|
-
if path.lower() == "stdout":
|
|
62
|
-
return path
|
|
63
|
-
if Path(path).is_absolute():
|
|
64
|
-
return path
|
|
65
|
-
# Windows drive-letter paths are also absolute
|
|
66
|
-
if len(path) > 1 and path[1] == ":":
|
|
61
|
+
if not output_dir or path.lower() == "stdout":
|
|
67
62
|
return path
|
|
68
|
-
return
|
|
63
|
+
return safe_output_path(path, output_dir)
|
|
69
64
|
|
|
70
65
|
|
|
71
66
|
# ---------------------------------------------------------------------------
|
|
@@ -212,12 +207,12 @@ def x_export(**kwargs: Any) -> None:
|
|
|
212
207
|
|
|
213
208
|
def x_export_query(**kwargs: Any) -> None:
|
|
214
209
|
select_stmt = kwargs["query"]
|
|
215
|
-
outfile = kwargs["filename"]
|
|
210
|
+
outfile = _apply_output_dir(kwargs["filename"])
|
|
216
211
|
description = kwargs["description"]
|
|
217
212
|
tee = bool(kwargs["tee"])
|
|
218
213
|
append = bool(kwargs["append"])
|
|
219
214
|
filefmt = kwargs["format"].lower()
|
|
220
|
-
zipfilename = kwargs["zipfilename"]
|
|
215
|
+
zipfilename = _apply_output_dir(kwargs["zipfilename"]) if kwargs["zipfilename"] else None
|
|
221
216
|
notype = bool(kwargs.get("notype"))
|
|
222
217
|
_check_zip_compat(outfile, filefmt, zipfilename)
|
|
223
218
|
check_dir(outfile)
|
|
@@ -242,13 +237,13 @@ def x_export_query(**kwargs: Any) -> None:
|
|
|
242
237
|
|
|
243
238
|
def x_export_query_with_template(**kwargs: Any) -> None:
|
|
244
239
|
select_stmt = kwargs["query"]
|
|
245
|
-
outfile = kwargs["filename"]
|
|
240
|
+
outfile = _apply_output_dir(kwargs["filename"])
|
|
246
241
|
template_file = kwargs["template"]
|
|
247
242
|
tee = kwargs["tee"]
|
|
248
243
|
tee = bool(tee)
|
|
249
244
|
append = kwargs["append"]
|
|
250
245
|
append = bool(append)
|
|
251
|
-
zipfilename = kwargs["zipfilename"]
|
|
246
|
+
zipfilename = _apply_output_dir(kwargs["zipfilename"]) if kwargs["zipfilename"] else None
|
|
252
247
|
check_dir(outfile)
|
|
253
248
|
if tee and outfile.lower() != "stdout":
|
|
254
249
|
prettyprint_query(select_stmt, _state.dbs.current(), "stdout", False)
|
|
@@ -262,13 +257,13 @@ def x_export_with_template(**kwargs: Any) -> None:
|
|
|
262
257
|
table = kwargs["table"]
|
|
263
258
|
queryname = _state.dbs.current().schema_qualified_table_name(schema, table)
|
|
264
259
|
select_stmt = f"select * from {queryname};"
|
|
265
|
-
outfile = kwargs["filename"]
|
|
260
|
+
outfile = _apply_output_dir(kwargs["filename"])
|
|
266
261
|
template_file = kwargs["template"]
|
|
267
262
|
tee = kwargs["tee"]
|
|
268
263
|
tee = bool(tee)
|
|
269
264
|
append = kwargs["append"]
|
|
270
265
|
append = bool(append)
|
|
271
|
-
zipfilename = kwargs["zipfilename"]
|
|
266
|
+
zipfilename = _apply_output_dir(kwargs["zipfilename"]) if kwargs["zipfilename"] else None
|
|
272
267
|
check_dir(outfile)
|
|
273
268
|
if tee and outfile.lower() != "stdout":
|
|
274
269
|
prettyprint_query(select_stmt, _state.dbs.current(), "stdout", False)
|
|
@@ -279,7 +274,7 @@ def x_export_with_template(**kwargs: Any) -> None:
|
|
|
279
274
|
|
|
280
275
|
def x_export_ods_multiple(**kwargs: Any) -> None:
|
|
281
276
|
table_list = kwargs["tables"]
|
|
282
|
-
outfile = kwargs["filename"]
|
|
277
|
+
outfile = _apply_output_dir(kwargs["filename"])
|
|
283
278
|
description = kwargs["description"]
|
|
284
279
|
tee = kwargs["tee"]
|
|
285
280
|
tee = bool(tee)
|
|
@@ -292,7 +287,7 @@ def x_export_ods_multiple(**kwargs: Any) -> None:
|
|
|
292
287
|
def x_export_xlsx_multiple(**kwargs: Any) -> None:
|
|
293
288
|
"""Export multiple tables to separate worksheets in a single XLSX workbook."""
|
|
294
289
|
table_list = kwargs["tables"]
|
|
295
|
-
outfile = kwargs["filename"]
|
|
290
|
+
outfile = _apply_output_dir(kwargs["filename"])
|
|
296
291
|
description = kwargs["description"]
|
|
297
292
|
tee = kwargs["tee"]
|
|
298
293
|
tee = bool(tee)
|
|
@@ -303,10 +298,10 @@ def x_export_xlsx_multiple(**kwargs: Any) -> None:
|
|
|
303
298
|
|
|
304
299
|
|
|
305
300
|
def x_export_metadata(**kwargs: Any) -> None:
|
|
306
|
-
outfile = kwargs["filename"]
|
|
301
|
+
outfile = _apply_output_dir(kwargs["filename"])
|
|
307
302
|
append = kwargs["append"] is not None
|
|
308
303
|
xall = kwargs["all"] is not None
|
|
309
|
-
zipfilename = kwargs["zipfilename"]
|
|
304
|
+
zipfilename = _apply_output_dir(kwargs["zipfilename"]) if kwargs["zipfilename"] else None
|
|
310
305
|
filefmt = kwargs["format"].lower()
|
|
311
306
|
if xall:
|
|
312
307
|
hdrs, rows = _state.export_metadata.get_all()
|
|
@@ -17,7 +17,6 @@ import execsql.state as _state
|
|
|
17
17
|
from execsql.exceptions import ErrInfo
|
|
18
18
|
from execsql.models import DataTable
|
|
19
19
|
from execsql.script import current_script_line
|
|
20
|
-
from execsql.types import dbt_firebird
|
|
21
20
|
from execsql.utils.errors import exception_desc
|
|
22
21
|
from execsql.utils.fileio import filewriter_close
|
|
23
22
|
from execsql.utils.strings import unquoted
|
|
@@ -95,7 +94,7 @@ def x_copy(**kwargs: Any) -> None:
|
|
|
95
94
|
except Exception:
|
|
96
95
|
_state.exec_log.log_status_info(f"Could not drop existing table ({tbl2}) for COPY metacommand")
|
|
97
96
|
db2.execute(create_tbl)
|
|
98
|
-
if db2.
|
|
97
|
+
if db2.needs_explicit_commit_after_ddl():
|
|
99
98
|
db2.execute("COMMIT;")
|
|
100
99
|
try:
|
|
101
100
|
hdrs, rows = db1.select_rowsource(select_stmt)
|
|
@@ -169,7 +168,7 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
169
168
|
except Exception:
|
|
170
169
|
_state.exec_log.log_status_info(f"Could not drop existing table ({tbl2}) for COPY metacommand")
|
|
171
170
|
db2.execute(create_tbl)
|
|
172
|
-
if db2.
|
|
171
|
+
if db2.needs_explicit_commit_after_ddl():
|
|
173
172
|
db2.execute("COMMIT;")
|
|
174
173
|
try:
|
|
175
174
|
hdrs, rows = db1.select_rowsource(select_stmt)
|
|
@@ -209,6 +208,13 @@ def x_zip_buffer_mb(**kwargs: Any) -> None:
|
|
|
209
208
|
def x_rm_file(**kwargs: Any) -> None:
|
|
210
209
|
import glob as _glob
|
|
211
210
|
|
|
211
|
+
if not getattr(_state.conf, "allow_rm_file", True):
|
|
212
|
+
raise ErrInfo(
|
|
213
|
+
type="cmd",
|
|
214
|
+
command_text=kwargs.get("metacommandline", "RM_FILE"),
|
|
215
|
+
other_msg="The RM_FILE metacommand is disabled (--no-rm-file).",
|
|
216
|
+
)
|
|
217
|
+
|
|
212
218
|
fn = kwargs["filename"].strip(' "')
|
|
213
219
|
fnlist = _glob.glob(fn)
|
|
214
220
|
for f in fnlist:
|
|
@@ -249,7 +255,19 @@ def x_hdf5_text_len(**kwargs: Any) -> None:
|
|
|
249
255
|
|
|
250
256
|
|
|
251
257
|
def x_serve(**kwargs: Any) -> None:
|
|
258
|
+
from execsql.utils.fileio import safe_output_path
|
|
259
|
+
|
|
260
|
+
if not getattr(_state.conf, "allow_serve", True):
|
|
261
|
+
raise ErrInfo(
|
|
262
|
+
type="cmd",
|
|
263
|
+
command_text=kwargs.get("metacommandline", "SERVE"),
|
|
264
|
+
other_msg="The SERVE metacommand is disabled (--no-serve).",
|
|
265
|
+
)
|
|
266
|
+
|
|
252
267
|
infname = kwargs["filename"]
|
|
268
|
+
serve_root = getattr(_state.conf, "serve_root", None)
|
|
269
|
+
if serve_root:
|
|
270
|
+
infname = safe_output_path(infname, serve_root)
|
|
253
271
|
fmt = kwargs["format"].lower()
|
|
254
272
|
if not Path(infname).is_file():
|
|
255
273
|
raise ErrInfo(
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
- ``x_import_xls`` / ``x_import_xls_pattern`` — same for XLS/XLSX.
|
|
9
9
|
- ``x_import_parquet`` — IMPORT … FROM PARQUET (via polars).
|
|
10
10
|
- ``x_import_feather`` — IMPORT … FROM FEATHER (via polars).
|
|
11
|
-
- ``x_import_json`` — IMPORT … FROM JSON (array of objects or
|
|
11
|
+
- ``x_import_json`` — IMPORT … FROM JSON (array of objects or JSON Lines).
|
|
12
12
|
- ``x_import_row_buffer`` — CONFIG IMPORT_ROW_BUFFER.
|
|
13
13
|
- ``x_show_progress`` — CONFIG SHOW_PROGRESS (toggle the import progress bar).
|
|
14
14
|
"""
|
|
@@ -163,7 +163,18 @@ def x_import_ods_pattern(**kwargs: Any) -> None:
|
|
|
163
163
|
is_new = 0
|
|
164
164
|
schemaname = kwargs["schema"]
|
|
165
165
|
filename = kwargs["filename"]
|
|
166
|
-
|
|
166
|
+
# B18/F012: surface a friendly error for malformed regex patterns
|
|
167
|
+
# rather than letting an uncaught re.error bubble up. (ReDoS via
|
|
168
|
+
# catastrophic backtracking remains a documented risk — re2 is
|
|
169
|
+
# not in stdlib so we can't enforce a complexity cap here.)
|
|
170
|
+
try:
|
|
171
|
+
rx = re.compile(kwargs["patn"], re.I)
|
|
172
|
+
except re.error as exc:
|
|
173
|
+
raise ErrInfo(
|
|
174
|
+
type="cmd",
|
|
175
|
+
command_text=kwargs.get("metacommandline", "IMPORT ODS PATTERN"),
|
|
176
|
+
other_msg=f"Invalid regular expression {kwargs['patn']!r}: {exc}",
|
|
177
|
+
) from exc
|
|
167
178
|
hdr_rows = kwargs["skip"]
|
|
168
179
|
if not hdr_rows:
|
|
169
180
|
hdr_rows = 0
|
|
@@ -260,7 +271,16 @@ def x_import_xls_pattern(**kwargs: Any) -> None:
|
|
|
260
271
|
is_new = 0
|
|
261
272
|
schemaname = kwargs["schema"]
|
|
262
273
|
filename = kwargs["filename"]
|
|
263
|
-
|
|
274
|
+
# B18/F012: surface a friendly error for malformed regex patterns
|
|
275
|
+
# (see x_import_ods_pattern for the full rationale).
|
|
276
|
+
try:
|
|
277
|
+
rx = re.compile(kwargs["patn"], re.I)
|
|
278
|
+
except re.error as exc:
|
|
279
|
+
raise ErrInfo(
|
|
280
|
+
type="cmd",
|
|
281
|
+
command_text=kwargs.get("metacommandline", "IMPORT XLS PATTERN"),
|
|
282
|
+
other_msg=f"Invalid regular expression {kwargs['patn']!r}: {exc}",
|
|
283
|
+
) from exc
|
|
264
284
|
hdr_rows = kwargs["skip"]
|
|
265
285
|
encoding = kwargs["encoding"]
|
|
266
286
|
if not hdr_rows:
|
|
@@ -3,22 +3,23 @@ from __future__ import annotations
|
|
|
3
3
|
"""
|
|
4
4
|
Script-block extension and dispatch handlers for execsql.
|
|
5
5
|
|
|
6
|
-
Handlers for the named-script
|
|
7
|
-
|
|
6
|
+
Handlers for the named-script extension metacommands invoked by the
|
|
7
|
+
AST executor:
|
|
8
8
|
|
|
9
|
-
- ``x_executescript`` — ``EXECUTE SCRIPT <name>`` / ``RUN SCRIPT <name>``
|
|
10
|
-
(look up a previously-registered ``BEGIN SCRIPT`` block and run it,
|
|
11
|
-
optionally with parameter bindings and a WHILE / UNTIL loop).
|
|
12
9
|
- ``x_extendscript`` — ``EXTEND SCRIPT <name> WITH SCRIPT|FILE …``
|
|
13
10
|
(append additional commands to an existing named script block from
|
|
14
11
|
an inline source).
|
|
15
12
|
- ``x_extendscript_metacommand`` — ``EXTEND SCRIPT … WITH METACOMMAND …``.
|
|
16
13
|
- ``x_extendscript_sql`` — ``EXTEND SCRIPT … WITH SQL …``.
|
|
14
|
+
- ``x_executescript`` — dispatch-table sentinel only. ``EXECUTE SCRIPT``
|
|
15
|
+
/ ``RUN SCRIPT`` are handled natively by the AST executor; this stub
|
|
16
|
+
raises ``ErrInfo`` if the parser ever fails to recognize them as
|
|
17
|
+
structural nodes.
|
|
17
18
|
|
|
18
19
|
Registration of ``BEGIN SCRIPT … END SCRIPT`` blocks themselves is
|
|
19
20
|
handled by the AST parser (block boundaries) and executor (registering
|
|
20
|
-
the block on ``ctx.ast_scripts``); this module is only the
|
|
21
|
-
extension handlers.
|
|
21
|
+
the block on ``ctx.ast_scripts``); this module is only the
|
|
22
|
+
extension-site handlers.
|
|
22
23
|
"""
|
|
23
24
|
|
|
24
25
|
import copy
|
execsql/script/ast.py
CHANGED
|
@@ -483,6 +483,14 @@ def _format_nodes(nodes: list[Node], lines: list[str], prefix: str) -> None:
|
|
|
483
483
|
_format_if_block(node, lines, child_prefix)
|
|
484
484
|
elif isinstance(node, (LoopBlock, BatchBlock, ScriptBlock, SqlBlock)):
|
|
485
485
|
_format_nodes(node.body, lines, child_prefix)
|
|
486
|
+
elif isinstance(node, (SqlStatement, MetaCommandStatement, Comment, IncludeDirective)):
|
|
487
|
+
# Leaf nodes — no children to render.
|
|
488
|
+
pass
|
|
489
|
+
else:
|
|
490
|
+
# Guard against silently skipping bodies of future block-type nodes.
|
|
491
|
+
raise NotImplementedError(
|
|
492
|
+
f"_format_nodes does not handle {type(node).__name__}",
|
|
493
|
+
)
|
|
486
494
|
|
|
487
495
|
|
|
488
496
|
def _format_if_block(node: IfBlock, lines: list[str], prefix: str) -> None:
|
execsql/script/engine.py
CHANGED
|
@@ -67,25 +67,18 @@ class MetaCommand:
|
|
|
67
67
|
rx: Any,
|
|
68
68
|
exec_func: Any,
|
|
69
69
|
description: str | None = None,
|
|
70
|
-
run_in_batch: bool = False,
|
|
71
|
-
run_when_false: bool = False,
|
|
72
70
|
set_error_flag: bool = True,
|
|
73
71
|
category: str | None = None,
|
|
74
72
|
) -> None:
|
|
75
73
|
self.rx = rx
|
|
76
74
|
self.exec_fn = exec_func
|
|
77
75
|
self.description = description
|
|
78
|
-
self.run_in_batch = run_in_batch
|
|
79
|
-
self.run_when_false = run_when_false
|
|
80
76
|
self.set_error_flag = set_error_flag
|
|
81
77
|
self.category = category
|
|
82
78
|
self.hitcount = 0
|
|
83
79
|
|
|
84
80
|
def __repr__(self) -> str:
|
|
85
|
-
return (
|
|
86
|
-
f"MetaCommand({self.rx.pattern!r}, {self.exec_fn!r}, {self.description!r}, "
|
|
87
|
-
f"{self.run_in_batch!r}, {self.run_when_false!r})"
|
|
88
|
-
)
|
|
81
|
+
return f"MetaCommand({self.rx.pattern!r}, {self.exec_fn!r}, {self.description!r})"
|
|
89
82
|
|
|
90
83
|
def run(self, cmd_str: str) -> tuple:
|
|
91
84
|
"""Match *cmd_str* against this entry's regex and, if it matches, invoke the handler.
|
|
@@ -171,8 +164,6 @@ class MetaCommandList:
|
|
|
171
164
|
matching_regexes: Any,
|
|
172
165
|
exec_func: Any,
|
|
173
166
|
description: str | None = None,
|
|
174
|
-
run_in_batch: bool = False,
|
|
175
|
-
run_when_false: bool = False,
|
|
176
167
|
set_error_flag: bool = True,
|
|
177
168
|
category: str | None = None,
|
|
178
169
|
) -> None:
|
|
@@ -193,8 +184,6 @@ class MetaCommandList:
|
|
|
193
184
|
rx,
|
|
194
185
|
exec_func,
|
|
195
186
|
description,
|
|
196
|
-
run_in_batch,
|
|
197
|
-
run_when_false,
|
|
198
187
|
set_error_flag,
|
|
199
188
|
category,
|
|
200
189
|
)
|
|
@@ -442,6 +431,14 @@ def set_system_vars(ctx: Any = None) -> None:
|
|
|
442
431
|
|
|
443
432
|
|
|
444
433
|
_MAX_SUBSTITUTION_DEPTH = 100
|
|
434
|
+
# B17/F013: output-byte ceiling for substitute_vars. The depth cap
|
|
435
|
+
# above stops cyclic references (a → !!b!!, b → !!a!!) but does not
|
|
436
|
+
# stop exponential expansion: a single variable referencing the same
|
|
437
|
+
# other variable N times grows ``2^N`` per iteration without ever
|
|
438
|
+
# looping. Track the rendered length and abort when it crosses
|
|
439
|
+
# _MAX_SUBSTITUTION_BYTES (default 10 MB — well above any legitimate
|
|
440
|
+
# SQL statement, comfortably below a memory-pressure failure mode).
|
|
441
|
+
_MAX_SUBSTITUTION_BYTES = 10 * 1024 * 1024
|
|
445
442
|
|
|
446
443
|
|
|
447
444
|
def substitute_vars(command_str: str, localvars: SubVarSet | None = None, ctx: Any = None) -> str:
|
|
@@ -452,12 +449,23 @@ def substitute_vars(command_str: str, localvars: SubVarSet | None = None, ctx: A
|
|
|
452
449
|
localvars: Optional local variable overlay to merge with globals.
|
|
453
450
|
ctx: Optional :class:`RuntimeContext`. When ``None``, falls through
|
|
454
451
|
to the global ``_state`` module (legacy behavior).
|
|
452
|
+
|
|
453
|
+
Raises:
|
|
454
|
+
ErrInfo: when the iteration count exceeds
|
|
455
|
+
:data:`_MAX_SUBSTITUTION_DEPTH` (cyclic reference) OR the
|
|
456
|
+
expanded output exceeds :data:`_MAX_SUBSTITUTION_BYTES`
|
|
457
|
+
(exponential expansion bomb).
|
|
455
458
|
"""
|
|
456
459
|
_s = ctx if ctx is not None else _state
|
|
457
460
|
if localvars is not None:
|
|
458
461
|
subs = _s.subvars.merge(localvars)
|
|
459
462
|
else:
|
|
460
463
|
subs = _s.subvars
|
|
464
|
+
# Allow runtime override of the byte cap via conf. None / missing
|
|
465
|
+
# → use the engine default (back-compat with users who legitimately
|
|
466
|
+
# render multi-MB SQL through substitution).
|
|
467
|
+
conf_max = getattr(_s.conf, "max_substitution_bytes", None)
|
|
468
|
+
max_bytes = conf_max if conf_max is not None else _MAX_SUBSTITUTION_BYTES
|
|
461
469
|
cmdstr = command_str
|
|
462
470
|
subs_made = True
|
|
463
471
|
iterations = 0
|
|
@@ -467,6 +475,19 @@ def substitute_vars(command_str: str, localvars: SubVarSet | None = None, ctx: A
|
|
|
467
475
|
cmdstr, any_subbed = _s.counters.substitute_all(cmdstr)
|
|
468
476
|
subs_made = subs_made or any_subbed
|
|
469
477
|
iterations += 1
|
|
478
|
+
# Only enforce the byte cap when expansion ACTUALLY happened
|
|
479
|
+
# this iteration. A user passing a large pre-existing literal
|
|
480
|
+
# with no !!var!! tokens shouldn't be rejected — the cap
|
|
481
|
+
# targets expansion-bomb growth, not literal input size.
|
|
482
|
+
if subs_made and len(cmdstr) > max_bytes:
|
|
483
|
+
raise ErrInfo(
|
|
484
|
+
type="error",
|
|
485
|
+
other_msg=(
|
|
486
|
+
f"Substitution variable expansion exceeded {max_bytes} bytes "
|
|
487
|
+
f"(possible exponential expansion bomb) while expanding: "
|
|
488
|
+
f"{command_str[:200]}"
|
|
489
|
+
),
|
|
490
|
+
)
|
|
470
491
|
if iterations >= _MAX_SUBSTITUTION_DEPTH:
|
|
471
492
|
raise ErrInfo(
|
|
472
493
|
type="error",
|
execsql/script/executor.py
CHANGED
|
@@ -783,6 +783,18 @@ def _execute_include_native(
|
|
|
783
783
|
if len(target) > 1 and target[0] == "~" and target[1] == os.sep:
|
|
784
784
|
target = str(Path.home() / target[2:])
|
|
785
785
|
|
|
786
|
+
# Optional containment: when conf.include_root is set, the resolved
|
|
787
|
+
# INCLUDE / EXECUTE SCRIPT target must live under that root.
|
|
788
|
+
include_root = getattr(ctx.conf, "include_root", None) if hasattr(ctx, "conf") else None
|
|
789
|
+
if include_root is None:
|
|
790
|
+
import execsql.state as _state
|
|
791
|
+
|
|
792
|
+
include_root = getattr(_state.conf, "include_root", None)
|
|
793
|
+
if include_root:
|
|
794
|
+
from execsql.utils.fileio import safe_output_path
|
|
795
|
+
|
|
796
|
+
target = safe_output_path(target, include_root)
|
|
797
|
+
|
|
786
798
|
target_path = Path(target)
|
|
787
799
|
|
|
788
800
|
# IF EXISTS handling
|