execsql2 2.37.0__py3-none-any.whl → 2.38.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +95 -45
- execsql/cli/commands/run.py +2 -2
- execsql/cli/lint.py +6 -1
- execsql/cli/run.py +5 -0
- execsql/config.py +7 -0
- execsql/data/execsql.conf.template +21 -0
- execsql/db/base.py +52 -24
- execsql/db/duckdb.py +43 -4
- execsql/db/firebird.py +8 -0
- execsql/db/mysql.py +8 -3
- execsql/db/oracle.py +5 -0
- execsql/db/postgres.py +2 -2
- execsql/db/sqlite.py +117 -10
- execsql/db/sqlserver.py +11 -7
- execsql/db/target_types.py +151 -4
- execsql/debug/repl.py +2 -1
- execsql/exceptions.py +14 -0
- execsql/exporters/html.py +26 -42
- execsql/exporters/ods.py +93 -21
- execsql/exporters/sqlite.py +82 -47
- execsql/format.py +12 -4
- execsql/gui/desktop.py +2 -1
- execsql/gui/tui.py +5 -2
- execsql/importers/base.py +9 -7
- execsql/importers/json.py +37 -13
- execsql/importers/ods.py +22 -2
- execsql/lsp/document.py +6 -1
- execsql/lsp/hover.py +2 -1
- execsql/metacommands/conditions.py +3 -1
- execsql/metacommands/connect.py +18 -3
- execsql/metacommands/dispatch.py +10 -0
- execsql/metacommands/io.py +1 -0
- execsql/metacommands/io_export.py +1 -2
- execsql/metacommands/io_fileops.py +26 -90
- execsql/metacommands/io_import.py +26 -8
- execsql/metacommands/reference.py +7 -0
- execsql/parser.py +14 -37
- execsql/script/control.py +17 -7
- execsql/script/executor.py +160 -34
- execsql/script/preparation.py +42 -0
- execsql/types.py +6 -5
- execsql/utils/errors.py +2 -2
- execsql/utils/fileio.py +342 -134
- execsql/utils/spreadsheet.py +74 -0
- execsql/utils/strings.py +13 -7
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/README.md +5 -5
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/example_config_prompt.sql +2 -2
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/METADATA +32 -119
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/RECORD +64 -62
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/WHEEL +1 -1
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/licenses/NOTICE +0 -0
execsql/importers/json.py
CHANGED
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
"""
|
|
4
4
|
JSON import for execsql.
|
|
5
5
|
|
|
6
|
-
Provides :func:`import_json`, used by ``IMPORT …
|
|
6
|
+
Provides :func:`import_json`, used by ``IMPORT … FROM JSON``.
|
|
7
7
|
Supports JSON arrays of objects (``[{…}, …]``) and `JSON Lines
|
|
8
8
|
<https://jsonlines.org/>`_ (JSONL, one object per line). Nested
|
|
9
9
|
objects are flattened with dot-separated keys; nested arrays and
|
|
@@ -12,6 +12,7 @@ maps to a scalar database value.
|
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
14
|
import json
|
|
15
|
+
from decimal import Decimal
|
|
15
16
|
from pathlib import Path
|
|
16
17
|
from typing import Any
|
|
17
18
|
|
|
@@ -22,30 +23,51 @@ from execsql.importers.base import import_data_table
|
|
|
22
23
|
__all__ = ["import_json"]
|
|
23
24
|
|
|
24
25
|
|
|
26
|
+
def _array_number(value: Any) -> Any:
|
|
27
|
+
"""Keep the established JSON text representation of numbers inside arrays."""
|
|
28
|
+
return float(value) if isinstance(value, Decimal) else str(value)
|
|
29
|
+
|
|
30
|
+
|
|
25
31
|
def _flatten(obj: Any, prefix: str = "", sep: str = ".") -> dict[str, Any]:
|
|
26
32
|
"""Recursively flatten a nested dict.
|
|
27
33
|
|
|
28
34
|
Nested dicts produce dot-separated keys. All other compound values
|
|
29
35
|
(lists, nested lists-of-dicts) are serialized as JSON strings so the
|
|
30
|
-
result is always ``{str: scalar}``.
|
|
36
|
+
result is always ``{str: scalar}``. Distinct paths that produce the
|
|
37
|
+
same column name raise ErrInfo instead of discarding a value.
|
|
31
38
|
"""
|
|
32
39
|
items: dict[str, Any] = {}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
40
|
+
source_paths: dict[str, tuple[str, ...]] = {}
|
|
41
|
+
|
|
42
|
+
def visit(current: Any, parent: str, path: tuple[str, ...]) -> None:
|
|
43
|
+
if not isinstance(current, dict):
|
|
44
|
+
return
|
|
45
|
+
for key, value in current.items():
|
|
46
|
+
column = f"{parent}{sep}{key}" if parent else key
|
|
47
|
+
source_path = (*path, key)
|
|
36
48
|
if isinstance(value, dict):
|
|
37
|
-
|
|
38
|
-
elif isinstance(value, list):
|
|
39
|
-
# Serialize arrays as JSON strings — tables are flat.
|
|
40
|
-
items[new_key] = json.dumps(value, default=str)
|
|
49
|
+
visit(value, column, source_path)
|
|
41
50
|
else:
|
|
42
|
-
|
|
51
|
+
if column in items:
|
|
52
|
+
raise ErrInfo(
|
|
53
|
+
type="error",
|
|
54
|
+
other_msg=(
|
|
55
|
+
f"JSON field paths {json.dumps(source_paths[column])} and {json.dumps(source_path)} "
|
|
56
|
+
f"flatten to the same column {column!r}. Rename the conflicting keys."
|
|
57
|
+
),
|
|
58
|
+
)
|
|
59
|
+
source_paths[column] = source_path
|
|
60
|
+
items[column] = json.dumps(value, default=_array_number) if isinstance(value, list) else value
|
|
61
|
+
|
|
62
|
+
visit(obj, prefix, ())
|
|
43
63
|
return items
|
|
44
64
|
|
|
45
65
|
|
|
46
66
|
def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
|
|
47
67
|
"""Read a JSON file and return a list of flat dicts.
|
|
48
68
|
|
|
69
|
+
Fractional and exponent scalar numbers retain their exact Decimal digits;
|
|
70
|
+
integers remain int. Arrays retain their established JSON text encoding.
|
|
49
71
|
Accepts either a JSON array of objects or `JSON Lines
|
|
50
72
|
<https://jsonlines.org/>`_ (JSONL). The JSONL path streams the
|
|
51
73
|
file line-by-line so the raw text isn't buffered alongside the
|
|
@@ -69,7 +91,7 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
|
|
|
69
91
|
# Standard JSON array — must buffer the whole file (json.loads
|
|
70
92
|
# has no streaming mode; ijson would be needed for that).
|
|
71
93
|
text = Path(filename).read_text(encoding=encoding)
|
|
72
|
-
raw = json.loads(text)
|
|
94
|
+
raw = json.loads(text, parse_float=Decimal)
|
|
73
95
|
if not isinstance(raw, list):
|
|
74
96
|
raise ErrInfo(type="error", other_msg="JSON file root is not an array of objects.")
|
|
75
97
|
records = raw
|
|
@@ -82,7 +104,7 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
|
|
|
82
104
|
if not line:
|
|
83
105
|
continue
|
|
84
106
|
try:
|
|
85
|
-
obj = json.loads(line)
|
|
107
|
+
obj = json.loads(line, parse_float=Decimal)
|
|
86
108
|
except json.JSONDecodeError as exc:
|
|
87
109
|
raise ErrInfo(
|
|
88
110
|
type="error",
|
|
@@ -126,7 +148,9 @@ def import_json(
|
|
|
126
148
|
|
|
127
149
|
Objects are flattened so that nested keys become dot-separated column
|
|
128
150
|
names (e.g. ``address.city``). Arrays within objects are stored as
|
|
129
|
-
JSON strings.
|
|
151
|
+
JSON strings. Colliding paths within a record are rejected before
|
|
152
|
+
any table is created, replaced or populated. Fractional and exponent scalar
|
|
153
|
+
numbers are parsed as Decimal before table inference and target conversion.
|
|
130
154
|
"""
|
|
131
155
|
from execsql.utils.errors import exception_desc
|
|
132
156
|
|
execsql/importers/ods.py
CHANGED
|
@@ -15,9 +15,23 @@ from execsql.exceptions import ErrInfo
|
|
|
15
15
|
from execsql.db.base import Database
|
|
16
16
|
from execsql.exporters.ods import OdsFile
|
|
17
17
|
from execsql.importers.base import import_data_table
|
|
18
|
+
from execsql.utils.fileio import check_zip_decompression_ratio
|
|
19
|
+
from execsql.utils.spreadsheet import SheetBudget
|
|
18
20
|
import execsql.state as _state
|
|
19
21
|
|
|
20
|
-
__all__ = ["importods", "ods_data"]
|
|
22
|
+
__all__ = ["check_ods_archive", "importods", "ods_data"]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def check_ods_archive(filename: str) -> None:
|
|
26
|
+
"""Apply ODS archive limits before any reader, including sheet-name discovery."""
|
|
27
|
+
budget = SheetBudget(_state.conf, filename, "<workbook>")
|
|
28
|
+
check_zip_decompression_ratio(
|
|
29
|
+
filename,
|
|
30
|
+
max_uncompressed_mb=budget.limits["ods_max_xml_mb"],
|
|
31
|
+
max_ratio=budget.limits["ods_max_compression_ratio"],
|
|
32
|
+
xml_only=True,
|
|
33
|
+
limit_names=("ods_max_xml_mb", "ods_max_compression_ratio"),
|
|
34
|
+
)
|
|
21
35
|
|
|
22
36
|
|
|
23
37
|
def ods_data(
|
|
@@ -30,15 +44,21 @@ def ods_data(
|
|
|
30
44
|
|
|
31
45
|
conf = _state.conf
|
|
32
46
|
|
|
47
|
+
budget = SheetBudget(conf, filename, sheetname)
|
|
48
|
+
check_ods_archive(filename)
|
|
33
49
|
wbk = OdsFile()
|
|
34
50
|
try:
|
|
35
51
|
wbk.open(filename)
|
|
36
52
|
except Exception as e:
|
|
37
53
|
raise ErrInfo(type="cmd", other_msg=f"{filename} is not a valid OpenDocument spreadsheet.") from e
|
|
38
54
|
try:
|
|
39
|
-
alldata = wbk.sheet_data(sheetname, junk_header_rows)
|
|
55
|
+
alldata = wbk.sheet_data(sheetname, junk_header_rows, budget=budget)
|
|
56
|
+
except ErrInfo:
|
|
57
|
+
raise
|
|
40
58
|
except Exception as e:
|
|
41
59
|
raise ErrInfo(type="cmd", other_msg=f"{sheetname} is not a worksheet in {filename}.") from e
|
|
60
|
+
finally:
|
|
61
|
+
wbk.close()
|
|
42
62
|
if not alldata:
|
|
43
63
|
raise ErrInfo(type="error", other_msg=f"The input file {filename}, sheet {sheetname} contains no data.")
|
|
44
64
|
colhdrs = alldata[0]
|
execsql/lsp/document.py
CHANGED
|
@@ -173,7 +173,12 @@ def file_facts(source: str, path: str | None, base_dir: str | None = None) -> Fi
|
|
|
173
173
|
steps: list[_Define | _ScriptDef | _ScriptCall | _Include] = []
|
|
174
174
|
reads_ini = False
|
|
175
175
|
lines = source.splitlines()
|
|
176
|
-
|
|
176
|
+
if path:
|
|
177
|
+
script_dir = Path(path).parent
|
|
178
|
+
elif base_dir:
|
|
179
|
+
script_dir = Path(base_dir)
|
|
180
|
+
else:
|
|
181
|
+
script_dir = None
|
|
177
182
|
tree = parse_string(source, path or "<untitled>", errors=[])
|
|
178
183
|
|
|
179
184
|
for node in tree.walk():
|
execsql/lsp/hover.py
CHANGED
|
@@ -106,6 +106,7 @@ def hover(index: ScriptIndex, source: str, line: int, character: int) -> types.H
|
|
|
106
106
|
start = m.end("lead")
|
|
107
107
|
written = re.match(r"\s*".join(map(re.escape, entry.keyword.split())), text[start:], re.I)
|
|
108
108
|
first_word = re.match(r"[A-Za-z_]+", text[start:])
|
|
109
|
-
|
|
109
|
+
keyword = written or first_word
|
|
110
|
+
keyword_end = start + (keyword.end() if keyword else 0)
|
|
110
111
|
body = f"**{entry.keyword}** — {entry.category} metacommand\n\n{entry.summary}\n\n{_syntax_block(entry.forms)}\n\n[Reference]({entry.url})"
|
|
111
112
|
return types.Hover(contents=markdown(body), range=span(line, start, keyword_end))
|
|
@@ -401,7 +401,9 @@ def xf_columnexists(**kwargs: Any) -> bool:
|
|
|
401
401
|
|
|
402
402
|
def xf_aliasdefined(**kwargs: Any) -> bool:
|
|
403
403
|
alias = kwargs["alias"]
|
|
404
|
-
|
|
404
|
+
# CONNECT stores the alias lowercased; match it case-insensitively,
|
|
405
|
+
# as USE and make_current already do.
|
|
406
|
+
return alias.lower() in _state.dbs.aliases()
|
|
405
407
|
|
|
406
408
|
|
|
407
409
|
def xf_metacommanderror(**kwargs: Any) -> bool:
|
execsql/metacommands/connect.py
CHANGED
|
@@ -442,6 +442,17 @@ def x_connect_dsn(**kwargs: Any) -> None:
|
|
|
442
442
|
return None
|
|
443
443
|
|
|
444
444
|
|
|
445
|
+
def _set_current_db_vars(db_alias: str) -> None:
|
|
446
|
+
"""Point $CURRENT_DBMS, $CURRENT_DATABASE and $DB_SERVER at *db_alias*.
|
|
447
|
+
|
|
448
|
+
Shared by ``USE`` and the ``DISCONNECT`` that switches the run back to
|
|
449
|
+
the initial database, so the variables always name the database in use.
|
|
450
|
+
"""
|
|
451
|
+
_state.subvars.add_substitution("$CURRENT_DBMS", _state.dbs.aliased_as(db_alias).type.dbms_id)
|
|
452
|
+
_state.subvars.add_substitution("$CURRENT_DATABASE", _state.dbs.aliased_as(db_alias).name())
|
|
453
|
+
_state.subvars.add_substitution("$DB_SERVER", _state.dbs.aliased_as(db_alias).server_name)
|
|
454
|
+
|
|
455
|
+
|
|
445
456
|
def x_use(**kwargs: Any) -> None:
|
|
446
457
|
db_alias = kwargs["db_alias"].lower()
|
|
447
458
|
if db_alias not in _state.dbs.aliases():
|
|
@@ -452,9 +463,7 @@ def x_use(**kwargs: Any) -> None:
|
|
|
452
463
|
)
|
|
453
464
|
_state.dbs.make_current(db_alias)
|
|
454
465
|
_state.exec_log.log_db_connect(_state.dbs.current())
|
|
455
|
-
|
|
456
|
-
_state.subvars.add_substitution("$CURRENT_DATABASE", _state.dbs.aliased_as(db_alias).name())
|
|
457
|
-
_state.subvars.add_substitution("$DB_SERVER", _state.dbs.aliased_as(db_alias).server_name)
|
|
466
|
+
_set_current_db_vars(db_alias)
|
|
458
467
|
return None
|
|
459
468
|
|
|
460
469
|
|
|
@@ -473,7 +482,13 @@ def x_disconnect(**kwargs: Any) -> None:
|
|
|
473
482
|
raise ErrInfo(type="cmd", other_msg="You may not disconnect from a database that is currently used in a batch.")
|
|
474
483
|
_state.exec_log.log_status_info(f"Disconnecting from database with alias '{alias}'")
|
|
475
484
|
if alias == current_alias:
|
|
485
|
+
# Switch the run back to the initial database before the entry is
|
|
486
|
+
# removed, and share USE's variable updates, so $CURRENT_DATABASE,
|
|
487
|
+
# $CURRENT_DBMS and $DB_SERVER name the database in use, not the
|
|
488
|
+
# one just closed, and the log's connection record moves too.
|
|
476
489
|
_state.dbs.make_current("initial")
|
|
490
|
+
_set_current_db_vars("initial")
|
|
491
|
+
_state.exec_log.log_db_connect(_state.dbs.current())
|
|
477
492
|
_state.dbs.disconnect(alias)
|
|
478
493
|
|
|
479
494
|
|
execsql/metacommands/dispatch.py
CHANGED
|
@@ -12,6 +12,8 @@ from __future__ import annotations
|
|
|
12
12
|
|
|
13
13
|
import re
|
|
14
14
|
|
|
15
|
+
from execsql.utils.spreadsheet import INPUT_LIMIT_DEFAULTS
|
|
16
|
+
|
|
15
17
|
# Handler imports — grouped by module for readability.
|
|
16
18
|
from execsql.metacommands.connect import (
|
|
17
19
|
x_autocommit_off,
|
|
@@ -125,6 +127,7 @@ from execsql.metacommands.io import (
|
|
|
125
127
|
x_import_ods_pattern,
|
|
126
128
|
x_import_parquet,
|
|
127
129
|
x_import_row_buffer,
|
|
130
|
+
x_input_limit,
|
|
128
131
|
x_import_xls,
|
|
129
132
|
x_import_xls_pattern,
|
|
130
133
|
x_include,
|
|
@@ -713,6 +716,13 @@ def build_dispatch_table() -> MetaCommandList:
|
|
|
713
716
|
description="MAX_INT",
|
|
714
717
|
category="config_option",
|
|
715
718
|
)
|
|
719
|
+
for name in INPUT_LIMIT_DEFAULTS:
|
|
720
|
+
mcl.add(
|
|
721
|
+
rf"^\s*CONFIG\s+(?P<name>{name.upper()})\s+(?P<value>[0-9]+)\s*$",
|
|
722
|
+
x_input_limit,
|
|
723
|
+
description=name.upper(),
|
|
724
|
+
category="config_option",
|
|
725
|
+
)
|
|
716
726
|
mcl.add(
|
|
717
727
|
r"^\s*CONFIG\s+IMPORT_ROW_BUFFER\s+(?P<rows>[1-9][0-9]*)\s*$",
|
|
718
728
|
x_import_row_buffer,
|
execsql/metacommands/io.py
CHANGED
|
@@ -11,8 +11,7 @@
|
|
|
11
11
|
- ``x_export_row_buffer`` — CONFIG EXPORT_ROW_BUFFER (set the per-export
|
|
12
12
|
row-buffer size).
|
|
13
13
|
|
|
14
|
-
Per-format dispatch happens inside ``
|
|
15
|
-
``if filefmt in (...): ... elif ...`` chain, which calls the
|
|
14
|
+
Per-format dispatch happens inside ``_dispatch_format()``, which calls the
|
|
16
15
|
appropriate writer in :mod:`execsql.exporters`.
|
|
17
16
|
"""
|
|
18
17
|
|
|
@@ -18,6 +18,7 @@ from shutil import copyfileobj
|
|
|
18
18
|
from typing import Any, Literal
|
|
19
19
|
|
|
20
20
|
import execsql.state as _state
|
|
21
|
+
from execsql.db.base import Database
|
|
21
22
|
from execsql.exceptions import ErrInfo
|
|
22
23
|
from execsql.importers.base import refuse_existing_table
|
|
23
24
|
from execsql.models import DataTable
|
|
@@ -102,73 +103,8 @@ def x_copy(**kwargs: Any) -> None:
|
|
|
102
103
|
db1 = _state.dbs.aliased_as(alias1)
|
|
103
104
|
db2 = _state.dbs.aliased_as(alias2)
|
|
104
105
|
tbl1 = db1.schema_qualified_table_name(schema1, table1)
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
# (PostgreSQL reads Orders as orders), and the SELECT below names a missing
|
|
108
|
-
# source in the database's own words.
|
|
109
|
-
if new_tbl2 and new_tbl2 == "new":
|
|
110
|
-
refuse_existing_table(db2, schema2, table2)
|
|
111
|
-
select_stmt = f"select * from {tbl1};"
|
|
112
|
-
|
|
113
|
-
get_ts_tablespec = None
|
|
114
|
-
rows_to_close = None
|
|
115
|
-
spool: _RowSpool | None = None
|
|
116
|
-
rows_for_insert: Any
|
|
117
|
-
|
|
118
|
-
if new_tbl2:
|
|
119
|
-
try:
|
|
120
|
-
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
121
|
-
except ErrInfo:
|
|
122
|
-
raise
|
|
123
|
-
except Exception as e:
|
|
124
|
-
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
125
|
-
rows_to_close = rows
|
|
126
|
-
# Values come from a database cursor, so a text value stays text.
|
|
127
|
-
spool = _RowSpool(rows)
|
|
128
|
-
get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
|
|
129
|
-
rows_for_insert = spool.replay()
|
|
130
|
-
tbl_desc = get_ts_tablespec
|
|
131
|
-
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
132
|
-
if new_tbl2 == "replacement":
|
|
133
|
-
try:
|
|
134
|
-
db2.drop_table_for_replacement(schema2, table2)
|
|
135
|
-
except ErrInfo:
|
|
136
|
-
raise
|
|
137
|
-
except Exception as e:
|
|
138
|
-
raise ErrInfo(
|
|
139
|
-
type="db",
|
|
140
|
-
other_msg=f"Could not drop existing table ({tbl2}) for COPY metacommand",
|
|
141
|
-
exception_msg=exception_desc(),
|
|
142
|
-
) from e
|
|
143
|
-
db2.execute(create_tbl)
|
|
144
|
-
if db2.needs_explicit_commit_after_ddl():
|
|
145
|
-
db2.execute("COMMIT;")
|
|
146
|
-
else:
|
|
147
|
-
try:
|
|
148
|
-
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
149
|
-
except ErrInfo:
|
|
150
|
-
raise
|
|
151
|
-
except Exception as e:
|
|
152
|
-
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
153
|
-
rows_to_close = rows
|
|
154
|
-
rows_for_insert = rows
|
|
155
|
-
|
|
156
|
-
def get_ts() -> DataTable:
|
|
157
|
-
nonlocal get_ts_tablespec
|
|
158
|
-
if get_ts_tablespec is None:
|
|
159
|
-
ts_hdrs, ts_rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
160
|
-
get_ts_tablespec = DataTable(ts_hdrs, ts_rows, infer_strings=False)
|
|
161
|
-
return get_ts_tablespec
|
|
162
|
-
|
|
163
|
-
try:
|
|
164
|
-
db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
|
|
165
|
-
db2.commit()
|
|
166
|
-
except BaseException:
|
|
167
|
-
_close_rowsource(rows_to_close)
|
|
168
|
-
raise
|
|
169
|
-
finally:
|
|
170
|
-
if spool is not None:
|
|
171
|
-
spool.close()
|
|
106
|
+
# Let the source database resolve unquoted names and report missing tables.
|
|
107
|
+
_copy_rows(db1, db2, f"select * from {tbl1};", schema2, table2, new_tbl2)
|
|
172
108
|
|
|
173
109
|
|
|
174
110
|
def x_copy_query(**kwargs: Any) -> None:
|
|
@@ -193,29 +129,38 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
193
129
|
)
|
|
194
130
|
db1 = _state.dbs.aliased_as(alias1)
|
|
195
131
|
db2 = _state.dbs.aliased_as(alias2)
|
|
132
|
+
_copy_rows(db1, db2, select_stmt, schema2, table2, new_tbl2)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _copy_rows(
|
|
136
|
+
db1: Database,
|
|
137
|
+
db2: Database,
|
|
138
|
+
select_stmt: str,
|
|
139
|
+
schema2: str | None,
|
|
140
|
+
table2: str,
|
|
141
|
+
new_tbl2: str | None,
|
|
142
|
+
) -> None:
|
|
143
|
+
"""Copy a query's rows, sharing table creation, type inference and spool cleanup."""
|
|
196
144
|
tbl2 = db2.schema_qualified_table_name(schema2, table2)
|
|
197
|
-
if new_tbl2
|
|
145
|
+
if new_tbl2 == "new":
|
|
198
146
|
refuse_existing_table(db2, schema2, table2)
|
|
199
147
|
|
|
200
|
-
get_ts_tablespec = None
|
|
201
|
-
rows_to_close = None
|
|
148
|
+
get_ts_tablespec: DataTable | None = None
|
|
202
149
|
spool: _RowSpool | None = None
|
|
203
|
-
|
|
150
|
+
try:
|
|
151
|
+
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
152
|
+
except ErrInfo:
|
|
153
|
+
raise
|
|
154
|
+
except Exception as e:
|
|
155
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
156
|
+
rows_for_insert: Iterator[Any] = rows
|
|
204
157
|
|
|
205
158
|
if new_tbl2:
|
|
206
|
-
try:
|
|
207
|
-
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
208
|
-
except ErrInfo:
|
|
209
|
-
raise
|
|
210
|
-
except Exception as e:
|
|
211
|
-
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
212
|
-
rows_to_close = rows
|
|
213
159
|
# Values come from a database cursor, so a text value stays text.
|
|
214
160
|
spool = _RowSpool(rows)
|
|
215
161
|
get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
|
|
216
162
|
rows_for_insert = spool.replay()
|
|
217
|
-
|
|
218
|
-
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
163
|
+
create_tbl = get_ts_tablespec.create_table(db2.type, schema2, table2)
|
|
219
164
|
if new_tbl2 == "replacement":
|
|
220
165
|
try:
|
|
221
166
|
db2.drop_table_for_replacement(schema2, table2)
|
|
@@ -230,15 +175,6 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
230
175
|
db2.execute(create_tbl)
|
|
231
176
|
if db2.needs_explicit_commit_after_ddl():
|
|
232
177
|
db2.execute("COMMIT;")
|
|
233
|
-
else:
|
|
234
|
-
try:
|
|
235
|
-
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
236
|
-
except ErrInfo:
|
|
237
|
-
raise
|
|
238
|
-
except Exception as e:
|
|
239
|
-
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
240
|
-
rows_to_close = rows
|
|
241
|
-
rows_for_insert = rows
|
|
242
178
|
|
|
243
179
|
def get_ts() -> DataTable:
|
|
244
180
|
nonlocal get_ts_tablespec
|
|
@@ -251,7 +187,7 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
251
187
|
db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
|
|
252
188
|
db2.commit()
|
|
253
189
|
except BaseException:
|
|
254
|
-
_close_rowsource(
|
|
190
|
+
_close_rowsource(rows)
|
|
255
191
|
raise
|
|
256
192
|
finally:
|
|
257
193
|
if spool is not None:
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"""Import metacommand handlers.
|
|
2
2
|
|
|
3
|
-
- ``x_import`` — IMPORT (delimited text / CSV;
|
|
4
|
-
for .ods/.xls/.xlsx in the same handler).
|
|
3
|
+
- ``x_import`` — IMPORT (delimited text / CSV; spreadsheet forms have separate handlers).
|
|
5
4
|
- ``x_import_file`` — IMPORT_FILE (insert a binary file blob as a row).
|
|
6
|
-
- ``x_import_ods`` / ``x_import_ods_pattern`` — IMPORT … FROM
|
|
7
|
-
|
|
8
|
-
- ``x_import_xls`` / ``x_import_xls_pattern`` —
|
|
5
|
+
- ``x_import_ods`` / ``x_import_ods_pattern`` — IMPORT … FROM <file>
|
|
6
|
+
SHEET <name> / SHEETS MATCHING <pattern> (ODS).
|
|
7
|
+
- ``x_import_xls`` / ``x_import_xls_pattern`` — IMPORT … FROM EXCEL <file>
|
|
8
|
+
SHEET <name> / SHEETS MATCHING <pattern> (XLS/XLSX).
|
|
9
9
|
- ``x_import_parquet`` — IMPORT … FROM PARQUET (via polars).
|
|
10
10
|
- ``x_import_feather`` — IMPORT … FROM FEATHER (via polars).
|
|
11
11
|
- ``x_import_json`` — IMPORT … FROM JSON (array of objects or JSON Lines).
|
|
@@ -22,9 +22,10 @@ from typing import Any
|
|
|
22
22
|
import execsql.state as _state
|
|
23
23
|
from execsql.exceptions import ErrInfo
|
|
24
24
|
from execsql.importers.csv import importfile, importtable
|
|
25
|
+
from execsql.utils.spreadsheet import INPUT_LIMIT_DEFAULTS, SheetBudget
|
|
25
26
|
from execsql.importers.feather import import_feather, import_parquet
|
|
26
27
|
from execsql.importers.json import import_json
|
|
27
|
-
from execsql.importers.ods import OdsFile, importods
|
|
28
|
+
from execsql.importers.ods import OdsFile, check_ods_archive, importods
|
|
28
29
|
from execsql.exporters.xls import XlsFile, XlsxFile
|
|
29
30
|
from execsql.importers.xls import importxls
|
|
30
31
|
from execsql.utils.errors import exception_desc
|
|
@@ -185,13 +186,21 @@ def x_import_ods_pattern(**kwargs: Any) -> None:
|
|
|
185
186
|
command_text=kwargs["metacommandline"],
|
|
186
187
|
other_msg="Input file does not exist",
|
|
187
188
|
)
|
|
189
|
+
check_ods_archive(filename)
|
|
188
190
|
wbk = OdsFile()
|
|
189
191
|
try:
|
|
190
192
|
wbk.open(filename)
|
|
191
193
|
except Exception as e:
|
|
192
194
|
raise ErrInfo(type="cmd", other_msg=f"{filename} is not a valid OpenDocument spreadsheet.") from e
|
|
193
|
-
|
|
194
|
-
|
|
195
|
+
try:
|
|
196
|
+
sheets = wbk.sheetnames()
|
|
197
|
+
impsheets = [s for s in sheets if rx.search(s)]
|
|
198
|
+
# Validate each selected sheet before the first target can be changed;
|
|
199
|
+
# discard the rows between sheets instead of retaining their expansion.
|
|
200
|
+
for sheet in impsheets:
|
|
201
|
+
wbk.sheet_data(sheet, hdr_rows, budget=SheetBudget(_state.conf, filename, sheet))
|
|
202
|
+
finally:
|
|
203
|
+
wbk.close()
|
|
195
204
|
tables = list(impsheets)
|
|
196
205
|
if _state.conf.clean_col_hdrs:
|
|
197
206
|
tables = clean_words(tables)
|
|
@@ -461,3 +470,12 @@ def x_import_row_buffer(**kwargs: Any) -> None:
|
|
|
461
470
|
def x_show_progress(**kwargs: Any) -> None:
|
|
462
471
|
setting = kwargs["setting"].lower()
|
|
463
472
|
_state.conf.show_progress = setting in ("yes", "on", "true", "1")
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def x_input_limit(**kwargs: Any) -> None:
|
|
476
|
+
"""Set a non-negative spreadsheet input budget for subsequent imports."""
|
|
477
|
+
name = kwargs["name"].lower()
|
|
478
|
+
value = int(kwargs["value"])
|
|
479
|
+
if name not in INPUT_LIMIT_DEFAULTS or value < 0:
|
|
480
|
+
raise ErrInfo(type="cmd", other_msg=f"Invalid spreadsheet input limit: {name}={value}.")
|
|
481
|
+
setattr(_state.conf, name, value)
|
|
@@ -260,6 +260,13 @@ _METACOMMANDS: tuple[Metacommand, ...] = (
|
|
|
260
260
|
"CONFIG GUI_LEVEL <n>",
|
|
261
261
|
"CONFIG HDF5_TEXT_LEN <n>",
|
|
262
262
|
"CONFIG IMPORT_ROW_BUFFER <n>",
|
|
263
|
+
"CONFIG ODS_MAX_XML_MB <n>",
|
|
264
|
+
"CONFIG ODS_MAX_COMPRESSION_RATIO <n>",
|
|
265
|
+
"CONFIG IMPORT_MAX_ROWS <n>",
|
|
266
|
+
"CONFIG IMPORT_MAX_COLUMNS <n>",
|
|
267
|
+
"CONFIG IMPORT_MAX_CELLS <n>",
|
|
268
|
+
"CONFIG IMPORT_MAX_CELL_CHARS <n>",
|
|
269
|
+
"CONFIG IMPORT_MAX_TEXT_CHARS <n>",
|
|
263
270
|
"CONFIG LOG_DATAVARS YES|NO",
|
|
264
271
|
"CONFIG LOG_SQL YES|NO",
|
|
265
272
|
"CONFIG LOG_WRITE_MESSAGES YES|NO",
|
execsql/parser.py
CHANGED
|
@@ -48,8 +48,6 @@ class SourceString:
|
|
|
48
48
|
|
|
49
49
|
def eoi(self) -> bool:
|
|
50
50
|
"""Return True if the entire source string has been consumed."""
|
|
51
|
-
# Returns True or False indicating whether or not there is any of
|
|
52
|
-
# the source string left to be consumed.
|
|
53
51
|
return self.currpos >= len(self.str)
|
|
54
52
|
|
|
55
53
|
def eat_whitespace(self) -> None:
|
|
@@ -59,55 +57,34 @@ class SourceString:
|
|
|
59
57
|
|
|
60
58
|
def match_str(self, str: str) -> str | None:
|
|
61
59
|
"""Match a string case-insensitively at the current position and advance."""
|
|
62
|
-
# Tries to match the 'str' argument at the current position in the
|
|
63
|
-
# source string. Matching is case-insensitive. If matching succeeds,
|
|
64
|
-
# the matched string is returned and the internal pointer is incremented.
|
|
65
|
-
# If matching fails, None is returned and the internal pointer is unchanged.
|
|
66
60
|
self.eat_whitespace()
|
|
67
|
-
if self.eoi():
|
|
61
|
+
if self.eoi() or not self.str.lower().startswith(str.lower(), self.currpos):
|
|
68
62
|
return None
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
matched = self.str[self.currpos : self.currpos + len(str)]
|
|
73
|
-
self.currpos += len(str)
|
|
74
|
-
return matched
|
|
75
|
-
else:
|
|
76
|
-
return None
|
|
63
|
+
matched = self.str[self.currpos : self.currpos + len(str)]
|
|
64
|
+
self.currpos += len(str)
|
|
65
|
+
return matched
|
|
77
66
|
|
|
78
67
|
def match_regex(self, regex: Any) -> dict | None:
|
|
79
68
|
"""Match a compiled regex at the current position and return named groups."""
|
|
80
|
-
# Tries to match the 'regex' argument at the current position in the
|
|
81
|
-
# source string. If it succeeds, a dictionary of all of the named
|
|
82
|
-
# groups is returned, and the internal pointer is incremented.
|
|
83
69
|
self.eat_whitespace()
|
|
84
70
|
if self.eoi():
|
|
85
71
|
return None
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
else:
|
|
92
|
-
return None
|
|
72
|
+
m = regex.match(self.str[self.currpos :])
|
|
73
|
+
if not m:
|
|
74
|
+
return None
|
|
75
|
+
self.currpos += m.end(0)
|
|
76
|
+
return m.groupdict() or {}
|
|
93
77
|
|
|
94
78
|
def match_metacommand(self, commandlist: Any) -> tuple | None:
|
|
95
79
|
"""Match a metacommand from the command list at the current position."""
|
|
96
|
-
# Tries to match text at the current position to any metacommand
|
|
97
|
-
# in the specified commandlist.
|
|
98
|
-
# If it succeeds, the return value is a tuple of the MetaCommand object
|
|
99
|
-
# and a dictionary of all of the named groups. The internal pointer is
|
|
100
|
-
# incremented past the match.
|
|
101
80
|
self.eat_whitespace()
|
|
102
81
|
if self.eoi():
|
|
103
82
|
return None
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
else:
|
|
110
|
-
return None
|
|
83
|
+
m = commandlist.get_match(self.str[self.currpos :])
|
|
84
|
+
if m is None:
|
|
85
|
+
return None
|
|
86
|
+
self.currpos += m[1].end(0)
|
|
87
|
+
return (m[0], m[1].groupdict() or {})
|
|
111
88
|
|
|
112
89
|
def remainder(self) -> str:
|
|
113
90
|
"""Return the unconsumed portion of the source string."""
|
execsql/script/control.py
CHANGED
|
@@ -89,15 +89,25 @@ class BatchLevels:
|
|
|
89
89
|
autocommit off, whose transaction is the script's) before the error
|
|
90
90
|
is raised: a commit that already succeeded
|
|
91
91
|
cannot be undone, but nothing else of the batch is left pending for
|
|
92
|
-
a later statement to commit.
|
|
92
|
+
a later statement to commit. An interrupt rolls back the database
|
|
93
|
+
it landed on as well, since it may have come before that database's
|
|
94
|
+
own rollback. The level leaves the stack last, so until then an
|
|
95
|
+
interrupt finds it there to abandon.
|
|
93
96
|
"""
|
|
94
|
-
b = self.batchlevels
|
|
95
|
-
|
|
96
|
-
|
|
97
|
+
b = self.batchlevels[-1]
|
|
98
|
+
committed = 0
|
|
99
|
+
try:
|
|
100
|
+
for db in b.dbs_used:
|
|
97
101
|
db.commit()
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
102
|
+
committed += 1
|
|
103
|
+
except Exception:
|
|
104
|
+
_roll_back(b.dbs_used[committed + 1 :])
|
|
105
|
+
raise
|
|
106
|
+
except BaseException:
|
|
107
|
+
_roll_back(b.dbs_used[committed:])
|
|
108
|
+
raise
|
|
109
|
+
finally:
|
|
110
|
+
self.batchlevels.pop()
|
|
101
111
|
|
|
102
112
|
def abandon_batch(self) -> None:
|
|
103
113
|
"""Pop the innermost batch level without committing it.
|