execsql2 2.37.0__py3-none-any.whl → 2.38.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. execsql/api.py +95 -45
  2. execsql/cli/commands/run.py +2 -2
  3. execsql/cli/lint.py +6 -1
  4. execsql/cli/run.py +5 -0
  5. execsql/config.py +7 -0
  6. execsql/data/execsql.conf.template +21 -0
  7. execsql/db/base.py +52 -24
  8. execsql/db/duckdb.py +43 -4
  9. execsql/db/firebird.py +8 -0
  10. execsql/db/mysql.py +8 -3
  11. execsql/db/oracle.py +5 -0
  12. execsql/db/postgres.py +2 -2
  13. execsql/db/sqlite.py +117 -10
  14. execsql/db/sqlserver.py +11 -7
  15. execsql/db/target_types.py +151 -4
  16. execsql/debug/repl.py +2 -1
  17. execsql/exceptions.py +14 -0
  18. execsql/exporters/html.py +26 -42
  19. execsql/exporters/ods.py +93 -21
  20. execsql/exporters/sqlite.py +82 -47
  21. execsql/format.py +12 -4
  22. execsql/gui/desktop.py +2 -1
  23. execsql/gui/tui.py +5 -2
  24. execsql/importers/base.py +9 -7
  25. execsql/importers/json.py +37 -13
  26. execsql/importers/ods.py +22 -2
  27. execsql/lsp/document.py +6 -1
  28. execsql/lsp/hover.py +2 -1
  29. execsql/metacommands/conditions.py +3 -1
  30. execsql/metacommands/connect.py +18 -3
  31. execsql/metacommands/dispatch.py +10 -0
  32. execsql/metacommands/io.py +1 -0
  33. execsql/metacommands/io_export.py +1 -2
  34. execsql/metacommands/io_fileops.py +26 -90
  35. execsql/metacommands/io_import.py +26 -8
  36. execsql/metacommands/reference.py +7 -0
  37. execsql/parser.py +14 -37
  38. execsql/script/control.py +17 -7
  39. execsql/script/executor.py +160 -34
  40. execsql/script/preparation.py +42 -0
  41. execsql/types.py +6 -5
  42. execsql/utils/errors.py +2 -2
  43. execsql/utils/fileio.py +342 -134
  44. execsql/utils/spreadsheet.py +74 -0
  45. execsql/utils/strings.py +13 -7
  46. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/README.md +5 -5
  47. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/example_config_prompt.sql +2 -2
  48. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/METADATA +32 -119
  49. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/RECORD +64 -62
  50. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/WHEEL +1 -1
  51. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
  52. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
  53. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_compare.sql +0 -0
  54. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
  55. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
  56. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
  57. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
  58. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/script_template.sql +0 -0
  59. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
  60. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
  61. {execsql2-2.37.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
  62. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/entry_points.txt +0 -0
  63. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/licenses/LICENSE.txt +0 -0
  64. {execsql2-2.37.0.dist-info → execsql2-2.38.0.dist-info}/licenses/NOTICE +0 -0
execsql/importers/json.py CHANGED
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  """
4
4
  JSON import for execsql.
5
5
 
6
- Provides :func:`import_json`, used by ``IMPORT … FORMAT json``.
6
+ Provides :func:`import_json`, used by ``IMPORT … FROM JSON``.
7
7
  Supports JSON arrays of objects (``[{…}, …]``) and `JSON Lines
8
8
  <https://jsonlines.org/>`_ (JSONL, one object per line). Nested
9
9
  objects are flattened with dot-separated keys; nested arrays and
@@ -12,6 +12,7 @@ maps to a scalar database value.
12
12
  """
13
13
 
14
14
  import json
15
+ from decimal import Decimal
15
16
  from pathlib import Path
16
17
  from typing import Any
17
18
 
@@ -22,30 +23,51 @@ from execsql.importers.base import import_data_table
22
23
  __all__ = ["import_json"]
23
24
 
24
25
 
26
+ def _array_number(value: Any) -> Any:
27
+ """Keep the established JSON text representation of numbers inside arrays."""
28
+ return float(value) if isinstance(value, Decimal) else str(value)
29
+
30
+
25
31
  def _flatten(obj: Any, prefix: str = "", sep: str = ".") -> dict[str, Any]:
26
32
  """Recursively flatten a nested dict.
27
33
 
28
34
  Nested dicts produce dot-separated keys. All other compound values
29
35
  (lists, nested lists-of-dicts) are serialized as JSON strings so the
30
- result is always ``{str: scalar}``.
36
+ result is always ``{str: scalar}``. Distinct paths that produce the
37
+ same column name raise ErrInfo instead of discarding a value.
31
38
  """
32
39
  items: dict[str, Any] = {}
33
- if isinstance(obj, dict):
34
- for key, value in obj.items():
35
- new_key = f"{prefix}{sep}{key}" if prefix else key
40
+ source_paths: dict[str, tuple[str, ...]] = {}
41
+
42
+ def visit(current: Any, parent: str, path: tuple[str, ...]) -> None:
43
+ if not isinstance(current, dict):
44
+ return
45
+ for key, value in current.items():
46
+ column = f"{parent}{sep}{key}" if parent else key
47
+ source_path = (*path, key)
36
48
  if isinstance(value, dict):
37
- items.update(_flatten(value, new_key, sep))
38
- elif isinstance(value, list):
39
- # Serialize arrays as JSON strings — tables are flat.
40
- items[new_key] = json.dumps(value, default=str)
49
+ visit(value, column, source_path)
41
50
  else:
42
- items[new_key] = value
51
+ if column in items:
52
+ raise ErrInfo(
53
+ type="error",
54
+ other_msg=(
55
+ f"JSON field paths {json.dumps(source_paths[column])} and {json.dumps(source_path)} "
56
+ f"flatten to the same column {column!r}. Rename the conflicting keys."
57
+ ),
58
+ )
59
+ source_paths[column] = source_path
60
+ items[column] = json.dumps(value, default=_array_number) if isinstance(value, list) else value
61
+
62
+ visit(obj, prefix, ())
43
63
  return items
44
64
 
45
65
 
46
66
  def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
47
67
  """Read a JSON file and return a list of flat dicts.
48
68
 
69
+ Fractional and exponent scalar numbers retain their exact Decimal digits;
70
+ integers remain int. Arrays retain their established JSON text encoding.
49
71
  Accepts either a JSON array of objects or `JSON Lines
50
72
  <https://jsonlines.org/>`_ (JSONL). The JSONL path streams the
51
73
  file line-by-line so the raw text isn't buffered alongside the
@@ -69,7 +91,7 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
69
91
  # Standard JSON array — must buffer the whole file (json.loads
70
92
  # has no streaming mode; ijson would be needed for that).
71
93
  text = Path(filename).read_text(encoding=encoding)
72
- raw = json.loads(text)
94
+ raw = json.loads(text, parse_float=Decimal)
73
95
  if not isinstance(raw, list):
74
96
  raise ErrInfo(type="error", other_msg="JSON file root is not an array of objects.")
75
97
  records = raw
@@ -82,7 +104,7 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
82
104
  if not line:
83
105
  continue
84
106
  try:
85
- obj = json.loads(line)
107
+ obj = json.loads(line, parse_float=Decimal)
86
108
  except json.JSONDecodeError as exc:
87
109
  raise ErrInfo(
88
110
  type="error",
@@ -126,7 +148,9 @@ def import_json(
126
148
 
127
149
  Objects are flattened so that nested keys become dot-separated column
128
150
  names (e.g. ``address.city``). Arrays within objects are stored as
129
- JSON strings.
151
+ JSON strings. Colliding paths within a record are rejected before
152
+ any table is created, replaced or populated. Fractional and exponent scalar
153
+ numbers are parsed as Decimal before table inference and target conversion.
130
154
  """
131
155
  from execsql.utils.errors import exception_desc
132
156
 
execsql/importers/ods.py CHANGED
@@ -15,9 +15,23 @@ from execsql.exceptions import ErrInfo
15
15
  from execsql.db.base import Database
16
16
  from execsql.exporters.ods import OdsFile
17
17
  from execsql.importers.base import import_data_table
18
+ from execsql.utils.fileio import check_zip_decompression_ratio
19
+ from execsql.utils.spreadsheet import SheetBudget
18
20
  import execsql.state as _state
19
21
 
20
- __all__ = ["importods", "ods_data"]
22
+ __all__ = ["check_ods_archive", "importods", "ods_data"]
23
+
24
+
25
+ def check_ods_archive(filename: str) -> None:
26
+ """Apply ODS archive limits before any reader, including sheet-name discovery."""
27
+ budget = SheetBudget(_state.conf, filename, "<workbook>")
28
+ check_zip_decompression_ratio(
29
+ filename,
30
+ max_uncompressed_mb=budget.limits["ods_max_xml_mb"],
31
+ max_ratio=budget.limits["ods_max_compression_ratio"],
32
+ xml_only=True,
33
+ limit_names=("ods_max_xml_mb", "ods_max_compression_ratio"),
34
+ )
21
35
 
22
36
 
23
37
  def ods_data(
@@ -30,15 +44,21 @@ def ods_data(
30
44
 
31
45
  conf = _state.conf
32
46
 
47
+ budget = SheetBudget(conf, filename, sheetname)
48
+ check_ods_archive(filename)
33
49
  wbk = OdsFile()
34
50
  try:
35
51
  wbk.open(filename)
36
52
  except Exception as e:
37
53
  raise ErrInfo(type="cmd", other_msg=f"{filename} is not a valid OpenDocument spreadsheet.") from e
38
54
  try:
39
- alldata = wbk.sheet_data(sheetname, junk_header_rows)
55
+ alldata = wbk.sheet_data(sheetname, junk_header_rows, budget=budget)
56
+ except ErrInfo:
57
+ raise
40
58
  except Exception as e:
41
59
  raise ErrInfo(type="cmd", other_msg=f"{sheetname} is not a worksheet in {filename}.") from e
60
+ finally:
61
+ wbk.close()
42
62
  if not alldata:
43
63
  raise ErrInfo(type="error", other_msg=f"The input file {filename}, sheet {sheetname} contains no data.")
44
64
  colhdrs = alldata[0]
execsql/lsp/document.py CHANGED
@@ -173,7 +173,12 @@ def file_facts(source: str, path: str | None, base_dir: str | None = None) -> Fi
173
173
  steps: list[_Define | _ScriptDef | _ScriptCall | _Include] = []
174
174
  reads_ini = False
175
175
  lines = source.splitlines()
176
- script_dir = Path(path).parent if path else (Path(base_dir) if base_dir else None)
176
+ if path:
177
+ script_dir = Path(path).parent
178
+ elif base_dir:
179
+ script_dir = Path(base_dir)
180
+ else:
181
+ script_dir = None
177
182
  tree = parse_string(source, path or "<untitled>", errors=[])
178
183
 
179
184
  for node in tree.walk():
execsql/lsp/hover.py CHANGED
@@ -106,6 +106,7 @@ def hover(index: ScriptIndex, source: str, line: int, character: int) -> types.H
106
106
  start = m.end("lead")
107
107
  written = re.match(r"\s*".join(map(re.escape, entry.keyword.split())), text[start:], re.I)
108
108
  first_word = re.match(r"[A-Za-z_]+", text[start:])
109
- keyword_end = start + (written.end() if written else first_word.end() if first_word else 0)
109
+ keyword = written or first_word
110
+ keyword_end = start + (keyword.end() if keyword else 0)
110
111
  body = f"**{entry.keyword}** — {entry.category} metacommand\n\n{entry.summary}\n\n{_syntax_block(entry.forms)}\n\n[Reference]({entry.url})"
111
112
  return types.Hover(contents=markdown(body), range=span(line, start, keyword_end))
@@ -401,7 +401,9 @@ def xf_columnexists(**kwargs: Any) -> bool:
401
401
 
402
402
  def xf_aliasdefined(**kwargs: Any) -> bool:
403
403
  alias = kwargs["alias"]
404
- return alias in _state.dbs.aliases()
404
+ # CONNECT stores the alias lowercased; match it case-insensitively,
405
+ # as USE and make_current already do.
406
+ return alias.lower() in _state.dbs.aliases()
405
407
 
406
408
 
407
409
  def xf_metacommanderror(**kwargs: Any) -> bool:
@@ -442,6 +442,17 @@ def x_connect_dsn(**kwargs: Any) -> None:
442
442
  return None
443
443
 
444
444
 
445
+ def _set_current_db_vars(db_alias: str) -> None:
446
+ """Point $CURRENT_DBMS, $CURRENT_DATABASE and $DB_SERVER at *db_alias*.
447
+
448
+ Shared by ``USE`` and the ``DISCONNECT`` that switches the run back to
449
+ the initial database, so the variables always name the database in use.
450
+ """
451
+ _state.subvars.add_substitution("$CURRENT_DBMS", _state.dbs.aliased_as(db_alias).type.dbms_id)
452
+ _state.subvars.add_substitution("$CURRENT_DATABASE", _state.dbs.aliased_as(db_alias).name())
453
+ _state.subvars.add_substitution("$DB_SERVER", _state.dbs.aliased_as(db_alias).server_name)
454
+
455
+
445
456
  def x_use(**kwargs: Any) -> None:
446
457
  db_alias = kwargs["db_alias"].lower()
447
458
  if db_alias not in _state.dbs.aliases():
@@ -452,9 +463,7 @@ def x_use(**kwargs: Any) -> None:
452
463
  )
453
464
  _state.dbs.make_current(db_alias)
454
465
  _state.exec_log.log_db_connect(_state.dbs.current())
455
- _state.subvars.add_substitution("$CURRENT_DBMS", _state.dbs.aliased_as(db_alias).type.dbms_id)
456
- _state.subvars.add_substitution("$CURRENT_DATABASE", _state.dbs.aliased_as(db_alias).name())
457
- _state.subvars.add_substitution("$DB_SERVER", _state.dbs.aliased_as(db_alias).server_name)
466
+ _set_current_db_vars(db_alias)
458
467
  return None
459
468
 
460
469
 
@@ -473,7 +482,13 @@ def x_disconnect(**kwargs: Any) -> None:
473
482
  raise ErrInfo(type="cmd", other_msg="You may not disconnect from a database that is currently used in a batch.")
474
483
  _state.exec_log.log_status_info(f"Disconnecting from database with alias '{alias}'")
475
484
  if alias == current_alias:
485
+ # Switch the run back to the initial database before the entry is
486
+ # removed, and share USE's variable updates, so $CURRENT_DATABASE,
487
+ # $CURRENT_DBMS and $DB_SERVER name the database in use, not the
488
+ # one just closed, and the log's connection record moves too.
476
489
  _state.dbs.make_current("initial")
490
+ _set_current_db_vars("initial")
491
+ _state.exec_log.log_db_connect(_state.dbs.current())
477
492
  _state.dbs.disconnect(alias)
478
493
 
479
494
 
@@ -12,6 +12,8 @@ from __future__ import annotations
12
12
 
13
13
  import re
14
14
 
15
+ from execsql.utils.spreadsheet import INPUT_LIMIT_DEFAULTS
16
+
15
17
  # Handler imports — grouped by module for readability.
16
18
  from execsql.metacommands.connect import (
17
19
  x_autocommit_off,
@@ -125,6 +127,7 @@ from execsql.metacommands.io import (
125
127
  x_import_ods_pattern,
126
128
  x_import_parquet,
127
129
  x_import_row_buffer,
130
+ x_input_limit,
128
131
  x_import_xls,
129
132
  x_import_xls_pattern,
130
133
  x_include,
@@ -713,6 +716,13 @@ def build_dispatch_table() -> MetaCommandList:
713
716
  description="MAX_INT",
714
717
  category="config_option",
715
718
  )
719
+ for name in INPUT_LIMIT_DEFAULTS:
720
+ mcl.add(
721
+ rf"^\s*CONFIG\s+(?P<name>{name.upper()})\s+(?P<value>[0-9]+)\s*$",
722
+ x_input_limit,
723
+ description=name.upper(),
724
+ category="config_option",
725
+ )
716
726
  mcl.add(
717
727
  r"^\s*CONFIG\s+IMPORT_ROW_BUFFER\s+(?P<rows>[1-9][0-9]*)\s*$",
718
728
  x_import_row_buffer,
@@ -39,6 +39,7 @@ from execsql.metacommands.io_import import ( # noqa: F401
39
39
  x_import_ods_pattern,
40
40
  x_import_parquet,
41
41
  x_import_row_buffer,
42
+ x_input_limit,
42
43
  x_import_xls,
43
44
  x_show_progress,
44
45
  x_import_xls_pattern,
@@ -11,8 +11,7 @@
11
11
  - ``x_export_row_buffer`` — CONFIG EXPORT_ROW_BUFFER (set the per-export
12
12
  row-buffer size).
13
13
 
14
- Per-format dispatch happens inside ``x_export``'s
15
- ``if filefmt in (...): ... elif ...`` chain, which calls the
14
+ Per-format dispatch happens inside ``_dispatch_format()``, which calls the
16
15
  appropriate writer in :mod:`execsql.exporters`.
17
16
  """
18
17
 
@@ -18,6 +18,7 @@ from shutil import copyfileobj
18
18
  from typing import Any, Literal
19
19
 
20
20
  import execsql.state as _state
21
+ from execsql.db.base import Database
21
22
  from execsql.exceptions import ErrInfo
22
23
  from execsql.importers.base import refuse_existing_table
23
24
  from execsql.models import DataTable
@@ -102,73 +103,8 @@ def x_copy(**kwargs: Any) -> None:
102
103
  db1 = _state.dbs.aliased_as(alias1)
103
104
  db2 = _state.dbs.aliased_as(alias2)
104
105
  tbl1 = db1.schema_qualified_table_name(schema1, table1)
105
- tbl2 = db2.schema_qualified_table_name(schema2, table2)
106
- # No check that the source exists: the database may fold an unquoted name
107
- # (PostgreSQL reads Orders as orders), and the SELECT below names a missing
108
- # source in the database's own words.
109
- if new_tbl2 and new_tbl2 == "new":
110
- refuse_existing_table(db2, schema2, table2)
111
- select_stmt = f"select * from {tbl1};"
112
-
113
- get_ts_tablespec = None
114
- rows_to_close = None
115
- spool: _RowSpool | None = None
116
- rows_for_insert: Any
117
-
118
- if new_tbl2:
119
- try:
120
- hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
121
- except ErrInfo:
122
- raise
123
- except Exception as e:
124
- raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
125
- rows_to_close = rows
126
- # Values come from a database cursor, so a text value stays text.
127
- spool = _RowSpool(rows)
128
- get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
129
- rows_for_insert = spool.replay()
130
- tbl_desc = get_ts_tablespec
131
- create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
132
- if new_tbl2 == "replacement":
133
- try:
134
- db2.drop_table_for_replacement(schema2, table2)
135
- except ErrInfo:
136
- raise
137
- except Exception as e:
138
- raise ErrInfo(
139
- type="db",
140
- other_msg=f"Could not drop existing table ({tbl2}) for COPY metacommand",
141
- exception_msg=exception_desc(),
142
- ) from e
143
- db2.execute(create_tbl)
144
- if db2.needs_explicit_commit_after_ddl():
145
- db2.execute("COMMIT;")
146
- else:
147
- try:
148
- hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
149
- except ErrInfo:
150
- raise
151
- except Exception as e:
152
- raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
153
- rows_to_close = rows
154
- rows_for_insert = rows
155
-
156
- def get_ts() -> DataTable:
157
- nonlocal get_ts_tablespec
158
- if get_ts_tablespec is None:
159
- ts_hdrs, ts_rows = db1.select_rowsource(select_stmt, decode_bytes=False)
160
- get_ts_tablespec = DataTable(ts_hdrs, ts_rows, infer_strings=False)
161
- return get_ts_tablespec
162
-
163
- try:
164
- db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
165
- db2.commit()
166
- except BaseException:
167
- _close_rowsource(rows_to_close)
168
- raise
169
- finally:
170
- if spool is not None:
171
- spool.close()
106
+ # Let the source database resolve unquoted names and report missing tables.
107
+ _copy_rows(db1, db2, f"select * from {tbl1};", schema2, table2, new_tbl2)
172
108
 
173
109
 
174
110
  def x_copy_query(**kwargs: Any) -> None:
@@ -193,29 +129,38 @@ def x_copy_query(**kwargs: Any) -> None:
193
129
  )
194
130
  db1 = _state.dbs.aliased_as(alias1)
195
131
  db2 = _state.dbs.aliased_as(alias2)
132
+ _copy_rows(db1, db2, select_stmt, schema2, table2, new_tbl2)
133
+
134
+
135
+ def _copy_rows(
136
+ db1: Database,
137
+ db2: Database,
138
+ select_stmt: str,
139
+ schema2: str | None,
140
+ table2: str,
141
+ new_tbl2: str | None,
142
+ ) -> None:
143
+ """Copy a query's rows, sharing table creation, type inference and spool cleanup."""
196
144
  tbl2 = db2.schema_qualified_table_name(schema2, table2)
197
- if new_tbl2 and new_tbl2 == "new":
145
+ if new_tbl2 == "new":
198
146
  refuse_existing_table(db2, schema2, table2)
199
147
 
200
- get_ts_tablespec = None
201
- rows_to_close = None
148
+ get_ts_tablespec: DataTable | None = None
202
149
  spool: _RowSpool | None = None
203
- rows_for_insert: Any
150
+ try:
151
+ hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
152
+ except ErrInfo:
153
+ raise
154
+ except Exception as e:
155
+ raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
156
+ rows_for_insert: Iterator[Any] = rows
204
157
 
205
158
  if new_tbl2:
206
- try:
207
- hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
208
- except ErrInfo:
209
- raise
210
- except Exception as e:
211
- raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
212
- rows_to_close = rows
213
159
  # Values come from a database cursor, so a text value stays text.
214
160
  spool = _RowSpool(rows)
215
161
  get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
216
162
  rows_for_insert = spool.replay()
217
- tbl_desc = get_ts_tablespec
218
- create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
163
+ create_tbl = get_ts_tablespec.create_table(db2.type, schema2, table2)
219
164
  if new_tbl2 == "replacement":
220
165
  try:
221
166
  db2.drop_table_for_replacement(schema2, table2)
@@ -230,15 +175,6 @@ def x_copy_query(**kwargs: Any) -> None:
230
175
  db2.execute(create_tbl)
231
176
  if db2.needs_explicit_commit_after_ddl():
232
177
  db2.execute("COMMIT;")
233
- else:
234
- try:
235
- hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
236
- except ErrInfo:
237
- raise
238
- except Exception as e:
239
- raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
240
- rows_to_close = rows
241
- rows_for_insert = rows
242
178
 
243
179
  def get_ts() -> DataTable:
244
180
  nonlocal get_ts_tablespec
@@ -251,7 +187,7 @@ def x_copy_query(**kwargs: Any) -> None:
251
187
  db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
252
188
  db2.commit()
253
189
  except BaseException:
254
- _close_rowsource(rows_to_close)
190
+ _close_rowsource(rows)
255
191
  raise
256
192
  finally:
257
193
  if spool is not None:
@@ -1,11 +1,11 @@
1
1
  """Import metacommand handlers.
2
2
 
3
- - ``x_import`` — IMPORT (delimited text / CSV; dispatches by extension
4
- for .ods/.xls/.xlsx in the same handler).
3
+ - ``x_import`` — IMPORT (delimited text / CSV; spreadsheet forms have separate handlers).
5
4
  - ``x_import_file`` — IMPORT_FILE (insert a binary file blob as a row).
6
- - ``x_import_ods`` / ``x_import_ods_pattern`` — IMPORT … FROM ODS,
7
- single-sheet and SHEETS MATCHING <pattern>.
8
- - ``x_import_xls`` / ``x_import_xls_pattern`` — same for XLS/XLSX.
5
+ - ``x_import_ods`` / ``x_import_ods_pattern`` — IMPORT … FROM <file>
6
+ SHEET <name> / SHEETS MATCHING <pattern> (ODS).
7
+ - ``x_import_xls`` / ``x_import_xls_pattern`` — IMPORT … FROM EXCEL <file>
8
+ SHEET <name> / SHEETS MATCHING <pattern> (XLS/XLSX).
9
9
  - ``x_import_parquet`` — IMPORT … FROM PARQUET (via polars).
10
10
  - ``x_import_feather`` — IMPORT … FROM FEATHER (via polars).
11
11
  - ``x_import_json`` — IMPORT … FROM JSON (array of objects or JSON Lines).
@@ -22,9 +22,10 @@ from typing import Any
22
22
  import execsql.state as _state
23
23
  from execsql.exceptions import ErrInfo
24
24
  from execsql.importers.csv import importfile, importtable
25
+ from execsql.utils.spreadsheet import INPUT_LIMIT_DEFAULTS, SheetBudget
25
26
  from execsql.importers.feather import import_feather, import_parquet
26
27
  from execsql.importers.json import import_json
27
- from execsql.importers.ods import OdsFile, importods
28
+ from execsql.importers.ods import OdsFile, check_ods_archive, importods
28
29
  from execsql.exporters.xls import XlsFile, XlsxFile
29
30
  from execsql.importers.xls import importxls
30
31
  from execsql.utils.errors import exception_desc
@@ -185,13 +186,21 @@ def x_import_ods_pattern(**kwargs: Any) -> None:
185
186
  command_text=kwargs["metacommandline"],
186
187
  other_msg="Input file does not exist",
187
188
  )
189
+ check_ods_archive(filename)
188
190
  wbk = OdsFile()
189
191
  try:
190
192
  wbk.open(filename)
191
193
  except Exception as e:
192
194
  raise ErrInfo(type="cmd", other_msg=f"{filename} is not a valid OpenDocument spreadsheet.") from e
193
- sheets = wbk.sheetnames()
194
- impsheets = [s for s in sheets if rx.search(s)]
195
+ try:
196
+ sheets = wbk.sheetnames()
197
+ impsheets = [s for s in sheets if rx.search(s)]
198
+ # Validate each selected sheet before the first target can be changed;
199
+ # discard the rows between sheets instead of retaining their expansion.
200
+ for sheet in impsheets:
201
+ wbk.sheet_data(sheet, hdr_rows, budget=SheetBudget(_state.conf, filename, sheet))
202
+ finally:
203
+ wbk.close()
195
204
  tables = list(impsheets)
196
205
  if _state.conf.clean_col_hdrs:
197
206
  tables = clean_words(tables)
@@ -461,3 +470,12 @@ def x_import_row_buffer(**kwargs: Any) -> None:
461
470
  def x_show_progress(**kwargs: Any) -> None:
462
471
  setting = kwargs["setting"].lower()
463
472
  _state.conf.show_progress = setting in ("yes", "on", "true", "1")
473
+
474
+
475
+ def x_input_limit(**kwargs: Any) -> None:
476
+ """Set a non-negative spreadsheet input budget for subsequent imports."""
477
+ name = kwargs["name"].lower()
478
+ value = int(kwargs["value"])
479
+ if name not in INPUT_LIMIT_DEFAULTS or value < 0:
480
+ raise ErrInfo(type="cmd", other_msg=f"Invalid spreadsheet input limit: {name}={value}.")
481
+ setattr(_state.conf, name, value)
@@ -260,6 +260,13 @@ _METACOMMANDS: tuple[Metacommand, ...] = (
260
260
  "CONFIG GUI_LEVEL <n>",
261
261
  "CONFIG HDF5_TEXT_LEN <n>",
262
262
  "CONFIG IMPORT_ROW_BUFFER <n>",
263
+ "CONFIG ODS_MAX_XML_MB <n>",
264
+ "CONFIG ODS_MAX_COMPRESSION_RATIO <n>",
265
+ "CONFIG IMPORT_MAX_ROWS <n>",
266
+ "CONFIG IMPORT_MAX_COLUMNS <n>",
267
+ "CONFIG IMPORT_MAX_CELLS <n>",
268
+ "CONFIG IMPORT_MAX_CELL_CHARS <n>",
269
+ "CONFIG IMPORT_MAX_TEXT_CHARS <n>",
263
270
  "CONFIG LOG_DATAVARS YES|NO",
264
271
  "CONFIG LOG_SQL YES|NO",
265
272
  "CONFIG LOG_WRITE_MESSAGES YES|NO",
execsql/parser.py CHANGED
@@ -48,8 +48,6 @@ class SourceString:
48
48
 
49
49
  def eoi(self) -> bool:
50
50
  """Return True if the entire source string has been consumed."""
51
- # Returns True or False indicating whether or not there is any of
52
- # the source string left to be consumed.
53
51
  return self.currpos >= len(self.str)
54
52
 
55
53
  def eat_whitespace(self) -> None:
@@ -59,55 +57,34 @@ class SourceString:
59
57
 
60
58
  def match_str(self, str: str) -> str | None:
61
59
  """Match a string case-insensitively at the current position and advance."""
62
- # Tries to match the 'str' argument at the current position in the
63
- # source string. Matching is case-insensitive. If matching succeeds,
64
- # the matched string is returned and the internal pointer is incremented.
65
- # If matching fails, None is returned and the internal pointer is unchanged.
66
60
  self.eat_whitespace()
67
- if self.eoi():
61
+ if self.eoi() or not self.str.lower().startswith(str.lower(), self.currpos):
68
62
  return None
69
- else:
70
- found = self.str.lower().startswith(str.lower(), self.currpos)
71
- if found:
72
- matched = self.str[self.currpos : self.currpos + len(str)]
73
- self.currpos += len(str)
74
- return matched
75
- else:
76
- return None
63
+ matched = self.str[self.currpos : self.currpos + len(str)]
64
+ self.currpos += len(str)
65
+ return matched
77
66
 
78
67
  def match_regex(self, regex: Any) -> dict | None:
79
68
  """Match a compiled regex at the current position and return named groups."""
80
- # Tries to match the 'regex' argument at the current position in the
81
- # source string. If it succeeds, a dictionary of all of the named
82
- # groups is returned, and the internal pointer is incremented.
83
69
  self.eat_whitespace()
84
70
  if self.eoi():
85
71
  return None
86
- else:
87
- m = regex.match(self.str[self.currpos :])
88
- if m:
89
- self.currpos += m.end(0)
90
- return m.groupdict() or {}
91
- else:
92
- return None
72
+ m = regex.match(self.str[self.currpos :])
73
+ if not m:
74
+ return None
75
+ self.currpos += m.end(0)
76
+ return m.groupdict() or {}
93
77
 
94
78
  def match_metacommand(self, commandlist: Any) -> tuple | None:
95
79
  """Match a metacommand from the command list at the current position."""
96
- # Tries to match text at the current position to any metacommand
97
- # in the specified commandlist.
98
- # If it succeeds, the return value is a tuple of the MetaCommand object
99
- # and a dictionary of all of the named groups. The internal pointer is
100
- # incremented past the match.
101
80
  self.eat_whitespace()
102
81
  if self.eoi():
103
82
  return None
104
- else:
105
- m = commandlist.get_match(self.str[self.currpos :])
106
- if m is not None:
107
- self.currpos += m[1].end(0)
108
- return (m[0], m[1].groupdict() or {})
109
- else:
110
- return None
83
+ m = commandlist.get_match(self.str[self.currpos :])
84
+ if m is None:
85
+ return None
86
+ self.currpos += m[1].end(0)
87
+ return (m[0], m[1].groupdict() or {})
111
88
 
112
89
  def remainder(self) -> str:
113
90
  """Return the unconsumed portion of the source string."""
execsql/script/control.py CHANGED
@@ -89,15 +89,25 @@ class BatchLevels:
89
89
  autocommit off, whose transaction is the script's) before the error
90
90
  is raised: a commit that already succeeded
91
91
  cannot be undone, but nothing else of the batch is left pending for
92
- a later statement to commit.
92
+ a later statement to commit. An interrupt rolls back the database
93
+ it landed on as well, since it may have come before that database's
94
+ own rollback. The level leaves the stack last, so until then an
95
+ interrupt finds it there to abandon.
93
96
  """
94
- b = self.batchlevels.pop()
95
- for i, db in enumerate(b.dbs_used):
96
- try:
97
+ b = self.batchlevels[-1]
98
+ committed = 0
99
+ try:
100
+ for db in b.dbs_used:
97
101
  db.commit()
98
- except BaseException:
99
- _roll_back(b.dbs_used[i + 1 :])
100
- raise
102
+ committed += 1
103
+ except Exception:
104
+ _roll_back(b.dbs_used[committed + 1 :])
105
+ raise
106
+ except BaseException:
107
+ _roll_back(b.dbs_used[committed:])
108
+ raise
109
+ finally:
110
+ self.batchlevels.pop()
101
111
 
102
112
  def abandon_batch(self) -> None:
103
113
  """Pop the innermost batch level without committing it.