execsql2 2.24.0__py3-none-any.whl → 2.25.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. execsql/api.py +169 -160
  2. execsql/cli/__init__.py +2 -0
  3. execsql/cli/commands/lint.py +2 -10
  4. execsql/cli/commands/run.py +35 -27
  5. execsql/cli/dsn.py +11 -4
  6. execsql/cli/lint.py +271 -28
  7. execsql/cli/options.py +1 -1
  8. execsql/cli/run.py +56 -19
  9. execsql/config.py +5 -19
  10. execsql/data/execsql.conf.template +14 -4
  11. execsql/db/access.py +29 -8
  12. execsql/db/base.py +40 -8
  13. execsql/db/dsn.py +13 -4
  14. execsql/db/duckdb.py +3 -1
  15. execsql/db/factory.py +11 -2
  16. execsql/db/firebird.py +21 -7
  17. execsql/db/mysql.py +37 -16
  18. execsql/db/oracle.py +49 -16
  19. execsql/db/postgres.py +30 -6
  20. execsql/db/sqlite.py +7 -9
  21. execsql/db/sqlserver.py +25 -8
  22. execsql/exporters/delimited.py +87 -28
  23. execsql/exporters/xlsx.py +24 -9
  24. execsql/exporters/zip.py +1 -2
  25. execsql/format.py +29 -2
  26. execsql/gui/desktop.py +1 -3
  27. execsql/gui/tui.py +2 -2
  28. execsql/importers/base.py +35 -1
  29. execsql/importers/csv.py +3 -1
  30. execsql/manifest.py +33 -22
  31. execsql/metacommands/conditions.py +6 -7
  32. execsql/metacommands/data.py +4 -1
  33. execsql/metacommands/debug.py +2 -4
  34. execsql/metacommands/io_fileops.py +62 -37
  35. execsql/metacommands/io_import.py +0 -1
  36. execsql/metacommands/prompt.py +2 -0
  37. execsql/metacommands/upsert.py +5 -2
  38. execsql/models.py +3 -2
  39. execsql/parser.py +14 -5
  40. execsql/plugins.py +24 -0
  41. execsql/script/engine.py +40 -11
  42. execsql/script/executor.py +9 -5
  43. execsql/script/parser.py +167 -118
  44. execsql/state.py +23 -9
  45. execsql/state.pyi +8 -0
  46. execsql/types.py +22 -13
  47. execsql/utils/auth.py +169 -24
  48. execsql/utils/datetime.py +55 -17
  49. execsql/utils/errors.py +71 -9
  50. execsql/utils/fileio.py +404 -117
  51. execsql/utils/gui.py +3 -0
  52. execsql/utils/mail.py +37 -12
  53. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/METADATA +18 -12
  54. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/RECORD +72 -72
  55. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/README.md +0 -0
  56. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
  57. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
  58. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
  59. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/md_compare.sql +0 -0
  60. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
  61. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
  62. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
  63. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
  64. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/pg_upsert.sql +0 -0
  65. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/script_template.sql +0 -0
  66. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
  67. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
  68. {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
  69. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/WHEEL +0 -0
  70. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/entry_points.txt +0 -0
  71. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/licenses/LICENSE.txt +0 -0
  72. {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/licenses/NOTICE +0 -0
execsql/format.py CHANGED
@@ -17,12 +17,12 @@ import re
17
17
  from collections.abc import Iterator
18
18
  from pathlib import Path
19
19
 
20
- __all__ = ["collect_paths", "format_file", "parse_keyword", "run_formatter"]
20
+ __all__ = ["collect_paths", "format_file", "open_dollar_quote", "parse_keyword", "run_formatter"]
21
21
 
22
22
 
23
23
  _SQLGLOT_MISSING_MSG = (
24
24
  "execsql format requires sqlglot for SQL reformatting.\n"
25
- " Install with: pip install execsql2[formatter]\n"
25
+ ' Install the formatter extra: uv tool install "execsql2[formatter]" (or pip install "execsql2[formatter]")\n'
26
26
  " Or skip SQL reformatting with the --no-sql flag."
27
27
  )
28
28
 
@@ -286,6 +286,33 @@ def _iter_sql_spans(sql: str) -> Iterator[tuple[int, int, str]]:
286
286
  i += 1
287
287
 
288
288
 
289
+ def open_dollar_quote(sql: str, open_tag: str | None = None) -> str | None:
290
+ """The dollar-quote tag (``$$`` or ``$tag$``) still open at the end of *sql*, or ``None``.
291
+
292
+ *open_tag* is a tag already open where *sql* starts, as when a script is
293
+ split at a ``;`` inside a dollar-quoted body and *sql* is a later piece.
294
+ Comments, quoted identifiers and ``'...'`` literals are skipped, so a
295
+ ``$$`` inside them is not taken for a marker.
296
+ """
297
+ start = 0
298
+ if open_tag is not None:
299
+ close = sql.find(open_tag)
300
+ if close < 0:
301
+ return open_tag
302
+ start = close + len(open_tag)
303
+ rest = sql[start:]
304
+ for span_start, span_stop, kind in _iter_sql_spans(rest):
305
+ if kind != "dollar":
306
+ continue
307
+ m = _DOLLAR_QUOTE_RE.match(rest, span_start)
308
+ assert m is not None
309
+ tag = m.group(0)
310
+ body = rest[span_start:span_stop]
311
+ if len(body) < 2 * len(tag) or not body.endswith(tag):
312
+ return tag
313
+ return None
314
+
315
+
289
316
  def _iter_sql_literals(sql: str) -> Iterator[tuple[int, int, str]]:
290
317
  """Yield ``(start, end, kind)`` spans for every string literal in *sql*.
291
318
 
execsql/gui/desktop.py CHANGED
@@ -162,7 +162,6 @@ class MsgDialog:
162
162
  class PauseDialog:
163
163
  def __init__(self, root: tk.Tk, args: dict) -> None:
164
164
  self.result: dict = {}
165
- self._quit = False
166
165
 
167
166
  win = tk.Toplevel(root)
168
167
  win.title(args.get("title", "Pause"))
@@ -177,7 +176,6 @@ class PauseDialog:
177
176
  )
178
177
 
179
178
  countdown = args.get("countdown")
180
- self._remaining = countdown
181
179
 
182
180
  status_var = tk.StringVar(value="")
183
181
  ttk.Label(frame, textvariable=status_var).pack()
@@ -897,7 +895,7 @@ class MapDialog:
897
895
  frame,
898
896
  text=(
899
897
  "(Interactive map requires the tkintermapview package — "
900
- "`pip install execsql2[map]` — showing tabular data)"
898
+ "install the map extra, execsql2[map] — showing tabular data)"
901
899
  ),
902
900
  wraplength=580,
903
901
  justify="left",
execsql/gui/tui.py CHANGED
@@ -12,7 +12,7 @@ SQL script in a background thread and keeps the main thread for the Textual
12
12
  ``_state.gui_manager_queue``; the conductor polls that queue and serves each
13
13
  request by pushing the appropriate ``ModalScreen`` via ``push_screen_wait()``.
14
14
 
15
- Install with: pip install "execsql2[tui]"
15
+ Textual is a dependency of execsql2, so it is installed with the package.
16
16
  """
17
17
 
18
18
  from __future__ import annotations
@@ -33,7 +33,7 @@ try:
33
33
  import textual # noqa: F401
34
34
  except ImportError as _e:
35
35
  raise ImportError(
36
- "Textual is not installed. Install it with: pip install 'execsql2[tui]'",
36
+ "Textual is not installed. It is a dependency of execsql2; reinstall execsql2 to restore it.",
37
37
  ) from _e
38
38
 
39
39
  from textual import on
execsql/importers/base.py CHANGED
@@ -16,7 +16,39 @@ from execsql.exceptions import ErrInfo
16
16
  from execsql.db.base import Database
17
17
  import execsql.state as _state
18
18
 
19
- __all__ = ["import_data_table"]
19
+ __all__ = ["import_data_table", "refuse_existing_table", "table_exists_if_known"]
20
+
21
+
22
+ def table_exists_if_known(db: Database, table_name: str, schema_name: str | None = None) -> bool | None:
23
+ """Whether the table exists, or ``None`` when the database cannot say (no catalog access, ...)."""
24
+ try:
25
+ return db.table_exists(table_name, schema_name)
26
+ except Exception:
27
+ return None
28
+
29
+
30
+ def refuse_existing_table(db: Database, schema_name: str | None, table_name: str) -> None:
31
+ """Stop a ``TO NEW`` before any DDL when the target table already exists.
32
+
33
+ Only the schema the CREATE TABLE will use counts: for an unqualified
34
+ name, the database's current schema (MySQL: database). Best effort:
35
+ when the database cannot say, the command goes ahead and the CREATE
36
+ TABLE reports any conflict itself.
37
+ """
38
+ lookup_schema = schema_name
39
+ if schema_name is None and not db.unqualified_lookup_is_scoped:
40
+ try:
41
+ lookup_schema = db.current_schema()
42
+ except Exception:
43
+ lookup_schema = None
44
+ if lookup_schema is None:
45
+ return # An unqualified lookup would search every schema.
46
+ if table_exists_if_known(db, table_name, lookup_schema):
47
+ name = db.schema_qualified_table_name(schema_name, table_name)
48
+ raise ErrInfo(
49
+ type="cmd",
50
+ other_msg=f"Table {name} already exists",
51
+ )
20
52
 
21
53
 
22
54
  def import_data_table(
@@ -74,6 +106,8 @@ def import_data_table(
74
106
  return tablespec_cache
75
107
 
76
108
  if is_new:
109
+ if is_new == 1:
110
+ refuse_existing_table(db, schemaname, tablename)
77
111
  if is_new == 2:
78
112
  tblspec = db.schema_qualified_table_name(schemaname, tablename)
79
113
  try:
execsql/importers/csv.py CHANGED
@@ -14,6 +14,7 @@ from typing import Any
14
14
 
15
15
  from execsql.exceptions import ErrInfo
16
16
  from execsql.db.base import Database
17
+ from execsql.importers.base import refuse_existing_table
17
18
  import execsql.state as _state
18
19
  from execsql.types import dbt_firebird
19
20
 
@@ -26,7 +27,6 @@ def importtable(
26
27
  tablename: str,
27
28
  filename: str,
28
29
  is_new: Any,
29
- skip_header_line: bool = True,
30
30
  quotechar: str | None = None,
31
31
  delimchar: str | None = None,
32
32
  encoding: str | None = None,
@@ -49,6 +49,8 @@ def importtable(
49
49
  quotechar = None
50
50
  inf.lineformat(delimchar, quotechar, None)
51
51
  if is_new in (1, 2):
52
+ if is_new == 1:
53
+ refuse_existing_table(db, schemaname, tablename)
52
54
  inf.evaluate_column_types()
53
55
  sql = inf.create_table(db.type, schemaname, tablename)
54
56
  if is_new == 2:
execsql/manifest.py CHANGED
@@ -32,12 +32,16 @@ from execsql import __version__
32
32
 
33
33
  __all__ = ["RunManifest", "check_writable", "current", "start"]
34
34
 
35
- _current: RunManifest | None = None
36
-
37
35
 
38
36
  def current() -> RunManifest | None:
39
- """The active manifest recorder, or ``None`` when ``--manifest`` was not given."""
40
- return _current
37
+ """The active run's manifest recorder, or ``None`` when ``--manifest`` was not given.
38
+
39
+ Held on the run's :class:`~execsql.state.RuntimeContext`, so a library
40
+ ``execsql.run()`` in the same process never records into a CLI manifest.
41
+ """
42
+ import execsql.state as _state
43
+
44
+ return _state.manifest
41
45
 
42
46
 
43
47
  def check_writable(path: str) -> str | None:
@@ -65,10 +69,12 @@ def start(path: str, script: str | None, variables: list[str]) -> RunManifest:
65
69
  If nothing else writes it (``exit_now`` on an error, the end of a run), an
66
70
  ``atexit`` fallback does.
67
71
  """
68
- global _current
69
- _current = RunManifest(path, script, variables)
70
- atexit.register(_current.finish, None)
71
- return _current
72
+ import execsql.state as _state
73
+
74
+ manifest = RunManifest(path, script, variables)
75
+ _state.manifest = manifest
76
+ atexit.register(manifest.finish, None)
77
+ return manifest
72
78
 
73
79
 
74
80
  def _now() -> datetime.datetime:
@@ -137,22 +143,26 @@ class RunManifest:
137
143
 
138
144
  # -- finishing -------------------------------------------------------------
139
145
 
146
+ def add_error(self, error: Any) -> None:
147
+ """Record *error* (an :class:`ErrInfo`) in the manifest's ``errors``."""
148
+ # str(ErrInfo) is its concise message; errmsg() is the multi-line
149
+ # console block with a timestamp, which a reader of JSON does not want.
150
+ self.errors.append(
151
+ {
152
+ "message": " ".join(str(error).split()),
153
+ "type": getattr(error, "type", None),
154
+ "source": getattr(error, "script_file", None),
155
+ "line": getattr(error, "script_line_no", None),
156
+ "command": getattr(error, "cmd", None) or getattr(error, "command", None),
157
+ },
158
+ )
159
+
140
160
  def finish(self, exit_status: int | None, error: Any = None) -> None:
141
161
  """Write the manifest once. Later calls (the atexit fallback) do nothing."""
142
162
  if self.written:
143
163
  return
144
164
  if error is not None:
145
- # str(ErrInfo) is its concise message; errmsg() is the multi-line
146
- # console block with a timestamp, which a reader of JSON does not want.
147
- self.errors.append(
148
- {
149
- "message": " ".join(str(error).split()),
150
- "type": getattr(error, "type", None),
151
- "source": getattr(error, "script_file", None),
152
- "line": getattr(error, "script_line_no", None),
153
- "command": getattr(error, "cmd", None) or getattr(error, "command", None),
154
- },
155
- )
165
+ self.add_error(error)
156
166
  finished = _now()
157
167
  data = {
158
168
  "execsql_version": __version__,
@@ -172,9 +182,10 @@ class RunManifest:
172
182
  "errors": self.errors,
173
183
  }
174
184
  self.written = True
175
- global _current
176
- if _current is self:
177
- _current = None
185
+ import execsql.state as _state
186
+
187
+ if _state.manifest is self:
188
+ _state.manifest = None
178
189
  atexit.unregister(self.finish)
179
190
  try:
180
191
  self.path.parent.mkdir(parents=True, exist_ok=True)
@@ -100,12 +100,11 @@ def xf_hasrows(**kwargs: Any) -> bool:
100
100
  return bool(nrows > 0)
101
101
 
102
102
 
103
- def _row_count(queryname: str, sql_context: str, metacommandline: str) -> int:
103
+ def _row_count(queryname: str, metacommandline: str) -> int:
104
104
  """Return the number of rows in *queryname*, raising ErrInfo on failure.
105
105
 
106
106
  Args:
107
107
  queryname: Table or view name to count rows in.
108
- sql_context: The SQL string to include in error messages.
109
108
  metacommandline: The full metacommand line for error context.
110
109
 
111
110
  Returns:
@@ -167,7 +166,7 @@ def xf_row_count_gt(**kwargs: Any) -> bool:
167
166
  queryname = kwargs["queryname"]
168
167
  mcl = kwargs["metacommandline"]
169
168
  n = _parse_row_count_n(kwargs["n"], mcl)
170
- return _row_count(queryname, f"select count(*) from {queryname};", mcl) > n
169
+ return _row_count(queryname, mcl) > n
171
170
 
172
171
 
173
172
  def xf_row_count_gte(**kwargs: Any) -> bool:
@@ -183,7 +182,7 @@ def xf_row_count_gte(**kwargs: Any) -> bool:
183
182
  queryname = kwargs["queryname"]
184
183
  mcl = kwargs["metacommandline"]
185
184
  n = _parse_row_count_n(kwargs["n"], mcl)
186
- return _row_count(queryname, f"select count(*) from {queryname};", mcl) >= n
185
+ return _row_count(queryname, mcl) >= n
187
186
 
188
187
 
189
188
  def xf_row_count_eq(**kwargs: Any) -> bool:
@@ -199,7 +198,7 @@ def xf_row_count_eq(**kwargs: Any) -> bool:
199
198
  queryname = kwargs["queryname"]
200
199
  mcl = kwargs["metacommandline"]
201
200
  n = _parse_row_count_n(kwargs["n"], mcl)
202
- return _row_count(queryname, f"select count(*) from {queryname};", mcl) == n
201
+ return _row_count(queryname, mcl) == n
203
202
 
204
203
 
205
204
  def xf_row_count_lt(**kwargs: Any) -> bool:
@@ -215,7 +214,7 @@ def xf_row_count_lt(**kwargs: Any) -> bool:
215
214
  queryname = kwargs["queryname"]
216
215
  mcl = kwargs["metacommandline"]
217
216
  n = _parse_row_count_n(kwargs["n"], mcl)
218
- return _row_count(queryname, f"select count(*) from {queryname};", mcl) < n
217
+ return _row_count(queryname, mcl) < n
219
218
 
220
219
 
221
220
  def xf_sqlerror(**kwargs: Any) -> bool:
@@ -428,7 +427,7 @@ def xf_newer_date(**kwargs: Any) -> bool:
428
427
  datestr = unquoted(kwargs["datestr"])
429
428
  if not Path(file1).exists():
430
429
  raise ErrInfo(type="cmd", other_msg=f"File {file1} does not exist.")
431
- dt_value = parse_datetime(datestr)
430
+ dt_value = parse_datetime(datestr, strict=False)
432
431
  if not dt_value:
433
432
  raise ErrInfo(type="cmd", other_msg=f"{datestr} can't be interpreted as a date/time.")
434
433
  return os.stat(file1).st_mtime > time.mktime(dt_value.timetuple())
@@ -36,6 +36,7 @@ from execsql.parser import NumericParser
36
36
  from execsql.script import current_script_line
37
37
  from execsql.utils.crypto import Encrypt
38
38
  from execsql.utils.errors import exception_desc, exit_now
39
+ from execsql.utils.fileio import register_secret
39
40
  from execsql.utils.gui import GUI_SELECTSUB, GuiSpec, enable_gui
40
41
  from execsql.utils.strings import get_subvarset, unquoted
41
42
 
@@ -126,7 +127,9 @@ def x_sub_encrypt(**kwargs: Any) -> None:
126
127
  def x_sub_decrypt(**kwargs: Any) -> None:
127
128
  varname = kwargs["match"]
128
129
  subvarset, varname = get_subvarset(varname, kwargs["metacommandline"])
129
- subvarset.add_substitution(varname, Encrypt().decrypt(kwargs["crypttext"]))
130
+ plaintext = Encrypt().decrypt(kwargs["crypttext"])
131
+ register_secret(plaintext)
132
+ subvarset.add_substitution(varname, plaintext)
130
133
  return None
131
134
 
132
135
 
@@ -52,10 +52,8 @@ def x_debug_commandliststack(**kwargs: Any) -> None:
52
52
  ``<main>`` script, ``EXECUTE SCRIPT`` calls, ``INCLUDE``'d files,
53
53
  ``IF``/``ELSEIF``/``ELSE`` branches, ``LOOP`` iterations (with iteration
54
54
  count), and ``BATCH`` blocks. Each frame shows source file and line.
55
-
56
- The legacy ``commandliststack`` is reported separately as a secondary view
57
- because it only records SCRIPT call frames; the unified ``ast_exec_stack``
58
- is the authoritative debugger view.
55
+ The metacommand keeps its upstream name; what it prints is the AST
56
+ executor's ``ast_exec_stack``.
59
57
  """
60
58
  stack = getattr(_state, "ast_exec_stack", None) or []
61
59
  _state.output.write(f"Execution Stack (depth: {len(stack)}):\n")
@@ -8,14 +8,18 @@ Implements ``x_include``, ``x_copy``, ``x_copy_query``, ``x_zip``,
8
8
  from __future__ import annotations
9
9
 
10
10
  import os
11
+ import pickle
12
+ import struct
11
13
  import sys
12
- from itertools import tee
14
+ import tempfile
15
+ from collections.abc import Iterator
13
16
  from pathlib import Path
14
17
  from shutil import copyfileobj
15
- from typing import Any, Literal, cast
18
+ from typing import Any, Literal
16
19
 
17
20
  import execsql.state as _state
18
21
  from execsql.exceptions import ErrInfo
22
+ from execsql.importers.base import refuse_existing_table
19
23
  from execsql.models import DataTable
20
24
  from execsql.script import current_script_line
21
25
  from execsql.utils.errors import exception_desc
@@ -23,6 +27,39 @@ from execsql.utils.fileio import filewriter_close
23
27
  from execsql.utils.strings import unquoted
24
28
 
25
29
 
30
+ class _RowSpool:
31
+ """Pass rows through while writing each to a temporary file, to be read back once more.
32
+
33
+ ``COPY ... TO NEW`` needs every row twice, once to describe the new table
34
+ and once to load it, but runs the source query once. The rows are kept
35
+ in memory up to *max_size* bytes and on disk beyond that. Each row is
36
+ pickled on its own, so neither pass keeps a reference to earlier rows.
37
+ """
38
+
39
+ _LENGTH = struct.Struct("<Q")
40
+
41
+ def __init__(self, rows: Any, max_size: int = 8 * 1024 * 1024) -> None:
42
+ self._rows = rows
43
+ self._file = tempfile.SpooledTemporaryFile(max_size=max_size) # noqa: SIM115 - closed by close()
44
+
45
+ def __iter__(self) -> Iterator[Any]:
46
+ for row in self._rows:
47
+ data = pickle.dumps(row, protocol=pickle.HIGHEST_PROTOCOL)
48
+ self._file.write(self._LENGTH.pack(len(data)))
49
+ self._file.write(data)
50
+ yield row
51
+
52
+ def replay(self) -> Iterator[Any]:
53
+ """Yield the rows passed through, in order."""
54
+ self._file.seek(0)
55
+ while header := self._file.read(self._LENGTH.size):
56
+ (size,) = self._LENGTH.unpack(header)
57
+ yield pickle.loads(self._file.read(size))
58
+
59
+ def close(self) -> None:
60
+ self._file.close()
61
+
62
+
26
63
  def _close_rowsource(rows: Any) -> None:
27
64
  close = getattr(rows, "close", None)
28
65
  if close:
@@ -63,29 +100,16 @@ def x_copy(**kwargs: Any) -> None:
63
100
  db2 = _state.dbs.aliased_as(alias2)
64
101
  tbl1 = db1.schema_qualified_table_name(schema1, table1)
65
102
  tbl2 = db2.schema_qualified_table_name(schema2, table2)
66
- try:
67
- if not db1.table_exists(table1, schema1):
68
- raise ErrInfo(
69
- type="cmd",
70
- command_text=kwargs["metacommandline"],
71
- other_msg=f"Table {tbl1} does not exist",
72
- )
73
- except Exception:
74
- pass # Best-effort check; some adapters lack information_schema.
103
+ # No check that the source exists: the database may fold an unquoted name
104
+ # (PostgreSQL reads Orders as orders), and the SELECT below names a missing
105
+ # source in the database's own words.
75
106
  if new_tbl2 and new_tbl2 == "new":
76
- try:
77
- if db2.table_exists(table2, schema2):
78
- raise ErrInfo(
79
- type="cmd",
80
- command_text=kwargs["metacommandline"],
81
- other_msg=f"Table {tbl2} already exists",
82
- )
83
- except Exception:
84
- pass # Best-effort check; some adapters lack information_schema.
107
+ refuse_existing_table(db2, schema2, table2)
85
108
  select_stmt = f"select * from {tbl1};"
86
109
 
87
110
  get_ts_tablespec = None
88
111
  rows_to_close = None
112
+ spool: _RowSpool | None = None
89
113
  rows_for_insert: Any
90
114
 
91
115
  if new_tbl2:
@@ -96,9 +120,10 @@ def x_copy(**kwargs: Any) -> None:
96
120
  except Exception as e:
97
121
  raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
98
122
  rows_to_close = rows
99
- rows = cast(Any, rows)
100
- rows_for_schema, rows_for_insert = tee(rows)
101
- get_ts_tablespec = DataTable(hdrs, rows_for_schema)
123
+ # Values come from a database cursor, so a text value stays text.
124
+ spool = _RowSpool(rows)
125
+ get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
126
+ rows_for_insert = spool.replay()
102
127
  tbl_desc = get_ts_tablespec
103
128
  create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
104
129
  if new_tbl2 == "replacement":
@@ -129,7 +154,7 @@ def x_copy(**kwargs: Any) -> None:
129
154
  nonlocal get_ts_tablespec
130
155
  if get_ts_tablespec is None:
131
156
  ts_hdrs, ts_rows = db1.select_rowsource(select_stmt)
132
- get_ts_tablespec = DataTable(ts_hdrs, ts_rows)
157
+ get_ts_tablespec = DataTable(ts_hdrs, ts_rows, infer_strings=False)
133
158
  return get_ts_tablespec
134
159
 
135
160
  try:
@@ -138,6 +163,9 @@ def x_copy(**kwargs: Any) -> None:
138
163
  except BaseException:
139
164
  _close_rowsource(rows_to_close)
140
165
  raise
166
+ finally:
167
+ if spool is not None:
168
+ spool.close()
141
169
 
142
170
 
143
171
  def x_copy_query(**kwargs: Any) -> None:
@@ -164,18 +192,11 @@ def x_copy_query(**kwargs: Any) -> None:
164
192
  db2 = _state.dbs.aliased_as(alias2)
165
193
  tbl2 = db2.schema_qualified_table_name(schema2, table2)
166
194
  if new_tbl2 and new_tbl2 == "new":
167
- try:
168
- if db2.table_exists(table2, schema2):
169
- raise ErrInfo(
170
- type="cmd",
171
- command_text=kwargs["metacommandline"],
172
- other_msg=f"Table {tbl2} already exists",
173
- )
174
- except Exception:
175
- pass # Best-effort check; some adapters lack information_schema.
195
+ refuse_existing_table(db2, schema2, table2)
176
196
 
177
197
  get_ts_tablespec = None
178
198
  rows_to_close = None
199
+ spool: _RowSpool | None = None
179
200
  rows_for_insert: Any
180
201
 
181
202
  if new_tbl2:
@@ -186,9 +207,10 @@ def x_copy_query(**kwargs: Any) -> None:
186
207
  except Exception as e:
187
208
  raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
188
209
  rows_to_close = rows
189
- rows = cast(Any, rows)
190
- rows_for_schema, rows_for_insert = tee(rows)
191
- get_ts_tablespec = DataTable(hdrs, rows_for_schema)
210
+ # Values come from a database cursor, so a text value stays text.
211
+ spool = _RowSpool(rows)
212
+ get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
213
+ rows_for_insert = spool.replay()
192
214
  tbl_desc = get_ts_tablespec
193
215
  create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
194
216
  if new_tbl2 == "replacement":
@@ -219,7 +241,7 @@ def x_copy_query(**kwargs: Any) -> None:
219
241
  nonlocal get_ts_tablespec
220
242
  if get_ts_tablespec is None:
221
243
  ts_hdrs, ts_rows = db1.select_rowsource(select_stmt)
222
- get_ts_tablespec = DataTable(ts_hdrs, ts_rows)
244
+ get_ts_tablespec = DataTable(ts_hdrs, ts_rows, infer_strings=False)
223
245
  return get_ts_tablespec
224
246
 
225
247
  try:
@@ -228,6 +250,9 @@ def x_copy_query(**kwargs: Any) -> None:
228
250
  except BaseException:
229
251
  _close_rowsource(rows_to_close)
230
252
  raise
253
+ finally:
254
+ if spool is not None:
255
+ spool.close()
231
256
 
232
257
 
233
258
  def x_zip(**kwargs: Any) -> None:
@@ -74,7 +74,6 @@ def x_import(**kwargs: Any) -> None:
74
74
  tablename,
75
75
  filename,
76
76
  is_new,
77
- skip_header_line=True,
78
77
  quotechar=quotechar,
79
78
  delimchar=delimchar,
80
79
  encoding=enc,
@@ -67,6 +67,7 @@ from execsql.utils.gui import (
67
67
  pause,
68
68
  pause_win,
69
69
  )
70
+ from execsql.utils.fileio import register_secret
70
71
  from execsql.utils.strings import unquoted
71
72
 
72
73
 
@@ -144,6 +145,7 @@ def x_prompt_enter(**kwargs: Any) -> None:
144
145
  subvarset.add_substitution(sub_var, txtval)
145
146
  script_name, lno = current_script_line()
146
147
  if as_pw:
148
+ register_secret(txtval)
147
149
  _state.exec_log.log_status_info(f"Password assigned to variable {{{sub_var}}} on line {lno}.")
148
150
  else:
149
151
  _state.exec_log.log_status_info(f"Variable {{{sub_var}}} set to {{{txtval}}} on line {lno}.")
@@ -4,7 +4,7 @@ Integrates pg-upsert (https://pg-upsert.readthedocs.io/) as an optional
4
4
  dependency, providing QA-checked, FK-dependency-ordered upserts from a
5
5
  staging schema to a base schema on PostgreSQL.
6
6
 
7
- Requires: ``pip install execsql2[upsert]``
7
+ Requires the ``upsert`` extra (``execsql2[upsert]``).
8
8
  """
9
9
 
10
10
  from __future__ import annotations
@@ -213,7 +213,10 @@ def _require_pg_upsert() -> None:
213
213
  except ImportError as exc:
214
214
  raise ErrInfo(
215
215
  "exception",
216
- other_msg=("PG_UPSERT requires the pg-upsert package. Install it with: pip install execsql2[upsert]"),
216
+ other_msg=(
217
+ 'PG_UPSERT requires the pg-upsert package. Install the upsert extra: uv tool install "execsql2[upsert]"'
218
+ ' (or pip install "execsql2[upsert]").'
219
+ ),
217
220
  ) from exc
218
221
 
219
222
 
execsql/models.py CHANGED
@@ -268,7 +268,6 @@ class DataTable:
268
268
 
269
269
  self.inputrows = 0 # Total number of rows in the row source.
270
270
  self.datarows = 0 # Number of non-empty rows (with data values).
271
- self.shortrows = 0 # Number of rows without as many data values as column names.
272
271
  self.cols: list = [] # List of Column objects.
273
272
  for n in column_names:
274
273
  self.cols.append(Column(n, infer_strings=infer_strings))
@@ -281,7 +280,6 @@ class DataTable:
281
280
  self.datarows += 1
282
281
  chkcols = len(self.cols)
283
282
  if dataitems < chkcols:
284
- self.shortrows += 1
285
283
  chkcols = len(datarow)
286
284
  else:
287
285
  if dataitems > chkcols:
@@ -307,6 +305,9 @@ class DataTable:
307
305
  raise DataTableError(errmsg)
308
306
  for i in range(chkcols):
309
307
  self.cols[i].eval_types(datarow[i])
308
+ # Trailing values left off a short row are imported as NULL.
309
+ for i in range(chkcols, len(self.cols)):
310
+ self.cols[i].eval_types(None)
310
311
  for col in self.cols:
311
312
  col.column_type()
312
313
 
execsql/parser.py CHANGED
@@ -215,10 +215,17 @@ class CondParser(CondTokens):
215
215
  """Recursive-descent parser for boolean conditional expressions."""
216
216
 
217
217
  # Takes a conditional expression string.
218
- def __init__(self, condexpr: str) -> None:
219
- """Initialise the parser with the conditional expression string."""
218
+ def __init__(self, condexpr: str, conditionallist: Any = None) -> None:
219
+ """Initialise the parser with the conditional expression string.
220
+
221
+ *conditionallist* is the table of conditional tests to match against;
222
+ ``None`` means the current run's ``conditionallist``. Passing one lets
223
+ a caller with no run in progress, such as ``execsql lint``, parse
224
+ (without evaluating) a condition.
225
+ """
220
226
  self.condexpr = condexpr
221
227
  self.cond_expr = SourceString(condexpr)
228
+ self.conditionallist = conditionallist
222
229
 
223
230
  def match_not(self) -> int | None:
224
231
  """Match a NOT operator and return its token type, or None."""
@@ -251,10 +258,12 @@ class CondParser(CondTokens):
251
258
  m1 = self.factor()
252
259
  return CondAstNode(self.NOT, m1, None)
253
260
  # Find the matching metacommand -- get a tuple consisting of (metacommand, groupdict)
254
- # conditionallist is a module-level global in the main execsql module
255
- import execsql.state as _state
261
+ conditionallist = self.conditionallist
262
+ if conditionallist is None:
263
+ import execsql.state as _state
256
264
 
257
- matched_metacommand = self.cond_expr.match_metacommand(_state.conditionallist)
265
+ conditionallist = _state.conditionallist
266
+ matched_metacommand = self.cond_expr.match_metacommand(conditionallist)
258
267
  if matched_metacommand is not None:
259
268
  matched_metacommand[1]["metacommandline"] = self.condexpr
260
269
  return CondAstNode(self.CONDITIONAL, matched_metacommand, None)