execsql2 2.24.0__py3-none-any.whl → 2.25.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +169 -160
- execsql/cli/__init__.py +2 -0
- execsql/cli/commands/lint.py +2 -10
- execsql/cli/commands/run.py +35 -27
- execsql/cli/dsn.py +11 -4
- execsql/cli/lint.py +271 -28
- execsql/cli/options.py +1 -1
- execsql/cli/run.py +56 -19
- execsql/config.py +5 -19
- execsql/data/execsql.conf.template +14 -4
- execsql/db/access.py +29 -8
- execsql/db/base.py +40 -8
- execsql/db/dsn.py +13 -4
- execsql/db/duckdb.py +3 -1
- execsql/db/factory.py +11 -2
- execsql/db/firebird.py +21 -7
- execsql/db/mysql.py +37 -16
- execsql/db/oracle.py +49 -16
- execsql/db/postgres.py +30 -6
- execsql/db/sqlite.py +7 -9
- execsql/db/sqlserver.py +25 -8
- execsql/exporters/delimited.py +87 -28
- execsql/exporters/xlsx.py +24 -9
- execsql/exporters/zip.py +1 -2
- execsql/format.py +29 -2
- execsql/gui/desktop.py +1 -3
- execsql/gui/tui.py +2 -2
- execsql/importers/base.py +35 -1
- execsql/importers/csv.py +3 -1
- execsql/manifest.py +33 -22
- execsql/metacommands/conditions.py +6 -7
- execsql/metacommands/data.py +4 -1
- execsql/metacommands/debug.py +2 -4
- execsql/metacommands/io_fileops.py +62 -37
- execsql/metacommands/io_import.py +0 -1
- execsql/metacommands/prompt.py +2 -0
- execsql/metacommands/upsert.py +5 -2
- execsql/models.py +3 -2
- execsql/parser.py +14 -5
- execsql/plugins.py +24 -0
- execsql/script/engine.py +40 -11
- execsql/script/executor.py +9 -5
- execsql/script/parser.py +167 -118
- execsql/state.py +23 -9
- execsql/state.pyi +8 -0
- execsql/types.py +22 -13
- execsql/utils/auth.py +169 -24
- execsql/utils/datetime.py +55 -17
- execsql/utils/errors.py +71 -9
- execsql/utils/fileio.py +404 -117
- execsql/utils/gui.py +3 -0
- execsql/utils/mail.py +37 -12
- {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/METADATA +18 -12
- {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/RECORD +72 -72
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/README.md +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/pg_upsert.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.24.0.data → execsql2-2.25.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/WHEEL +0 -0
- {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.24.0.dist-info → execsql2-2.25.0.dist-info}/licenses/NOTICE +0 -0
execsql/format.py
CHANGED
|
@@ -17,12 +17,12 @@ import re
|
|
|
17
17
|
from collections.abc import Iterator
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
|
|
20
|
-
__all__ = ["collect_paths", "format_file", "parse_keyword", "run_formatter"]
|
|
20
|
+
__all__ = ["collect_paths", "format_file", "open_dollar_quote", "parse_keyword", "run_formatter"]
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
_SQLGLOT_MISSING_MSG = (
|
|
24
24
|
"execsql format requires sqlglot for SQL reformatting.\n"
|
|
25
|
-
|
|
25
|
+
' Install the formatter extra: uv tool install "execsql2[formatter]" (or pip install "execsql2[formatter]")\n'
|
|
26
26
|
" Or skip SQL reformatting with the --no-sql flag."
|
|
27
27
|
)
|
|
28
28
|
|
|
@@ -286,6 +286,33 @@ def _iter_sql_spans(sql: str) -> Iterator[tuple[int, int, str]]:
|
|
|
286
286
|
i += 1
|
|
287
287
|
|
|
288
288
|
|
|
289
|
+
def open_dollar_quote(sql: str, open_tag: str | None = None) -> str | None:
|
|
290
|
+
"""The dollar-quote tag (``$$`` or ``$tag$``) still open at the end of *sql*, or ``None``.
|
|
291
|
+
|
|
292
|
+
*open_tag* is a tag already open where *sql* starts, as when a script is
|
|
293
|
+
split at a ``;`` inside a dollar-quoted body and *sql* is a later piece.
|
|
294
|
+
Comments, quoted identifiers and ``'...'`` literals are skipped, so a
|
|
295
|
+
``$$`` inside them is not taken for a marker.
|
|
296
|
+
"""
|
|
297
|
+
start = 0
|
|
298
|
+
if open_tag is not None:
|
|
299
|
+
close = sql.find(open_tag)
|
|
300
|
+
if close < 0:
|
|
301
|
+
return open_tag
|
|
302
|
+
start = close + len(open_tag)
|
|
303
|
+
rest = sql[start:]
|
|
304
|
+
for span_start, span_stop, kind in _iter_sql_spans(rest):
|
|
305
|
+
if kind != "dollar":
|
|
306
|
+
continue
|
|
307
|
+
m = _DOLLAR_QUOTE_RE.match(rest, span_start)
|
|
308
|
+
assert m is not None
|
|
309
|
+
tag = m.group(0)
|
|
310
|
+
body = rest[span_start:span_stop]
|
|
311
|
+
if len(body) < 2 * len(tag) or not body.endswith(tag):
|
|
312
|
+
return tag
|
|
313
|
+
return None
|
|
314
|
+
|
|
315
|
+
|
|
289
316
|
def _iter_sql_literals(sql: str) -> Iterator[tuple[int, int, str]]:
|
|
290
317
|
"""Yield ``(start, end, kind)`` spans for every string literal in *sql*.
|
|
291
318
|
|
execsql/gui/desktop.py
CHANGED
|
@@ -162,7 +162,6 @@ class MsgDialog:
|
|
|
162
162
|
class PauseDialog:
|
|
163
163
|
def __init__(self, root: tk.Tk, args: dict) -> None:
|
|
164
164
|
self.result: dict = {}
|
|
165
|
-
self._quit = False
|
|
166
165
|
|
|
167
166
|
win = tk.Toplevel(root)
|
|
168
167
|
win.title(args.get("title", "Pause"))
|
|
@@ -177,7 +176,6 @@ class PauseDialog:
|
|
|
177
176
|
)
|
|
178
177
|
|
|
179
178
|
countdown = args.get("countdown")
|
|
180
|
-
self._remaining = countdown
|
|
181
179
|
|
|
182
180
|
status_var = tk.StringVar(value="")
|
|
183
181
|
ttk.Label(frame, textvariable=status_var).pack()
|
|
@@ -897,7 +895,7 @@ class MapDialog:
|
|
|
897
895
|
frame,
|
|
898
896
|
text=(
|
|
899
897
|
"(Interactive map requires the tkintermapview package — "
|
|
900
|
-
"
|
|
898
|
+
"install the map extra, execsql2[map] — showing tabular data)"
|
|
901
899
|
),
|
|
902
900
|
wraplength=580,
|
|
903
901
|
justify="left",
|
execsql/gui/tui.py
CHANGED
|
@@ -12,7 +12,7 @@ SQL script in a background thread and keeps the main thread for the Textual
|
|
|
12
12
|
``_state.gui_manager_queue``; the conductor polls that queue and serves each
|
|
13
13
|
request by pushing the appropriate ``ModalScreen`` via ``push_screen_wait()``.
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
Textual is a dependency of execsql2, so it is installed with the package.
|
|
16
16
|
"""
|
|
17
17
|
|
|
18
18
|
from __future__ import annotations
|
|
@@ -33,7 +33,7 @@ try:
|
|
|
33
33
|
import textual # noqa: F401
|
|
34
34
|
except ImportError as _e:
|
|
35
35
|
raise ImportError(
|
|
36
|
-
"Textual is not installed.
|
|
36
|
+
"Textual is not installed. It is a dependency of execsql2; reinstall execsql2 to restore it.",
|
|
37
37
|
) from _e
|
|
38
38
|
|
|
39
39
|
from textual import on
|
execsql/importers/base.py
CHANGED
|
@@ -16,7 +16,39 @@ from execsql.exceptions import ErrInfo
|
|
|
16
16
|
from execsql.db.base import Database
|
|
17
17
|
import execsql.state as _state
|
|
18
18
|
|
|
19
|
-
__all__ = ["import_data_table"]
|
|
19
|
+
__all__ = ["import_data_table", "refuse_existing_table", "table_exists_if_known"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def table_exists_if_known(db: Database, table_name: str, schema_name: str | None = None) -> bool | None:
|
|
23
|
+
"""Whether the table exists, or ``None`` when the database cannot say (no catalog access, ...)."""
|
|
24
|
+
try:
|
|
25
|
+
return db.table_exists(table_name, schema_name)
|
|
26
|
+
except Exception:
|
|
27
|
+
return None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def refuse_existing_table(db: Database, schema_name: str | None, table_name: str) -> None:
|
|
31
|
+
"""Stop a ``TO NEW`` before any DDL when the target table already exists.
|
|
32
|
+
|
|
33
|
+
Only the schema the CREATE TABLE will use counts: for an unqualified
|
|
34
|
+
name, the database's current schema (MySQL: database). Best effort:
|
|
35
|
+
when the database cannot say, the command goes ahead and the CREATE
|
|
36
|
+
TABLE reports any conflict itself.
|
|
37
|
+
"""
|
|
38
|
+
lookup_schema = schema_name
|
|
39
|
+
if schema_name is None and not db.unqualified_lookup_is_scoped:
|
|
40
|
+
try:
|
|
41
|
+
lookup_schema = db.current_schema()
|
|
42
|
+
except Exception:
|
|
43
|
+
lookup_schema = None
|
|
44
|
+
if lookup_schema is None:
|
|
45
|
+
return # An unqualified lookup would search every schema.
|
|
46
|
+
if table_exists_if_known(db, table_name, lookup_schema):
|
|
47
|
+
name = db.schema_qualified_table_name(schema_name, table_name)
|
|
48
|
+
raise ErrInfo(
|
|
49
|
+
type="cmd",
|
|
50
|
+
other_msg=f"Table {name} already exists",
|
|
51
|
+
)
|
|
20
52
|
|
|
21
53
|
|
|
22
54
|
def import_data_table(
|
|
@@ -74,6 +106,8 @@ def import_data_table(
|
|
|
74
106
|
return tablespec_cache
|
|
75
107
|
|
|
76
108
|
if is_new:
|
|
109
|
+
if is_new == 1:
|
|
110
|
+
refuse_existing_table(db, schemaname, tablename)
|
|
77
111
|
if is_new == 2:
|
|
78
112
|
tblspec = db.schema_qualified_table_name(schemaname, tablename)
|
|
79
113
|
try:
|
execsql/importers/csv.py
CHANGED
|
@@ -14,6 +14,7 @@ from typing import Any
|
|
|
14
14
|
|
|
15
15
|
from execsql.exceptions import ErrInfo
|
|
16
16
|
from execsql.db.base import Database
|
|
17
|
+
from execsql.importers.base import refuse_existing_table
|
|
17
18
|
import execsql.state as _state
|
|
18
19
|
from execsql.types import dbt_firebird
|
|
19
20
|
|
|
@@ -26,7 +27,6 @@ def importtable(
|
|
|
26
27
|
tablename: str,
|
|
27
28
|
filename: str,
|
|
28
29
|
is_new: Any,
|
|
29
|
-
skip_header_line: bool = True,
|
|
30
30
|
quotechar: str | None = None,
|
|
31
31
|
delimchar: str | None = None,
|
|
32
32
|
encoding: str | None = None,
|
|
@@ -49,6 +49,8 @@ def importtable(
|
|
|
49
49
|
quotechar = None
|
|
50
50
|
inf.lineformat(delimchar, quotechar, None)
|
|
51
51
|
if is_new in (1, 2):
|
|
52
|
+
if is_new == 1:
|
|
53
|
+
refuse_existing_table(db, schemaname, tablename)
|
|
52
54
|
inf.evaluate_column_types()
|
|
53
55
|
sql = inf.create_table(db.type, schemaname, tablename)
|
|
54
56
|
if is_new == 2:
|
execsql/manifest.py
CHANGED
|
@@ -32,12 +32,16 @@ from execsql import __version__
|
|
|
32
32
|
|
|
33
33
|
__all__ = ["RunManifest", "check_writable", "current", "start"]
|
|
34
34
|
|
|
35
|
-
_current: RunManifest | None = None
|
|
36
|
-
|
|
37
35
|
|
|
38
36
|
def current() -> RunManifest | None:
|
|
39
|
-
"""The active manifest recorder, or ``None`` when ``--manifest`` was not given.
|
|
40
|
-
|
|
37
|
+
"""The active run's manifest recorder, or ``None`` when ``--manifest`` was not given.
|
|
38
|
+
|
|
39
|
+
Held on the run's :class:`~execsql.state.RuntimeContext`, so a library
|
|
40
|
+
``execsql.run()`` in the same process never records into a CLI manifest.
|
|
41
|
+
"""
|
|
42
|
+
import execsql.state as _state
|
|
43
|
+
|
|
44
|
+
return _state.manifest
|
|
41
45
|
|
|
42
46
|
|
|
43
47
|
def check_writable(path: str) -> str | None:
|
|
@@ -65,10 +69,12 @@ def start(path: str, script: str | None, variables: list[str]) -> RunManifest:
|
|
|
65
69
|
If nothing else writes it (``exit_now`` on an error, the end of a run), an
|
|
66
70
|
``atexit`` fallback does.
|
|
67
71
|
"""
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
+
import execsql.state as _state
|
|
73
|
+
|
|
74
|
+
manifest = RunManifest(path, script, variables)
|
|
75
|
+
_state.manifest = manifest
|
|
76
|
+
atexit.register(manifest.finish, None)
|
|
77
|
+
return manifest
|
|
72
78
|
|
|
73
79
|
|
|
74
80
|
def _now() -> datetime.datetime:
|
|
@@ -137,22 +143,26 @@ class RunManifest:
|
|
|
137
143
|
|
|
138
144
|
# -- finishing -------------------------------------------------------------
|
|
139
145
|
|
|
146
|
+
def add_error(self, error: Any) -> None:
|
|
147
|
+
"""Record *error* (an :class:`ErrInfo`) in the manifest's ``errors``."""
|
|
148
|
+
# str(ErrInfo) is its concise message; errmsg() is the multi-line
|
|
149
|
+
# console block with a timestamp, which a reader of JSON does not want.
|
|
150
|
+
self.errors.append(
|
|
151
|
+
{
|
|
152
|
+
"message": " ".join(str(error).split()),
|
|
153
|
+
"type": getattr(error, "type", None),
|
|
154
|
+
"source": getattr(error, "script_file", None),
|
|
155
|
+
"line": getattr(error, "script_line_no", None),
|
|
156
|
+
"command": getattr(error, "cmd", None) or getattr(error, "command", None),
|
|
157
|
+
},
|
|
158
|
+
)
|
|
159
|
+
|
|
140
160
|
def finish(self, exit_status: int | None, error: Any = None) -> None:
|
|
141
161
|
"""Write the manifest once. Later calls (the atexit fallback) do nothing."""
|
|
142
162
|
if self.written:
|
|
143
163
|
return
|
|
144
164
|
if error is not None:
|
|
145
|
-
|
|
146
|
-
# console block with a timestamp, which a reader of JSON does not want.
|
|
147
|
-
self.errors.append(
|
|
148
|
-
{
|
|
149
|
-
"message": " ".join(str(error).split()),
|
|
150
|
-
"type": getattr(error, "type", None),
|
|
151
|
-
"source": getattr(error, "script_file", None),
|
|
152
|
-
"line": getattr(error, "script_line_no", None),
|
|
153
|
-
"command": getattr(error, "cmd", None) or getattr(error, "command", None),
|
|
154
|
-
},
|
|
155
|
-
)
|
|
165
|
+
self.add_error(error)
|
|
156
166
|
finished = _now()
|
|
157
167
|
data = {
|
|
158
168
|
"execsql_version": __version__,
|
|
@@ -172,9 +182,10 @@ class RunManifest:
|
|
|
172
182
|
"errors": self.errors,
|
|
173
183
|
}
|
|
174
184
|
self.written = True
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
185
|
+
import execsql.state as _state
|
|
186
|
+
|
|
187
|
+
if _state.manifest is self:
|
|
188
|
+
_state.manifest = None
|
|
178
189
|
atexit.unregister(self.finish)
|
|
179
190
|
try:
|
|
180
191
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -100,12 +100,11 @@ def xf_hasrows(**kwargs: Any) -> bool:
|
|
|
100
100
|
return bool(nrows > 0)
|
|
101
101
|
|
|
102
102
|
|
|
103
|
-
def _row_count(queryname: str,
|
|
103
|
+
def _row_count(queryname: str, metacommandline: str) -> int:
|
|
104
104
|
"""Return the number of rows in *queryname*, raising ErrInfo on failure.
|
|
105
105
|
|
|
106
106
|
Args:
|
|
107
107
|
queryname: Table or view name to count rows in.
|
|
108
|
-
sql_context: The SQL string to include in error messages.
|
|
109
108
|
metacommandline: The full metacommand line for error context.
|
|
110
109
|
|
|
111
110
|
Returns:
|
|
@@ -167,7 +166,7 @@ def xf_row_count_gt(**kwargs: Any) -> bool:
|
|
|
167
166
|
queryname = kwargs["queryname"]
|
|
168
167
|
mcl = kwargs["metacommandline"]
|
|
169
168
|
n = _parse_row_count_n(kwargs["n"], mcl)
|
|
170
|
-
return _row_count(queryname,
|
|
169
|
+
return _row_count(queryname, mcl) > n
|
|
171
170
|
|
|
172
171
|
|
|
173
172
|
def xf_row_count_gte(**kwargs: Any) -> bool:
|
|
@@ -183,7 +182,7 @@ def xf_row_count_gte(**kwargs: Any) -> bool:
|
|
|
183
182
|
queryname = kwargs["queryname"]
|
|
184
183
|
mcl = kwargs["metacommandline"]
|
|
185
184
|
n = _parse_row_count_n(kwargs["n"], mcl)
|
|
186
|
-
return _row_count(queryname,
|
|
185
|
+
return _row_count(queryname, mcl) >= n
|
|
187
186
|
|
|
188
187
|
|
|
189
188
|
def xf_row_count_eq(**kwargs: Any) -> bool:
|
|
@@ -199,7 +198,7 @@ def xf_row_count_eq(**kwargs: Any) -> bool:
|
|
|
199
198
|
queryname = kwargs["queryname"]
|
|
200
199
|
mcl = kwargs["metacommandline"]
|
|
201
200
|
n = _parse_row_count_n(kwargs["n"], mcl)
|
|
202
|
-
return _row_count(queryname,
|
|
201
|
+
return _row_count(queryname, mcl) == n
|
|
203
202
|
|
|
204
203
|
|
|
205
204
|
def xf_row_count_lt(**kwargs: Any) -> bool:
|
|
@@ -215,7 +214,7 @@ def xf_row_count_lt(**kwargs: Any) -> bool:
|
|
|
215
214
|
queryname = kwargs["queryname"]
|
|
216
215
|
mcl = kwargs["metacommandline"]
|
|
217
216
|
n = _parse_row_count_n(kwargs["n"], mcl)
|
|
218
|
-
return _row_count(queryname,
|
|
217
|
+
return _row_count(queryname, mcl) < n
|
|
219
218
|
|
|
220
219
|
|
|
221
220
|
def xf_sqlerror(**kwargs: Any) -> bool:
|
|
@@ -428,7 +427,7 @@ def xf_newer_date(**kwargs: Any) -> bool:
|
|
|
428
427
|
datestr = unquoted(kwargs["datestr"])
|
|
429
428
|
if not Path(file1).exists():
|
|
430
429
|
raise ErrInfo(type="cmd", other_msg=f"File {file1} does not exist.")
|
|
431
|
-
dt_value = parse_datetime(datestr)
|
|
430
|
+
dt_value = parse_datetime(datestr, strict=False)
|
|
432
431
|
if not dt_value:
|
|
433
432
|
raise ErrInfo(type="cmd", other_msg=f"{datestr} can't be interpreted as a date/time.")
|
|
434
433
|
return os.stat(file1).st_mtime > time.mktime(dt_value.timetuple())
|
execsql/metacommands/data.py
CHANGED
|
@@ -36,6 +36,7 @@ from execsql.parser import NumericParser
|
|
|
36
36
|
from execsql.script import current_script_line
|
|
37
37
|
from execsql.utils.crypto import Encrypt
|
|
38
38
|
from execsql.utils.errors import exception_desc, exit_now
|
|
39
|
+
from execsql.utils.fileio import register_secret
|
|
39
40
|
from execsql.utils.gui import GUI_SELECTSUB, GuiSpec, enable_gui
|
|
40
41
|
from execsql.utils.strings import get_subvarset, unquoted
|
|
41
42
|
|
|
@@ -126,7 +127,9 @@ def x_sub_encrypt(**kwargs: Any) -> None:
|
|
|
126
127
|
def x_sub_decrypt(**kwargs: Any) -> None:
|
|
127
128
|
varname = kwargs["match"]
|
|
128
129
|
subvarset, varname = get_subvarset(varname, kwargs["metacommandline"])
|
|
129
|
-
|
|
130
|
+
plaintext = Encrypt().decrypt(kwargs["crypttext"])
|
|
131
|
+
register_secret(plaintext)
|
|
132
|
+
subvarset.add_substitution(varname, plaintext)
|
|
130
133
|
return None
|
|
131
134
|
|
|
132
135
|
|
execsql/metacommands/debug.py
CHANGED
|
@@ -52,10 +52,8 @@ def x_debug_commandliststack(**kwargs: Any) -> None:
|
|
|
52
52
|
``<main>`` script, ``EXECUTE SCRIPT`` calls, ``INCLUDE``'d files,
|
|
53
53
|
``IF``/``ELSEIF``/``ELSE`` branches, ``LOOP`` iterations (with iteration
|
|
54
54
|
count), and ``BATCH`` blocks. Each frame shows source file and line.
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
because it only records SCRIPT call frames; the unified ``ast_exec_stack``
|
|
58
|
-
is the authoritative debugger view.
|
|
55
|
+
The metacommand keeps its upstream name; what it prints is the AST
|
|
56
|
+
executor's ``ast_exec_stack``.
|
|
59
57
|
"""
|
|
60
58
|
stack = getattr(_state, "ast_exec_stack", None) or []
|
|
61
59
|
_state.output.write(f"Execution Stack (depth: {len(stack)}):\n")
|
|
@@ -8,14 +8,18 @@ Implements ``x_include``, ``x_copy``, ``x_copy_query``, ``x_zip``,
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import os
|
|
11
|
+
import pickle
|
|
12
|
+
import struct
|
|
11
13
|
import sys
|
|
12
|
-
|
|
14
|
+
import tempfile
|
|
15
|
+
from collections.abc import Iterator
|
|
13
16
|
from pathlib import Path
|
|
14
17
|
from shutil import copyfileobj
|
|
15
|
-
from typing import Any, Literal
|
|
18
|
+
from typing import Any, Literal
|
|
16
19
|
|
|
17
20
|
import execsql.state as _state
|
|
18
21
|
from execsql.exceptions import ErrInfo
|
|
22
|
+
from execsql.importers.base import refuse_existing_table
|
|
19
23
|
from execsql.models import DataTable
|
|
20
24
|
from execsql.script import current_script_line
|
|
21
25
|
from execsql.utils.errors import exception_desc
|
|
@@ -23,6 +27,39 @@ from execsql.utils.fileio import filewriter_close
|
|
|
23
27
|
from execsql.utils.strings import unquoted
|
|
24
28
|
|
|
25
29
|
|
|
30
|
+
class _RowSpool:
|
|
31
|
+
"""Pass rows through while writing each to a temporary file, to be read back once more.
|
|
32
|
+
|
|
33
|
+
``COPY ... TO NEW`` needs every row twice, once to describe the new table
|
|
34
|
+
and once to load it, but runs the source query once. The rows are kept
|
|
35
|
+
in memory up to *max_size* bytes and on disk beyond that. Each row is
|
|
36
|
+
pickled on its own, so neither pass keeps a reference to earlier rows.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
_LENGTH = struct.Struct("<Q")
|
|
40
|
+
|
|
41
|
+
def __init__(self, rows: Any, max_size: int = 8 * 1024 * 1024) -> None:
|
|
42
|
+
self._rows = rows
|
|
43
|
+
self._file = tempfile.SpooledTemporaryFile(max_size=max_size) # noqa: SIM115 - closed by close()
|
|
44
|
+
|
|
45
|
+
def __iter__(self) -> Iterator[Any]:
|
|
46
|
+
for row in self._rows:
|
|
47
|
+
data = pickle.dumps(row, protocol=pickle.HIGHEST_PROTOCOL)
|
|
48
|
+
self._file.write(self._LENGTH.pack(len(data)))
|
|
49
|
+
self._file.write(data)
|
|
50
|
+
yield row
|
|
51
|
+
|
|
52
|
+
def replay(self) -> Iterator[Any]:
|
|
53
|
+
"""Yield the rows passed through, in order."""
|
|
54
|
+
self._file.seek(0)
|
|
55
|
+
while header := self._file.read(self._LENGTH.size):
|
|
56
|
+
(size,) = self._LENGTH.unpack(header)
|
|
57
|
+
yield pickle.loads(self._file.read(size))
|
|
58
|
+
|
|
59
|
+
def close(self) -> None:
|
|
60
|
+
self._file.close()
|
|
61
|
+
|
|
62
|
+
|
|
26
63
|
def _close_rowsource(rows: Any) -> None:
|
|
27
64
|
close = getattr(rows, "close", None)
|
|
28
65
|
if close:
|
|
@@ -63,29 +100,16 @@ def x_copy(**kwargs: Any) -> None:
|
|
|
63
100
|
db2 = _state.dbs.aliased_as(alias2)
|
|
64
101
|
tbl1 = db1.schema_qualified_table_name(schema1, table1)
|
|
65
102
|
tbl2 = db2.schema_qualified_table_name(schema2, table2)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
type="cmd",
|
|
70
|
-
command_text=kwargs["metacommandline"],
|
|
71
|
-
other_msg=f"Table {tbl1} does not exist",
|
|
72
|
-
)
|
|
73
|
-
except Exception:
|
|
74
|
-
pass # Best-effort check; some adapters lack information_schema.
|
|
103
|
+
# No check that the source exists: the database may fold an unquoted name
|
|
104
|
+
# (PostgreSQL reads Orders as orders), and the SELECT below names a missing
|
|
105
|
+
# source in the database's own words.
|
|
75
106
|
if new_tbl2 and new_tbl2 == "new":
|
|
76
|
-
|
|
77
|
-
if db2.table_exists(table2, schema2):
|
|
78
|
-
raise ErrInfo(
|
|
79
|
-
type="cmd",
|
|
80
|
-
command_text=kwargs["metacommandline"],
|
|
81
|
-
other_msg=f"Table {tbl2} already exists",
|
|
82
|
-
)
|
|
83
|
-
except Exception:
|
|
84
|
-
pass # Best-effort check; some adapters lack information_schema.
|
|
107
|
+
refuse_existing_table(db2, schema2, table2)
|
|
85
108
|
select_stmt = f"select * from {tbl1};"
|
|
86
109
|
|
|
87
110
|
get_ts_tablespec = None
|
|
88
111
|
rows_to_close = None
|
|
112
|
+
spool: _RowSpool | None = None
|
|
89
113
|
rows_for_insert: Any
|
|
90
114
|
|
|
91
115
|
if new_tbl2:
|
|
@@ -96,9 +120,10 @@ def x_copy(**kwargs: Any) -> None:
|
|
|
96
120
|
except Exception as e:
|
|
97
121
|
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
98
122
|
rows_to_close = rows
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
get_ts_tablespec = DataTable(hdrs,
|
|
123
|
+
# Values come from a database cursor, so a text value stays text.
|
|
124
|
+
spool = _RowSpool(rows)
|
|
125
|
+
get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
|
|
126
|
+
rows_for_insert = spool.replay()
|
|
102
127
|
tbl_desc = get_ts_tablespec
|
|
103
128
|
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
104
129
|
if new_tbl2 == "replacement":
|
|
@@ -129,7 +154,7 @@ def x_copy(**kwargs: Any) -> None:
|
|
|
129
154
|
nonlocal get_ts_tablespec
|
|
130
155
|
if get_ts_tablespec is None:
|
|
131
156
|
ts_hdrs, ts_rows = db1.select_rowsource(select_stmt)
|
|
132
|
-
get_ts_tablespec = DataTable(ts_hdrs, ts_rows)
|
|
157
|
+
get_ts_tablespec = DataTable(ts_hdrs, ts_rows, infer_strings=False)
|
|
133
158
|
return get_ts_tablespec
|
|
134
159
|
|
|
135
160
|
try:
|
|
@@ -138,6 +163,9 @@ def x_copy(**kwargs: Any) -> None:
|
|
|
138
163
|
except BaseException:
|
|
139
164
|
_close_rowsource(rows_to_close)
|
|
140
165
|
raise
|
|
166
|
+
finally:
|
|
167
|
+
if spool is not None:
|
|
168
|
+
spool.close()
|
|
141
169
|
|
|
142
170
|
|
|
143
171
|
def x_copy_query(**kwargs: Any) -> None:
|
|
@@ -164,18 +192,11 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
164
192
|
db2 = _state.dbs.aliased_as(alias2)
|
|
165
193
|
tbl2 = db2.schema_qualified_table_name(schema2, table2)
|
|
166
194
|
if new_tbl2 and new_tbl2 == "new":
|
|
167
|
-
|
|
168
|
-
if db2.table_exists(table2, schema2):
|
|
169
|
-
raise ErrInfo(
|
|
170
|
-
type="cmd",
|
|
171
|
-
command_text=kwargs["metacommandline"],
|
|
172
|
-
other_msg=f"Table {tbl2} already exists",
|
|
173
|
-
)
|
|
174
|
-
except Exception:
|
|
175
|
-
pass # Best-effort check; some adapters lack information_schema.
|
|
195
|
+
refuse_existing_table(db2, schema2, table2)
|
|
176
196
|
|
|
177
197
|
get_ts_tablespec = None
|
|
178
198
|
rows_to_close = None
|
|
199
|
+
spool: _RowSpool | None = None
|
|
179
200
|
rows_for_insert: Any
|
|
180
201
|
|
|
181
202
|
if new_tbl2:
|
|
@@ -186,9 +207,10 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
186
207
|
except Exception as e:
|
|
187
208
|
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
188
209
|
rows_to_close = rows
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
get_ts_tablespec = DataTable(hdrs,
|
|
210
|
+
# Values come from a database cursor, so a text value stays text.
|
|
211
|
+
spool = _RowSpool(rows)
|
|
212
|
+
get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
|
|
213
|
+
rows_for_insert = spool.replay()
|
|
192
214
|
tbl_desc = get_ts_tablespec
|
|
193
215
|
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
194
216
|
if new_tbl2 == "replacement":
|
|
@@ -219,7 +241,7 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
219
241
|
nonlocal get_ts_tablespec
|
|
220
242
|
if get_ts_tablespec is None:
|
|
221
243
|
ts_hdrs, ts_rows = db1.select_rowsource(select_stmt)
|
|
222
|
-
get_ts_tablespec = DataTable(ts_hdrs, ts_rows)
|
|
244
|
+
get_ts_tablespec = DataTable(ts_hdrs, ts_rows, infer_strings=False)
|
|
223
245
|
return get_ts_tablespec
|
|
224
246
|
|
|
225
247
|
try:
|
|
@@ -228,6 +250,9 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
228
250
|
except BaseException:
|
|
229
251
|
_close_rowsource(rows_to_close)
|
|
230
252
|
raise
|
|
253
|
+
finally:
|
|
254
|
+
if spool is not None:
|
|
255
|
+
spool.close()
|
|
231
256
|
|
|
232
257
|
|
|
233
258
|
def x_zip(**kwargs: Any) -> None:
|
execsql/metacommands/prompt.py
CHANGED
|
@@ -67,6 +67,7 @@ from execsql.utils.gui import (
|
|
|
67
67
|
pause,
|
|
68
68
|
pause_win,
|
|
69
69
|
)
|
|
70
|
+
from execsql.utils.fileio import register_secret
|
|
70
71
|
from execsql.utils.strings import unquoted
|
|
71
72
|
|
|
72
73
|
|
|
@@ -144,6 +145,7 @@ def x_prompt_enter(**kwargs: Any) -> None:
|
|
|
144
145
|
subvarset.add_substitution(sub_var, txtval)
|
|
145
146
|
script_name, lno = current_script_line()
|
|
146
147
|
if as_pw:
|
|
148
|
+
register_secret(txtval)
|
|
147
149
|
_state.exec_log.log_status_info(f"Password assigned to variable {{{sub_var}}} on line {lno}.")
|
|
148
150
|
else:
|
|
149
151
|
_state.exec_log.log_status_info(f"Variable {{{sub_var}}} set to {{{txtval}}} on line {lno}.")
|
execsql/metacommands/upsert.py
CHANGED
|
@@ -4,7 +4,7 @@ Integrates pg-upsert (https://pg-upsert.readthedocs.io/) as an optional
|
|
|
4
4
|
dependency, providing QA-checked, FK-dependency-ordered upserts from a
|
|
5
5
|
staging schema to a base schema on PostgreSQL.
|
|
6
6
|
|
|
7
|
-
Requires
|
|
7
|
+
Requires the ``upsert`` extra (``execsql2[upsert]``).
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
10
|
from __future__ import annotations
|
|
@@ -213,7 +213,10 @@ def _require_pg_upsert() -> None:
|
|
|
213
213
|
except ImportError as exc:
|
|
214
214
|
raise ErrInfo(
|
|
215
215
|
"exception",
|
|
216
|
-
other_msg=(
|
|
216
|
+
other_msg=(
|
|
217
|
+
'PG_UPSERT requires the pg-upsert package. Install the upsert extra: uv tool install "execsql2[upsert]"'
|
|
218
|
+
' (or pip install "execsql2[upsert]").'
|
|
219
|
+
),
|
|
217
220
|
) from exc
|
|
218
221
|
|
|
219
222
|
|
execsql/models.py
CHANGED
|
@@ -268,7 +268,6 @@ class DataTable:
|
|
|
268
268
|
|
|
269
269
|
self.inputrows = 0 # Total number of rows in the row source.
|
|
270
270
|
self.datarows = 0 # Number of non-empty rows (with data values).
|
|
271
|
-
self.shortrows = 0 # Number of rows without as many data values as column names.
|
|
272
271
|
self.cols: list = [] # List of Column objects.
|
|
273
272
|
for n in column_names:
|
|
274
273
|
self.cols.append(Column(n, infer_strings=infer_strings))
|
|
@@ -281,7 +280,6 @@ class DataTable:
|
|
|
281
280
|
self.datarows += 1
|
|
282
281
|
chkcols = len(self.cols)
|
|
283
282
|
if dataitems < chkcols:
|
|
284
|
-
self.shortrows += 1
|
|
285
283
|
chkcols = len(datarow)
|
|
286
284
|
else:
|
|
287
285
|
if dataitems > chkcols:
|
|
@@ -307,6 +305,9 @@ class DataTable:
|
|
|
307
305
|
raise DataTableError(errmsg)
|
|
308
306
|
for i in range(chkcols):
|
|
309
307
|
self.cols[i].eval_types(datarow[i])
|
|
308
|
+
# Trailing values left off a short row are imported as NULL.
|
|
309
|
+
for i in range(chkcols, len(self.cols)):
|
|
310
|
+
self.cols[i].eval_types(None)
|
|
310
311
|
for col in self.cols:
|
|
311
312
|
col.column_type()
|
|
312
313
|
|
execsql/parser.py
CHANGED
|
@@ -215,10 +215,17 @@ class CondParser(CondTokens):
|
|
|
215
215
|
"""Recursive-descent parser for boolean conditional expressions."""
|
|
216
216
|
|
|
217
217
|
# Takes a conditional expression string.
|
|
218
|
-
def __init__(self, condexpr: str) -> None:
|
|
219
|
-
"""Initialise the parser with the conditional expression string.
|
|
218
|
+
def __init__(self, condexpr: str, conditionallist: Any = None) -> None:
|
|
219
|
+
"""Initialise the parser with the conditional expression string.
|
|
220
|
+
|
|
221
|
+
*conditionallist* is the table of conditional tests to match against;
|
|
222
|
+
``None`` means the current run's ``conditionallist``. Passing one lets
|
|
223
|
+
a caller with no run in progress, such as ``execsql lint``, parse
|
|
224
|
+
(without evaluating) a condition.
|
|
225
|
+
"""
|
|
220
226
|
self.condexpr = condexpr
|
|
221
227
|
self.cond_expr = SourceString(condexpr)
|
|
228
|
+
self.conditionallist = conditionallist
|
|
222
229
|
|
|
223
230
|
def match_not(self) -> int | None:
|
|
224
231
|
"""Match a NOT operator and return its token type, or None."""
|
|
@@ -251,10 +258,12 @@ class CondParser(CondTokens):
|
|
|
251
258
|
m1 = self.factor()
|
|
252
259
|
return CondAstNode(self.NOT, m1, None)
|
|
253
260
|
# Find the matching metacommand -- get a tuple consisting of (metacommand, groupdict)
|
|
254
|
-
|
|
255
|
-
|
|
261
|
+
conditionallist = self.conditionallist
|
|
262
|
+
if conditionallist is None:
|
|
263
|
+
import execsql.state as _state
|
|
256
264
|
|
|
257
|
-
|
|
265
|
+
conditionallist = _state.conditionallist
|
|
266
|
+
matched_metacommand = self.cond_expr.match_metacommand(conditionallist)
|
|
258
267
|
if matched_metacommand is not None:
|
|
259
268
|
matched_metacommand[1]["metacommandline"] = self.condexpr
|
|
260
269
|
return CondAstNode(self.CONDITIONAL, matched_metacommand, None)
|