execsql2 2.17.2__py3-none-any.whl → 2.18.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/cli/__init__.py +13 -1
- execsql/cli/lint.py +16 -565
- execsql/cli/run.py +29 -2
- execsql/config.py +20 -0
- execsql/db/access.py +6 -0
- execsql/db/base.py +57 -1
- execsql/db/dsn.py +19 -9
- execsql/db/firebird.py +6 -0
- execsql/db/mysql.py +81 -0
- execsql/db/oracle.py +6 -0
- execsql/db/sqlite.py +37 -18
- execsql/db/sqlserver.py +31 -6
- execsql/exporters/base.py +1 -1
- execsql/exporters/duckdb.py +8 -4
- execsql/exporters/ods.py +11 -0
- execsql/exporters/sqlite.py +10 -3
- execsql/exporters/templates.py +10 -0
- execsql/exporters/xls.py +4 -0
- execsql/exporters/xlsx.py +9 -0
- execsql/importers/json.py +49 -32
- execsql/metacommands/conditions.py +7 -2
- execsql/metacommands/io_export.py +21 -26
- execsql/metacommands/io_fileops.py +21 -3
- execsql/metacommands/io_import.py +23 -3
- execsql/script/ast.py +8 -0
- execsql/script/engine.py +32 -0
- execsql/script/executor.py +12 -0
- execsql/script/variables.py +41 -15
- execsql/utils/auth.py +49 -1
- execsql/utils/fileio.py +120 -0
- execsql/utils/gui.py +11 -1
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/md_compare.sql +12 -12
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/md_glossary.sql +5 -5
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/md_upsert.sql +13 -13
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/pg_compare.sql +24 -24
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/pg_glossary.sql +5 -5
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/pg_upsert.sql +29 -29
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/script_template.sql +2 -2
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/ss_compare.sql +24 -24
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/ss_glossary.sql +6 -6
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/ss_upsert.sql +2917 -2917
- {execsql2-2.17.2.dist-info → execsql2-2.18.0.dist-info}/METADATA +8 -3
- {execsql2-2.17.2.dist-info → execsql2-2.18.0.dist-info}/RECORD +52 -52
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/README.md +0 -0
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/execsql.conf +0 -0
- {execsql2-2.17.2.data → execsql2-2.18.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.17.2.dist-info → execsql2-2.18.0.dist-info}/WHEEL +0 -0
- {execsql2-2.17.2.dist-info → execsql2-2.18.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.17.2.dist-info → execsql2-2.18.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.17.2.dist-info → execsql2-2.18.0.dist-info}/licenses/NOTICE +0 -0
execsql/cli/run.py
CHANGED
|
@@ -197,6 +197,11 @@ def _ping_db(db: Any) -> None:
|
|
|
197
197
|
# ---------------------------------------------------------------------------
|
|
198
198
|
|
|
199
199
|
|
|
200
|
+
# B12/F014: shared sensitive-name filter used by env-var seeding, -a
|
|
201
|
+
# value logging, and any future credential-redaction sites.
|
|
202
|
+
_SENSITIVE_SUBSTRINGS = ("SECRET", "TOKEN", "PASSWORD", "PASSWD", "PRIVATE_KEY", "CREDENTIAL")
|
|
203
|
+
|
|
204
|
+
|
|
200
205
|
def _seed_early_subvars() -> SubVarSet:
|
|
201
206
|
"""Create and populate the initial substitution variable pool.
|
|
202
207
|
|
|
@@ -205,7 +210,6 @@ def _seed_early_subvars() -> SubVarSet:
|
|
|
205
210
|
"""
|
|
206
211
|
subvars = SubVarSet()
|
|
207
212
|
|
|
208
|
-
_SENSITIVE_SUBSTRINGS = ("SECRET", "TOKEN", "PASSWORD", "PASSWD", "PRIVATE_KEY", "CREDENTIAL")
|
|
209
213
|
for k in os.environ:
|
|
210
214
|
if any(s in k.upper() for s in _SENSITIVE_SUBSTRINGS):
|
|
211
215
|
continue
|
|
@@ -310,6 +314,14 @@ def _apply_dsn(dsn: str, conf: ConfigData, db_type: str | None) -> tuple[str | N
|
|
|
310
314
|
if parsed["password"]:
|
|
311
315
|
conf.db_password = parsed["password"]
|
|
312
316
|
conf.passwd_prompt = False
|
|
317
|
+
# B12/F038: embedding the password in the --dsn URL leaks it to
|
|
318
|
+
# `ps`, shell history, and any logging that captures argv. Warn
|
|
319
|
+
# so the user knows to prefer keyring or a password-less DSN.
|
|
320
|
+
_err_console.print(
|
|
321
|
+
"[bold yellow]Warning:[/bold yellow] --dsn URL contains a password; "
|
|
322
|
+
"it may be visible in `ps`, shell history, and process accounting. "
|
|
323
|
+
"Consider keyring or a password-less DSN.",
|
|
324
|
+
)
|
|
313
325
|
port = parsed["port"] # may be None
|
|
314
326
|
return db_type, user, port
|
|
315
327
|
|
|
@@ -490,8 +502,15 @@ def _setup_logging(
|
|
|
490
502
|
for n, repl in enumerate(sub_vars):
|
|
491
503
|
var = f"$ARG_{n + 1}"
|
|
492
504
|
subvars.add_substitution(var, repl)
|
|
505
|
+
# B12/F014: -a values are user input that may contain
|
|
506
|
+
# secrets. Redact the value in the log line when the
|
|
507
|
+
# surrounding -a payload looks sensitive (matches the
|
|
508
|
+
# existing env-var filter at _seed_early_subvars).
|
|
509
|
+
display_repl = repl
|
|
510
|
+
if any(s in str(repl).upper() for s in _SENSITIVE_SUBSTRINGS):
|
|
511
|
+
display_repl = "***"
|
|
493
512
|
logger.log_status_info(
|
|
494
|
-
f"Command-line substitution variable assignment: {var} set to {{{
|
|
513
|
+
f"Command-line substitution variable assignment: {var} set to {{{display_repl}}}",
|
|
495
514
|
)
|
|
496
515
|
|
|
497
516
|
return logger
|
|
@@ -533,6 +552,8 @@ def _run(
|
|
|
533
552
|
lint: bool = False,
|
|
534
553
|
debug: bool = False,
|
|
535
554
|
no_system_cmd: bool = False,
|
|
555
|
+
no_rm_file: bool = False,
|
|
556
|
+
no_serve: bool = False,
|
|
536
557
|
config_file: str | None = None,
|
|
537
558
|
) -> None:
|
|
538
559
|
"""Initialise state, connect to the database, load the script, and run it.
|
|
@@ -741,6 +762,12 @@ def _run(
|
|
|
741
762
|
if no_system_cmd:
|
|
742
763
|
conf.allow_system_cmd = False
|
|
743
764
|
|
|
765
|
+
if no_rm_file:
|
|
766
|
+
conf.allow_rm_file = False
|
|
767
|
+
|
|
768
|
+
if no_serve:
|
|
769
|
+
conf.allow_serve = False
|
|
770
|
+
|
|
744
771
|
if _ast_tree is not None:
|
|
745
772
|
_execute_script_ast(_ast_tree, conf, profile=profile, profile_limit=profile_limit)
|
|
746
773
|
|
execsql/config.py
CHANGED
|
@@ -315,6 +315,12 @@ class ConfigData:
|
|
|
315
315
|
self._get_bool(cp, self._CONFIG_SECTION, "log_sql", "log_sql")
|
|
316
316
|
self._get_int(cp, self._CONFIG_SECTION, "max_log_size_mb", "max_log_size_mb")
|
|
317
317
|
self._get_bool(cp, self._CONFIG_SECTION, "allow_system_cmd", "allow_system_cmd")
|
|
318
|
+
self._get_bool(cp, self._CONFIG_SECTION, "allow_rm_file", "allow_rm_file")
|
|
319
|
+
self._get_bool(cp, self._CONFIG_SECTION, "allow_serve", "allow_serve")
|
|
320
|
+
self._get_str(cp, self._CONFIG_SECTION, "include_root", "include_root")
|
|
321
|
+
self._get_str(cp, self._CONFIG_SECTION, "serve_root", "serve_root")
|
|
322
|
+
self._get_str(cp, self._CONFIG_SECTION, "template_root", "template_root")
|
|
323
|
+
self._get_int(cp, self._CONFIG_SECTION, "max_substitution_bytes", "max_substitution_bytes")
|
|
318
324
|
# --- [email] ---
|
|
319
325
|
self._get_str(cp, self._EMAIL_SECTION, "host", "smtp_host")
|
|
320
326
|
self._get_int(cp, self._EMAIL_SECTION, "port", "smtp_port")
|
|
@@ -426,6 +432,20 @@ class ConfigData:
|
|
|
426
432
|
self.export_output_dir: str | None = None
|
|
427
433
|
self.dao_flush_delay_secs = 5.0
|
|
428
434
|
self.allow_system_cmd = True
|
|
435
|
+
# B05: path-containment controls. ``*_root`` keys, when set, force
|
|
436
|
+
# the corresponding handler to confine resolved paths under the
|
|
437
|
+
# named root and reject anything that escapes via ``..``,
|
|
438
|
+
# absolute paths, drive letters, or UNCs. ``allow_*`` toggles
|
|
439
|
+
# disable the handler entirely (symmetric with allow_system_cmd).
|
|
440
|
+
self.include_root: str | None = None
|
|
441
|
+
self.serve_root: str | None = None
|
|
442
|
+
self.template_root: str | None = None
|
|
443
|
+
self.allow_rm_file = True
|
|
444
|
+
self.allow_serve = True
|
|
445
|
+
# B17/F013: byte ceiling on substitute_vars() expansion to
|
|
446
|
+
# defeat exponential-expansion bombs. None = use the engine
|
|
447
|
+
# default (10 MB).
|
|
448
|
+
self.max_substitution_bytes: int | None = None
|
|
429
449
|
self.zip_buffer_mb = 10
|
|
430
450
|
if os.name == "posix":
|
|
431
451
|
sys_config_file = str(Path("/etc") / self.config_file_name)
|
execsql/db/access.py
CHANGED
|
@@ -88,6 +88,12 @@ class AccessDatabase(Database):
|
|
|
88
88
|
def __repr__(self) -> str:
|
|
89
89
|
return f"AccessDatabase({self.db_name}, {self.encoding})"
|
|
90
90
|
|
|
91
|
+
def auto_commits_ddl(self) -> bool:
|
|
92
|
+
"""MS Access (via Jet/ACE on pyodbc) implicitly commits DDL —
|
|
93
|
+
``rollback()`` is a silent no-op for any transaction whose
|
|
94
|
+
boundary the DDL crossed."""
|
|
95
|
+
return True
|
|
96
|
+
|
|
91
97
|
def open_db(self) -> None:
|
|
92
98
|
"""Open an ODBC connection to the Access database."""
|
|
93
99
|
# Open an ODBC connection.
|
execsql/db/base.py
CHANGED
|
@@ -153,9 +153,41 @@ class Database(ABC):
|
|
|
153
153
|
|
|
154
154
|
def quote_identifier(self, identifier: str) -> str:
|
|
155
155
|
"""Return *identifier* wrapped in double-quotes with any embedded
|
|
156
|
-
double-quotes escaped (standard SQL identifier quoting).
|
|
156
|
+
double-quotes escaped (standard SQL identifier quoting).
|
|
157
|
+
|
|
158
|
+
Override in subclasses for adapters whose native identifier quote
|
|
159
|
+
differs (MySQL uses backticks, SQL Server uses square brackets).
|
|
160
|
+
"""
|
|
157
161
|
return '"' + identifier.replace('"', '""') + '"'
|
|
158
162
|
|
|
163
|
+
def quote_qualified_identifier(self, *parts: str) -> str:
|
|
164
|
+
"""Quote each non-empty part of a possibly-multi-segment identifier
|
|
165
|
+
and join them with ``.`` (e.g. ``"schema"."table"``).
|
|
166
|
+
|
|
167
|
+
Skips ``None`` or empty parts so callers don't need to special-case
|
|
168
|
+
schemaless databases (SQLite, Firebird).
|
|
169
|
+
"""
|
|
170
|
+
return ".".join(self.quote_identifier(p) for p in parts if p)
|
|
171
|
+
|
|
172
|
+
def quote_literal(self, value: Any) -> str:
|
|
173
|
+
"""Return *value* as a SQL string literal, safely escaped.
|
|
174
|
+
|
|
175
|
+
Default ANSI/Postgres behaviour: wrap in single quotes, double
|
|
176
|
+
embedded apostrophes, escape backslashes (so a value containing
|
|
177
|
+
``\\'`` cannot terminate the literal on MySQL default mode or
|
|
178
|
+
PostgreSQL E-strings), and reject embedded NUL bytes (most
|
|
179
|
+
wire protocols truncate or reject them).
|
|
180
|
+
|
|
181
|
+
Override per-DBMS only when the wire protocol requires a different
|
|
182
|
+
literal form.
|
|
183
|
+
"""
|
|
184
|
+
if value is None:
|
|
185
|
+
return "NULL"
|
|
186
|
+
s = str(value)
|
|
187
|
+
if "\x00" in s:
|
|
188
|
+
raise ValueError("SQL literal contains a NUL byte; refusing to quote.")
|
|
189
|
+
return "'" + s.replace("\\", "\\\\").replace("'", "''") + "'"
|
|
190
|
+
|
|
159
191
|
def paramsubs(self, paramcount: int) -> str:
|
|
160
192
|
"""Return a comma-separated string of *paramcount* parameter placeholders."""
|
|
161
193
|
return ",".join((self.paramstr,) * paramcount)
|
|
@@ -210,6 +242,30 @@ class Database(ABC):
|
|
|
210
242
|
except Exception:
|
|
211
243
|
pass # Best-effort; connection may already be closed.
|
|
212
244
|
|
|
245
|
+
def needs_explicit_commit_after_ddl(self) -> bool:
|
|
246
|
+
"""Return True if this adapter's driver does NOT auto-commit DDL.
|
|
247
|
+
|
|
248
|
+
Firebird is the notable case: a CREATE TABLE issued via the
|
|
249
|
+
driver remains pending until commit, so callers that issue DDL
|
|
250
|
+
followed by DML on a fresh table must commit in between. Most
|
|
251
|
+
other adapters either auto-commit on DDL or run DDL inside the
|
|
252
|
+
current transaction with no special handling required.
|
|
253
|
+
"""
|
|
254
|
+
return False
|
|
255
|
+
|
|
256
|
+
def auto_commits_ddl(self) -> bool:
|
|
257
|
+
"""Return True if this adapter's driver implicitly commits DDL.
|
|
258
|
+
|
|
259
|
+
Oracle, MySQL, SQL Server, and MS Access all auto-commit DDL —
|
|
260
|
+
``rollback()`` is a silent no-op for any transaction whose
|
|
261
|
+
boundary the DDL crossed. Callers that wrap DDL inside an
|
|
262
|
+
explicit ``BEGIN BATCH … END BATCH`` block on these adapters
|
|
263
|
+
get weaker rollback guarantees than on PostgreSQL / SQLite, and
|
|
264
|
+
should be aware of the asymmetry. See
|
|
265
|
+
``docs/about/divergence.md`` for the full per-DBMS matrix.
|
|
266
|
+
"""
|
|
267
|
+
return False
|
|
268
|
+
|
|
213
269
|
def schema_qualified_table_name(self, schema_name: str | None, table_name: str) -> str:
|
|
214
270
|
"""Return the quoted, optionally schema-qualified form of *table_name*."""
|
|
215
271
|
table_name = self.type.quoted(table_name)
|
execsql/db/dsn.py
CHANGED
|
@@ -70,17 +70,27 @@ class DsnDatabase(Database):
|
|
|
70
70
|
if self.need_passwd and self.user and self.password is None:
|
|
71
71
|
self.password = get_password("DSN", self.db_name, self.user)
|
|
72
72
|
|
|
73
|
+
def _odbc_quote(value: str) -> str:
|
|
74
|
+
"""Brace-quote *value* for an ODBC connection-string attribute.
|
|
75
|
+
|
|
76
|
+
ODBC attribute lists are ``;``-delimited; a malicious value
|
|
77
|
+
containing ``;`` can inject attributes (CWE-91). Wrap every
|
|
78
|
+
user-supplied value in ``{…}`` and double any embedded ``}``
|
|
79
|
+
so ``pa}ss`` becomes ``{pa}}ss}``.
|
|
80
|
+
"""
|
|
81
|
+
return "{" + str(value).replace("}", "}}") + "}"
|
|
82
|
+
|
|
73
83
|
def _dsn_connect(autocommit: bool = False):
|
|
74
|
-
|
|
84
|
+
# Build a connection string with brace-quoted attribute values
|
|
85
|
+
# so that ``;``, ``=`` and other ODBC delimiters in user-
|
|
86
|
+
# supplied credentials cannot inject additional attributes.
|
|
87
|
+
parts = [f"DSN={_odbc_quote(self.db_name)}"]
|
|
75
88
|
if self.need_passwd:
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
else:
|
|
82
|
-
kwargs = {"autocommit": autocommit} if autocommit else {}
|
|
83
|
-
self.conn = pyodbc.connect(cs % self.db_name, **kwargs)
|
|
89
|
+
parts.append(f"UID={_odbc_quote(self.user)}")
|
|
90
|
+
parts.append(f"PWD={_odbc_quote(self.password)}")
|
|
91
|
+
connstr = ";".join(parts) + ";"
|
|
92
|
+
kwargs = {"autocommit": autocommit} if autocommit else {}
|
|
93
|
+
self.conn = pyodbc.connect(connstr, **kwargs)
|
|
84
94
|
|
|
85
95
|
def _try_connect():
|
|
86
96
|
try:
|
execsql/db/firebird.py
CHANGED
|
@@ -59,6 +59,12 @@ class FirebirdDatabase(Database):
|
|
|
59
59
|
f"{self.need_passwd!r}, {self.port!r}, {self.encoding!r})"
|
|
60
60
|
)
|
|
61
61
|
|
|
62
|
+
def needs_explicit_commit_after_ddl(self) -> bool:
|
|
63
|
+
"""Firebird leaves DDL pending until commit — callers issuing
|
|
64
|
+
``CREATE TABLE`` then ``INSERT`` on a fresh table must commit
|
|
65
|
+
in between."""
|
|
66
|
+
return True
|
|
67
|
+
|
|
62
68
|
def open_db(self) -> None:
|
|
63
69
|
"""Open a connection to the Firebird database."""
|
|
64
70
|
import fdb as firebird_lib
|
execsql/db/mysql.py
CHANGED
|
@@ -73,6 +73,87 @@ class MySQLDatabase(Database):
|
|
|
73
73
|
f"{self.need_passwd!r}, {self.port!r}, {self.encoding!r})"
|
|
74
74
|
)
|
|
75
75
|
|
|
76
|
+
def auto_commits_ddl(self) -> bool:
|
|
77
|
+
"""MySQL / MariaDB implicitly commit DDL — ``rollback()`` is a
|
|
78
|
+
silent no-op for any transaction whose boundary the DDL
|
|
79
|
+
crossed. See ``docs/about/divergence.md``."""
|
|
80
|
+
return True
|
|
81
|
+
|
|
82
|
+
def quote_identifier(self, identifier: str) -> str:
|
|
83
|
+
"""MySQL / MariaDB native identifier quoting uses backticks.
|
|
84
|
+
|
|
85
|
+
Override of :meth:`Database.quote_identifier` for B07a/F021: the
|
|
86
|
+
base ANSI ``"…"`` form only works after the connection sets
|
|
87
|
+
``sql_mode=ANSI`` / ``ANSI_QUOTES`` (the adapter does this at
|
|
88
|
+
open_db, but user SQL that resets the mode would silently break
|
|
89
|
+
execsql-generated DDL).
|
|
90
|
+
"""
|
|
91
|
+
return "`" + identifier.replace("`", "``") + "`"
|
|
92
|
+
|
|
93
|
+
# B10/F069: MySQL's ``lower_case_table_names`` server variable
|
|
94
|
+
# controls whether identifier comparisons against information_schema
|
|
95
|
+
# rows are case-sensitive. On Linux the default is 0 (case-sensitive
|
|
96
|
+
# storage, case-sensitive compare); on Windows/macOS the default is
|
|
97
|
+
# 1 (lowercased storage, case-insensitive compare); ``2`` stores
|
|
98
|
+
# as-created but compares case-insensitively. The base
|
|
99
|
+
# ``information_schema.tables`` lookups in Database compare
|
|
100
|
+
# literally, so ``table_exists("MyTable")`` returned False on
|
|
101
|
+
# case-sensitive Linux servers even when ``MyTable`` actually
|
|
102
|
+
# existed. Fold the input name when the server is case-insensitive.
|
|
103
|
+
|
|
104
|
+
def _lower_case_table_names(self) -> int:
|
|
105
|
+
"""Return the server-level ``@@lower_case_table_names`` value, cached.
|
|
106
|
+
|
|
107
|
+
Looks up the system variable on first call; subsequent calls
|
|
108
|
+
return the cached value. Safe to call before ``open_db()``
|
|
109
|
+
returns — falls back to ``0`` (case-sensitive) if the lookup
|
|
110
|
+
fails for any reason.
|
|
111
|
+
"""
|
|
112
|
+
cached = getattr(self, "_cached_lctn", None)
|
|
113
|
+
if cached is not None:
|
|
114
|
+
return cached
|
|
115
|
+
try:
|
|
116
|
+
_, rows = self.select_data("SELECT @@lower_case_table_names;")
|
|
117
|
+
value = int(rows[0][0]) if rows else 0
|
|
118
|
+
except Exception:
|
|
119
|
+
value = 0
|
|
120
|
+
self._cached_lctn = value
|
|
121
|
+
return value
|
|
122
|
+
|
|
123
|
+
def _fold_identifier(self, name: str | None) -> str | None:
|
|
124
|
+
"""Lowercase *name* when the server is case-insensitive (LCTN 1/2)."""
|
|
125
|
+
if name is None:
|
|
126
|
+
return None
|
|
127
|
+
return name.lower() if self._lower_case_table_names() in (1, 2) else name
|
|
128
|
+
|
|
129
|
+
# NB: schema_exists is overridden below to return False unconditionally
|
|
130
|
+
# — MySQL's pre-existing behavior. The case-folding only matters for
|
|
131
|
+
# adapters where schema lookups are meaningful, which MySQL skips.
|
|
132
|
+
|
|
133
|
+
def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
|
|
134
|
+
return super().table_exists(
|
|
135
|
+
self._fold_identifier(table_name),
|
|
136
|
+
self._fold_identifier(schema_name),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
def column_exists(
|
|
140
|
+
self,
|
|
141
|
+
table_name: str,
|
|
142
|
+
column_name: str,
|
|
143
|
+
schema_name: str | None = None,
|
|
144
|
+
) -> bool:
|
|
145
|
+
return super().column_exists(
|
|
146
|
+
self._fold_identifier(table_name),
|
|
147
|
+
self._fold_identifier(column_name),
|
|
148
|
+
self._fold_identifier(schema_name),
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
|
|
152
|
+
return super().view_exists(
|
|
153
|
+
self._fold_identifier(view_name),
|
|
154
|
+
self._fold_identifier(schema_name),
|
|
155
|
+
)
|
|
156
|
+
|
|
76
157
|
def open_db(self) -> None:
|
|
77
158
|
"""Open a connection to the MySQL or MariaDB server."""
|
|
78
159
|
import pymysql as mysql_lib
|
execsql/db/oracle.py
CHANGED
|
@@ -61,6 +61,12 @@ class OracleDatabase(Database):
|
|
|
61
61
|
f"{self.need_passwd!r}, {self.port!r}, {self.encoding!r})"
|
|
62
62
|
)
|
|
63
63
|
|
|
64
|
+
def auto_commits_ddl(self) -> bool:
|
|
65
|
+
"""Oracle implicitly commits DDL — ``rollback()`` is a silent
|
|
66
|
+
no-op for any transaction whose boundary the DDL crossed.
|
|
67
|
+
See ``docs/about/divergence.md`` for the per-DBMS matrix."""
|
|
68
|
+
return True
|
|
69
|
+
|
|
64
70
|
def open_db(self) -> None:
|
|
65
71
|
"""Open a connection to the Oracle database."""
|
|
66
72
|
import cx_Oracle
|
execsql/db/sqlite.py
CHANGED
|
@@ -189,6 +189,38 @@ class SQLiteDatabase(Database):
|
|
|
189
189
|
sql = f"insert into {sq_name} ({colspec}) values ({paramspec});"
|
|
190
190
|
curs = self.cursor()
|
|
191
191
|
total_rows = 0
|
|
192
|
+
# B08/F019: batch rows and use cursor.executemany instead of one
|
|
193
|
+
# cursor.execute per row. The pre-fix code was 10–100× slower on
|
|
194
|
+
# the most common open-source target. Honour import_row_buffer
|
|
195
|
+
# (default 1000) so the batch never blows memory on huge imports.
|
|
196
|
+
batch_size = max(1, int(getattr(_state.conf, "import_row_buffer", 1000) or 1000))
|
|
197
|
+
batch: list[list[Any]] = []
|
|
198
|
+
|
|
199
|
+
def _flush(last_line: list[Any]) -> None:
|
|
200
|
+
nonlocal total_rows
|
|
201
|
+
if not batch:
|
|
202
|
+
return
|
|
203
|
+
try:
|
|
204
|
+
curs.executemany(sql, batch)
|
|
205
|
+
except ErrInfo:
|
|
206
|
+
raise
|
|
207
|
+
except Exception as e:
|
|
208
|
+
self.rollback()
|
|
209
|
+
raise ErrInfo(
|
|
210
|
+
type="db",
|
|
211
|
+
command_text=sql,
|
|
212
|
+
exception_msg=exception_desc(),
|
|
213
|
+
other_msg=f"Can't load data into table {sq_name} from line {{{last_line}}}",
|
|
214
|
+
) from e
|
|
215
|
+
total_rows += len(batch)
|
|
216
|
+
batch.clear()
|
|
217
|
+
interval = getattr(_state.conf, "import_progress_interval", 0)
|
|
218
|
+
if _state.exec_log and interval > 0 and total_rows % interval == 0:
|
|
219
|
+
_state.exec_log.log_status_info(
|
|
220
|
+
f"IMPORT into {sq_name}: {total_rows} rows imported so far.",
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
last_line: list[Any] = []
|
|
192
224
|
try:
|
|
193
225
|
for datalineno, line in enumerate(rowsource):
|
|
194
226
|
# Skip empty rows.
|
|
@@ -216,24 +248,11 @@ class SQLiteDatabase(Database):
|
|
|
216
248
|
if not _state.conf.empty_rows:
|
|
217
249
|
add_line = not all(c is None for c in linedata)
|
|
218
250
|
if add_line:
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
self.rollback()
|
|
225
|
-
raise ErrInfo(
|
|
226
|
-
type="db",
|
|
227
|
-
command_text=sql,
|
|
228
|
-
exception_msg=exception_desc(),
|
|
229
|
-
other_msg=f"Can't load data into table {sq_name} from line {{{line}}}",
|
|
230
|
-
) from e
|
|
231
|
-
total_rows += 1
|
|
232
|
-
interval = getattr(_state.conf, "import_progress_interval", 0)
|
|
233
|
-
if _state.exec_log and interval > 0 and total_rows % interval == 0:
|
|
234
|
-
_state.exec_log.log_status_info(
|
|
235
|
-
f"IMPORT into {sq_name}: {total_rows} rows imported so far.",
|
|
236
|
-
)
|
|
251
|
+
batch.append(linedata)
|
|
252
|
+
last_line = line
|
|
253
|
+
if len(batch) >= batch_size:
|
|
254
|
+
_flush(last_line)
|
|
255
|
+
_flush(last_line)
|
|
237
256
|
finally:
|
|
238
257
|
curs.close()
|
|
239
258
|
if _state.exec_log:
|
execsql/db/sqlserver.py
CHANGED
|
@@ -58,6 +58,24 @@ class SqlServerDatabase(Database):
|
|
|
58
58
|
f"{self.need_passwd!r}, {self.port!r}, {self.encoding!r})"
|
|
59
59
|
)
|
|
60
60
|
|
|
61
|
+
def auto_commits_ddl(self) -> bool:
|
|
62
|
+
"""SQL Server implicitly commits DDL on Microsoft's pyodbc
|
|
63
|
+
driver in autocommit mode — ``rollback()`` is a silent no-op
|
|
64
|
+
for any transaction whose boundary the DDL crossed."""
|
|
65
|
+
return True
|
|
66
|
+
|
|
67
|
+
def quote_identifier(self, identifier: str) -> str:
|
|
68
|
+
"""SQL Server native identifier quoting uses square brackets.
|
|
69
|
+
|
|
70
|
+
Override of :meth:`Database.quote_identifier` for B07a/F021: the
|
|
71
|
+
base ANSI ``"…"`` form only works while ``QUOTED_IDENTIFIER`` is
|
|
72
|
+
``ON`` (the adapter sets this at open_db, but user SQL that
|
|
73
|
+
toggles the session setting would silently break
|
|
74
|
+
execsql-generated DDL). ``]`` inside an identifier is escaped to
|
|
75
|
+
``]]``.
|
|
76
|
+
"""
|
|
77
|
+
return "[" + identifier.replace("]", "]]") + "]"
|
|
78
|
+
|
|
61
79
|
def open_db(self) -> None:
|
|
62
80
|
"""Open a connection to the SQL Server database via pyodbc."""
|
|
63
81
|
import pyodbc
|
|
@@ -133,13 +151,20 @@ class SqlServerDatabase(Database):
|
|
|
133
151
|
type="error",
|
|
134
152
|
other_msg=f"Can't open SQL Server database {self.db_name} on {self.server_name}",
|
|
135
153
|
)
|
|
154
|
+
# B11/F022: ensure the session-setup cursor is closed even
|
|
155
|
+
# if one of the SET statements raises. The previous code
|
|
156
|
+
# left a raw cursor open across pyodbc's connection,
|
|
157
|
+
# leaking a server-side handle for the connection's life.
|
|
136
158
|
curs = self.conn.cursor()
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
159
|
+
try:
|
|
160
|
+
curs.execute("SET IMPLICIT_TRANSACTIONS OFF;")
|
|
161
|
+
curs.execute("SET ANSI_NULLS ON;")
|
|
162
|
+
curs.execute("SET ANSI_PADDING ON;")
|
|
163
|
+
curs.execute("SET ANSI_WARNINGS ON;")
|
|
164
|
+
curs.execute("SET QUOTED_IDENTIFIER ON;")
|
|
165
|
+
self.conn.commit()
|
|
166
|
+
finally:
|
|
167
|
+
curs.close()
|
|
143
168
|
|
|
144
169
|
def exec_cmd(self, querycommand: str) -> None:
|
|
145
170
|
"""Execute a stored procedure by name."""
|
execsql/exporters/base.py
CHANGED
|
@@ -22,7 +22,7 @@ from execsql.script import current_script_line
|
|
|
22
22
|
from execsql.utils.errors import file_size_date
|
|
23
23
|
from execsql.utils.gui import ConsoleUIError
|
|
24
24
|
|
|
25
|
-
__all__ = ["
|
|
25
|
+
__all__ = ["ExportMetadata", "ExportRecord", "WriteSpec"]
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
class ExportRecord:
|
execsql/exporters/duckdb.py
CHANGED
|
@@ -37,21 +37,25 @@ def export_duckdb(
|
|
|
37
37
|
from execsql.models import DataTable
|
|
38
38
|
|
|
39
39
|
chunksize = 10000
|
|
40
|
+
# B07a/F026: tablename and catalog are substituted from user input.
|
|
41
|
+
# Identifier-quote the table-name interpolation site and use
|
|
42
|
+
# placeholders for the information_schema literal-value query.
|
|
43
|
+
qtable = dbt_duckdb.quoted(tablename)
|
|
40
44
|
pre_exist = Path(outfile).is_file()
|
|
41
45
|
ddb = duckdb.connect(outfile, read_only=False)
|
|
42
46
|
if pre_exist:
|
|
43
47
|
catalog = Path(outfile).stem
|
|
44
48
|
curs = ddb.cursor()
|
|
45
49
|
res = curs.execute(
|
|
46
|
-
|
|
47
|
-
|
|
50
|
+
"select count(*) as rows from information_schema.tables where table_catalog = ? and table_name = ?;",
|
|
51
|
+
(catalog, tablename),
|
|
48
52
|
)
|
|
49
53
|
rv = res.fetchone()
|
|
50
54
|
if not (rv is None or rv[0] == 0):
|
|
51
55
|
if append:
|
|
52
56
|
raise ErrInfo(type="error", other_msg=f"The table {tablename} already exists in {outfile}.")
|
|
53
57
|
else:
|
|
54
|
-
curs.execute(f"drop table {
|
|
58
|
+
curs.execute(f"drop table {qtable};")
|
|
55
59
|
curs.close()
|
|
56
60
|
# Construct and run the CREATE TABLE statement
|
|
57
61
|
rowdata = list(rows)
|
|
@@ -63,7 +67,7 @@ def export_duckdb(
|
|
|
63
67
|
columns = [dbt_duckdb.quoted(col) for col in hdrs]
|
|
64
68
|
colspec = ",".join(columns)
|
|
65
69
|
paramspec = ",".join(("?",) * len(columns))
|
|
66
|
-
sql = f"insert into {
|
|
70
|
+
sql = f"insert into {qtable} ({colspec}) values ({paramspec});"
|
|
67
71
|
n_chunks = math.ceil(len(rowdata) / chunksize)
|
|
68
72
|
curs.execute("BEGIN TRANSACTION;")
|
|
69
73
|
for i in range(n_chunks):
|
execsql/exporters/ods.py
CHANGED
|
@@ -35,6 +35,17 @@ class OdsFile:
|
|
|
35
35
|
def __init__(self) -> None:
|
|
36
36
|
"""Import odfpy and initialise the workbook state."""
|
|
37
37
|
global of
|
|
38
|
+
# B15/F030: defuse stdlib XML parsers before odfpy uses them.
|
|
39
|
+
# odfpy parses .ods through xml.dom.minidom and friends — a
|
|
40
|
+
# malicious .ods can carry a billion-laughs or external-entity
|
|
41
|
+
# bomb. defuse_stdlib() is a global patch and is idempotent.
|
|
42
|
+
try:
|
|
43
|
+
import defusedxml
|
|
44
|
+
|
|
45
|
+
defusedxml.defuse_stdlib()
|
|
46
|
+
except ImportError:
|
|
47
|
+
# defusedxml is in the [ods] extra; absence is non-fatal.
|
|
48
|
+
pass
|
|
38
49
|
try:
|
|
39
50
|
import odf as of # noqa: F401 — submodule imports below register on the `of` alias
|
|
40
51
|
import odf.opendocument # noqa: F401
|
execsql/exporters/sqlite.py
CHANGED
|
@@ -30,20 +30,27 @@ def export_sqlite(
|
|
|
30
30
|
from execsql.models import DataTable
|
|
31
31
|
|
|
32
32
|
chunksize = 10000
|
|
33
|
+
# B07a/F025: tablename is substituted from user input — use the
|
|
34
|
+
# existing dbt_sqlite.quoted helper to identifier-quote every site
|
|
35
|
+
# rather than f-string-interpolating into SQL.
|
|
36
|
+
qtable = dbt_sqlite.quoted(tablename)
|
|
33
37
|
pre_exist = Path(outfile).is_file()
|
|
34
38
|
sdb = sqlite3.connect(outfile)
|
|
35
39
|
try:
|
|
36
40
|
if pre_exist:
|
|
37
41
|
curs = sdb.cursor()
|
|
42
|
+
# Existence check: the table name is a value here (not an
|
|
43
|
+
# identifier), so use a parameter rather than string concat.
|
|
38
44
|
res = curs.execute(
|
|
39
|
-
|
|
45
|
+
"select name from sqlite_master where type='table' and name=?;",
|
|
46
|
+
(tablename,),
|
|
40
47
|
)
|
|
41
48
|
rv = res.fetchone()
|
|
42
49
|
if not (rv is None or rv[0] == 0):
|
|
43
50
|
if append:
|
|
44
51
|
raise ErrInfo(type="error", other_msg=f"The table {tablename} already exists in {outfile}.")
|
|
45
52
|
else:
|
|
46
|
-
curs.execute(f"drop table {
|
|
53
|
+
curs.execute(f"drop table {qtable};")
|
|
47
54
|
curs.close()
|
|
48
55
|
# Construct and run the CREATE TABLE statement
|
|
49
56
|
rowdata = list(rows)
|
|
@@ -55,7 +62,7 @@ def export_sqlite(
|
|
|
55
62
|
columns = [dbt_sqlite.quoted(col) for col in hdrs]
|
|
56
63
|
colspec = ",".join(columns)
|
|
57
64
|
paramspec = ",".join(("?",) * len(columns))
|
|
58
|
-
sql = f"insert into {
|
|
65
|
+
sql = f"insert into {qtable} ({colspec}) values ({paramspec});"
|
|
59
66
|
n_chunks = math.ceil(len(rowdata) / chunksize)
|
|
60
67
|
for i in range(n_chunks):
|
|
61
68
|
start = i * chunksize
|
execsql/exporters/templates.py
CHANGED
|
@@ -28,6 +28,11 @@ class StrTemplateReport:
|
|
|
28
28
|
def __init__(self, template_file: str) -> None:
|
|
29
29
|
"""Load and compile the template from the given file path."""
|
|
30
30
|
conf = _state.conf
|
|
31
|
+
template_root = getattr(conf, "template_root", None)
|
|
32
|
+
if template_root:
|
|
33
|
+
from execsql.utils.fileio import safe_output_path
|
|
34
|
+
|
|
35
|
+
template_file = safe_output_path(template_file, template_root)
|
|
31
36
|
self.infname = template_file
|
|
32
37
|
from execsql.utils.fileio import EncodedFile
|
|
33
38
|
|
|
@@ -89,6 +94,11 @@ class JinjaTemplateReport:
|
|
|
89
94
|
"The jinja2 library is required to produce reports with the Jinja2 templating system. See http://jinja.pocoo.org/",
|
|
90
95
|
)
|
|
91
96
|
conf = _state.conf
|
|
97
|
+
template_root = getattr(conf, "template_root", None)
|
|
98
|
+
if template_root:
|
|
99
|
+
from execsql.utils.fileio import safe_output_path
|
|
100
|
+
|
|
101
|
+
template_file = safe_output_path(template_file, template_root)
|
|
92
102
|
self.infname = template_file
|
|
93
103
|
from execsql.utils.fileio import EncodedFile
|
|
94
104
|
|
execsql/exporters/xls.py
CHANGED
|
@@ -203,6 +203,10 @@ class XlsxFile:
|
|
|
203
203
|
self.encoding = encoding
|
|
204
204
|
self.read_only = read_only
|
|
205
205
|
if Path(filename).is_file():
|
|
206
|
+
# B15/F031: defend against zip-bomb XLSX before openpyxl parses it.
|
|
207
|
+
from execsql.utils.fileio import check_zip_decompression_ratio
|
|
208
|
+
|
|
209
|
+
check_zip_decompression_ratio(filename)
|
|
206
210
|
if read_only:
|
|
207
211
|
self.wbk = self._openpyxl.load_workbook(filename, read_only=True)
|
|
208
212
|
else:
|
execsql/exporters/xlsx.py
CHANGED
|
@@ -61,6 +61,8 @@ def _cell_value(item: Any) -> Any:
|
|
|
61
61
|
if isinstance(item, datetime.time):
|
|
62
62
|
# openpyxl has no native time-only type; store as HH:MM:SS string.
|
|
63
63
|
return item.strftime("%H:%M:%S")
|
|
64
|
+
if isinstance(item, str):
|
|
65
|
+
return item
|
|
64
66
|
return str(item)
|
|
65
67
|
|
|
66
68
|
|
|
@@ -103,6 +105,10 @@ def write_query_to_xlsx(
|
|
|
103
105
|
# Determine sheet name and open/create workbook
|
|
104
106
|
# ------------------------------------------------------------------
|
|
105
107
|
if append and Path(outfile).is_file():
|
|
108
|
+
# B15/F031: defend against zip-bomb XLSX before openpyxl parses it.
|
|
109
|
+
from execsql.utils.fileio import check_zip_decompression_ratio
|
|
110
|
+
|
|
111
|
+
check_zip_decompression_ratio(outfile)
|
|
106
112
|
wb = openpyxl.load_workbook(outfile)
|
|
107
113
|
existing_names = wb.sheetnames
|
|
108
114
|
base = sheetname or "Sheet"
|
|
@@ -223,6 +229,9 @@ def write_queries_to_xlsx(
|
|
|
223
229
|
os.unlink(outfile)
|
|
224
230
|
|
|
225
231
|
if Path(outfile).is_file():
|
|
232
|
+
from execsql.utils.fileio import check_zip_decompression_ratio
|
|
233
|
+
|
|
234
|
+
check_zip_decompression_ratio(outfile)
|
|
226
235
|
wb = openpyxl.load_workbook(outfile)
|
|
227
236
|
else:
|
|
228
237
|
wb = openpyxl.Workbook()
|