execsql2 2.22.9__py3-none-any.whl → 2.23.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +71 -0
- execsql/config.py +2 -0
- execsql/data/execsql.conf.template +5 -0
- execsql/db/access.py +34 -10
- execsql/db/base.py +120 -2
- execsql/db/dsn.py +14 -7
- execsql/db/duckdb.py +102 -10
- execsql/db/firebird.py +12 -7
- execsql/db/mysql.py +55 -10
- execsql/db/oracle.py +24 -11
- execsql/db/postgres.py +17 -9
- execsql/db/sqlite.py +23 -8
- execsql/db/sqlserver.py +7 -7
- execsql/db/tiers.py +79 -0
- execsql/exporters/delimited.py +15 -1
- execsql/exporters/html.py +8 -2
- execsql/exporters/latex.py +35 -2
- execsql/exporters/markdown.py +9 -3
- execsql/exporters/ods.py +22 -4
- execsql/exporters/pretty.py +13 -0
- execsql/exporters/templates.py +18 -1
- execsql/exporters/values.py +25 -4
- execsql/exporters/xlsx.py +7 -1
- execsql/state.py +5 -0
- execsql/state.pyi +3 -1
- execsql/types.py +4 -1
- execsql/utils/fileio.py +47 -2
- {execsql2-2.22.9.dist-info → execsql2-2.23.1.dist-info}/METADATA +22 -13
- {execsql2-2.22.9.dist-info → execsql2-2.23.1.dist-info}/RECORD +47 -46
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/README.md +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/pg_upsert.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.22.9.data → execsql2-2.23.1.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.22.9.dist-info → execsql2-2.23.1.dist-info}/WHEEL +0 -0
- {execsql2-2.22.9.dist-info → execsql2-2.23.1.dist-info}/entry_points.txt +0 -0
- {execsql2-2.22.9.dist-info → execsql2-2.23.1.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.22.9.dist-info → execsql2-2.23.1.dist-info}/licenses/NOTICE +0 -0
execsql/api.py
CHANGED
|
@@ -29,6 +29,7 @@ Usage::
|
|
|
29
29
|
|
|
30
30
|
from __future__ import annotations
|
|
31
31
|
|
|
32
|
+
import atexit
|
|
32
33
|
import dataclasses
|
|
33
34
|
import datetime
|
|
34
35
|
import io
|
|
@@ -511,6 +512,11 @@ def run(
|
|
|
511
512
|
errors: list[ScriptError] = []
|
|
512
513
|
t0 = time.perf_counter()
|
|
513
514
|
|
|
515
|
+
# WRITE ... TO <file> and TEE are handled by a FileWriter subprocess,
|
|
516
|
+
# and fileio drops every write when none is running. Start one here so
|
|
517
|
+
# the API behaves like the CLI.
|
|
518
|
+
_ensure_filewriter(conf)
|
|
519
|
+
|
|
514
520
|
try:
|
|
515
521
|
execute(tree, ctx=ctx)
|
|
516
522
|
except SystemExit:
|
|
@@ -538,6 +544,17 @@ def run(
|
|
|
538
544
|
|
|
539
545
|
elapsed = time.perf_counter() - t0
|
|
540
546
|
|
|
547
|
+
# Flush and close every file the script wrote, so the caller can read
|
|
548
|
+
# them the moment run() returns. The subprocess itself is left running
|
|
549
|
+
# for reuse by a later run() and is reaped by the atexit handler; a
|
|
550
|
+
# writer the caller started is theirs and is never shut down here.
|
|
551
|
+
from execsql.utils.fileio import filewriter_close_all_after_write
|
|
552
|
+
|
|
553
|
+
try:
|
|
554
|
+
filewriter_close_all_after_write()
|
|
555
|
+
except Exception:
|
|
556
|
+
pass # Best-effort: a failed flush must not mask a script error.
|
|
557
|
+
|
|
541
558
|
# ------------------------------------------------------------------
|
|
542
559
|
# Collect results
|
|
543
560
|
# ------------------------------------------------------------------
|
|
@@ -571,6 +588,60 @@ def run(
|
|
|
571
588
|
# ---------------------------------------------------------------------------
|
|
572
589
|
# Helpers
|
|
573
590
|
# ---------------------------------------------------------------------------
|
|
591
|
+
def _ensure_filewriter(conf: Any) -> bool:
|
|
592
|
+
"""Start the FileWriter subprocess if one is not already running.
|
|
593
|
+
|
|
594
|
+
``WRITE ... TO <file>`` and ``TEE`` hand their output to a FileWriter
|
|
595
|
+
subprocess, and every entry point in :mod:`execsql.utils.fileio` guards on
|
|
596
|
+
the subprocess being alive — dropping the write when it is not, so that a
|
|
597
|
+
dead writer cannot deadlock the caller on a full pipe. The CLI starts one
|
|
598
|
+
during startup; :func:`run` did not, so every file write under the API was
|
|
599
|
+
silently discarded and the run still reported success.
|
|
600
|
+
|
|
601
|
+
A writer started here outlives the call and is reused by a later
|
|
602
|
+
:func:`run`, exactly as the CLI keeps one for the life of the process; it
|
|
603
|
+
is reaped by an ``atexit`` handler. A writer the caller started themselves
|
|
604
|
+
is left alone.
|
|
605
|
+
|
|
606
|
+
Returns:
|
|
607
|
+
``True`` if this call started the writer, ``False`` if one was already
|
|
608
|
+
running.
|
|
609
|
+
"""
|
|
610
|
+
import execsql.state as _state
|
|
611
|
+
import execsql.utils.fileio as _fileio
|
|
612
|
+
from execsql.utils.fileio import FileWriter, filewriter_end
|
|
613
|
+
|
|
614
|
+
if _fileio.filewriter is not None and _fileio.filewriter.is_alive():
|
|
615
|
+
return False
|
|
616
|
+
|
|
617
|
+
# Drain stale messages so a previously-dead subprocess cannot leak
|
|
618
|
+
# responses or unconsumed commands into the new one. On macOS (`spawn`)
|
|
619
|
+
# the OS pipe buffer is small enough that retained entries would deadlock
|
|
620
|
+
# the next put().
|
|
621
|
+
for q in (_fileio.fw_input, _fileio.fw_output):
|
|
622
|
+
try:
|
|
623
|
+
while True:
|
|
624
|
+
q.get_nowait()
|
|
625
|
+
except Exception:
|
|
626
|
+
pass
|
|
627
|
+
|
|
628
|
+
try:
|
|
629
|
+
_fileio.filewriter = _state.filewriter = FileWriter(
|
|
630
|
+
_fileio.fw_input,
|
|
631
|
+
_fileio.fw_output,
|
|
632
|
+
file_encoding=conf.output_encoding,
|
|
633
|
+
open_timeout=getattr(conf, "outfile_open_timeout", 10),
|
|
634
|
+
)
|
|
635
|
+
_state.filewriter.start()
|
|
636
|
+
except Exception:
|
|
637
|
+
# Under the `spawn` start method (macOS, Windows) the subprocess
|
|
638
|
+
# re-imports the parent's __main__, which fails from a REPL, a
|
|
639
|
+
# notebook, or `python -c`. Leave the writer unset: fileio's guards
|
|
640
|
+
# then warn on the first dropped write rather than deadlocking.
|
|
641
|
+
_fileio.filewriter = _state.filewriter = None
|
|
642
|
+
return False
|
|
643
|
+
atexit.register(filewriter_end)
|
|
644
|
+
return True
|
|
574
645
|
|
|
575
646
|
|
|
576
647
|
def _capture_errors(ctx: RuntimeContext, errors: list[ScriptError]) -> None:
|
execsql/config.py
CHANGED
|
@@ -316,6 +316,7 @@ class ConfigData:
|
|
|
316
316
|
self._get_int(cp, self._OUTPUT_SECTION, "zip_buffer_mb", "zip_buffer_mb")
|
|
317
317
|
# --- [interface] ---
|
|
318
318
|
self._get_bool(cp, self._INTERFACE_SECTION, "write_warnings", "write_warnings")
|
|
319
|
+
self._get_bool(cp, self._INTERFACE_SECTION, "support_tier_notice", "support_tier_notice")
|
|
319
320
|
self._get_int(cp, self._INTERFACE_SECTION, "console_height", "gui_console_height", min_val=5)
|
|
320
321
|
self._get_int(cp, self._INTERFACE_SECTION, "console_width", "gui_console_width", min_val=20)
|
|
321
322
|
self._get_bool(cp, self._INTERFACE_SECTION, "console_wait_when_done", "gui_wait_on_exit")
|
|
@@ -409,6 +410,7 @@ class ConfigData:
|
|
|
409
410
|
self.scan_lines = 100
|
|
410
411
|
self.hdf5_text_len = 1000
|
|
411
412
|
self.write_warnings = False
|
|
413
|
+
self.support_tier_notice = True
|
|
412
414
|
self.write_prefix: str | None = None
|
|
413
415
|
self.write_suffix: str | None = None
|
|
414
416
|
self.gui_level: int = 0
|
|
@@ -231,6 +231,11 @@
|
|
|
231
231
|
# Values: Yes or No. Default: No.
|
|
232
232
|
#write_warnings=No
|
|
233
233
|
|
|
234
|
+
# Whether opening a best-effort database connection (MS Access, Firebird,
|
|
235
|
+
# Oracle, ODBC DSN) notes on stderr that the adapter is not verified in CI.
|
|
236
|
+
# Values: Yes or No. Default: Yes.
|
|
237
|
+
#support_tier_notice=Yes
|
|
238
|
+
|
|
234
239
|
# Text to prefix to WRITE output (with a space separator). "clear" removes a previous prefix.
|
|
235
240
|
#write_prefix=
|
|
236
241
|
|
execsql/db/access.py
CHANGED
|
@@ -14,7 +14,8 @@ import time
|
|
|
14
14
|
from pathlib import Path
|
|
15
15
|
from typing import Any, cast
|
|
16
16
|
|
|
17
|
-
from execsql.db.base import Database
|
|
17
|
+
from execsql.db.base import Database, _fetched
|
|
18
|
+
from execsql.db.tiers import SupportTier
|
|
18
19
|
from execsql.exceptions import ErrInfo
|
|
19
20
|
from execsql.utils.errors import exception_desc, fatal_error
|
|
20
21
|
from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
|
|
@@ -26,6 +27,11 @@ __all__ = ["AccessDatabase"]
|
|
|
26
27
|
class AccessDatabase(Database):
|
|
27
28
|
"""MS Access adapter connecting to .mdb/.accdb files via DAO (win32com) with pyodbc fallback."""
|
|
28
29
|
|
|
30
|
+
#: The Access Database Engine install on the Windows CI runner is
|
|
31
|
+
#: best-effort, so a run can silently skip these tests.
|
|
32
|
+
support_tier = SupportTier.BEST_EFFORT
|
|
33
|
+
support_tier_name = "MS Access"
|
|
34
|
+
|
|
29
35
|
# Regex for the 'create temporary view' SQL extension
|
|
30
36
|
temp_rx = re.compile(
|
|
31
37
|
r"^\s*create(?:\s+or\s+replace)?(\s+temp(?:orary)?)?\s+(?:(view|query))\s+(\w+) as\s+",
|
|
@@ -58,13 +64,15 @@ class AccessDatabase(Database):
|
|
|
58
64
|
fatal_error("The pyodbc module is required. See http://github.com/mkleehammer/pyodbc")
|
|
59
65
|
from execsql.types import dbt_access
|
|
60
66
|
|
|
67
|
+
super().__init__(
|
|
68
|
+
server_name=None,
|
|
69
|
+
db_name=Access_fn,
|
|
70
|
+
user_name=user_name,
|
|
71
|
+
need_passwd=need_passwd,
|
|
72
|
+
)
|
|
61
73
|
self.type = dbt_access
|
|
62
|
-
self.server_name = None
|
|
63
|
-
self.db_name = Access_fn
|
|
64
74
|
# The following assignment is tentative and may be changed when the connection is made.
|
|
65
75
|
self.jet4 = len(Access_fn) > 6 and Access_fn.lower()[-6:] == ".accdb"
|
|
66
|
-
self.user = user_name
|
|
67
|
-
self.need_passwd = need_passwd
|
|
68
76
|
self.password = password
|
|
69
77
|
# Encoding is only applicable to Jet < 4.0: non-accdb databases.
|
|
70
78
|
self.encoding = encoding or "windows-1252"
|
|
@@ -223,7 +231,13 @@ class AccessDatabase(Database):
|
|
|
223
231
|
if time.time() - self.last_dao_time < 5.0:
|
|
224
232
|
time.sleep(5 - (time.time() - self.last_dao_time))
|
|
225
233
|
|
|
226
|
-
def execute(
|
|
234
|
+
def execute(
|
|
235
|
+
self,
|
|
236
|
+
sqlcmd: Any,
|
|
237
|
+
paramlist: list | None = None,
|
|
238
|
+
*,
|
|
239
|
+
fetch: bool = False,
|
|
240
|
+
) -> tuple[list[str], list] | None:
|
|
227
241
|
"""Execute a SQL command, handling encoding, DAO flush, and temporary queries."""
|
|
228
242
|
|
|
229
243
|
# A shortcut to self.cursor().execute() that handles encoding and that
|
|
@@ -231,9 +245,15 @@ class AccessDatabase(Database):
|
|
|
231
245
|
# to allow Jet's read buffer to be flushed (see https://support.microsoft.com/en-us/kb/225048).
|
|
232
246
|
# This also handles the 'CREATE TEMPORARY QUERY' extension to Access.
|
|
233
247
|
# For Access, commands in a tuple (batch) are executed singly.
|
|
234
|
-
def exec1(sql: str, paramlist: list | None) -> None:
|
|
248
|
+
def exec1(sql: str, paramlist: list | None) -> tuple[list[str], list] | None:
|
|
235
249
|
tqd = self.temp_rx.match(sql)
|
|
236
250
|
if tqd:
|
|
251
|
+
if self.conn is not None and self.holding():
|
|
252
|
+
self.warn_once(
|
|
253
|
+
"temporary query",
|
|
254
|
+
"CREATE TEMPORARY QUERY reopens the Access connection, which rolls back statements "
|
|
255
|
+
"not yet committed under AUTOCOMMIT OFF or BEGIN BATCH.",
|
|
256
|
+
)
|
|
237
257
|
qn = tqd.group(3)
|
|
238
258
|
qsql = sql[tqd.end() :]
|
|
239
259
|
if self.dao_conn is None:
|
|
@@ -250,7 +270,9 @@ class AccessDatabase(Database):
|
|
|
250
270
|
self.conn = None
|
|
251
271
|
if tqd.group(1) and tqd.group(1).strip().lower()[:4] == "temp" and qn not in self.temp_query_names:
|
|
252
272
|
self.temp_query_names.append(qn)
|
|
273
|
+
return None
|
|
253
274
|
else:
|
|
275
|
+
self.before_statement(sql)
|
|
254
276
|
self.dao_flush_check()
|
|
255
277
|
with self._cursor() as curs:
|
|
256
278
|
encoded_sql: Any
|
|
@@ -263,12 +285,14 @@ class AccessDatabase(Database):
|
|
|
263
285
|
else:
|
|
264
286
|
curs.execute(encoded_sql, paramlist)
|
|
265
287
|
_state.subvars.add_substitution("$LAST_ROWCOUNT", curs.rowcount)
|
|
288
|
+
return _fetched(curs) if fetch else None
|
|
266
289
|
|
|
267
290
|
if type(sqlcmd) in (list, tuple):
|
|
291
|
+
result = None
|
|
268
292
|
for sql in sqlcmd:
|
|
269
|
-
exec1(sql, paramlist)
|
|
270
|
-
|
|
271
|
-
|
|
293
|
+
result = exec1(sql, paramlist)
|
|
294
|
+
return result
|
|
295
|
+
return exec1(sqlcmd, paramlist)
|
|
272
296
|
|
|
273
297
|
def exec_cmd(self, querycommand: str) -> None:
|
|
274
298
|
"""Execute a stored query command via DAO."""
|
execsql/db/base.py
CHANGED
|
@@ -19,15 +19,31 @@ import datetime
|
|
|
19
19
|
import re
|
|
20
20
|
from abc import ABC, abstractmethod
|
|
21
21
|
from decimal import Decimal
|
|
22
|
-
from typing import Any
|
|
22
|
+
from typing import Any, ClassVar
|
|
23
23
|
from collections.abc import Callable, Generator, Iterator
|
|
24
24
|
|
|
25
25
|
from execsql.exceptions import ErrInfo
|
|
26
26
|
from execsql.utils.errors import exception_desc
|
|
27
|
+
from execsql.db.tiers import SupportTier, announce_tier
|
|
27
28
|
import execsql.state as _state
|
|
28
29
|
|
|
29
30
|
__all__ = ["Database", "DatabasePool"]
|
|
30
31
|
|
|
32
|
+
# Leading whitespace and comments, skipped to find a statement's first words.
|
|
33
|
+
_LEADING_NOISE_RX = re.compile(r"(?:\s+|--[^\n]*(?:\n|$)|/\*.*?\*/)*", re.S)
|
|
34
|
+
|
|
35
|
+
# Statements that manage a transaction themselves; one is never opened in front of them.
|
|
36
|
+
_TRANSACTION_CONTROL_RX = re.compile(
|
|
37
|
+
r"(?:BEGIN|START\s+TRANSACTION|COMMIT|END|ROLLBACK|ABORT|SAVEPOINT|RELEASE)\b",
|
|
38
|
+
re.I,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def statement_start(sql: str) -> str:
|
|
43
|
+
"""*sql* from its first keyword on: leading whitespace and comments removed."""
|
|
44
|
+
match = _LEADING_NOISE_RX.match(sql)
|
|
45
|
+
return sql[match.end() :] if match else sql
|
|
46
|
+
|
|
31
47
|
|
|
32
48
|
def _default_dt_cast() -> dict[type, Callable]:
|
|
33
49
|
"""Build the default type-cast mapping used by all database backends."""
|
|
@@ -45,6 +61,13 @@ def _default_dt_cast() -> dict[type, Callable]:
|
|
|
45
61
|
}
|
|
46
62
|
|
|
47
63
|
|
|
64
|
+
def _fetched(curs: Any) -> tuple[list[str], list] | None:
|
|
65
|
+
"""The rows a just-executed cursor produced, as ``(column_names, rows)``; ``None`` if none."""
|
|
66
|
+
if curs.description is None:
|
|
67
|
+
return None
|
|
68
|
+
return [d[0] for d in curs.description], curs.fetchall()
|
|
69
|
+
|
|
70
|
+
|
|
48
71
|
class Database(ABC):
|
|
49
72
|
"""Abstract base class for every DBMS adapter.
|
|
50
73
|
|
|
@@ -80,6 +103,15 @@ class Database(ABC):
|
|
|
80
103
|
conn: Any
|
|
81
104
|
autocommit: bool
|
|
82
105
|
|
|
106
|
+
#: How thoroughly this adapter is verified. Subclasses that are not
|
|
107
|
+
#: exercised against a live server in CI override this with
|
|
108
|
+
#: ``SupportTier.BEST_EFFORT``; see :mod:`execsql.db.tiers`.
|
|
109
|
+
support_tier: ClassVar[SupportTier] = SupportTier.SUPPORTED
|
|
110
|
+
|
|
111
|
+
#: Name used in the best-effort notice. Defaults to the class name minus
|
|
112
|
+
#: the ``Database`` suffix (``FirebirdDatabase`` -> ``Firebird``).
|
|
113
|
+
support_tier_name: ClassVar[str | None] = None
|
|
114
|
+
|
|
83
115
|
_dt_cast: dict[type, Callable] | None = None
|
|
84
116
|
|
|
85
117
|
@property
|
|
@@ -205,13 +237,17 @@ class Database(ABC):
|
|
|
205
237
|
"""Return a comma-separated string of *paramcount* parameter placeholders."""
|
|
206
238
|
return ",".join((self.paramstr,) * paramcount)
|
|
207
239
|
|
|
208
|
-
def execute(self, sql: Any, paramlist: list | None = None) -> None:
|
|
240
|
+
def execute(self, sql: Any, paramlist: list | None = None, *, fetch: bool = False) -> tuple[list[str], list] | None:
|
|
209
241
|
"""Execute *sql* (optionally with *paramlist*), updating ``$LAST_ROWCOUNT``.
|
|
210
242
|
|
|
243
|
+
With *fetch*, return ``(column_names, rows)`` when the statement
|
|
244
|
+
produces rows (a prompt shows them), else ``None``.
|
|
245
|
+
|
|
211
246
|
Rolls back the current transaction and re-raises on any driver error.
|
|
212
247
|
"""
|
|
213
248
|
if type(sql) in (tuple, list):
|
|
214
249
|
sql = " ".join(sql)
|
|
250
|
+
self.before_statement(sql)
|
|
215
251
|
try:
|
|
216
252
|
with self._cursor() as curs:
|
|
217
253
|
if paramlist is None:
|
|
@@ -222,6 +258,7 @@ class Database(ABC):
|
|
|
222
258
|
_state.subvars.add_substitution("$LAST_ROWCOUNT", curs.rowcount)
|
|
223
259
|
except Exception:
|
|
224
260
|
pass # Non-critical: some drivers lack rowcount support.
|
|
261
|
+
return _fetched(curs) if fetch else None
|
|
225
262
|
except Exception:
|
|
226
263
|
try:
|
|
227
264
|
self.rollback()
|
|
@@ -234,6 +271,81 @@ class Database(ABC):
|
|
|
234
271
|
"""Execute a stored procedure or function by name."""
|
|
235
272
|
...
|
|
236
273
|
|
|
274
|
+
# ------------------------------------------------------------------
|
|
275
|
+
# Holding statements until commit (AUTOCOMMIT OFF, BEGIN BATCH)
|
|
276
|
+
# ------------------------------------------------------------------
|
|
277
|
+
|
|
278
|
+
#: Statements the DBMS commits on its own, with everything run before
|
|
279
|
+
#: them, even inside a transaction (MySQL and Oracle DDL). Matched against
|
|
280
|
+
#: the start of the statement; a warning names each one run while holding.
|
|
281
|
+
implicit_commit_rx: ClassVar[re.Pattern[str] | None] = None
|
|
282
|
+
|
|
283
|
+
#: Statements that cannot run inside a transaction on this DBMS; one is
|
|
284
|
+
#: not opened in front of them.
|
|
285
|
+
no_transaction_rx: ClassVar[re.Pattern[str] | None] = None
|
|
286
|
+
|
|
287
|
+
#: Set on a connection whose driver could only connect in autocommit mode
|
|
288
|
+
#: (an ODBC DSN without transactions): nothing can be held back.
|
|
289
|
+
driver_autocommits: bool = False
|
|
290
|
+
|
|
291
|
+
def holding(self) -> bool:
|
|
292
|
+
"""Whether statements on this connection wait for a commit.
|
|
293
|
+
|
|
294
|
+
True while AUTOCOMMIT is OFF for this database, or inside BEGIN BATCH
|
|
295
|
+
(which holds every database).
|
|
296
|
+
"""
|
|
297
|
+
if not self.autocommit:
|
|
298
|
+
return True
|
|
299
|
+
status = _state.status
|
|
300
|
+
return status is not None and status.batch.in_batch()
|
|
301
|
+
|
|
302
|
+
def before_statement(self, sql: str) -> None:
|
|
303
|
+
"""Make sure *sql* is held until commit when execsql is holding statements.
|
|
304
|
+
|
|
305
|
+
Most drivers already hold every statement: they open a transaction
|
|
306
|
+
before the first one and keep it open until commit or rollback. An
|
|
307
|
+
adapter whose driver does not (SQLite for DDL, DuckDB for everything)
|
|
308
|
+
overrides :meth:`begin_transaction`. Where the DBMS itself commits a
|
|
309
|
+
statement regardless, a warning says so, since the work cannot be
|
|
310
|
+
rolled back.
|
|
311
|
+
"""
|
|
312
|
+
if not self.holding():
|
|
313
|
+
return
|
|
314
|
+
if self.driver_autocommits:
|
|
315
|
+
self.warn_once(
|
|
316
|
+
"driver",
|
|
317
|
+
f"{self.name()} cannot hold statements until COMMIT: its driver commits each one as it runs. "
|
|
318
|
+
"AUTOCOMMIT OFF and BEGIN BATCH have no effect on it.",
|
|
319
|
+
)
|
|
320
|
+
return
|
|
321
|
+
start = statement_start(sql)
|
|
322
|
+
if self.implicit_commit_rx is not None and (m := self.implicit_commit_rx.match(start)):
|
|
323
|
+
from execsql.utils.errors import write_warning
|
|
324
|
+
|
|
325
|
+
write_warning(
|
|
326
|
+
f"{self.type.dbms_id} commits {' '.join(m.group(0).upper().split())} immediately, together with "
|
|
327
|
+
"everything run before it; it cannot be rolled back.",
|
|
328
|
+
always=True,
|
|
329
|
+
)
|
|
330
|
+
if _TRANSACTION_CONTROL_RX.match(start):
|
|
331
|
+
return
|
|
332
|
+
if self.no_transaction_rx is not None and self.no_transaction_rx.match(start):
|
|
333
|
+
return
|
|
334
|
+
self.begin_transaction()
|
|
335
|
+
|
|
336
|
+
def begin_transaction(self) -> None:
|
|
337
|
+
"""Open a transaction if none is open. Most drivers do this themselves; a no-op here."""
|
|
338
|
+
|
|
339
|
+
def warn_once(self, key: str, message: str) -> None:
|
|
340
|
+
"""Write *message* as a warning the first time *key* comes up on this connection."""
|
|
341
|
+
warned: set[str] = self.__dict__.setdefault("_warned", set())
|
|
342
|
+
if key in warned:
|
|
343
|
+
return
|
|
344
|
+
warned.add(key)
|
|
345
|
+
from execsql.utils.errors import write_warning
|
|
346
|
+
|
|
347
|
+
write_warning(message, always=True)
|
|
348
|
+
|
|
237
349
|
def autocommit_on(self) -> None:
|
|
238
350
|
"""Enable autocommit mode so each statement is committed immediately."""
|
|
239
351
|
self.autocommit = True
|
|
@@ -552,6 +664,7 @@ class Database(ABC):
|
|
|
552
664
|
rows = iter(rowsource)
|
|
553
665
|
eof = False
|
|
554
666
|
total_rows = 0
|
|
667
|
+
self.before_statement(sql)
|
|
555
668
|
|
|
556
669
|
# Optional rich progress bar for long-running imports.
|
|
557
670
|
use_progress = getattr(_state.conf, "show_progress", False)
|
|
@@ -735,6 +848,7 @@ class Database(ABC):
|
|
|
735
848
|
sq_name = self.schema_qualified_table_name(schema_name, table_name)
|
|
736
849
|
quoted_col = self.quote_identifier(column_name)
|
|
737
850
|
sql = f"insert into {sq_name} ({quoted_col}) values ({self.paramsubs(1)});"
|
|
851
|
+
self.before_statement(sql)
|
|
738
852
|
with self._cursor() as curs:
|
|
739
853
|
curs.execute(sql, (filedata,))
|
|
740
854
|
|
|
@@ -778,6 +892,10 @@ class DatabasePool:
|
|
|
778
892
|
)
|
|
779
893
|
self.pool[db_alias].close()
|
|
780
894
|
self.pool[db_alias] = db_obj
|
|
895
|
+
announce_tier(
|
|
896
|
+
db_obj.support_tier,
|
|
897
|
+
db_obj.support_tier_name or type(db_obj).__name__.removesuffix("Database"),
|
|
898
|
+
)
|
|
781
899
|
# Refresh static system vars so $DB_NAME, $DB_USER, etc. reflect the new connection.
|
|
782
900
|
try:
|
|
783
901
|
from execsql.script.engine import set_static_system_vars
|
execsql/db/dsn.py
CHANGED
|
@@ -10,6 +10,7 @@ the CLI.
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
from execsql.db.base import Database
|
|
13
|
+
from execsql.db.tiers import SupportTier
|
|
13
14
|
from execsql.exceptions import ErrInfo
|
|
14
15
|
from execsql.utils.errors import exception_desc, fatal_error
|
|
15
16
|
from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
|
|
@@ -21,6 +22,11 @@ __all__ = ["DsnDatabase"]
|
|
|
21
22
|
class DsnDatabase(Database):
|
|
22
23
|
"""Generic ODBC adapter that connects to any data source registered as an ODBC DSN via pyodbc."""
|
|
23
24
|
|
|
25
|
+
#: CI covers DSN string parsing only — no live DSN is ever opened,
|
|
26
|
+
#: and a DSN may front any DBMS at all.
|
|
27
|
+
support_tier = SupportTier.BEST_EFFORT
|
|
28
|
+
support_tier_name = "ODBC DSN"
|
|
29
|
+
|
|
24
30
|
# There's no telling what is actually connected to a DSN, so this uses
|
|
25
31
|
# generic Database methods almost exclusively. Only 'exec_cmd()' is
|
|
26
32
|
# overridden, and that uses the method for SQL Server because the DAO
|
|
@@ -41,18 +47,17 @@ class DsnDatabase(Database):
|
|
|
41
47
|
fatal_error("The pyodbc module is required. See http://github.com/mkleehammer/pyodbc")
|
|
42
48
|
from execsql.types import dbt_dsn
|
|
43
49
|
|
|
50
|
+
super().__init__(
|
|
51
|
+
server_name=None,
|
|
52
|
+
db_name=dsn_name,
|
|
53
|
+
user_name=user_name,
|
|
54
|
+
need_passwd=need_passwd,
|
|
55
|
+
)
|
|
44
56
|
self.type = dbt_dsn
|
|
45
|
-
self.server_name = None
|
|
46
|
-
self.db_name = dsn_name
|
|
47
|
-
self.user = user_name
|
|
48
|
-
self.need_passwd = need_passwd
|
|
49
57
|
self.password = password
|
|
50
|
-
self.port = None
|
|
51
58
|
self.encoding = encoding
|
|
52
59
|
self.encode_commands = True
|
|
53
60
|
self.paramstr = "?"
|
|
54
|
-
self.conn = None
|
|
55
|
-
self.autocommit = True
|
|
56
61
|
self.open_db()
|
|
57
62
|
self.password = None # Clear cleartext password after successful connection
|
|
58
63
|
|
|
@@ -104,6 +109,8 @@ class DsnDatabase(Database):
|
|
|
104
109
|
if "Optional feature not implemented" in excdesc:
|
|
105
110
|
try:
|
|
106
111
|
_dsn_connect(autocommit=True)
|
|
112
|
+
# This ODBC driver has no transactions: nothing can be held.
|
|
113
|
+
self.driver_autocommits = True
|
|
107
114
|
except Exception as e:
|
|
108
115
|
raise ErrInfo(
|
|
109
116
|
type="exception",
|
execsql/db/duckdb.py
CHANGED
|
@@ -8,18 +8,55 @@ analytics databases via the ``duckdb`` package. Corresponds to ``-t k``
|
|
|
8
8
|
on the CLI.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
+
import re
|
|
11
12
|
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
12
14
|
|
|
13
|
-
from execsql.db.base import Database
|
|
15
|
+
from execsql.db.base import Database, statement_start
|
|
14
16
|
from execsql.exceptions import ErrInfo
|
|
15
17
|
from execsql.utils.errors import exception_desc, fatal_error
|
|
16
18
|
import execsql.state as _state
|
|
17
19
|
|
|
18
20
|
__all__ = ["DuckDBDatabase"]
|
|
19
21
|
|
|
22
|
+
# Statements that return rows; any other statement DuckDB answers with a
|
|
23
|
+
# one-column "Count" result (rows inserted, updated or deleted).
|
|
24
|
+
_QUERY_RX = re.compile(r"(?:SELECT|WITH|VALUES|FROM|TABLE|SHOW|DESCRIBE|SUMMARIZE|EXPLAIN|PRAGMA|CALL)\b|\(", re.I)
|
|
25
|
+
|
|
26
|
+
# A statement that opens a transaction, or one that ends it.
|
|
27
|
+
_TRANSACTION_RX = re.compile(r"(?P<begin>BEGIN|START\s+TRANSACTION)\b|(?P<end>COMMIT|END|ROLLBACK|ABORT)\b", re.I)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _SharedCursor:
|
|
31
|
+
"""The connection, used as a cursor so a statement joins its open transaction.
|
|
32
|
+
|
|
33
|
+
A DuckDB cursor is a separate connection with a transaction of its own,
|
|
34
|
+
and closing it rolls that transaction back. ``close()`` therefore leaves
|
|
35
|
+
the connection open.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, conn: Any) -> None:
|
|
39
|
+
self._conn = conn
|
|
40
|
+
|
|
41
|
+
def __getattr__(self, name: str) -> Any:
|
|
42
|
+
return getattr(self._conn, name)
|
|
43
|
+
|
|
44
|
+
def close(self) -> None:
|
|
45
|
+
pass
|
|
46
|
+
|
|
20
47
|
|
|
21
48
|
class DuckDBDatabase(Database):
|
|
22
|
-
"""DuckDB in-process analytics adapter using the duckdb package.
|
|
49
|
+
"""DuckDB in-process analytics adapter using the duckdb package.
|
|
50
|
+
|
|
51
|
+
The duckdb driver commits each statement unless a transaction was begun,
|
|
52
|
+
and has no way to ask whether one is open, so this adapter begins one
|
|
53
|
+
itself while statements are held (AUTOCOMMIT OFF, BEGIN BATCH) and keeps
|
|
54
|
+
track of it. While it is open, every statement runs on the connection
|
|
55
|
+
itself rather than on a cursor (see :class:`_SharedCursor`).
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
#: A transaction this adapter or a script's own BEGIN opened, not yet ended.
|
|
59
|
+
in_transaction: bool = False
|
|
23
60
|
|
|
24
61
|
def __init__(self, DuckDB_fn: str) -> None:
|
|
25
62
|
try:
|
|
@@ -28,22 +65,77 @@ class DuckDBDatabase(Database):
|
|
|
28
65
|
fatal_error("The duckdb module is required.")
|
|
29
66
|
from execsql.types import dbt_duckdb
|
|
30
67
|
|
|
68
|
+
super().__init__(
|
|
69
|
+
server_name=None,
|
|
70
|
+
db_name=DuckDB_fn,
|
|
71
|
+
user_name=None,
|
|
72
|
+
need_passwd=False,
|
|
73
|
+
encoding="UTF-8",
|
|
74
|
+
)
|
|
31
75
|
self.type = dbt_duckdb
|
|
32
|
-
self.server_name = None
|
|
33
|
-
self.db_name = DuckDB_fn
|
|
34
76
|
self.catalog_name = Path(DuckDB_fn).stem
|
|
35
|
-
self.user = None
|
|
36
|
-
self.need_passwd = False
|
|
37
|
-
self.encoding = "UTF-8"
|
|
38
77
|
self.encode_commands = False
|
|
39
|
-
self.paramstr = "?"
|
|
40
|
-
self.conn = None
|
|
41
|
-
self.autocommit = True
|
|
42
78
|
self.open_db()
|
|
43
79
|
|
|
44
80
|
def __repr__(self) -> str:
|
|
45
81
|
return f"DuckDBDatabase({self.db_name!r})"
|
|
46
82
|
|
|
83
|
+
def cursor(self) -> Any:
|
|
84
|
+
"""A cursor, or while a transaction is open, the connection itself."""
|
|
85
|
+
if self.conn is None:
|
|
86
|
+
self.open_db()
|
|
87
|
+
if self.in_transaction:
|
|
88
|
+
return _SharedCursor(self.conn)
|
|
89
|
+
return self.conn.cursor()
|
|
90
|
+
|
|
91
|
+
def begin_transaction(self) -> None:
|
|
92
|
+
"""Open a transaction if none is open."""
|
|
93
|
+
if self.conn is None:
|
|
94
|
+
self.open_db()
|
|
95
|
+
if not self.in_transaction:
|
|
96
|
+
# A second BEGIN would abort the open transaction, hence the tracking.
|
|
97
|
+
self.conn.execute("BEGIN TRANSACTION")
|
|
98
|
+
self.in_transaction = True
|
|
99
|
+
|
|
100
|
+
def execute(self, sql: Any, paramlist: list | None = None, *, fetch: bool = False) -> tuple[list[str], list] | None:
|
|
101
|
+
"""Execute *sql*, noting a transaction the statement itself begins or ends.
|
|
102
|
+
|
|
103
|
+
The driver's ``rowcount`` is always -1: DuckDB answers an INSERT,
|
|
104
|
+
UPDATE or DELETE with a one-column ``Count`` result instead, which is
|
|
105
|
+
read here to set ``$LAST_ROWCOUNT`` as other drivers do.
|
|
106
|
+
"""
|
|
107
|
+
text = " ".join(sql) if type(sql) in (tuple, list) else sql
|
|
108
|
+
start = statement_start(text)
|
|
109
|
+
m = _TRANSACTION_RX.match(start)
|
|
110
|
+
if m and m.group("begin"):
|
|
111
|
+
self.in_transaction = True # so the BEGIN itself runs on the connection
|
|
112
|
+
query = bool(_QUERY_RX.match(start))
|
|
113
|
+
result = super().execute(sql, paramlist, fetch=fetch or not query)
|
|
114
|
+
if m and m.group("end"):
|
|
115
|
+
self.in_transaction = False
|
|
116
|
+
if not query and result is not None and result[0] == ["Count"]:
|
|
117
|
+
rows = result[1]
|
|
118
|
+
if _state.subvars is not None:
|
|
119
|
+
_state.subvars.add_substitution("$LAST_ROWCOUNT", rows[0][0] if rows else -1)
|
|
120
|
+
return None
|
|
121
|
+
return result if fetch else None
|
|
122
|
+
|
|
123
|
+
def commit(self) -> None:
|
|
124
|
+
"""Commit the open transaction if autocommit is enabled."""
|
|
125
|
+
super().commit()
|
|
126
|
+
if self.autocommit:
|
|
127
|
+
self.in_transaction = False
|
|
128
|
+
|
|
129
|
+
def rollback(self) -> None:
|
|
130
|
+
"""Roll back the open transaction."""
|
|
131
|
+
super().rollback()
|
|
132
|
+
self.in_transaction = False
|
|
133
|
+
|
|
134
|
+
def close(self) -> None:
|
|
135
|
+
"""Close the connection; an open transaction is rolled back."""
|
|
136
|
+
super().close()
|
|
137
|
+
self.in_transaction = False
|
|
138
|
+
|
|
47
139
|
def open_db(self) -> None:
|
|
48
140
|
"""Open a connection to the DuckDB database file."""
|
|
49
141
|
import duckdb
|
execsql/db/firebird.py
CHANGED
|
@@ -9,6 +9,7 @@ via the ``firebird-driver`` package. Corresponds to ``-t f`` on the CLI.
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
from execsql.db.base import Database
|
|
12
|
+
from execsql.db.tiers import SupportTier
|
|
12
13
|
from execsql.exceptions import ErrInfo
|
|
13
14
|
from execsql.utils.errors import exception_desc, fatal_error
|
|
14
15
|
from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
|
|
@@ -20,6 +21,10 @@ __all__ = ["FirebirdDatabase"]
|
|
|
20
21
|
class FirebirdDatabase(Database):
|
|
21
22
|
"""Firebird adapter using the firebird-driver package."""
|
|
22
23
|
|
|
24
|
+
#: No Firebird server runs in CI; this adapter has no tests.
|
|
25
|
+
support_tier = SupportTier.BEST_EFFORT
|
|
26
|
+
support_tier_name = "Firebird"
|
|
27
|
+
|
|
23
28
|
def __init__(
|
|
24
29
|
self,
|
|
25
30
|
server_name: str,
|
|
@@ -39,18 +44,18 @@ class FirebirdDatabase(Database):
|
|
|
39
44
|
)
|
|
40
45
|
from execsql.types import dbt_firebird
|
|
41
46
|
|
|
47
|
+
super().__init__(
|
|
48
|
+
server_name=str(server_name),
|
|
49
|
+
db_name=str(db_name),
|
|
50
|
+
user_name=str(user_name),
|
|
51
|
+
need_passwd=need_passwd,
|
|
52
|
+
port=port if port else 3050,
|
|
53
|
+
)
|
|
42
54
|
self.type = dbt_firebird
|
|
43
|
-
self.server_name = str(server_name)
|
|
44
|
-
self.db_name = str(db_name)
|
|
45
|
-
self.user = str(user_name)
|
|
46
|
-
self.need_passwd = need_passwd
|
|
47
55
|
self.password = password
|
|
48
|
-
self.port = port if port else 3050
|
|
49
56
|
self.encoding = encoding or "latin1"
|
|
50
57
|
self.encode_commands = True
|
|
51
58
|
self.paramstr = "?"
|
|
52
|
-
self.conn = None
|
|
53
|
-
self.autocommit = True
|
|
54
59
|
self.open_db()
|
|
55
60
|
self.password = None # Clear cleartext password after successful connection
|
|
56
61
|
|