execsql2 2.35.0__py3-none-any.whl → 2.38.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +111 -53
- execsql/cli/commands/run.py +2 -2
- execsql/cli/lint.py +60 -7
- execsql/cli/run.py +10 -10
- execsql/config.py +7 -0
- execsql/data/execsql.conf.template +31 -7
- execsql/db/base.py +64 -26
- execsql/db/duckdb.py +51 -5
- execsql/db/firebird.py +8 -0
- execsql/db/mysql.py +8 -3
- execsql/db/oracle.py +5 -0
- execsql/db/postgres.py +2 -2
- execsql/db/sqlite.py +117 -10
- execsql/db/sqlserver.py +11 -7
- execsql/db/target_types.py +151 -4
- execsql/debug/repl.py +4 -3
- execsql/exceptions.py +14 -0
- execsql/exporters/html.py +26 -42
- execsql/exporters/ods.py +93 -21
- execsql/exporters/sqlite.py +82 -47
- execsql/format.py +12 -4
- execsql/gui/desktop.py +2 -1
- execsql/gui/tui.py +5 -2
- execsql/importers/base.py +9 -7
- execsql/importers/json.py +37 -13
- execsql/importers/ods.py +22 -2
- execsql/lsp/document.py +6 -1
- execsql/lsp/hover.py +2 -1
- execsql/metacommands/conditions.py +3 -1
- execsql/metacommands/connect.py +28 -12
- execsql/metacommands/dispatch.py +10 -0
- execsql/metacommands/io.py +1 -0
- execsql/metacommands/io_export.py +1 -2
- execsql/metacommands/io_fileops.py +32 -93
- execsql/metacommands/io_import.py +26 -8
- execsql/metacommands/io_write.py +3 -1
- execsql/metacommands/reference.py +7 -0
- execsql/metacommands/script_ext.py +13 -10
- execsql/metacommands/system.py +13 -6
- execsql/parser.py +14 -37
- execsql/script/ast.py +21 -0
- execsql/script/control.py +73 -3
- execsql/script/executor.py +503 -227
- execsql/script/parser.py +102 -52
- execsql/script/preparation.py +42 -0
- execsql/state.py +23 -0
- execsql/types.py +6 -5
- execsql/utils/errors.py +24 -6
- execsql/utils/fileio.py +342 -134
- execsql/utils/sigterm.py +154 -0
- execsql/utils/spreadsheet.py +74 -0
- execsql/utils/strings.py +13 -7
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/README.md +5 -5
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/example_config_prompt.sql +2 -2
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/METADATA +32 -119
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/RECORD +71 -68
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/WHEEL +1 -1
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/licenses/NOTICE +0 -0
execsql/db/base.py
CHANGED
|
@@ -24,7 +24,8 @@ from collections.abc import Callable, Generator, Iterator
|
|
|
24
24
|
|
|
25
25
|
from execsql.exceptions import ErrInfo
|
|
26
26
|
from execsql.utils.errors import exception_desc
|
|
27
|
-
from execsql.db.target_types import ColumnKinds, existing_table_converter, kind_for_python_type
|
|
27
|
+
from execsql.db.target_types import ColumnKinds, ColumnMetadata, existing_table_converter, kind_for_python_type
|
|
28
|
+
from execsql.types import DT_Decimal
|
|
28
29
|
from execsql.db.tiers import SupportTier, announce_tier
|
|
29
30
|
from execsql.types import DataType
|
|
30
31
|
import execsql.state as _state
|
|
@@ -285,6 +286,9 @@ class Database(ABC):
|
|
|
285
286
|
#: (an ODBC DSN without transactions): nothing can be held back.
|
|
286
287
|
driver_autocommits: bool = False
|
|
287
288
|
|
|
289
|
+
#: Native Decimal parameter capacity for exact binding; None leaves it unchanged.
|
|
290
|
+
decimal_binding_precision: int | None = None
|
|
291
|
+
|
|
288
292
|
def holding(self) -> bool:
|
|
289
293
|
"""Whether statements on this connection wait for a commit.
|
|
290
294
|
|
|
@@ -352,9 +356,19 @@ class Database(ABC):
|
|
|
352
356
|
self.autocommit = False
|
|
353
357
|
|
|
354
358
|
def commit(self) -> None:
|
|
355
|
-
"""Commit the current transaction if autocommit is enabled.
|
|
359
|
+
"""Commit the current transaction if autocommit is enabled.
|
|
360
|
+
|
|
361
|
+
A commit that fails (a deferred constraint, a lock) rolls the
|
|
362
|
+
transaction back before the error is raised: some DBMSs (SQLite)
|
|
363
|
+
keep it open, and the next commit on the connection would store
|
|
364
|
+
what was just reported as failed.
|
|
365
|
+
"""
|
|
356
366
|
if self.conn and self.autocommit:
|
|
357
|
-
|
|
367
|
+
try:
|
|
368
|
+
self.conn.commit()
|
|
369
|
+
except BaseException:
|
|
370
|
+
self.rollback()
|
|
371
|
+
raise
|
|
358
372
|
|
|
359
373
|
def rollback(self) -> None:
|
|
360
374
|
"""Roll back the current transaction; swallows errors (best-effort)."""
|
|
@@ -362,13 +376,15 @@ class Database(ABC):
|
|
|
362
376
|
try:
|
|
363
377
|
self.conn.rollback()
|
|
364
378
|
except Exception as e:
|
|
365
|
-
# Best-effort; connection may already be closed.
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
379
|
+
# Best-effort; connection may already be closed.
|
|
380
|
+
self._log_failed_rollback(e)
|
|
381
|
+
|
|
382
|
+
def _log_failed_rollback(self, e: Exception) -> None:
|
|
383
|
+
"""Log a rollback that failed, so a cascading failure isn't invisible in a CI / cron log."""
|
|
384
|
+
if _state.exec_log is not None:
|
|
385
|
+
_state.exec_log.log_status_info(
|
|
386
|
+
f"Rollback failed on {self.__class__.__name__}: {e!r}",
|
|
387
|
+
)
|
|
372
388
|
|
|
373
389
|
def needs_explicit_commit_after_ddl(self) -> bool:
|
|
374
390
|
"""Return True if this adapter's driver does NOT auto-commit DDL.
|
|
@@ -612,16 +628,24 @@ class Database(ABC):
|
|
|
612
628
|
*sq_name* is schema-qualified and quoted, as for an ``INSERT``, and
|
|
613
629
|
is looked up the way the ``INSERT`` finds it. Keys are lower-cased
|
|
614
630
|
column names; a column with no fitting execsql type maps to ``None``.
|
|
615
|
-
|
|
616
|
-
|
|
631
|
+
The dict-compatible result also carries known decimal scales in
|
|
632
|
+
``ColumnMetadata.decimal_scales``. Legacy plain-dict providers remain
|
|
633
|
+
supported with unknown scales. This default reads the driver's empty
|
|
634
|
+
query description; adapters with exact catalog types override it.
|
|
617
635
|
"""
|
|
618
636
|
description, _rows = self._column_type_query(sq_name, f"select * from {sq_name} where 1=0;")
|
|
619
|
-
return {d[0].lower(): self.
|
|
637
|
+
return ColumnMetadata({d[0].lower(): self._described_column(d) for d in description})
|
|
620
638
|
|
|
621
639
|
def _described_kind(self, description: Any) -> type[DataType] | None:
|
|
622
640
|
"""The execsql type for one column of a cursor description (a DB-API 7-item sequence)."""
|
|
623
641
|
return kind_for_python_type(description[1], description[5])
|
|
624
642
|
|
|
643
|
+
def _described_column(self, description: Any) -> tuple[type[DataType] | None, int | None]:
|
|
644
|
+
"""Read a kind and decimal scale from a DB-API description."""
|
|
645
|
+
kind = self._described_kind(description)
|
|
646
|
+
scale = description[5]
|
|
647
|
+
return kind, scale if kind is DT_Decimal and type(scale) is int else None
|
|
648
|
+
|
|
625
649
|
def _column_type_query(self, sq_name: str, sql: str, params: list | None = None) -> tuple[Any, list]:
|
|
626
650
|
"""Run *sql* for :meth:`column_kinds` of *sq_name*; return the cursor description and the rows."""
|
|
627
651
|
with self._cursor() as curs:
|
|
@@ -734,7 +758,7 @@ class Database(ABC):
|
|
|
734
758
|
|
|
735
759
|
*rowsource* must be a generator yielding lists of values in column order.
|
|
736
760
|
*tablespec_src* is a zero-argument callable that returns the table's type specification.
|
|
737
|
-
|
|
761
|
+
Only selected columns are converted to the type scanned for each column; with
|
|
738
762
|
*existing_table* (the table was there before the IMPORT or COPY), the
|
|
739
763
|
type of the column it goes into decides instead (see
|
|
740
764
|
``db.target_types.existing_table_converter``).
|
|
@@ -760,9 +784,18 @@ class Database(ABC):
|
|
|
760
784
|
converters = [self._scanned_type_converter(col.column_type()[1]) for col in tablespec.cols]
|
|
761
785
|
if existing_table:
|
|
762
786
|
kinds = self.column_kinds(sq_name)
|
|
787
|
+
scales = kinds.decimal_scales if isinstance(kinds, ColumnMetadata) else {}
|
|
763
788
|
converters = [
|
|
764
|
-
existing_table_converter(
|
|
765
|
-
|
|
789
|
+
existing_table_converter(
|
|
790
|
+
kinds.get(col.name.lower()),
|
|
791
|
+
col.column_type()[1],
|
|
792
|
+
convert,
|
|
793
|
+
col.name,
|
|
794
|
+
sq_name,
|
|
795
|
+
decimal_binding_precision=self.decimal_binding_precision,
|
|
796
|
+
decimal_scale=scales.get(col.name.lower()) if included else None,
|
|
797
|
+
)
|
|
798
|
+
for col, convert, included in zip(tablespec.cols, converters, incl_col)
|
|
766
799
|
]
|
|
767
800
|
# Construct INSERT statement.
|
|
768
801
|
columns = [self.type.quoted(col) for col in sel_cols]
|
|
@@ -861,11 +894,7 @@ class Database(ABC):
|
|
|
861
894
|
line[i] = None
|
|
862
895
|
# Trailing values left off the line are NULL.
|
|
863
896
|
line = list(line) + [None] * (len(ts_colnames) - len(line))
|
|
864
|
-
|
|
865
|
-
row = []
|
|
866
|
-
for i, v in enumerate(lt):
|
|
867
|
-
if incl_col[i]:
|
|
868
|
-
row.append(v)
|
|
897
|
+
row = [converters[i](val) for i, val in enumerate(line) if incl_col[i]]
|
|
869
898
|
add_line = True
|
|
870
899
|
if not _state.conf.empty_rows:
|
|
871
900
|
add_line = not all(c is None for c in row)
|
|
@@ -1066,8 +1095,8 @@ class DatabasePool:
|
|
|
1066
1095
|
return self.pool[self.initial_db]
|
|
1067
1096
|
|
|
1068
1097
|
def aliased_as(self, db_alias: str) -> Database:
|
|
1069
|
-
"""Return the ``Database`` registered under *db_alias
|
|
1070
|
-
return self.pool[db_alias]
|
|
1098
|
+
"""Return the ``Database`` registered under *db_alias* (any letter case)."""
|
|
1099
|
+
return self.pool[db_alias.lower()]
|
|
1071
1100
|
|
|
1072
1101
|
def make_current(self, db_alias: str) -> None:
|
|
1073
1102
|
"""Set the active database to *db_alias*; raises ``ErrInfo`` if the alias is unknown."""
|
|
@@ -1080,7 +1109,8 @@ class DatabasePool:
|
|
|
1080
1109
|
self.current_db = db_alias
|
|
1081
1110
|
|
|
1082
1111
|
def disconnect(self, alias: str) -> None:
|
|
1083
|
-
"""Close and remove the connection registered under *alias* from the pool."""
|
|
1112
|
+
"""Close and remove the connection registered under *alias* (any letter case) from the pool."""
|
|
1113
|
+
alias = alias.lower()
|
|
1084
1114
|
if alias == self.current_db or (alias == "initial" and "initial" in self.pool):
|
|
1085
1115
|
raise ErrInfo(
|
|
1086
1116
|
type="error",
|
|
@@ -1090,9 +1120,17 @@ class DatabasePool:
|
|
|
1090
1120
|
self.pool[alias].close()
|
|
1091
1121
|
del self.pool[alias]
|
|
1092
1122
|
|
|
1093
|
-
def closeall(self) -> None:
|
|
1094
|
-
"""Roll back and close
|
|
1123
|
+
def closeall(self, *, except_db: Database | None = None) -> None:
|
|
1124
|
+
"""Roll back and close the pool's connections except *except_db*.
|
|
1125
|
+
|
|
1126
|
+
The exception is matched by object identity under every alias and is
|
|
1127
|
+
neither rolled back nor closed. All aliases are removed, including
|
|
1128
|
+
aliases of the excluded caller-owned connection.
|
|
1129
|
+
Rollback follows the pool's ``do_rollback`` setting.
|
|
1130
|
+
"""
|
|
1095
1131
|
for alias, db in self.pool.items():
|
|
1132
|
+
if db is except_db:
|
|
1133
|
+
continue
|
|
1096
1134
|
nm = db.name()
|
|
1097
1135
|
try:
|
|
1098
1136
|
if self.do_rollback:
|
execsql/db/duckdb.py
CHANGED
|
@@ -9,6 +9,7 @@ on the CLI.
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import importlib.util
|
|
12
|
+
from decimal import Decimal
|
|
12
13
|
import re
|
|
13
14
|
import sys
|
|
14
15
|
from collections import deque
|
|
@@ -18,8 +19,8 @@ from pathlib import Path
|
|
|
18
19
|
from typing import Any
|
|
19
20
|
|
|
20
21
|
from execsql.db.base import Database, statement_start
|
|
21
|
-
from execsql.db.target_types import kind_for_type_name
|
|
22
|
-
from execsql.types import DataType
|
|
22
|
+
from execsql.db.target_types import kind_for_type_name, kind_and_scale_for_type_name, decimal_parameter
|
|
23
|
+
from execsql.types import DataType, DT_Decimal
|
|
23
24
|
from execsql.exceptions import ErrInfo
|
|
24
25
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
25
26
|
import execsql.state as _state
|
|
@@ -172,6 +173,9 @@ class DuckDBDatabase(Database):
|
|
|
172
173
|
and sees the run's temporary tables.
|
|
173
174
|
"""
|
|
174
175
|
|
|
176
|
+
#: Keep Decimal parameters exact across exponent/width conversion in drivers.
|
|
177
|
+
decimal_binding_precision = 38
|
|
178
|
+
|
|
175
179
|
#: A transaction this adapter or a script's own BEGIN opened, not yet ended.
|
|
176
180
|
in_transaction: bool = False
|
|
177
181
|
#: The cursor whose result the connection holds, if any.
|
|
@@ -253,19 +257,42 @@ class DuckDBDatabase(Database):
|
|
|
253
257
|
|
|
254
258
|
def commit(self) -> None:
|
|
255
259
|
"""Commit the open transaction if autocommit is enabled."""
|
|
256
|
-
|
|
260
|
+
try:
|
|
261
|
+
# Reading ahead another cursor's result can raise (DuckDB reports
|
|
262
|
+
# some errors at fetch time); roll back as a failed commit does.
|
|
263
|
+
self._take_connection()
|
|
264
|
+
except BaseException:
|
|
265
|
+
if self.autocommit and self.in_transaction:
|
|
266
|
+
self.rollback()
|
|
267
|
+
raise
|
|
257
268
|
super().commit()
|
|
258
269
|
if self.autocommit:
|
|
259
270
|
self.in_transaction = False
|
|
260
271
|
|
|
261
272
|
def rollback(self) -> None:
|
|
262
|
-
"""Roll back the open transaction.
|
|
273
|
+
"""Roll back the open transaction, if one is open.
|
|
274
|
+
|
|
275
|
+
The driver decides whether one is: ``in_transaction`` misses a
|
|
276
|
+
transaction begun partway through a statement. Outside a transaction
|
|
277
|
+
DuckDB has committed each statement as it ran, and its "no transaction
|
|
278
|
+
is active" error is not logged as a failed rollback.
|
|
279
|
+
"""
|
|
280
|
+
import duckdb
|
|
281
|
+
|
|
263
282
|
try:
|
|
264
283
|
self._take_connection()
|
|
265
284
|
except Exception: # best effort, as the rollback is: a failed transaction may not give up its rows
|
|
266
285
|
self._holder = None
|
|
267
|
-
super().rollback()
|
|
268
286
|
self.in_transaction = False
|
|
287
|
+
if self.conn is None:
|
|
288
|
+
return
|
|
289
|
+
try:
|
|
290
|
+
self.conn.rollback()
|
|
291
|
+
except duckdb.TransactionException as e:
|
|
292
|
+
if "no transaction is active" not in str(e):
|
|
293
|
+
self._log_failed_rollback(e)
|
|
294
|
+
except Exception as e:
|
|
295
|
+
self._log_failed_rollback(e)
|
|
269
296
|
|
|
270
297
|
def close(self) -> None:
|
|
271
298
|
"""Close the connection; an open transaction is rolled back."""
|
|
@@ -326,6 +353,25 @@ class DuckDBDatabase(Database):
|
|
|
326
353
|
"""DuckDB describes a column by its type's name (``DECIMAL(18,3)``, ``TIMESTAMP WITH TIME ZONE``)."""
|
|
327
354
|
return kind_for_type_name(str(description[1]))
|
|
328
355
|
|
|
356
|
+
def _scanned_type_converter(self, data_type: type[DataType]) -> Callable[[Any], Any]:
|
|
357
|
+
convert = super()._scanned_type_converter(data_type)
|
|
358
|
+
if data_type is not DT_Decimal:
|
|
359
|
+
return convert
|
|
360
|
+
|
|
361
|
+
def bind(value: Any) -> Any:
|
|
362
|
+
result = convert(value)
|
|
363
|
+
if isinstance(result, Decimal) and result.is_finite():
|
|
364
|
+
exponent = result.as_tuple().exponent
|
|
365
|
+
if isinstance(exponent, int) and exponent > 0:
|
|
366
|
+
return decimal_parameter(result, self.decimal_binding_precision, "value", "target table")
|
|
367
|
+
return result
|
|
368
|
+
|
|
369
|
+
return bind
|
|
370
|
+
|
|
371
|
+
def _described_column(self, description: Any) -> tuple[type[DataType] | None, int | None]:
|
|
372
|
+
_kind, scale = kind_and_scale_for_type_name(str(description[1]))
|
|
373
|
+
return self._described_kind(description), scale
|
|
374
|
+
|
|
329
375
|
def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
|
|
330
376
|
"""Return True if the named view exists in the DuckDB database."""
|
|
331
377
|
# DuckDB information_schema has no 'views' table; views are listed in 'tables'
|
execsql/db/firebird.py
CHANGED
|
@@ -8,7 +8,10 @@ via the ``firebird-driver`` package. Corresponds to ``-t f`` on the CLI.
|
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
10
|
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
11
13
|
from execsql.db.base import Database
|
|
14
|
+
from execsql.types import DataType, DT_Decimal
|
|
12
15
|
from execsql.db.tiers import SupportTier
|
|
13
16
|
from execsql.exceptions import ErrInfo
|
|
14
17
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
@@ -32,6 +35,11 @@ class FirebirdDatabase(Database):
|
|
|
32
35
|
support_tier = SupportTier.BEST_EFFORT
|
|
33
36
|
support_tier_name = "Firebird"
|
|
34
37
|
|
|
38
|
+
def _described_column(self, description: Any) -> tuple[type[DataType] | None, int | None]:
|
|
39
|
+
kind, scale = super()._described_column(description)
|
|
40
|
+
# firebird-driver exposes the engine's exponent, the negative SQL scale.
|
|
41
|
+
return kind, -scale if kind is DT_Decimal and scale is not None else None
|
|
42
|
+
|
|
35
43
|
def __init__(
|
|
36
44
|
self,
|
|
37
45
|
server_name: str,
|
execsql/db/mysql.py
CHANGED
|
@@ -13,7 +13,7 @@ import tempfile
|
|
|
13
13
|
from typing import Any, cast
|
|
14
14
|
|
|
15
15
|
from execsql.db.base import Database
|
|
16
|
-
from execsql.db.target_types import ColumnKinds,
|
|
16
|
+
from execsql.db.target_types import ColumnKinds, ColumnMetadata, kind_and_scale_for_type_name
|
|
17
17
|
from execsql.exceptions import ErrInfo
|
|
18
18
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
19
19
|
from execsql.utils.auth import (
|
|
@@ -87,7 +87,12 @@ def _data_length(filename: str) -> int | None:
|
|
|
87
87
|
except OSError:
|
|
88
88
|
return None
|
|
89
89
|
# Keep the line ending of the last line of data.
|
|
90
|
-
|
|
90
|
+
if ending == b"\r\n":
|
|
91
|
+
keep = content_end + 2
|
|
92
|
+
elif ending[:1] in (b"\r", b"\n"):
|
|
93
|
+
keep = content_end + 1
|
|
94
|
+
else:
|
|
95
|
+
keep = content_end
|
|
91
96
|
return keep if keep < size else None
|
|
92
97
|
|
|
93
98
|
|
|
@@ -248,7 +253,7 @@ class MySQLDatabase(Database):
|
|
|
248
253
|
def column_kinds(self, sq_name: str) -> ColumnKinds:
|
|
249
254
|
"""The execsql type of each column of *sq_name*, from ``SHOW COLUMNS`` (which tells BOOLEAN from TINYINT)."""
|
|
250
255
|
_description, rows = self._column_type_query(sq_name, f"show columns from {sq_name};")
|
|
251
|
-
return {str(row[0]).lower():
|
|
256
|
+
return ColumnMetadata({str(row[0]).lower(): kind_and_scale_for_type_name(_text(row[1])) for row in rows})
|
|
252
257
|
|
|
253
258
|
def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
|
|
254
259
|
return super().view_exists(
|
execsql/db/oracle.py
CHANGED
|
@@ -304,6 +304,11 @@ class OracleDatabase(Database):
|
|
|
304
304
|
type_name = _ORACLE_TYPE_NAMES.get(name, "")
|
|
305
305
|
return kind_for_type_name(type_name, oracle=True)
|
|
306
306
|
|
|
307
|
+
def _described_column(self, description: Any) -> tuple[type[DataType] | None, int | None]:
|
|
308
|
+
kind, scale = super()._described_column(description)
|
|
309
|
+
# oracledb uses -127 for unconstrained NUMBER and FLOAT, not rounding.
|
|
310
|
+
return kind, None if description[5] == -127 else scale
|
|
311
|
+
|
|
307
312
|
def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
|
|
308
313
|
"""Return True if the named view exists in the Oracle database."""
|
|
309
314
|
# Oracle folds unquoted identifiers to uppercase.
|
execsql/db/postgres.py
CHANGED
|
@@ -14,7 +14,7 @@ import re
|
|
|
14
14
|
from typing import Any
|
|
15
15
|
|
|
16
16
|
from execsql.db.base import Database
|
|
17
|
-
from execsql.db.target_types import ColumnKinds,
|
|
17
|
+
from execsql.db.target_types import ColumnKinds, ColumnMetadata, kind_and_scale_for_type_name
|
|
18
18
|
from execsql.exceptions import ErrInfo
|
|
19
19
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
20
20
|
from execsql.utils.auth import (
|
|
@@ -233,7 +233,7 @@ class PostgresDatabase(Database):
|
|
|
233
233
|
"where attrelid = %s::regclass and attnum > 0 and not attisdropped order by attnum;",
|
|
234
234
|
[sq_name],
|
|
235
235
|
)
|
|
236
|
-
return {name.lower():
|
|
236
|
+
return ColumnMetadata({name.lower(): kind_and_scale_for_type_name(type_name) for name, type_name in rows})
|
|
237
237
|
|
|
238
238
|
def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
|
|
239
239
|
"""Return True if the named view exists in the PostgreSQL database."""
|
execsql/db/sqlite.py
CHANGED
|
@@ -9,20 +9,70 @@ files via the Python standard library ``sqlite3`` module. Corresponds to
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import datetime
|
|
12
|
+
import math
|
|
12
13
|
import re
|
|
13
|
-
from
|
|
14
|
+
from collections.abc import Callable
|
|
15
|
+
from decimal import Decimal, InvalidOperation
|
|
14
16
|
from typing import Any
|
|
15
17
|
|
|
16
18
|
from execsql.db.base import Database
|
|
19
|
+
from execsql.db.target_types import ColumnKinds, _NUMBER_TEXT, _whole_number_converter, kind_for_type_name
|
|
20
|
+
from execsql.types import DT_Long
|
|
17
21
|
from execsql.exceptions import ErrInfo
|
|
18
22
|
from execsql.utils.errors import exception_desc, fatal_error
|
|
19
23
|
import execsql.state as _state
|
|
20
24
|
|
|
21
|
-
__all__ = ["SQLiteDatabase"]
|
|
25
|
+
__all__ = ["SQLiteDatabase", "sqlite_affinity"]
|
|
22
26
|
|
|
23
27
|
DEFAULT_CONNECT_TIMEOUT = 30 # seconds
|
|
24
28
|
|
|
25
29
|
|
|
30
|
+
def sqlite_affinity(declaration: str) -> str:
|
|
31
|
+
"""Return SQLite's affinity, observing its ordered type-name rules."""
|
|
32
|
+
name = declaration.upper()
|
|
33
|
+
if "INT" in name:
|
|
34
|
+
return "INTEGER"
|
|
35
|
+
if any(part in name for part in ("CHAR", "CLOB", "TEXT")):
|
|
36
|
+
return "TEXT"
|
|
37
|
+
if not name or "BLOB" in name:
|
|
38
|
+
return "BLOB"
|
|
39
|
+
if any(part in name for part in ("REAL", "FLOA", "DOUB")):
|
|
40
|
+
return "REAL"
|
|
41
|
+
return "NUMERIC"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _integer_converter(column: str, table: str) -> Callable[[Any], Any]:
|
|
45
|
+
"""Use the shared whole-number rule with SQLite's signed int64 bounds."""
|
|
46
|
+
whole_number = _whole_number_converter(column, table, lambda value: value)
|
|
47
|
+
|
|
48
|
+
def range_error(value: Any) -> ErrInfo:
|
|
49
|
+
return ErrInfo(
|
|
50
|
+
type="error",
|
|
51
|
+
other_msg=(
|
|
52
|
+
f"Can't load {value!r} into column {column} of {table}: the column holds whole numbers "
|
|
53
|
+
"in SQLite's signed 64-bit integer range."
|
|
54
|
+
),
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
def convert(value: Any) -> Any:
|
|
58
|
+
number = None
|
|
59
|
+
try:
|
|
60
|
+
if (
|
|
61
|
+
isinstance(value, (int, float, Decimal))
|
|
62
|
+
and not isinstance(value, bool)
|
|
63
|
+
or isinstance(value, str)
|
|
64
|
+
and _NUMBER_TEXT.fullmatch(value)
|
|
65
|
+
):
|
|
66
|
+
number = Decimal(value)
|
|
67
|
+
except InvalidOperation as exc:
|
|
68
|
+
raise range_error(value) from exc
|
|
69
|
+
if number is not None and number.is_finite() and not -(2**63) <= number <= 2**63 - 1:
|
|
70
|
+
raise range_error(value)
|
|
71
|
+
return whole_number(value)
|
|
72
|
+
|
|
73
|
+
return convert
|
|
74
|
+
|
|
75
|
+
|
|
26
76
|
class SQLiteDatabase(Database):
|
|
27
77
|
"""SQLite adapter using the Python standard-library sqlite3 module."""
|
|
28
78
|
|
|
@@ -98,11 +148,12 @@ class SQLiteDatabase(Database):
|
|
|
98
148
|
unqualified_lookup_is_scoped = True
|
|
99
149
|
|
|
100
150
|
def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
|
|
101
|
-
"""
|
|
151
|
+
"""Look up a table in *schema_name* (an attached database), or main."""
|
|
102
152
|
# SQLite stores names as-created; match case-insensitively so a lookup
|
|
103
153
|
# of "mytable" finds the catalog entry for MyTable.
|
|
104
154
|
with self._cursor() as curs:
|
|
105
|
-
|
|
155
|
+
catalog = self.schema_qualified_table_name(schema_name or "main", "sqlite_master")
|
|
156
|
+
sql = f"select name from {catalog} where type='table' and lower(name)=lower(?);"
|
|
106
157
|
try:
|
|
107
158
|
curs.execute(sql, (table_name,))
|
|
108
159
|
except ErrInfo:
|
|
@@ -129,9 +180,9 @@ class SQLiteDatabase(Database):
|
|
|
129
180
|
return column_name in cols
|
|
130
181
|
|
|
131
182
|
def table_columns(self, table_name: str, schema_name: str | None = None) -> list[str]:
|
|
132
|
-
"""Return
|
|
183
|
+
"""Return column names from the optionally database-qualified table."""
|
|
133
184
|
with self._cursor() as curs:
|
|
134
|
-
quoted_tbl = self.
|
|
185
|
+
quoted_tbl = self.schema_qualified_table_name(schema_name, table_name)
|
|
135
186
|
sql = f"select * from {quoted_tbl} where 1=0;"
|
|
136
187
|
try:
|
|
137
188
|
curs.execute(sql)
|
|
@@ -147,6 +198,28 @@ class SQLiteDatabase(Database):
|
|
|
147
198
|
) from e
|
|
148
199
|
return [d[0] for d in curs.description]
|
|
149
200
|
|
|
201
|
+
def column_kinds(self, sq_name: str) -> ColumnKinds:
|
|
202
|
+
"""Read declared types from the database INSERT resolves for *sq_name*.
|
|
203
|
+
|
|
204
|
+
Use execsql's type names, including scale-zero DECIMAL, rather than
|
|
205
|
+
SQLite's broader INTEGER-affinity substring rule. An unqualified name
|
|
206
|
+
keeps SQLite's normal temporary/main/attached lookup order.
|
|
207
|
+
"""
|
|
208
|
+
return {
|
|
209
|
+
name: kind_for_type_name(declaration) for name, declaration in self._column_declarations(sq_name).items()
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
def _column_declarations(self, sq_name: str) -> dict[str, str]:
|
|
213
|
+
"""Read declared column types from the same qualified table INSERT uses."""
|
|
214
|
+
identifier = r'(?:"(?:[^"]|"")*"|\w+)'
|
|
215
|
+
name = re.fullmatch(rf"(?:(?P<schema>{identifier})\.)?(?P<table>{identifier})", sq_name)
|
|
216
|
+
if name is None:
|
|
217
|
+
raise ErrInfo(type="error", other_msg=f"Invalid SQLite table name: {sq_name}")
|
|
218
|
+
prefix = f"{name['schema']}." if name["schema"] else ""
|
|
219
|
+
sql = f"PRAGMA {prefix}table_xinfo({name['table']})"
|
|
220
|
+
_description, rows = self._column_type_query(sq_name, sql)
|
|
221
|
+
return {row[1].lower(): row[2] for row in rows}
|
|
222
|
+
|
|
150
223
|
def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
|
|
151
224
|
"""Return True if the named view exists in the SQLite database."""
|
|
152
225
|
# Match case-insensitively — see table_exists().
|
|
@@ -187,12 +260,14 @@ class SQLiteDatabase(Database):
|
|
|
187
260
|
) -> None:
|
|
188
261
|
"""Populate a SQLite table from a row source generator.
|
|
189
262
|
|
|
190
|
-
|
|
191
|
-
|
|
263
|
+
Existing integer-kind columns reject fractions and values outside the
|
|
264
|
+
signed 64-bit range. Other columns retain their pass-through behavior
|
|
265
|
+
(dates, times and decimals as text). Finite decimals cannot overflow
|
|
266
|
+
a numeric-affinity column to infinity. A failed load is rolled back.
|
|
192
267
|
"""
|
|
193
268
|
# The rowsource argument must be a generator yielding a list of values for the columns of the table.
|
|
194
269
|
# The column_list argument must an iterable containing column names in the same order as produced by the rowsource.
|
|
195
|
-
sq_name = self.schema_qualified_table_name(
|
|
270
|
+
sq_name = self.schema_qualified_table_name(schema_name, table_name)
|
|
196
271
|
# Check specified column names.
|
|
197
272
|
tablespec = tablespec_src()
|
|
198
273
|
ts_colnames = [col.name for col in tablespec.cols]
|
|
@@ -205,6 +280,22 @@ class SQLiteDatabase(Database):
|
|
|
205
280
|
# Get column indexes for selected column names.
|
|
206
281
|
columns = column_list
|
|
207
282
|
data_indexes = [ts_colnames.index(col) for col in columns]
|
|
283
|
+
declarations = self._column_declarations(sq_name)
|
|
284
|
+
kinds = (
|
|
285
|
+
{name: kind_for_type_name(declaration) for name, declaration in declarations.items()}
|
|
286
|
+
if existing_table
|
|
287
|
+
else {}
|
|
288
|
+
)
|
|
289
|
+
numeric_columns = {
|
|
290
|
+
i
|
|
291
|
+
for i, column in enumerate(columns)
|
|
292
|
+
if sqlite_affinity(declarations.get(column.lower(), "")) in ("NUMERIC", "REAL", "INTEGER")
|
|
293
|
+
}
|
|
294
|
+
converters = {
|
|
295
|
+
i: _integer_converter(column, sq_name)
|
|
296
|
+
for i, column in enumerate(columns)
|
|
297
|
+
if kinds.get(column.lower()) is DT_Long
|
|
298
|
+
}
|
|
208
299
|
# Construct prepared SQL statement
|
|
209
300
|
colspec = ",".join([self.type.quoted(c) for c in columns])
|
|
210
301
|
paramspec = ",".join(["?" for c in columns])
|
|
@@ -227,7 +318,6 @@ class SQLiteDatabase(Database):
|
|
|
227
318
|
except ErrInfo:
|
|
228
319
|
raise
|
|
229
320
|
except Exception as e:
|
|
230
|
-
self.rollback()
|
|
231
321
|
raise ErrInfo(
|
|
232
322
|
type="db",
|
|
233
323
|
command_text=sql,
|
|
@@ -244,6 +334,8 @@ class SQLiteDatabase(Database):
|
|
|
244
334
|
|
|
245
335
|
last_line: list[Any] = []
|
|
246
336
|
try:
|
|
337
|
+
if converters or numeric_columns:
|
|
338
|
+
self.begin_transaction()
|
|
247
339
|
for line in rowsource:
|
|
248
340
|
# Skip empty rows.
|
|
249
341
|
if not (len(line) == 1 and line[0] is None):
|
|
@@ -259,6 +351,18 @@ class SQLiteDatabase(Database):
|
|
|
259
351
|
if not _state.conf.empty_strings and line[i].strip() == "":
|
|
260
352
|
line[i] = None
|
|
261
353
|
linedata = [line[ix] for ix in data_indexes]
|
|
354
|
+
for i, convert in converters.items():
|
|
355
|
+
linedata[i] = convert(linedata[i])
|
|
356
|
+
for i in numeric_columns:
|
|
357
|
+
value = linedata[i]
|
|
358
|
+
if isinstance(value, Decimal) and value.is_finite() and math.isinf(float(value)):
|
|
359
|
+
raise ErrInfo(
|
|
360
|
+
type="error",
|
|
361
|
+
other_msg=(
|
|
362
|
+
f"Can't load {value!r} into column {columns[i]} of {sq_name}: "
|
|
363
|
+
"SQLite would store infinity instead of this finite value."
|
|
364
|
+
),
|
|
365
|
+
)
|
|
262
366
|
# Convert datetime, time, and Decimal values to strings.
|
|
263
367
|
for i in range(len(linedata)):
|
|
264
368
|
if type(linedata[i]) in (datetime.datetime, datetime.time, Decimal):
|
|
@@ -272,6 +376,9 @@ class SQLiteDatabase(Database):
|
|
|
272
376
|
if len(batch) >= batch_size:
|
|
273
377
|
_flush(last_line)
|
|
274
378
|
_flush(last_line)
|
|
379
|
+
except Exception:
|
|
380
|
+
self.rollback()
|
|
381
|
+
raise
|
|
275
382
|
finally:
|
|
276
383
|
curs.close()
|
|
277
384
|
if _state.exec_log:
|
execsql/db/sqlserver.py
CHANGED
|
@@ -8,7 +8,7 @@ Server via ``pyodbc``. Corresponds to ``-t s`` on the CLI.
|
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
10
|
from execsql.db.base import Database
|
|
11
|
-
from execsql.db.target_types import ColumnKinds,
|
|
11
|
+
from execsql.db.target_types import ColumnKinds, ColumnMetadata, kind_and_scale_for_type_name
|
|
12
12
|
from execsql.db.odbc import connection_string, fresh_sessions, loggable
|
|
13
13
|
from execsql.exceptions import ErrInfo
|
|
14
14
|
from execsql.utils.errors import fatal_error, install_hint
|
|
@@ -245,12 +245,16 @@ class SqlServerDatabase(Database):
|
|
|
245
245
|
)
|
|
246
246
|
if not rows:
|
|
247
247
|
return super().column_kinds(sq_name)
|
|
248
|
-
return
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
248
|
+
return ColumnMetadata(
|
|
249
|
+
{
|
|
250
|
+
name.lower(): kind_and_scale_for_type_name(
|
|
251
|
+
f"{type_name}({precision},{scale})"
|
|
252
|
+
if type_name in ("decimal", "numeric", "money", "smallmoney")
|
|
253
|
+
else type_name,
|
|
254
|
+
)
|
|
255
|
+
for name, type_name, precision, scale in rows
|
|
256
|
+
},
|
|
257
|
+
)
|
|
254
258
|
|
|
255
259
|
def drop_table(self, tablename: str) -> None:
|
|
256
260
|
"""Drop *tablename*, schema-qualified and quoted by the caller."""
|