execsql2 2.31.1__py3-none-any.whl → 2.32.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/db/access.py +3 -1
- execsql/db/base.py +19 -1
- execsql/db/dsn.py +3 -0
- execsql/db/duckdb.py +60 -1
- execsql/db/odbc.py +18 -1
- execsql/db/sqlserver.py +3 -1
- execsql/exporters/duckdb.py +2 -10
- execsql/models.py +73 -48
- execsql/types.py +14 -0
- execsql/utils/datetime.py +16 -2
- {execsql2-2.31.1.dist-info → execsql2-2.32.0.dist-info}/METADATA +2 -2
- {execsql2-2.31.1.dist-info → execsql2-2.32.0.dist-info}/RECORD +29 -29
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/README.md +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.31.1.dist-info → execsql2-2.32.0.dist-info}/WHEEL +0 -0
- {execsql2-2.31.1.dist-info → execsql2-2.32.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.31.1.dist-info → execsql2-2.32.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.31.1.dist-info → execsql2-2.32.0.dist-info}/licenses/NOTICE +0 -0
execsql/db/access.py
CHANGED
|
@@ -15,7 +15,7 @@ from pathlib import Path
|
|
|
15
15
|
from typing import Any, cast
|
|
16
16
|
|
|
17
17
|
from execsql.db.base import Database, _fetched
|
|
18
|
-
from execsql.db.odbc import odbc_value
|
|
18
|
+
from execsql.db.odbc import fresh_sessions, odbc_value
|
|
19
19
|
from execsql.db.tiers import SupportTier
|
|
20
20
|
from execsql.exceptions import ErrInfo
|
|
21
21
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
@@ -108,6 +108,8 @@ class AccessDatabase(Database):
|
|
|
108
108
|
# Open an ODBC connection.
|
|
109
109
|
import pyodbc
|
|
110
110
|
|
|
111
|
+
fresh_sessions(pyodbc)
|
|
112
|
+
|
|
111
113
|
if self.conn is not None:
|
|
112
114
|
self.conn.close()
|
|
113
115
|
self.conn = None
|
execsql/db/base.py
CHANGED
|
@@ -665,6 +665,23 @@ class Database(ABC):
|
|
|
665
665
|
self.execute(f"drop table if exists {tablename} cascade;")
|
|
666
666
|
self.commit()
|
|
667
667
|
|
|
668
|
+
def row_inserter(self, curs: Any, table: str, columns: list[str]) -> Callable[[list[list[Any]]], None]:
|
|
669
|
+
"""Return a function that inserts one batch of rows into *table* through *curs*.
|
|
670
|
+
|
|
671
|
+
*table* is schema-qualified and quoted; *columns* are the quoted
|
|
672
|
+
column names, in the order of each row's values. IMPORT and COPY call
|
|
673
|
+
the function once per batch, all inside one transaction. This one
|
|
674
|
+
runs ``executemany``. An adapter may override it with a faster bulk
|
|
675
|
+
path only if that path stores exactly what ``executemany`` stores, for
|
|
676
|
+
every value type and column type: a faster load must not change data.
|
|
677
|
+
"""
|
|
678
|
+
sql = f"insert into {table} ({','.join(columns)}) values ({self.paramsubs(len(columns))});"
|
|
679
|
+
|
|
680
|
+
def insert(rows: list[list[Any]]) -> None:
|
|
681
|
+
curs.executemany(sql, rows)
|
|
682
|
+
|
|
683
|
+
return insert
|
|
684
|
+
|
|
668
685
|
def populate_table(
|
|
669
686
|
self,
|
|
670
687
|
schema_name: str | None,
|
|
@@ -740,6 +757,7 @@ class Database(ABC):
|
|
|
740
757
|
|
|
741
758
|
def _import_loop(curs) -> int:
|
|
742
759
|
nonlocal eof, total_rows, task_id
|
|
760
|
+
insert = self.row_inserter(curs, sq_name, columns)
|
|
743
761
|
while True:
|
|
744
762
|
b = []
|
|
745
763
|
for _j in range(_state.conf.import_row_buffer):
|
|
@@ -809,7 +827,7 @@ class Database(ABC):
|
|
|
809
827
|
b.append(row)
|
|
810
828
|
if len(b) > 0:
|
|
811
829
|
try:
|
|
812
|
-
|
|
830
|
+
insert(b)
|
|
813
831
|
except ErrInfo:
|
|
814
832
|
raise
|
|
815
833
|
except Exception as e:
|
execsql/db/dsn.py
CHANGED
|
@@ -10,6 +10,7 @@ the CLI.
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
from execsql.db.base import Database
|
|
13
|
+
from execsql.db.odbc import fresh_sessions
|
|
13
14
|
from execsql.db.tiers import SupportTier
|
|
14
15
|
from execsql.exceptions import ErrInfo
|
|
15
16
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
@@ -75,6 +76,8 @@ class DsnDatabase(Database):
|
|
|
75
76
|
# Open an ODBC connection using a DSN.
|
|
76
77
|
import pyodbc
|
|
77
78
|
|
|
79
|
+
fresh_sessions(pyodbc)
|
|
80
|
+
|
|
78
81
|
if self.conn is not None:
|
|
79
82
|
self.conn.close()
|
|
80
83
|
self.conn = None
|
execsql/db/duckdb.py
CHANGED
|
@@ -8,8 +8,12 @@ analytics databases via the ``duckdb`` package. Corresponds to ``-t k``
|
|
|
8
8
|
on the CLI.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
+
import importlib.util
|
|
11
12
|
import re
|
|
13
|
+
import sys
|
|
12
14
|
from collections import deque
|
|
15
|
+
from collections.abc import Callable, Iterator
|
|
16
|
+
from contextlib import contextmanager
|
|
13
17
|
from pathlib import Path
|
|
14
18
|
from typing import Any
|
|
15
19
|
|
|
@@ -18,7 +22,7 @@ from execsql.exceptions import ErrInfo
|
|
|
18
22
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
19
23
|
import execsql.state as _state
|
|
20
24
|
|
|
21
|
-
__all__ = ["DuckDBDatabase"]
|
|
25
|
+
__all__ = ["DuckDBDatabase", "insert_rows"]
|
|
22
26
|
|
|
23
27
|
# Statements that return rows; any other statement DuckDB answers with a
|
|
24
28
|
# one-column "Count" result (rows inserted, updated or deleted).
|
|
@@ -28,6 +32,53 @@ _QUERY_RX = re.compile(r"(?:SELECT|WITH|VALUES|FROM|TABLE|SHOW|DESCRIBE|SUMMARIZ
|
|
|
28
32
|
_TRANSACTION_RX = re.compile(r"(?P<begin>BEGIN|START\s+TRANSACTION)\b|(?P<end>COMMIT|END|ROLLBACK|ABORT)\b", re.I)
|
|
29
33
|
|
|
30
34
|
|
|
35
|
+
# Parameters bound by one INSERT ... VALUES statement in insert_rows(); a
|
|
36
|
+
# wide table gets fewer rows per statement.
|
|
37
|
+
_PARAMS_PER_STATEMENT = 10_000
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@contextmanager
|
|
41
|
+
def _pandas_lookup_skipped() -> Iterator[None]:
|
|
42
|
+
"""Make DuckDB's check for pandas fail at once while it binds rows.
|
|
43
|
+
|
|
44
|
+
The duckdb package asks Python's import system for ``pandas`` twice for
|
|
45
|
+
every row it binds, and when pandas is not installed nothing remembers
|
|
46
|
+
the answer, so each row searches ``sys.path`` again: about three
|
|
47
|
+
quarters of an INSERT's time. A ``None`` in ``sys.modules`` makes that
|
|
48
|
+
import fail without the search. It is set only when pandas is not
|
|
49
|
+
installed, so no import that could have succeeded fails, and it is
|
|
50
|
+
removed afterwards.
|
|
51
|
+
"""
|
|
52
|
+
if "pandas" in sys.modules or importlib.util.find_spec("pandas") is not None:
|
|
53
|
+
yield
|
|
54
|
+
return
|
|
55
|
+
sys.modules["pandas"] = None # type: ignore[assignment]
|
|
56
|
+
try:
|
|
57
|
+
yield
|
|
58
|
+
finally:
|
|
59
|
+
if "pandas" in sys.modules and sys.modules["pandas"] is None:
|
|
60
|
+
del sys.modules["pandas"]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def insert_rows(curs: Any, table: str, columns: list[str], rows: list[Any]) -> None:
|
|
64
|
+
"""Insert *rows* into *table* with multi-row ``INSERT ... VALUES`` statements.
|
|
65
|
+
|
|
66
|
+
*table* is quoted (and schema-qualified as needed); *columns* are the
|
|
67
|
+
quoted column names, in the order of each row's values. DuckDB's
|
|
68
|
+
``executemany`` runs one row at a time; binding many rows in one
|
|
69
|
+
statement is several times faster and stores the same values.
|
|
70
|
+
"""
|
|
71
|
+
if not rows:
|
|
72
|
+
return
|
|
73
|
+
per_statement = max(1, _PARAMS_PER_STATEMENT // max(1, len(columns)))
|
|
74
|
+
head = f"insert into {table} ({','.join(columns)}) values "
|
|
75
|
+
row_marks = "(" + ",".join("?" * len(columns)) + ")"
|
|
76
|
+
with _pandas_lookup_skipped():
|
|
77
|
+
for start in range(0, len(rows), per_statement):
|
|
78
|
+
chunk = rows[start : start + per_statement]
|
|
79
|
+
curs.execute(head + ",".join([row_marks] * len(chunk)), [value for row in chunk for value in row])
|
|
80
|
+
|
|
81
|
+
|
|
31
82
|
class _SharedCursor:
|
|
32
83
|
"""A cursor that runs on the database's one connection.
|
|
33
84
|
|
|
@@ -261,6 +312,14 @@ class DuckDBDatabase(Database):
|
|
|
261
312
|
curs.execute(sql, params)
|
|
262
313
|
return len(curs.fetchall()) > 0
|
|
263
314
|
|
|
315
|
+
def row_inserter(self, curs: Any, table: str, columns: list[str]) -> Callable[[list[list[Any]]], None]:
|
|
316
|
+
"""Insert each batch with multi-row ``INSERT ... VALUES`` (:func:`insert_rows`)."""
|
|
317
|
+
|
|
318
|
+
def insert(rows: list[list[Any]]) -> None:
|
|
319
|
+
insert_rows(curs, table, columns, rows)
|
|
320
|
+
|
|
321
|
+
return insert
|
|
322
|
+
|
|
264
323
|
def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
|
|
265
324
|
"""Return True if the named view exists in the DuckDB database."""
|
|
266
325
|
# DuckDB information_schema has no 'views' table; views are listed in 'tables'
|
execsql/db/odbc.py
CHANGED
|
@@ -7,15 +7,32 @@ taken from a substitution variable) would end its attribute early and
|
|
|
7
7
|
start another. :func:`odbc_value` brace-quotes such values;
|
|
8
8
|
:func:`connection_string` joins the attributes, and :func:`loggable`
|
|
9
9
|
gives the same string with the password masked for the log.
|
|
10
|
+
:func:`fresh_sessions` makes each pyodbc connection a new database session.
|
|
10
11
|
"""
|
|
11
12
|
|
|
12
13
|
from __future__ import annotations
|
|
13
14
|
|
|
14
|
-
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
__all__ = ["connection_string", "fresh_sessions", "loggable", "odbc_value"]
|
|
15
18
|
|
|
16
19
|
_SECRET_KEYS = frozenset({"pwd", "password"})
|
|
17
20
|
|
|
18
21
|
|
|
22
|
+
def fresh_sessions(pyodbc: Any) -> None:
|
|
23
|
+
"""Turn off pyodbc's connection pooling, so each connection is a new session.
|
|
24
|
+
|
|
25
|
+
With pooling on, closing a connection hands its session back to the ODBC
|
|
26
|
+
driver manager, and unixODBC gives it unchanged to the next connection to
|
|
27
|
+
the same server: temp tables and ``SET`` options from one ``CONNECT`` or
|
|
28
|
+
``execsql.run()`` were still there in the next. pyodbc applies the
|
|
29
|
+
setting when the process makes its first pyodbc connection, so it covers
|
|
30
|
+
every pyodbc connection in the process, and has no effect if one was
|
|
31
|
+
made before execsql's first.
|
|
32
|
+
"""
|
|
33
|
+
pyodbc.pooling = False
|
|
34
|
+
|
|
35
|
+
|
|
19
36
|
def odbc_value(value: object) -> str:
|
|
20
37
|
"""Return *value* as an ODBC attribute value, brace-quoted when it needs it.
|
|
21
38
|
|
execsql/db/sqlserver.py
CHANGED
|
@@ -8,7 +8,7 @@ Server via ``pyodbc``. Corresponds to ``-t s`` on the CLI.
|
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
10
|
from execsql.db.base import Database
|
|
11
|
-
from execsql.db.odbc import connection_string, loggable
|
|
11
|
+
from execsql.db.odbc import connection_string, fresh_sessions, loggable
|
|
12
12
|
from execsql.exceptions import ErrInfo
|
|
13
13
|
from execsql.utils.errors import fatal_error, install_hint
|
|
14
14
|
from execsql.utils.auth import (
|
|
@@ -98,6 +98,8 @@ class SqlServerDatabase(Database):
|
|
|
98
98
|
"""Open a connection to the SQL Server database via pyodbc."""
|
|
99
99
|
import pyodbc
|
|
100
100
|
|
|
101
|
+
fresh_sessions(pyodbc)
|
|
102
|
+
|
|
101
103
|
if self.conn is None:
|
|
102
104
|
begin_login()
|
|
103
105
|
if self.user and self.need_passwd and not self.password:
|
execsql/exporters/duckdb.py
CHANGED
|
@@ -10,7 +10,6 @@ Requires the ``execsql2[duckdb]`` extra.
|
|
|
10
10
|
|
|
11
11
|
import datetime as _datetime
|
|
12
12
|
import decimal as _decimal
|
|
13
|
-
import math
|
|
14
13
|
from pathlib import Path
|
|
15
14
|
from typing import Any
|
|
16
15
|
|
|
@@ -38,9 +37,9 @@ def export_duckdb(
|
|
|
38
37
|
fatal_error(f"The duckdb module is required to export data in that format. {install_hint('duckdb')}")
|
|
39
38
|
return
|
|
40
39
|
|
|
40
|
+
from execsql.db.duckdb import insert_rows
|
|
41
41
|
from execsql.models import DataTable
|
|
42
42
|
|
|
43
|
-
chunksize = 10000
|
|
44
43
|
# B07a/F026: tablename and catalog are substituted from user input.
|
|
45
44
|
# Identifier-quote the table-name interpolation site and use
|
|
46
45
|
# placeholders for the information_schema literal-value query.
|
|
@@ -70,15 +69,8 @@ def export_duckdb(
|
|
|
70
69
|
curs.execute(sql)
|
|
71
70
|
# Export all rows of data
|
|
72
71
|
columns = [dbt_duckdb.quoted(col) for col in hdrs]
|
|
73
|
-
colspec = ",".join(columns)
|
|
74
|
-
paramspec = ",".join(("?",) * len(columns))
|
|
75
|
-
sql = f"insert into {qtable} ({colspec}) values ({paramspec});"
|
|
76
|
-
n_chunks = math.ceil(len(rowdata) / chunksize)
|
|
77
72
|
curs.execute("BEGIN TRANSACTION;")
|
|
78
|
-
|
|
79
|
-
start = i * chunksize
|
|
80
|
-
end = start + chunksize
|
|
81
|
-
curs.executemany(sql, rowdata[start:end])
|
|
73
|
+
insert_rows(curs, qtable, columns, rowdata)
|
|
82
74
|
curs.execute("COMMIT;")
|
|
83
75
|
curs.close()
|
|
84
76
|
ddb.close()
|
execsql/models.py
CHANGED
|
@@ -50,6 +50,16 @@ __all__ = [
|
|
|
50
50
|
ColumnType = tuple[str, type[DataType], int | None, bool, int | None, int | None]
|
|
51
51
|
|
|
52
52
|
|
|
53
|
+
def _text_length(value: Any) -> int:
|
|
54
|
+
"""Return the length of *value* written as text (a column may turn out to be text)."""
|
|
55
|
+
if isinstance(value, str):
|
|
56
|
+
return len(value)
|
|
57
|
+
try:
|
|
58
|
+
return len(str(value))
|
|
59
|
+
except Exception:
|
|
60
|
+
return len(bytes(value))
|
|
61
|
+
|
|
62
|
+
|
|
53
63
|
class Column:
|
|
54
64
|
"""Compile data-type match statistics for a single column of imported data."""
|
|
55
65
|
|
|
@@ -84,40 +94,39 @@ class Column:
|
|
|
84
94
|
"""Test whether a non-null value matches this data type and update statistics."""
|
|
85
95
|
# datavalue must be non-null
|
|
86
96
|
if not self.failed:
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
self.count += 1
|
|
90
|
-
if isinstance(datavalue, str):
|
|
91
|
-
vlen = len(datavalue)
|
|
92
|
-
else:
|
|
93
|
-
# This column may turn out to have to be text, so we need the maximum length
|
|
94
|
-
# of any data value when represented as text.
|
|
95
|
-
try:
|
|
96
|
-
vlen = len(str(datavalue))
|
|
97
|
-
except Exception:
|
|
98
|
-
vlen = len(bytes(datavalue))
|
|
99
|
-
if self.maxlen > 0 and vlen != self.maxlen:
|
|
100
|
-
self.varlen = True
|
|
101
|
-
if vlen > self.maxlen:
|
|
102
|
-
self.maxlen = vlen
|
|
103
|
-
if self.dt.precision is not None and self.dt.scale is not None:
|
|
104
|
-
int_digits = self.dt.precision - self.dt.scale
|
|
105
|
-
if self.maxprecision is None:
|
|
106
|
-
self.maxprecision = self.dt.precision
|
|
107
|
-
self.scale = self.dt.scale
|
|
108
|
-
self.maxscale = self.dt.scale
|
|
109
|
-
self.max_int_digits = int_digits
|
|
110
|
-
else:
|
|
111
|
-
assert self.maxscale is not None
|
|
112
|
-
assert self.max_int_digits is not None
|
|
113
|
-
self.maxprecision = max(self.dt.precision, self.maxprecision)
|
|
114
|
-
if self.dt.scale != self.scale:
|
|
115
|
-
self.varscale = True
|
|
116
|
-
self.maxscale = max(self.dt.scale, self.maxscale)
|
|
117
|
-
self.max_int_digits = max(int_digits, self.max_int_digits)
|
|
97
|
+
if self.dt.matches(datavalue):
|
|
98
|
+
self.record(_text_length(datavalue))
|
|
118
99
|
else:
|
|
119
100
|
self.failed = True
|
|
120
101
|
|
|
102
|
+
def record(self, vlen: int) -> None:
|
|
103
|
+
"""Count a value this data type matched, *vlen* characters long as text.
|
|
104
|
+
|
|
105
|
+
Call it right after :meth:`DataType.matches` accepted the value:
|
|
106
|
+
a decimal type's precision and scale are those of the value it
|
|
107
|
+
last matched.
|
|
108
|
+
"""
|
|
109
|
+
self.count += 1
|
|
110
|
+
if self.maxlen > 0 and vlen != self.maxlen:
|
|
111
|
+
self.varlen = True
|
|
112
|
+
if vlen > self.maxlen:
|
|
113
|
+
self.maxlen = vlen
|
|
114
|
+
if self.dt.precision is not None and self.dt.scale is not None:
|
|
115
|
+
int_digits = self.dt.precision - self.dt.scale
|
|
116
|
+
if self.maxprecision is None:
|
|
117
|
+
self.maxprecision = self.dt.precision
|
|
118
|
+
self.scale = self.dt.scale
|
|
119
|
+
self.maxscale = self.dt.scale
|
|
120
|
+
self.max_int_digits = int_digits
|
|
121
|
+
else:
|
|
122
|
+
assert self.maxscale is not None
|
|
123
|
+
assert self.max_int_digits is not None
|
|
124
|
+
self.maxprecision = max(self.dt.precision, self.maxprecision)
|
|
125
|
+
if self.dt.scale != self.scale:
|
|
126
|
+
self.varscale = True
|
|
127
|
+
self.maxscale = max(self.dt.scale, self.maxscale)
|
|
128
|
+
self.max_int_digits = max(int_digits, self.max_int_digits)
|
|
129
|
+
|
|
121
130
|
def __init__(self, colname: str, infer_strings: bool = True) -> None:
|
|
122
131
|
"""Create a column characteriser for the named column.
|
|
123
132
|
|
|
@@ -177,6 +186,15 @@ class Column:
|
|
|
177
186
|
self.str_accums = tuple(ac for ac in self.accums if isinstance(ac.dt, (DT_Character, DT_Varchar, DT_Text)))
|
|
178
187
|
# A bytes value is checked as binary or text only: int(b"12") is 12.
|
|
179
188
|
self.bytes_accums = tuple(ac for ac in self.accums if isinstance(ac.dt, DT_Binary)) + self.str_accums
|
|
189
|
+
# A value is tried only against the types that matched every value
|
|
190
|
+
# before it: a type that failed once cannot be the column's type.
|
|
191
|
+
self._live = list(self.accums)
|
|
192
|
+
self._live_str = list(self.str_accums)
|
|
193
|
+
self._live_bytes = list(self.bytes_accums)
|
|
194
|
+
# The settings the scan reads for every value, read once.
|
|
195
|
+
self._trim_strings = conf.trim_strings
|
|
196
|
+
self._replace_newlines = conf.replace_newlines
|
|
197
|
+
self._empty_strings = conf.empty_strings
|
|
180
198
|
self.dt_eval = False
|
|
181
199
|
# self.dt is a tuple of: 0: column name; 1: data type class; 2: max length or None;
|
|
182
200
|
# 3: bool indicating any null values; 4: precision or None; 5: scale or None.
|
|
@@ -187,31 +205,38 @@ class Column:
|
|
|
187
205
|
|
|
188
206
|
def eval_types(self, column_value: Any) -> None:
|
|
189
207
|
"""Evaluate which data types the value matches and update counters."""
|
|
190
|
-
# Evaluate which data type(s) the value matches, and increment the appropriate counter(s).
|
|
191
|
-
import execsql.state as _state
|
|
192
|
-
|
|
193
208
|
self.dt_eval = False
|
|
194
209
|
self.rowcount += 1
|
|
195
|
-
|
|
196
|
-
if
|
|
197
|
-
if
|
|
210
|
+
is_str = isinstance(column_value, str)
|
|
211
|
+
if is_str:
|
|
212
|
+
if self._trim_strings:
|
|
198
213
|
column_value = column_value.strip()
|
|
199
|
-
if
|
|
214
|
+
if self._replace_newlines:
|
|
200
215
|
column_value = re.sub(r"[\s\t]*[\r\n]+[\s\t]*", " ", column_value)
|
|
201
|
-
if column_value is None or (
|
|
202
|
-
not conf.empty_strings and isinstance(column_value, str) and len(column_value.strip()) == 0
|
|
203
|
-
):
|
|
216
|
+
if column_value is None or (not self._empty_strings and is_str and len(column_value.strip()) == 0):
|
|
204
217
|
self.nullrows += 1
|
|
205
218
|
return
|
|
206
|
-
if not self.infer_strings and
|
|
207
|
-
|
|
208
|
-
dt.check(column_value)
|
|
219
|
+
if not self.infer_strings and is_str:
|
|
220
|
+
live = self._live_str
|
|
209
221
|
elif not self.infer_strings and isinstance(column_value, (bytes, bytearray, memoryview)):
|
|
210
|
-
|
|
211
|
-
dt.check(column_value)
|
|
222
|
+
live = self._live_bytes
|
|
212
223
|
else:
|
|
213
|
-
|
|
214
|
-
|
|
224
|
+
live = self._live
|
|
225
|
+
vlen = None
|
|
226
|
+
any_failed = False
|
|
227
|
+
# The value is not NULL, the one case DataType.matches() adds to _is_match().
|
|
228
|
+
for ac in live:
|
|
229
|
+
if ac.dt._is_match(column_value):
|
|
230
|
+
if vlen is None:
|
|
231
|
+
vlen = _text_length(column_value)
|
|
232
|
+
ac.record(vlen)
|
|
233
|
+
else:
|
|
234
|
+
ac.failed = True
|
|
235
|
+
any_failed = True
|
|
236
|
+
if any_failed:
|
|
237
|
+
self._live = [ac for ac in self._live if not ac.failed]
|
|
238
|
+
self._live_str = [ac for ac in self._live_str if not ac.failed]
|
|
239
|
+
self._live_bytes = [ac for ac in self._live_bytes if not ac.failed]
|
|
215
240
|
|
|
216
241
|
def column_type(self) -> ColumnType:
|
|
217
242
|
"""Return the inferred type of this column as a 6-tuple."""
|
execsql/types.py
CHANGED
|
@@ -550,6 +550,8 @@ class DT_Character(DataType):
|
|
|
550
550
|
return "DT_Character()"
|
|
551
551
|
|
|
552
552
|
def _is_match(self, data: object) -> bool:
|
|
553
|
+
if isinstance(data, str): # what _from_data() returns a str as, unchanged
|
|
554
|
+
return True
|
|
553
555
|
if isinstance(data, bytearray):
|
|
554
556
|
return False
|
|
555
557
|
return super()._is_match(data)
|
|
@@ -582,6 +584,8 @@ class DT_Varchar(DataType):
|
|
|
582
584
|
return "DT_Varchar()"
|
|
583
585
|
|
|
584
586
|
def _is_match(self, data: object) -> bool:
|
|
587
|
+
if isinstance(data, str): # the length check _from_data() makes
|
|
588
|
+
return len(data) <= 255
|
|
585
589
|
if isinstance(data, bytearray):
|
|
586
590
|
return False
|
|
587
591
|
return super()._is_match(data)
|
|
@@ -609,6 +613,8 @@ class DT_Text(DataType):
|
|
|
609
613
|
return "DT_Text()"
|
|
610
614
|
|
|
611
615
|
def _is_match(self, data: object) -> bool:
|
|
616
|
+
if isinstance(data, str): # what _from_data() returns a str as, unchanged
|
|
617
|
+
return True
|
|
612
618
|
if isinstance(data, bytearray):
|
|
613
619
|
return False
|
|
614
620
|
return super()._is_match(data)
|
|
@@ -634,6 +640,14 @@ class DT_Binary(DataType):
|
|
|
634
640
|
def __repr__(self) -> str:
|
|
635
641
|
return "DT_Binary()"
|
|
636
642
|
|
|
643
|
+
def _is_match(self, data: object) -> bool:
|
|
644
|
+
# bytearray(n) is n zero bytes, so an int converts when it is not
|
|
645
|
+
# negative. Building it to find out allocated n bytes per value: a
|
|
646
|
+
# column of epoch milliseconds asked for a terabyte each.
|
|
647
|
+
if isinstance(data, int):
|
|
648
|
+
return data >= 0
|
|
649
|
+
return super()._is_match(data)
|
|
650
|
+
|
|
637
651
|
|
|
638
652
|
class DbType:
|
|
639
653
|
"""Map Python DataType subclasses to DBMS-specific SQL type names."""
|
execsql/utils/datetime.py
CHANGED
|
@@ -14,7 +14,9 @@ today's date. ``1-2``, ``5/6`` and ``March 5`` are not timestamps.
|
|
|
14
14
|
"""
|
|
15
15
|
|
|
16
16
|
import datetime
|
|
17
|
+
import functools
|
|
17
18
|
import re
|
|
19
|
+
import time
|
|
18
20
|
from typing import Any, cast
|
|
19
21
|
|
|
20
22
|
from dateutil import parser as _dateutil_parser
|
|
@@ -58,9 +60,21 @@ _DEFAULT_B = datetime.datetime(2002, 2, 2)
|
|
|
58
60
|
|
|
59
61
|
def _parse(datestr: str, strict: bool) -> datetime.datetime | None:
|
|
60
62
|
"""Parse *datestr* with dateutil; when *strict*, only a fully written date."""
|
|
61
|
-
|
|
62
|
-
|
|
63
|
+
if not strict:
|
|
64
|
+
try:
|
|
63
65
|
return cast(datetime.datetime, _dateutil_parser.parse(datestr))
|
|
66
|
+
except (ValueError, OverflowError, TypeError):
|
|
67
|
+
return None
|
|
68
|
+
# A strict parse depends only on the string and on the local zone names
|
|
69
|
+
# dateutil matches a written zone against, so repeated values (dates
|
|
70
|
+
# recur, and type inference parses each one for both timestamp types)
|
|
71
|
+
# are parsed once.
|
|
72
|
+
return _parse_strict(datestr, time.tzname)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@functools.lru_cache(maxsize=4096)
|
|
76
|
+
def _parse_strict(datestr: str, _local_zone_names: tuple[str, str]) -> datetime.datetime | None:
|
|
77
|
+
try:
|
|
64
78
|
if not _FULL_YEAR.search(datestr):
|
|
65
79
|
short = _SHORT_YEAR.match(datestr)
|
|
66
80
|
# Month first only: dateutil would read 13-05-24 day-first,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: execsql2
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.32.0
|
|
4
4
|
Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
|
|
5
5
|
Project-URL: Homepage, https://execsql2.readthedocs.io/en/stable/
|
|
6
6
|
Project-URL: Repository, https://github.com/geocoug/execsql
|
|
@@ -441,7 +441,7 @@ execsql format --no-sql --in-place scripts/
|
|
|
441
441
|
```yaml
|
|
442
442
|
repos:
|
|
443
443
|
- repo: https://github.com/geocoug/execsql
|
|
444
|
-
rev: v2.
|
|
444
|
+
rev: v2.32.0
|
|
445
445
|
hooks:
|
|
446
446
|
- id: execsql-format
|
|
447
447
|
- id: execsql-lint
|
|
@@ -7,13 +7,13 @@ execsql/exceptions.py,sha256=aHavdblmP7m5ZLxyyKKmPSwwbdmP962j4ShP1Gd9r64,9651
|
|
|
7
7
|
execsql/format.py,sha256=z-KZ8VY1muz8S6iaErZX9E_7wE_VtjEcUB7e_KyhfyI,59565
|
|
8
8
|
execsql/interactive.py,sha256=7vm7NAlcarBJJOIffcLOcJqH9ZCInvHB_CmC2sQsXf8,8241
|
|
9
9
|
execsql/manifest.py,sha256=eJp39R8bYuFuJiGaBbDsk_lua13uJHXPOdwWdUfE-CQ,9027
|
|
10
|
-
execsql/models.py,sha256=
|
|
10
|
+
execsql/models.py,sha256=heq4wrTJHOHF5sCIbvnCiaNHteeT8G06aBoRAsF9vuI,17191
|
|
11
11
|
execsql/parser.py,sha256=bw26e1lD-PZscV11VtBdqSLkjpcYfNLQC2EYfYD5UwU,16012
|
|
12
12
|
execsql/plugins.py,sha256=RykMlIjTyWcnQc9ygjLErS3wdcsZSsG9QTIvg4DwfAI,13626
|
|
13
13
|
execsql/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
14
|
execsql/state.py,sha256=rXhxx58dVFjhG58HGOXm3ewloVlvpcaYnW1rij4qjkM,23478
|
|
15
15
|
execsql/state.pyi,sha256=ZDHZ1DPKYu4Qs4bcJT31XNv4GOt6O4s_zH4UHCowovw,4344
|
|
16
|
-
execsql/types.py,sha256=
|
|
16
|
+
execsql/types.py,sha256=XmLfqjXSxBJamZbz0JPodipXi1YzcN_2MjooPyzJsyY,33819
|
|
17
17
|
execsql/cli/__init__.py,sha256=FHg2lm2gt4L-XBnlVORn3qWHZ4739Ao-uwBIURfVIjg,2798
|
|
18
18
|
execsql/cli/application.py,sha256=tsSYE8cerJxY6y73GM_Xr1WBwc6A0_al8y969B6lPP8,12542
|
|
19
19
|
execsql/cli/dispatch.py,sha256=deyN2QQYvdzMSqGGa3CSMpaqdLNeMgc5Uke0mUbl764,2704
|
|
@@ -34,25 +34,25 @@ execsql/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
34
34
|
execsql/data/execsql.conf.template,sha256=Mz8wZxu0U2bfL-fsDpRHvdLkwNpfuZrl4QUMjp7SXyk,12770
|
|
35
35
|
execsql/data/script_header.sql,sha256=a7WKScekNgJoxVEmMRTkWv8NsAzW1jJ4vJfGy9HVy0g,349
|
|
36
36
|
execsql/db/__init__.py,sha256=jTbuafuKOqYtXFR1wvCOoKK5Lr3l1uErfaIbIr6UywI,1063
|
|
37
|
-
execsql/db/access.py,sha256=
|
|
38
|
-
execsql/db/base.py,sha256=
|
|
39
|
-
execsql/db/dsn.py,sha256=
|
|
40
|
-
execsql/db/duckdb.py,sha256=
|
|
37
|
+
execsql/db/access.py,sha256=fxiC3rtoOJzrNXNxau-w7oh1fd9tyh2_PMliBbFc164,21427
|
|
38
|
+
execsql/db/base.py,sha256=WdF1X0xG01ZKaYNkxcyUSKJCV04cceThPkvERlN-iu0,44619
|
|
39
|
+
execsql/db/dsn.py,sha256=Evrc6MUllJVFDn0jZB_iks-aKCsXuNF9XYjAkttqgGQ,6995
|
|
40
|
+
execsql/db/duckdb.py,sha256=6vD5b_tpvfByoFEajZNY5HLzVUnH1eOuPpzNGAKFsIQ,13624
|
|
41
41
|
execsql/db/factory.py,sha256=Uz7Xcbff6ZZbtLM55FhHCwGdo4KKfqU_1kyjvOHn-HY,5691
|
|
42
42
|
execsql/db/firebird.py,sha256=CoNnzFtOFAkp8OJByXXXclitCPAAjM23eFJGfy3C7IE,10171
|
|
43
43
|
execsql/db/mysql.py,sha256=FVId78NifBmNfEI8zmqjRMJ3FUy7hskv411LqABAAOQ,24198
|
|
44
|
-
execsql/db/odbc.py,sha256=
|
|
44
|
+
execsql/db/odbc.py,sha256=l03UNHi8sFtjHbgOKUNA2teKOisNk-8laQGCsHZs2eo,2526
|
|
45
45
|
execsql/db/oracle.py,sha256=i5qPahsy_z1AuZT7l54Hzixknkvbzd-SCQgiVcTN94w,13796
|
|
46
46
|
execsql/db/postgres.py,sha256=sX2azdSt9N6jm_I47Zg8fukdYExGHRq_7XwVRZcfgAo,23647
|
|
47
47
|
execsql/db/sqlite.py,sha256=fEpNPkfJGnqm1Wx_rZK5SEqaWsBqLy85Jxk_5NsnqgM,12109
|
|
48
|
-
execsql/db/sqlserver.py,sha256=
|
|
48
|
+
execsql/db/sqlserver.py,sha256=6-wySx9BrSTAyXl5BnBLjRYEpl4_GubDjiGuvpehgco,10528
|
|
49
49
|
execsql/db/tiers.py,sha256=PH2sUYwlYqcANGVKXprWQKVIlM6fpWU9q74djn8oFwc,2924
|
|
50
50
|
execsql/debug/__init__.py,sha256=j6EGUR0dHzUhWN1mHHtf1-Lhjq3Sb1V-vmnq2Ztgj1M,178
|
|
51
51
|
execsql/debug/repl.py,sha256=jnwrqRac4krc1pES7Ni-j1Dyw3mJIcWl5LQQIX1GkRw,20753
|
|
52
52
|
execsql/exporters/__init__.py,sha256=-Cnji-OgodJV8ftcDcOyTof0kQMy9J5kKVC8GVFpc3o,670
|
|
53
53
|
execsql/exporters/base.py,sha256=_GoO0YqwlTxrjogxHI3CxxRR64sMgIEjRogt0KCqzoY,6402
|
|
54
54
|
execsql/exporters/delimited.py,sha256=N0cP_CNlb_LLCUqbHmTQsOGufWnD_ddGdeFnXPE8Erc,35606
|
|
55
|
-
execsql/exporters/duckdb.py,sha256=
|
|
55
|
+
execsql/exporters/duckdb.py,sha256=IQp7ayIJHB2lf6mBSxl6k5R2AeUEvgD75Q5Y55mH8jI,3214
|
|
56
56
|
execsql/exporters/feather.py,sha256=tah48aep0cQRikZcmADXUPTzFddgDUT78Ndw_l0Hsj4,4178
|
|
57
57
|
execsql/exporters/html.py,sha256=hwf9UHs7EZ7xiP3p9ilBIsAukXcDu-KO2dcQHVh2z30,10111
|
|
58
58
|
execsql/exporters/json.py,sha256=wWGT6zjvIdWlO5FmXbaEw1xOhy23f1YY6bm5uQD_NS0,4393
|
|
@@ -125,7 +125,7 @@ execsql/utils/__init__.py,sha256=0uR6JwVJQRX3vceByNBduCAf5dd5assKjeqJUWvpZoA,278
|
|
|
125
125
|
execsql/utils/auth.py,sha256=0LYKdNMhodq9R0B8WyKp69a-xlP9wOR6Ua9Lgo2aJnQ,14868
|
|
126
126
|
execsql/utils/color.py,sha256=goQqhnLg8yERpFolQikA9zYSVcuAm0unKTJiWTbQm-U,634
|
|
127
127
|
execsql/utils/crypto.py,sha256=KlT6lPk-v0-qQQfAMfWvT9Q6kRdCD3_5JVaLajNocKM,3051
|
|
128
|
-
execsql/utils/datetime.py,sha256=
|
|
128
|
+
execsql/utils/datetime.py,sha256=9ZizCFBYut5gC4gU3Cmegfddv1HLoU5xVQWm9thy5EI,6127
|
|
129
129
|
execsql/utils/errors.py,sha256=WKojSMjNecL9riRo9TnZ4fXofZr2uDo2_4B6n-zP31M,11693
|
|
130
130
|
execsql/utils/fileio.py,sha256=IqIEhun9aOnr1oBCWK2z2VAPVo95bZ54cVglLW4rV9M,48208
|
|
131
131
|
execsql/utils/gui.py,sha256=1iiupP9AsLdR6ST0gcLE143o6y6YGUFlNv_pdVPxfCE,23943
|
|
@@ -135,22 +135,22 @@ execsql/utils/paths.py,sha256=7IIHT6vqXTfs6Ltfrzm_I2ZV4pU8IuedOLkHo6a1_K8,1175
|
|
|
135
135
|
execsql/utils/regex.py,sha256=DGzPBl0Kt4--RbwPhoZI6Bm6N_CBUvI0DbqE4EF6qp8,3718
|
|
136
136
|
execsql/utils/strings.py,sha256=UQNjpRCEFa1UO6feU-M-9e24wWAvizs_iu_4fFusLxo,8516
|
|
137
137
|
execsql/utils/timer.py,sha256=eDYf5VzCNFk7oo90InJucUm3XcBdhYMogjZMqeg9xzc,1899
|
|
138
|
-
execsql2-2.
|
|
139
|
-
execsql2-2.
|
|
140
|
-
execsql2-2.
|
|
141
|
-
execsql2-2.
|
|
142
|
-
execsql2-2.
|
|
143
|
-
execsql2-2.
|
|
144
|
-
execsql2-2.
|
|
145
|
-
execsql2-2.
|
|
146
|
-
execsql2-2.
|
|
147
|
-
execsql2-2.
|
|
148
|
-
execsql2-2.
|
|
149
|
-
execsql2-2.
|
|
150
|
-
execsql2-2.
|
|
151
|
-
execsql2-2.
|
|
152
|
-
execsql2-2.
|
|
153
|
-
execsql2-2.
|
|
154
|
-
execsql2-2.
|
|
155
|
-
execsql2-2.
|
|
156
|
-
execsql2-2.
|
|
138
|
+
execsql2-2.32.0.data/data/execsql2_extras/README.md,sha256=6f5bWiBlKRn8s6GVrU423-HFrU1Jwbk2AabxraLu3AY,5398
|
|
139
|
+
execsql2-2.32.0.data/data/execsql2_extras/config_settings.sqlite,sha256=aY5cxR7Q7J6zJ4bC9lu5mHUrhy211Cq3MNKPQVCt02E,20480
|
|
140
|
+
execsql2-2.32.0.data/data/execsql2_extras/example_config_prompt.sql,sha256=2e8KzzVWhho8KxYVHETSVmZdhW7wodioDsqBLSL6m4s,7487
|
|
141
|
+
execsql2-2.32.0.data/data/execsql2_extras/make_config_db.sql,sha256=WwyC6dK-Eh5CAVppiBCDHqiI1_wEI9U95Ytpr4lsZkg,8726
|
|
142
|
+
execsql2-2.32.0.data/data/execsql2_extras/md_compare.sql,sha256=ARYEcOlxWLxfw-KCKDeJRdCqnuCb9PVwGaHtITR2sYU,26535
|
|
143
|
+
execsql2-2.32.0.data/data/execsql2_extras/md_glossary.sql,sha256=gu2PDTm295YvxR7cDp25Mf-xXHly0BvlMudoNOrep5Q,10931
|
|
144
|
+
execsql2-2.32.0.data/data/execsql2_extras/md_upsert.sql,sha256=YNUTFS6yB4bZXs-wRDn6dN8jL5wIumzZyE2uo5arIos,113407
|
|
145
|
+
execsql2-2.32.0.data/data/execsql2_extras/pg_compare.sql,sha256=WcupGxQoWtMwHVPuy42xCpP4gAV4Ttb1W-_hwh-3xb4,24529
|
|
146
|
+
execsql2-2.32.0.data/data/execsql2_extras/pg_glossary.sql,sha256=CfQz2HZj2nMvJfYfQrAmNYdx0uRrOIWsCXwVo5gKv_k,9953
|
|
147
|
+
execsql2-2.32.0.data/data/execsql2_extras/script_template.sql,sha256=s5i0HULxp49N2Cpd24iALleM_dBq5oLuN3Arx_4Z7-o,11140
|
|
148
|
+
execsql2-2.32.0.data/data/execsql2_extras/ss_compare.sql,sha256=PpU-vPyqqC1faBoo5zwVZlZHYKkT79QZisWVlk4o-ss,26415
|
|
149
|
+
execsql2-2.32.0.data/data/execsql2_extras/ss_glossary.sql,sha256=_oTiZHpadPXWrNs8VoRe9PZgX9XdK490NMVWVTVtjl4,13110
|
|
150
|
+
execsql2-2.32.0.data/data/execsql2_extras/ss_upsert.sql,sha256=Dvst877h4KHn8ayYF9EXI_OxqLQ89wyfcnGYYT4szro,118304
|
|
151
|
+
execsql2-2.32.0.dist-info/METADATA,sha256=q3H3q9BleScKMNArYi3SplieuzY-X6flVjmfU5KHib8,26927
|
|
152
|
+
execsql2-2.32.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
153
|
+
execsql2-2.32.0.dist-info/entry_points.txt,sha256=jOf53-4IVGjEZeNYUOF1GGaiPNfVZcNKxP3vDFKEyMA,58
|
|
154
|
+
execsql2-2.32.0.dist-info/licenses/LICENSE.txt,sha256=LBdhuxejF8_bLCHZ2kWfmDXpDGUu914Gbd6_3JjCRe0,676
|
|
155
|
+
execsql2-2.32.0.dist-info/licenses/NOTICE,sha256=McYzgxYav3U1OaVsY4Su1sfBrfmplpRdA9b6-gCDQCg,342
|
|
156
|
+
execsql2-2.32.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{execsql2-2.31.1.data → execsql2-2.32.0.data}/data/execsql2_extras/example_config_prompt.sql
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|