execsql2 2.33.0__py3-none-any.whl → 2.35.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +1 -0
- execsql/config.py +2 -0
- execsql/data/execsql.conf.template +7 -0
- execsql/db/base.py +71 -8
- execsql/db/duckdb.py +6 -0
- execsql/db/mysql.py +54 -2
- execsql/db/oracle.py +36 -0
- execsql/db/postgres.py +17 -1
- execsql/db/sqlite.py +7 -1
- execsql/db/sqlserver.py +24 -0
- execsql/db/target_types.py +277 -0
- execsql/exporters/json.py +51 -7
- execsql/exporters/protocol.py +1 -0
- execsql/importers/base.py +1 -1
- execsql/importers/csv.py +1 -1
- execsql/metacommands/__init__.py +17 -1
- execsql/metacommands/data.py +7 -1
- execsql/metacommands/dispatch.py +7 -0
- execsql/metacommands/io_export.py +3 -1
- execsql/metacommands/io_fileops.py +2 -2
- execsql/metacommands/reference.py +1 -0
- execsql/models.py +30 -4
- execsql/script/executor.py +10 -10
- execsql/script/variables.py +4 -0
- execsql/types.py +2 -2
- execsql/utils/errors.py +2 -2
- {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/METADATA +3 -3
- {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/RECORD +45 -44
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/README.md +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/WHEEL +0 -0
- {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"""The execsql data type of each column of a table that IMPORT or COPY loads.
|
|
2
|
+
|
|
3
|
+
A load into a table that already exists converts each value by the type of
|
|
4
|
+
the column it goes into, not by the type scanned from the source data
|
|
5
|
+
(issue #100). The adapters report each column's type: by the name their
|
|
6
|
+
catalog gives it (PostgreSQL, MySQL, SQL Server, DuckDB, Oracle), or by the
|
|
7
|
+
Python type their driver describes it with (Access, Firebird, ODBC DSNs).
|
|
8
|
+
The functions here turn that
|
|
9
|
+
into one of execsql's data types, or ``None`` when execsql has no
|
|
10
|
+
conversion that fits (arrays, JSON, UUIDs, enums, geometry, intervals, …).
|
|
11
|
+
|
|
12
|
+
Only the types that change how a value is converted are told apart: every
|
|
13
|
+
character type is :class:`DT_Text`, every integer type :class:`DT_Long`,
|
|
14
|
+
and so on. A ``DECIMAL``/``NUMERIC``/``NUMBER`` with scale 0 holds whole
|
|
15
|
+
numbers only, so it is :class:`DT_Long` too.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import datetime
|
|
21
|
+
import math
|
|
22
|
+
import re
|
|
23
|
+
from collections.abc import Callable
|
|
24
|
+
from decimal import Decimal
|
|
25
|
+
from typing import Any, TypeAlias
|
|
26
|
+
|
|
27
|
+
from execsql.exceptions import ErrInfo
|
|
28
|
+
from execsql.types import (
|
|
29
|
+
DataType,
|
|
30
|
+
DT_Binary,
|
|
31
|
+
DT_Boolean,
|
|
32
|
+
DT_Date,
|
|
33
|
+
DT_Decimal,
|
|
34
|
+
DT_Float,
|
|
35
|
+
DT_Long,
|
|
36
|
+
DT_Text,
|
|
37
|
+
DT_Time,
|
|
38
|
+
DT_Timestamp,
|
|
39
|
+
DT_TimestampTZ,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
__all__ = ["ColumnKinds", "existing_table_converter", "kind_for_python_type", "kind_for_type_name"]
|
|
43
|
+
|
|
44
|
+
#: Lower-cased column name -> the execsql type values for it convert to, or None.
|
|
45
|
+
#: A string, not ``dict[...]``: on Python 3.10 that object passes
|
|
46
|
+
#: ``isinstance(x, type)``, so code that scans the adapter modules for classes
|
|
47
|
+
#: would take it for one.
|
|
48
|
+
ColumnKinds: TypeAlias = "dict[str, type[DataType] | None]"
|
|
49
|
+
|
|
50
|
+
_TEXT = {
|
|
51
|
+
"char",
|
|
52
|
+
"character",
|
|
53
|
+
"varchar",
|
|
54
|
+
"character varying",
|
|
55
|
+
"varchar2",
|
|
56
|
+
"nchar",
|
|
57
|
+
"national character",
|
|
58
|
+
"nvarchar",
|
|
59
|
+
"nvarchar2",
|
|
60
|
+
"national character varying",
|
|
61
|
+
"bpchar",
|
|
62
|
+
"text",
|
|
63
|
+
"ntext",
|
|
64
|
+
"tinytext",
|
|
65
|
+
"mediumtext",
|
|
66
|
+
"longtext",
|
|
67
|
+
"string",
|
|
68
|
+
"clob",
|
|
69
|
+
"nclob",
|
|
70
|
+
}
|
|
71
|
+
_INTEGER = {
|
|
72
|
+
"integer",
|
|
73
|
+
"int",
|
|
74
|
+
"int1",
|
|
75
|
+
"int2",
|
|
76
|
+
"int4",
|
|
77
|
+
"int8",
|
|
78
|
+
"tinyint",
|
|
79
|
+
"smallint",
|
|
80
|
+
"mediumint",
|
|
81
|
+
"bigint",
|
|
82
|
+
"hugeint",
|
|
83
|
+
"utinyint",
|
|
84
|
+
"usmallint",
|
|
85
|
+
"uinteger",
|
|
86
|
+
"ubigint",
|
|
87
|
+
"uhugeint",
|
|
88
|
+
"serial",
|
|
89
|
+
"smallserial",
|
|
90
|
+
"bigserial",
|
|
91
|
+
}
|
|
92
|
+
_DECIMAL = {"decimal", "numeric", "number", "money", "smallmoney"}
|
|
93
|
+
_FLOAT = {"float", "float4", "float8", "real", "double", "double precision", "binary_float", "binary_double"}
|
|
94
|
+
_BOOLEAN = {"boolean", "bool"}
|
|
95
|
+
_TIMESTAMP = {
|
|
96
|
+
"timestamp",
|
|
97
|
+
"timestamp without time zone",
|
|
98
|
+
"datetime",
|
|
99
|
+
"datetime2",
|
|
100
|
+
"smalldatetime",
|
|
101
|
+
"timestamp_s",
|
|
102
|
+
"timestamp_ms",
|
|
103
|
+
"timestamp_ns",
|
|
104
|
+
}
|
|
105
|
+
_TIMESTAMPTZ = {"timestamptz", "timestamp with time zone", "timestamp with local time zone", "datetimeoffset"}
|
|
106
|
+
_TIME = {"time", "time without time zone"}
|
|
107
|
+
_BINARY = {
|
|
108
|
+
"bytea",
|
|
109
|
+
"blob",
|
|
110
|
+
"tinyblob",
|
|
111
|
+
"mediumblob",
|
|
112
|
+
"longblob",
|
|
113
|
+
"binary",
|
|
114
|
+
"varbinary",
|
|
115
|
+
"binary varying",
|
|
116
|
+
"image",
|
|
117
|
+
"raw",
|
|
118
|
+
"long raw",
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
_PARAMS = re.compile(r"\(([^)]*)\)")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def kind_for_type_name(type_name: str, *, oracle: bool = False) -> type[DataType] | None:
|
|
125
|
+
"""The execsql type for a column declared as *type_name* (as the DBMS's catalog spells it).
|
|
126
|
+
|
|
127
|
+
*oracle*: Oracle's ``DATE`` holds a time of day too, so it is a timestamp there.
|
|
128
|
+
"""
|
|
129
|
+
name = type_name.strip().lower()
|
|
130
|
+
if not name or "[" in name or name.endswith(" array"):
|
|
131
|
+
return None
|
|
132
|
+
params = _PARAMS.search(name)
|
|
133
|
+
args = [a.strip() for a in params.group(1).split(",")] if params else []
|
|
134
|
+
base = " ".join(_PARAMS.sub(" ", name).replace(" unsigned", " ").replace(" zerofill", " ").split())
|
|
135
|
+
if base == "tinyint" and args == ["1"]:
|
|
136
|
+
return DT_Boolean # MySQL's BOOLEAN
|
|
137
|
+
if base == "bit":
|
|
138
|
+
# SQL Server's BIT; a wider BIT(n) is a bit string.
|
|
139
|
+
return DT_Boolean if args in ([], ["1"]) else None
|
|
140
|
+
if base in _TEXT:
|
|
141
|
+
return DT_Text
|
|
142
|
+
if base in _INTEGER:
|
|
143
|
+
return DT_Long
|
|
144
|
+
if base in _DECIMAL:
|
|
145
|
+
if args[1:] == ["0"] or len(args) == 1:
|
|
146
|
+
return DT_Long # DECIMAL(p,0), NUMERIC(p), NUMBER(p): whole numbers
|
|
147
|
+
return DT_Decimal
|
|
148
|
+
if base in _FLOAT:
|
|
149
|
+
return DT_Float
|
|
150
|
+
if base in _BOOLEAN:
|
|
151
|
+
return DT_Boolean
|
|
152
|
+
if base == "date":
|
|
153
|
+
return DT_Timestamp if oracle else DT_Date
|
|
154
|
+
if base in _TIMESTAMP:
|
|
155
|
+
return DT_Timestamp
|
|
156
|
+
if base in _TIMESTAMPTZ:
|
|
157
|
+
return DT_TimestampTZ
|
|
158
|
+
if base in _TIME:
|
|
159
|
+
return DT_Time
|
|
160
|
+
if base in _BINARY:
|
|
161
|
+
return DT_Binary
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def kind_for_python_type(type_code: object, scale: object = None) -> type[DataType] | None:
|
|
166
|
+
"""The execsql type for a column a driver describes by the Python type it returns (pyodbc, firebird-driver).
|
|
167
|
+
|
|
168
|
+
Binary columns are left out: pyodbc describes a SQL Server
|
|
169
|
+
``datetimeoffset`` as ``bytearray`` too.
|
|
170
|
+
"""
|
|
171
|
+
if type_code is bool:
|
|
172
|
+
return DT_Boolean
|
|
173
|
+
if type_code is int:
|
|
174
|
+
return DT_Long
|
|
175
|
+
if type_code is float:
|
|
176
|
+
return DT_Float
|
|
177
|
+
if type_code is Decimal:
|
|
178
|
+
return DT_Long if scale == 0 else DT_Decimal
|
|
179
|
+
if type_code is str:
|
|
180
|
+
return DT_Text
|
|
181
|
+
if type_code is datetime.datetime:
|
|
182
|
+
return DT_Timestamp
|
|
183
|
+
if type_code is datetime.date:
|
|
184
|
+
return DT_Date
|
|
185
|
+
if type_code is datetime.time:
|
|
186
|
+
return DT_Time
|
|
187
|
+
return None
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# ---------------------------------------------------------------------------
|
|
191
|
+
# Converting a value for the column it goes into
|
|
192
|
+
# ---------------------------------------------------------------------------
|
|
193
|
+
|
|
194
|
+
_INTEGER_TEXT = re.compile(r"^\s*[+-]?[0-9]+\s*$")
|
|
195
|
+
_NUMBER_TEXT = re.compile(r"^\s*[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?\s*$")
|
|
196
|
+
# A whole number longer than this fits no integer column; it goes to the
|
|
197
|
+
# database as given, which rejects it, instead of being expanded here.
|
|
198
|
+
_MAX_INTEGER_DIGITS = 40
|
|
199
|
+
|
|
200
|
+
Converter = Callable[[Any], Any]
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _as_given(value: Any) -> Any:
|
|
204
|
+
return value
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _whole_number_converter(column: str, table: str, default: Converter) -> Converter:
|
|
208
|
+
"""Convert a value for integer column *column*: a fraction is an error, never rounded.
|
|
209
|
+
|
|
210
|
+
A value that is not a number (NULL, a date, a word) gets *default*, the
|
|
211
|
+
conversion it had before, and the database takes or rejects that.
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
def convert(value: Any) -> Any:
|
|
215
|
+
if isinstance(value, bool):
|
|
216
|
+
return default(value)
|
|
217
|
+
if isinstance(value, int):
|
|
218
|
+
return value
|
|
219
|
+
if isinstance(value, str):
|
|
220
|
+
if _INTEGER_TEXT.match(value):
|
|
221
|
+
return int(value)
|
|
222
|
+
if not _NUMBER_TEXT.match(value):
|
|
223
|
+
return default(value)
|
|
224
|
+
number = Decimal(value)
|
|
225
|
+
elif isinstance(value, Decimal):
|
|
226
|
+
number = value
|
|
227
|
+
elif isinstance(value, float):
|
|
228
|
+
if not math.isfinite(value):
|
|
229
|
+
return default(value)
|
|
230
|
+
number = Decimal(value)
|
|
231
|
+
else:
|
|
232
|
+
return default(value)
|
|
233
|
+
if not number.is_finite() or number.adjusted() >= _MAX_INTEGER_DIGITS:
|
|
234
|
+
return default(value)
|
|
235
|
+
if number == number.to_integral_value():
|
|
236
|
+
return int(number)
|
|
237
|
+
raise ErrInfo(
|
|
238
|
+
type="error",
|
|
239
|
+
other_msg=(
|
|
240
|
+
f"Can't load {value!r} into column {column} of {table}: the column holds whole numbers, "
|
|
241
|
+
"and the database would round the value."
|
|
242
|
+
),
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
return convert
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _true_false_as_number(default: Converter) -> Converter:
|
|
249
|
+
"""Convert a value scanned as true/false for a number column: 1 or 0, read as before (``yes`` is 1)."""
|
|
250
|
+
|
|
251
|
+
def convert(value: Any) -> Any:
|
|
252
|
+
flag = default(value)
|
|
253
|
+
return int(flag) if isinstance(flag, bool) else flag
|
|
254
|
+
|
|
255
|
+
return convert
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def existing_table_converter(
|
|
259
|
+
kind: type[DataType] | None,
|
|
260
|
+
scanned: type[DataType],
|
|
261
|
+
default: Converter,
|
|
262
|
+
column: str,
|
|
263
|
+
table: str,
|
|
264
|
+
) -> Converter:
|
|
265
|
+
"""How a value for *column* of existing table *table* is converted.
|
|
266
|
+
|
|
267
|
+
*kind* is the column's type (``None`` if execsql has none that fits),
|
|
268
|
+
*scanned* the type the source column scanned as, and *default* the
|
|
269
|
+
conversion to that scanned type, which a new table's values get.
|
|
270
|
+
"""
|
|
271
|
+
if kind is DT_Text:
|
|
272
|
+
return _as_given
|
|
273
|
+
if kind in (DT_Long, DT_Decimal, DT_Float) and scanned is DT_Boolean:
|
|
274
|
+
return _true_false_as_number(default)
|
|
275
|
+
if kind is DT_Long:
|
|
276
|
+
return _whole_number_converter(column, table, default)
|
|
277
|
+
return default
|
execsql/exporters/json.py
CHANGED
|
@@ -3,9 +3,11 @@ from __future__ import annotations
|
|
|
3
3
|
"""
|
|
4
4
|
JSON export for execsql.
|
|
5
5
|
|
|
6
|
-
Provides :func:`write_query_to_json` (standard JSON array of objects)
|
|
7
|
-
:func:`
|
|
8
|
-
|
|
6
|
+
Provides :func:`write_query_to_json` (standard JSON array of objects),
|
|
7
|
+
:func:`write_query_to_jsonl` (`JSON Lines <https://jsonlines.org/>`_, one
|
|
8
|
+
object per line) and :func:`write_query_to_json_ts` (JSON with a top-level
|
|
9
|
+
timestamp wrapper), all of which serialize a query result set to a file or
|
|
10
|
+
stream.
|
|
9
11
|
"""
|
|
10
12
|
|
|
11
13
|
import json
|
|
@@ -18,7 +20,13 @@ from execsql.models import DataTable, to_json_type
|
|
|
18
20
|
from execsql.utils.errors import exception_desc
|
|
19
21
|
from execsql.utils.fileio import filewriter_close
|
|
20
22
|
|
|
21
|
-
__all__ = ["write_query_to_json", "write_query_to_json_ts"]
|
|
23
|
+
__all__ = ["write_query_to_json", "write_query_to_json_ts", "write_query_to_jsonl"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _row_json(hdrs: list[str], row: Any) -> str:
|
|
27
|
+
"""One row as a compact JSON object; JSON and JSONL write rows alike."""
|
|
28
|
+
dictdata = dict(zip(hdrs, [str(v) if isinstance(v, str) else v for v in row]))
|
|
29
|
+
return json.dumps(dictdata, separators=(",", ":"), default=str)
|
|
22
30
|
|
|
23
31
|
|
|
24
32
|
def write_query_to_json(
|
|
@@ -60,14 +68,50 @@ def write_query_to_json(
|
|
|
60
68
|
else:
|
|
61
69
|
f.write(",\n")
|
|
62
70
|
first = False
|
|
63
|
-
|
|
64
|
-
jsondata = json.dumps(dictdata, separators=(",", ":"), default=str)
|
|
65
|
-
f.write(str(jsondata))
|
|
71
|
+
f.write(_row_json(uhdrs, row))
|
|
66
72
|
f.write("\n]\n")
|
|
67
73
|
finally:
|
|
68
74
|
f.close()
|
|
69
75
|
|
|
70
76
|
|
|
77
|
+
def write_query_to_jsonl(
|
|
78
|
+
select_stmt: str,
|
|
79
|
+
db: Any,
|
|
80
|
+
outfile: str,
|
|
81
|
+
append: bool = False,
|
|
82
|
+
desc: str | None = None,
|
|
83
|
+
zipfile: str | None = None,
|
|
84
|
+
) -> None:
|
|
85
|
+
"""Execute a SELECT and write the result set as JSON Lines: one JSON object per row, one row per line.
|
|
86
|
+
|
|
87
|
+
Values are written as :func:`write_query_to_json` writes them. A line
|
|
88
|
+
break inside a value is escaped (``\\n``), so each row stays on one line.
|
|
89
|
+
With *append* the rows are added to the end of the file, which stays
|
|
90
|
+
valid JSON Lines.
|
|
91
|
+
"""
|
|
92
|
+
conf = _state.conf
|
|
93
|
+
try:
|
|
94
|
+
hdrs, rows = db.select_rowsource(select_stmt)
|
|
95
|
+
except ErrInfo:
|
|
96
|
+
raise
|
|
97
|
+
except Exception as e:
|
|
98
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
99
|
+
f: Any
|
|
100
|
+
if zipfile is None:
|
|
101
|
+
filewriter_close(outfile)
|
|
102
|
+
from execsql.utils.fileio import EncodedFile
|
|
103
|
+
|
|
104
|
+
f = EncodedFile(outfile, conf.output_encoding).open("at" if append else "wt")
|
|
105
|
+
else:
|
|
106
|
+
f = ZipWriter(zipfile, outfile, append)
|
|
107
|
+
try:
|
|
108
|
+
uhdrs = [str(h) for h in hdrs]
|
|
109
|
+
for row in rows:
|
|
110
|
+
f.write(_row_json(uhdrs, row) + "\n")
|
|
111
|
+
finally:
|
|
112
|
+
f.close()
|
|
113
|
+
|
|
114
|
+
|
|
71
115
|
def write_query_to_json_ts(
|
|
72
116
|
select_stmt: str,
|
|
73
117
|
db: Any,
|
execsql/exporters/protocol.py
CHANGED
execsql/importers/base.py
CHANGED
|
@@ -142,7 +142,7 @@ def import_data_table(
|
|
|
142
142
|
)
|
|
143
143
|
import_cols = hdrs
|
|
144
144
|
try:
|
|
145
|
-
db.populate_table(schemaname, tablename, data, import_cols, get_ts)
|
|
145
|
+
db.populate_table(schemaname, tablename, data, import_cols, get_ts, existing_table=not is_new)
|
|
146
146
|
db.commit()
|
|
147
147
|
except ErrInfo:
|
|
148
148
|
raise
|
execsql/importers/csv.py
CHANGED
|
@@ -91,7 +91,7 @@ def importtable(
|
|
|
91
91
|
other_msg=f"Non-existent table name ({tablename}) used with the IMPORT_FILE metacommand",
|
|
92
92
|
)
|
|
93
93
|
try:
|
|
94
|
-
db.import_tabular_file(schemaname, tablename, inf, skipheader=True)
|
|
94
|
+
db.import_tabular_file(schemaname, tablename, inf, skipheader=True, existing_table=not is_new)
|
|
95
95
|
db.commit()
|
|
96
96
|
except ErrInfo:
|
|
97
97
|
raise
|
execsql/metacommands/__init__.py
CHANGED
|
@@ -474,13 +474,29 @@ JSON_VARIANT_FORMATS = ["JSON_TS", "JSON_TABLESCHEMA"]
|
|
|
474
474
|
QUERY_EXPORT_FORMATS = (
|
|
475
475
|
DELIMITED_FORMATS
|
|
476
476
|
+ TEXT_FORMATS
|
|
477
|
-
+ [
|
|
477
|
+
+ [
|
|
478
|
+
"ODS",
|
|
479
|
+
"XLSX",
|
|
480
|
+
"JSON",
|
|
481
|
+
"JSONL",
|
|
482
|
+
"HTML",
|
|
483
|
+
"CGI-HTML",
|
|
484
|
+
"VALUES",
|
|
485
|
+
"LATEX",
|
|
486
|
+
"RAW",
|
|
487
|
+
"B64",
|
|
488
|
+
"FEATHER",
|
|
489
|
+
"YAML",
|
|
490
|
+
"MARKDOWN",
|
|
491
|
+
"MD",
|
|
492
|
+
]
|
|
478
493
|
)
|
|
479
494
|
TABLE_EXPORT_FORMATS = (
|
|
480
495
|
DELIMITED_FORMATS
|
|
481
496
|
+ TEXT_FORMATS
|
|
482
497
|
+ [
|
|
483
498
|
"JSON",
|
|
499
|
+
"JSONL",
|
|
484
500
|
"XML",
|
|
485
501
|
"VALUES",
|
|
486
502
|
"HTML",
|
execsql/metacommands/data.py
CHANGED
|
@@ -21,7 +21,7 @@ What's actually here:
|
|
|
21
21
|
``x_set_counter``.
|
|
22
22
|
- IMPORT / scanning configuration metacommands (the ``CONFIG …`` form):
|
|
23
23
|
``x_empty_strings``, ``x_trim_strings``, ``x_replace_newlines``,
|
|
24
|
-
``x_empty_rows``, ``x_only_strings``, ``x_boolean_int``,
|
|
24
|
+
``x_empty_rows``, ``x_exact_decimals``, ``x_only_strings``, ``x_boolean_int``,
|
|
25
25
|
``x_boolean_words``, ``x_fold_col_hdrs``, ``x_trim_col_hdrs``,
|
|
26
26
|
``x_clean_col_hdrs``, ``x_del_empty_cols``, ``x_create_col_hdrs``,
|
|
27
27
|
``x_dedup_col_hdrs``, ``x_import_common_cols_only``, ``x_max_int``.
|
|
@@ -301,6 +301,12 @@ def x_only_strings(**kwargs: Any) -> None:
|
|
|
301
301
|
return None
|
|
302
302
|
|
|
303
303
|
|
|
304
|
+
def x_exact_decimals(**kwargs: Any) -> None:
|
|
305
|
+
flag = kwargs["yesno"].lower()
|
|
306
|
+
_state.conf.exact_decimals = flag in ("yes", "on", "true", "1")
|
|
307
|
+
return None
|
|
308
|
+
|
|
309
|
+
|
|
304
310
|
def x_boolean_int(**kwargs: Any) -> None:
|
|
305
311
|
flag = kwargs["yesno"].lower()
|
|
306
312
|
_state.conf.boolean_int = flag in ("yes", "on", "true", "1")
|
execsql/metacommands/dispatch.py
CHANGED
|
@@ -64,6 +64,7 @@ from execsql.metacommands.data import (
|
|
|
64
64
|
x_dedup_col_hdrs,
|
|
65
65
|
x_empty_rows,
|
|
66
66
|
x_empty_strings,
|
|
67
|
+
x_exact_decimals,
|
|
67
68
|
x_fold_col_hdrs,
|
|
68
69
|
x_import_common_cols_only,
|
|
69
70
|
x_max_int,
|
|
@@ -768,6 +769,12 @@ def build_dispatch_table() -> MetaCommandList:
|
|
|
768
769
|
description="EMPTY_ROWS",
|
|
769
770
|
category="config_option",
|
|
770
771
|
)
|
|
772
|
+
mcl.add(
|
|
773
|
+
r"^\s*CONFIG\s+EXACT_DECIMALS\s+(?P<yesno>YES|NO|ON|OFF|TRUE|FALSE|0|1)\s*$",
|
|
774
|
+
x_exact_decimals,
|
|
775
|
+
description="EXACT_DECIMALS",
|
|
776
|
+
category="config_option",
|
|
777
|
+
)
|
|
771
778
|
mcl.add(
|
|
772
779
|
r"^\s*CONFIG\s+ONLY_STRINGS\s+(?P<yesno>YES|NO|ON|OFF|TRUE|FALSE|0|1)\s*$",
|
|
773
780
|
x_only_strings,
|
|
@@ -27,7 +27,7 @@ from execsql.exporters.delimited import write_delimited_file
|
|
|
27
27
|
from execsql.exporters.duckdb import write_query_to_duckdb
|
|
28
28
|
from execsql.exporters.feather import write_query_to_feather, write_query_to_hdf5
|
|
29
29
|
from execsql.exporters.html import write_query_to_cgi_html, write_query_to_html
|
|
30
|
-
from execsql.exporters.json import write_query_to_json, write_query_to_json_ts
|
|
30
|
+
from execsql.exporters.json import write_query_to_json, write_query_to_json_ts, write_query_to_jsonl
|
|
31
31
|
from execsql.exporters.latex import write_query_to_latex
|
|
32
32
|
from execsql.exporters.ods import write_queries_to_ods, write_query_to_ods
|
|
33
33
|
from execsql.exporters.xlsx import write_queries_to_xlsx, write_query_to_xlsx
|
|
@@ -123,6 +123,8 @@ def _dispatch_format(
|
|
|
123
123
|
write_query_to_xml(select_stmt, xml_table, db, outfile, append, desc=description, zipfile=zipfilename)
|
|
124
124
|
elif filefmt == "json":
|
|
125
125
|
write_query_to_json(select_stmt, db, outfile, append, desc=description, zipfile=zipfilename)
|
|
126
|
+
elif filefmt == "jsonl":
|
|
127
|
+
write_query_to_jsonl(select_stmt, db, outfile, append, desc=description, zipfile=zipfilename)
|
|
126
128
|
elif filefmt in ("json_ts", "json_tableschema"):
|
|
127
129
|
write_query_to_json_ts(select_stmt, db, outfile, append, not notype, desc=description, zipfile=zipfilename)
|
|
128
130
|
elif filefmt == "values":
|
|
@@ -158,7 +158,7 @@ def x_copy(**kwargs: Any) -> None:
|
|
|
158
158
|
return get_ts_tablespec
|
|
159
159
|
|
|
160
160
|
try:
|
|
161
|
-
db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts)
|
|
161
|
+
db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
|
|
162
162
|
db2.commit()
|
|
163
163
|
except BaseException:
|
|
164
164
|
_close_rowsource(rows_to_close)
|
|
@@ -245,7 +245,7 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
245
245
|
return get_ts_tablespec
|
|
246
246
|
|
|
247
247
|
try:
|
|
248
|
-
db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts)
|
|
248
|
+
db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
|
|
249
249
|
db2.commit()
|
|
250
250
|
except BaseException:
|
|
251
251
|
_close_rowsource(rows_to_close)
|
|
@@ -254,6 +254,7 @@ _METACOMMANDS: tuple[Metacommand, ...] = (
|
|
|
254
254
|
"CONFIG DELETE_EMPTY_COLUMNS YES|NO",
|
|
255
255
|
"CONFIG EMPTY_ROWS YES|NO",
|
|
256
256
|
"CONFIG EMPTY_STRINGS YES|NO",
|
|
257
|
+
"CONFIG EXACT_DECIMALS YES|NO",
|
|
257
258
|
"CONFIG EXPORT_ROW_BUFFER <n>",
|
|
258
259
|
"CONFIG FOLD_COLUMN_HEADERS NO|LOWER|UPPER",
|
|
259
260
|
"CONFIG GUI_LEVEL <n>",
|
execsql/models.py
CHANGED
|
@@ -49,6 +49,10 @@ __all__ = [
|
|
|
49
49
|
|
|
50
50
|
ColumnType = tuple[str, type[DataType], int | None, bool, int | None, int | None]
|
|
51
51
|
|
|
52
|
+
# Every decimal of up to this many significant digits reads back unchanged
|
|
53
|
+
# from an IEEE double (DBL_DIG).
|
|
54
|
+
_DOUBLE_EXACT_DIGITS = 15
|
|
55
|
+
|
|
52
56
|
|
|
53
57
|
def _text_length(value: Any) -> int:
|
|
54
58
|
"""Return the length of *value* written as text (a column may turn out to be text)."""
|
|
@@ -71,9 +75,14 @@ class Column:
|
|
|
71
75
|
"""Accumulate match counts and length statistics for a single data type."""
|
|
72
76
|
|
|
73
77
|
# Accumulates the count of matches for each data type, plus the maximum length if appropriate.
|
|
74
|
-
def __init__(self, data_type_obj: DataType) -> None:
|
|
75
|
-
"""Initialise the accumulator for the given data type.
|
|
78
|
+
def __init__(self, data_type_obj: DataType, *, exact_decimals: bool = False) -> None:
|
|
79
|
+
"""Initialise the accumulator for the given data type.
|
|
80
|
+
|
|
81
|
+
*exact_decimals*: the ``exact_decimals`` setting; see
|
|
82
|
+
:meth:`chooses_float`.
|
|
83
|
+
"""
|
|
76
84
|
self.dt = data_type_obj
|
|
85
|
+
self.exact_decimals = exact_decimals
|
|
77
86
|
self.failed = False
|
|
78
87
|
self.count = 0
|
|
79
88
|
self.maxlen = 0
|
|
@@ -127,6 +136,23 @@ class Column:
|
|
|
127
136
|
self.maxscale = max(self.dt.scale, self.maxscale)
|
|
128
137
|
self.max_int_digits = max(int_digits, self.max_int_digits)
|
|
129
138
|
|
|
139
|
+
def chooses_float(self) -> bool:
|
|
140
|
+
"""Whether a decimal column gives way to a float column.
|
|
141
|
+
|
|
142
|
+
A column of decimals with different scales (``1.50``, ``2``,
|
|
143
|
+
``-3.25``) is a float unless the ``exact_decimals`` setting is on,
|
|
144
|
+
or a value has more digits than a double holds exactly: every
|
|
145
|
+
decimal of up to 15 significant digits reads back unchanged from
|
|
146
|
+
a double, so such a column loses nothing, and a longer one
|
|
147
|
+
(``1234567890.1234567891``) stays ``DECIMAL(p, s)``.
|
|
148
|
+
"""
|
|
149
|
+
return (
|
|
150
|
+
self.varscale
|
|
151
|
+
and not self.exact_decimals
|
|
152
|
+
and self.maxprecision is not None
|
|
153
|
+
and self.maxprecision <= _DOUBLE_EXACT_DIGITS
|
|
154
|
+
)
|
|
155
|
+
|
|
130
156
|
def __init__(self, colname: str, infer_strings: bool = True) -> None:
|
|
131
157
|
"""Create a column characteriser for the named column.
|
|
132
158
|
|
|
@@ -169,7 +195,7 @@ class Column:
|
|
|
169
195
|
self.Accum(DT_Boolean()),
|
|
170
196
|
self.Accum(DT_Integer()),
|
|
171
197
|
self.Accum(DT_Long()),
|
|
172
|
-
self.Accum(DT_Decimal()),
|
|
198
|
+
self.Accum(DT_Decimal(), exact_decimals=conf.exact_decimals),
|
|
173
199
|
self.Accum(DT_Float()),
|
|
174
200
|
self.Accum(DT_Character()),
|
|
175
201
|
self.Accum(DT_Varchar()),
|
|
@@ -262,7 +288,7 @@ class Column:
|
|
|
262
288
|
break
|
|
263
289
|
else:
|
|
264
290
|
if ac.dt.precspec:
|
|
265
|
-
if ac.dt.precision is not None and ac.dt.scale is not None:
|
|
291
|
+
if ac.dt.precision is not None and ac.dt.scale is not None and not ac.chooses_float():
|
|
266
292
|
sel_type = ac
|
|
267
293
|
break
|
|
268
294
|
else:
|
execsql/script/executor.py
CHANGED
|
@@ -85,6 +85,14 @@ _DEFER_RX = re.compile(r"!\{([$@&~#+]?\w+)\}!")
|
|
|
85
85
|
_VARLIKE = re.compile(r"!![$@&~#]?\w+!!", re.I)
|
|
86
86
|
|
|
87
87
|
|
|
88
|
+
def _warn_unsubstituted(cmd: str) -> None:
|
|
89
|
+
"""Log a command that still looks like it holds a variable; print it too if ``write_warnings`` is on."""
|
|
90
|
+
from execsql.utils.errors import write_warning
|
|
91
|
+
from execsql.utils.fileio import redact
|
|
92
|
+
|
|
93
|
+
write_warning(f"There is a potential un-substituted variable in the command\n {redact(cmd)}")
|
|
94
|
+
|
|
95
|
+
|
|
88
96
|
def _stack_localvars(ctx: RuntimeContext) -> SubVarSet | None:
|
|
89
97
|
"""Build the merged ``~`` local + ``#`` param overlay for the current scope.
|
|
90
98
|
|
|
@@ -254,11 +262,7 @@ def _exec_sql(
|
|
|
254
262
|
effective_locals = _stack_localvars(ctx) or localvars
|
|
255
263
|
cmd = substitute_vars(text, effective_locals, ctx=ctx)
|
|
256
264
|
if _VARLIKE.search(cmd):
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
output.write(
|
|
260
|
-
f"Warning: There is a potential un-substituted variable in the command\n {redact(cmd)}\n",
|
|
261
|
-
)
|
|
265
|
+
_warn_unsubstituted(cmd)
|
|
262
266
|
e = None
|
|
263
267
|
result = None
|
|
264
268
|
try:
|
|
@@ -323,11 +327,7 @@ def _exec_metacommand(
|
|
|
323
327
|
assert subvars is not None
|
|
324
328
|
|
|
325
329
|
if _VARLIKE.search(cmd):
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
output.write(
|
|
329
|
-
f"Warning: There is a potential un-substituted variable in the command\n {redact(cmd)}\n",
|
|
330
|
-
)
|
|
330
|
+
_warn_unsubstituted(cmd)
|
|
331
331
|
e = None
|
|
332
332
|
try:
|
|
333
333
|
applies, result = metacommandlist.eval(cmd)
|
execsql/script/variables.py
CHANGED
|
@@ -317,6 +317,10 @@ class SubVarSet:
|
|
|
317
317
|
Uses case-insensitive string search instead of per-variable regex
|
|
318
318
|
compilation to avoid O(N) ``re.compile`` calls on every invocation.
|
|
319
319
|
"""
|
|
320
|
+
# Every token below starts with one of these, so text without them
|
|
321
|
+
# can't match; skip the scan over every defined variable.
|
|
322
|
+
if "!!" not in command_str and "!'!" not in command_str and '!"!' not in command_str:
|
|
323
|
+
return command_str, False
|
|
320
324
|
cmd_lower = command_str.lower()
|
|
321
325
|
|
|
322
326
|
def _check_nul(name: str, value: str) -> None:
|
execsql/types.py
CHANGED
|
@@ -855,8 +855,8 @@ dbt_dsn.name_datatype(DT_Binary, "LONGBINARY")
|
|
|
855
855
|
|
|
856
856
|
dbt_mysql = DbType("MySQL")
|
|
857
857
|
dbt_mysql.name_datatype(DT_TimestampTZ, "varchar", True, "char")
|
|
858
|
-
dbt_mysql.name_datatype(DT_Timestamp, "datetime"
|
|
859
|
-
dbt_mysql.name_datatype(DT_Date, "date"
|
|
858
|
+
dbt_mysql.name_datatype(DT_Timestamp, "datetime")
|
|
859
|
+
dbt_mysql.name_datatype(DT_Date, "date")
|
|
860
860
|
dbt_mysql.name_datatype(DT_Time, "time")
|
|
861
861
|
dbt_mysql.name_datatype(DT_Integer, "integer", False, "signed integer")
|
|
862
862
|
dbt_mysql.name_datatype(DT_Long, "bigint", False, "signed integer")
|
execsql/utils/errors.py
CHANGED
|
@@ -274,8 +274,8 @@ def write_warning(warning_msg: str, *, always: bool = False) -> None:
|
|
|
274
274
|
warning_msg: The warning text to emit.
|
|
275
275
|
always: When ``True``, always write to stderr regardless of the
|
|
276
276
|
``conf.write_warnings`` setting. Use this for structural warnings
|
|
277
|
-
(e.g. IF-level mismatch
|
|
278
|
-
|
|
277
|
+
(e.g. IF-level mismatch) that should always be visible. When
|
|
278
|
+
``False`` (default), stderr output is gated by
|
|
279
279
|
``conf.write_warnings``.
|
|
280
280
|
"""
|
|
281
281
|
if _state.exec_log is not None:
|