execsql2 2.33.0__py3-none-any.whl → 2.35.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. execsql/api.py +1 -0
  2. execsql/config.py +2 -0
  3. execsql/data/execsql.conf.template +7 -0
  4. execsql/db/base.py +71 -8
  5. execsql/db/duckdb.py +6 -0
  6. execsql/db/mysql.py +54 -2
  7. execsql/db/oracle.py +36 -0
  8. execsql/db/postgres.py +17 -1
  9. execsql/db/sqlite.py +7 -1
  10. execsql/db/sqlserver.py +24 -0
  11. execsql/db/target_types.py +277 -0
  12. execsql/exporters/json.py +51 -7
  13. execsql/exporters/protocol.py +1 -0
  14. execsql/importers/base.py +1 -1
  15. execsql/importers/csv.py +1 -1
  16. execsql/metacommands/__init__.py +17 -1
  17. execsql/metacommands/data.py +7 -1
  18. execsql/metacommands/dispatch.py +7 -0
  19. execsql/metacommands/io_export.py +3 -1
  20. execsql/metacommands/io_fileops.py +2 -2
  21. execsql/metacommands/reference.py +1 -0
  22. execsql/models.py +30 -4
  23. execsql/script/executor.py +10 -10
  24. execsql/script/variables.py +4 -0
  25. execsql/types.py +2 -2
  26. execsql/utils/errors.py +2 -2
  27. {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/METADATA +3 -3
  28. {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/RECORD +45 -44
  29. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/README.md +0 -0
  30. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
  31. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
  32. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
  33. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/md_compare.sql +0 -0
  34. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
  35. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
  36. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
  37. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
  38. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/script_template.sql +0 -0
  39. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
  40. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
  41. {execsql2-2.33.0.data → execsql2-2.35.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
  42. {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/WHEEL +0 -0
  43. {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/entry_points.txt +0 -0
  44. {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/licenses/LICENSE.txt +0 -0
  45. {execsql2-2.33.0.dist-info → execsql2-2.35.0.dist-info}/licenses/NOTICE +0 -0
@@ -0,0 +1,277 @@
1
+ """The execsql data type of each column of a table that IMPORT or COPY loads.
2
+
3
+ A load into a table that already exists converts each value by the type of
4
+ the column it goes into, not by the type scanned from the source data
5
+ (issue #100). The adapters report each column's type: by the name their
6
+ catalog gives it (PostgreSQL, MySQL, SQL Server, DuckDB, Oracle), or by the
7
+ Python type their driver describes it with (Access, Firebird, ODBC DSNs).
8
+ The functions here turn that
9
+ into one of execsql's data types, or ``None`` when execsql has no
10
+ conversion that fits (arrays, JSON, UUIDs, enums, geometry, intervals, …).
11
+
12
+ Only the types that change how a value is converted are told apart: every
13
+ character type is :class:`DT_Text`, every integer type :class:`DT_Long`,
14
+ and so on. A ``DECIMAL``/``NUMERIC``/``NUMBER`` with scale 0 holds whole
15
+ numbers only, so it is :class:`DT_Long` too.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import datetime
21
+ import math
22
+ import re
23
+ from collections.abc import Callable
24
+ from decimal import Decimal
25
+ from typing import Any, TypeAlias
26
+
27
+ from execsql.exceptions import ErrInfo
28
+ from execsql.types import (
29
+ DataType,
30
+ DT_Binary,
31
+ DT_Boolean,
32
+ DT_Date,
33
+ DT_Decimal,
34
+ DT_Float,
35
+ DT_Long,
36
+ DT_Text,
37
+ DT_Time,
38
+ DT_Timestamp,
39
+ DT_TimestampTZ,
40
+ )
41
+
42
+ __all__ = ["ColumnKinds", "existing_table_converter", "kind_for_python_type", "kind_for_type_name"]
43
+
44
+ #: Lower-cased column name -> the execsql type values for it convert to, or None.
45
+ #: A string, not ``dict[...]``: on Python 3.10 that object passes
46
+ #: ``isinstance(x, type)``, so code that scans the adapter modules for classes
47
+ #: would take it for one.
48
+ ColumnKinds: TypeAlias = "dict[str, type[DataType] | None]"
49
+
50
+ _TEXT = {
51
+ "char",
52
+ "character",
53
+ "varchar",
54
+ "character varying",
55
+ "varchar2",
56
+ "nchar",
57
+ "national character",
58
+ "nvarchar",
59
+ "nvarchar2",
60
+ "national character varying",
61
+ "bpchar",
62
+ "text",
63
+ "ntext",
64
+ "tinytext",
65
+ "mediumtext",
66
+ "longtext",
67
+ "string",
68
+ "clob",
69
+ "nclob",
70
+ }
71
+ _INTEGER = {
72
+ "integer",
73
+ "int",
74
+ "int1",
75
+ "int2",
76
+ "int4",
77
+ "int8",
78
+ "tinyint",
79
+ "smallint",
80
+ "mediumint",
81
+ "bigint",
82
+ "hugeint",
83
+ "utinyint",
84
+ "usmallint",
85
+ "uinteger",
86
+ "ubigint",
87
+ "uhugeint",
88
+ "serial",
89
+ "smallserial",
90
+ "bigserial",
91
+ }
92
+ _DECIMAL = {"decimal", "numeric", "number", "money", "smallmoney"}
93
+ _FLOAT = {"float", "float4", "float8", "real", "double", "double precision", "binary_float", "binary_double"}
94
+ _BOOLEAN = {"boolean", "bool"}
95
+ _TIMESTAMP = {
96
+ "timestamp",
97
+ "timestamp without time zone",
98
+ "datetime",
99
+ "datetime2",
100
+ "smalldatetime",
101
+ "timestamp_s",
102
+ "timestamp_ms",
103
+ "timestamp_ns",
104
+ }
105
+ _TIMESTAMPTZ = {"timestamptz", "timestamp with time zone", "timestamp with local time zone", "datetimeoffset"}
106
+ _TIME = {"time", "time without time zone"}
107
+ _BINARY = {
108
+ "bytea",
109
+ "blob",
110
+ "tinyblob",
111
+ "mediumblob",
112
+ "longblob",
113
+ "binary",
114
+ "varbinary",
115
+ "binary varying",
116
+ "image",
117
+ "raw",
118
+ "long raw",
119
+ }
120
+
121
+ _PARAMS = re.compile(r"\(([^)]*)\)")
122
+
123
+
124
+ def kind_for_type_name(type_name: str, *, oracle: bool = False) -> type[DataType] | None:
125
+ """The execsql type for a column declared as *type_name* (as the DBMS's catalog spells it).
126
+
127
+ *oracle*: Oracle's ``DATE`` holds a time of day too, so it is a timestamp there.
128
+ """
129
+ name = type_name.strip().lower()
130
+ if not name or "[" in name or name.endswith(" array"):
131
+ return None
132
+ params = _PARAMS.search(name)
133
+ args = [a.strip() for a in params.group(1).split(",")] if params else []
134
+ base = " ".join(_PARAMS.sub(" ", name).replace(" unsigned", " ").replace(" zerofill", " ").split())
135
+ if base == "tinyint" and args == ["1"]:
136
+ return DT_Boolean # MySQL's BOOLEAN
137
+ if base == "bit":
138
+ # SQL Server's BIT; a wider BIT(n) is a bit string.
139
+ return DT_Boolean if args in ([], ["1"]) else None
140
+ if base in _TEXT:
141
+ return DT_Text
142
+ if base in _INTEGER:
143
+ return DT_Long
144
+ if base in _DECIMAL:
145
+ if args[1:] == ["0"] or len(args) == 1:
146
+ return DT_Long # DECIMAL(p,0), NUMERIC(p), NUMBER(p): whole numbers
147
+ return DT_Decimal
148
+ if base in _FLOAT:
149
+ return DT_Float
150
+ if base in _BOOLEAN:
151
+ return DT_Boolean
152
+ if base == "date":
153
+ return DT_Timestamp if oracle else DT_Date
154
+ if base in _TIMESTAMP:
155
+ return DT_Timestamp
156
+ if base in _TIMESTAMPTZ:
157
+ return DT_TimestampTZ
158
+ if base in _TIME:
159
+ return DT_Time
160
+ if base in _BINARY:
161
+ return DT_Binary
162
+ return None
163
+
164
+
165
+ def kind_for_python_type(type_code: object, scale: object = None) -> type[DataType] | None:
166
+ """The execsql type for a column a driver describes by the Python type it returns (pyodbc, firebird-driver).
167
+
168
+ Binary columns are left out: pyodbc describes a SQL Server
169
+ ``datetimeoffset`` as ``bytearray`` too.
170
+ """
171
+ if type_code is bool:
172
+ return DT_Boolean
173
+ if type_code is int:
174
+ return DT_Long
175
+ if type_code is float:
176
+ return DT_Float
177
+ if type_code is Decimal:
178
+ return DT_Long if scale == 0 else DT_Decimal
179
+ if type_code is str:
180
+ return DT_Text
181
+ if type_code is datetime.datetime:
182
+ return DT_Timestamp
183
+ if type_code is datetime.date:
184
+ return DT_Date
185
+ if type_code is datetime.time:
186
+ return DT_Time
187
+ return None
188
+
189
+
190
+ # ---------------------------------------------------------------------------
191
+ # Converting a value for the column it goes into
192
+ # ---------------------------------------------------------------------------
193
+
194
+ _INTEGER_TEXT = re.compile(r"^\s*[+-]?[0-9]+\s*$")
195
+ _NUMBER_TEXT = re.compile(r"^\s*[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?\s*$")
196
+ # A whole number longer than this fits no integer column; it goes to the
197
+ # database as given, which rejects it, instead of being expanded here.
198
+ _MAX_INTEGER_DIGITS = 40
199
+
200
+ Converter = Callable[[Any], Any]
201
+
202
+
203
+ def _as_given(value: Any) -> Any:
204
+ return value
205
+
206
+
207
+ def _whole_number_converter(column: str, table: str, default: Converter) -> Converter:
208
+ """Convert a value for integer column *column*: a fraction is an error, never rounded.
209
+
210
+ A value that is not a number (NULL, a date, a word) gets *default*, the
211
+ conversion it had before, and the database takes or rejects that.
212
+ """
213
+
214
+ def convert(value: Any) -> Any:
215
+ if isinstance(value, bool):
216
+ return default(value)
217
+ if isinstance(value, int):
218
+ return value
219
+ if isinstance(value, str):
220
+ if _INTEGER_TEXT.match(value):
221
+ return int(value)
222
+ if not _NUMBER_TEXT.match(value):
223
+ return default(value)
224
+ number = Decimal(value)
225
+ elif isinstance(value, Decimal):
226
+ number = value
227
+ elif isinstance(value, float):
228
+ if not math.isfinite(value):
229
+ return default(value)
230
+ number = Decimal(value)
231
+ else:
232
+ return default(value)
233
+ if not number.is_finite() or number.adjusted() >= _MAX_INTEGER_DIGITS:
234
+ return default(value)
235
+ if number == number.to_integral_value():
236
+ return int(number)
237
+ raise ErrInfo(
238
+ type="error",
239
+ other_msg=(
240
+ f"Can't load {value!r} into column {column} of {table}: the column holds whole numbers, "
241
+ "and the database would round the value."
242
+ ),
243
+ )
244
+
245
+ return convert
246
+
247
+
248
+ def _true_false_as_number(default: Converter) -> Converter:
249
+ """Convert a value scanned as true/false for a number column: 1 or 0, read as before (``yes`` is 1)."""
250
+
251
+ def convert(value: Any) -> Any:
252
+ flag = default(value)
253
+ return int(flag) if isinstance(flag, bool) else flag
254
+
255
+ return convert
256
+
257
+
258
+ def existing_table_converter(
259
+ kind: type[DataType] | None,
260
+ scanned: type[DataType],
261
+ default: Converter,
262
+ column: str,
263
+ table: str,
264
+ ) -> Converter:
265
+ """How a value for *column* of existing table *table* is converted.
266
+
267
+ *kind* is the column's type (``None`` if execsql has none that fits),
268
+ *scanned* the type the source column scanned as, and *default* the
269
+ conversion to that scanned type, which a new table's values get.
270
+ """
271
+ if kind is DT_Text:
272
+ return _as_given
273
+ if kind in (DT_Long, DT_Decimal, DT_Float) and scanned is DT_Boolean:
274
+ return _true_false_as_number(default)
275
+ if kind is DT_Long:
276
+ return _whole_number_converter(column, table, default)
277
+ return default
execsql/exporters/json.py CHANGED
@@ -3,9 +3,11 @@ from __future__ import annotations
3
3
  """
4
4
  JSON export for execsql.
5
5
 
6
- Provides :func:`write_query_to_json` (standard JSON array of objects) and
7
- :func:`write_query_to_json_ts` (JSON with a top-level timestamp wrapper),
8
- both of which serialize a query result set to a file or stream.
6
+ Provides :func:`write_query_to_json` (standard JSON array of objects),
7
+ :func:`write_query_to_jsonl` (`JSON Lines <https://jsonlines.org/>`_, one
8
+ object per line) and :func:`write_query_to_json_ts` (JSON with a top-level
9
+ timestamp wrapper), all of which serialize a query result set to a file or
10
+ stream.
9
11
  """
10
12
 
11
13
  import json
@@ -18,7 +20,13 @@ from execsql.models import DataTable, to_json_type
18
20
  from execsql.utils.errors import exception_desc
19
21
  from execsql.utils.fileio import filewriter_close
20
22
 
21
- __all__ = ["write_query_to_json", "write_query_to_json_ts"]
23
+ __all__ = ["write_query_to_json", "write_query_to_json_ts", "write_query_to_jsonl"]
24
+
25
+
26
+ def _row_json(hdrs: list[str], row: Any) -> str:
27
+ """One row as a compact JSON object; JSON and JSONL write rows alike."""
28
+ dictdata = dict(zip(hdrs, [str(v) if isinstance(v, str) else v for v in row]))
29
+ return json.dumps(dictdata, separators=(",", ":"), default=str)
22
30
 
23
31
 
24
32
  def write_query_to_json(
@@ -60,14 +68,50 @@ def write_query_to_json(
60
68
  else:
61
69
  f.write(",\n")
62
70
  first = False
63
- dictdata = dict(zip(uhdrs, [str(v) if isinstance(v, str) else v for v in row]))
64
- jsondata = json.dumps(dictdata, separators=(",", ":"), default=str)
65
- f.write(str(jsondata))
71
+ f.write(_row_json(uhdrs, row))
66
72
  f.write("\n]\n")
67
73
  finally:
68
74
  f.close()
69
75
 
70
76
 
77
+ def write_query_to_jsonl(
78
+ select_stmt: str,
79
+ db: Any,
80
+ outfile: str,
81
+ append: bool = False,
82
+ desc: str | None = None,
83
+ zipfile: str | None = None,
84
+ ) -> None:
85
+ """Execute a SELECT and write the result set as JSON Lines: one JSON object per row, one row per line.
86
+
87
+ Values are written as :func:`write_query_to_json` writes them. A line
88
+ break inside a value is escaped (``\\n``), so each row stays on one line.
89
+ With *append* the rows are added to the end of the file, which stays
90
+ valid JSON Lines.
91
+ """
92
+ conf = _state.conf
93
+ try:
94
+ hdrs, rows = db.select_rowsource(select_stmt)
95
+ except ErrInfo:
96
+ raise
97
+ except Exception as e:
98
+ raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
99
+ f: Any
100
+ if zipfile is None:
101
+ filewriter_close(outfile)
102
+ from execsql.utils.fileio import EncodedFile
103
+
104
+ f = EncodedFile(outfile, conf.output_encoding).open("at" if append else "wt")
105
+ else:
106
+ f = ZipWriter(zipfile, outfile, append)
107
+ try:
108
+ uhdrs = [str(h) for h in hdrs]
109
+ for row in rows:
110
+ f.write(_row_json(uhdrs, row) + "\n")
111
+ finally:
112
+ f.close()
113
+
114
+
71
115
  def write_query_to_json_ts(
72
116
  select_stmt: str,
73
117
  db: Any,
@@ -51,6 +51,7 @@ class QueryExporter(Protocol):
51
51
  Conforming functions
52
52
  --------------------
53
53
  - ``write_query_to_json``
54
+ - ``write_query_to_jsonl``
54
55
  - ``write_query_to_html``
55
56
  - ``write_query_to_cgi_html``
56
57
  - ``write_query_to_latex``
execsql/importers/base.py CHANGED
@@ -142,7 +142,7 @@ def import_data_table(
142
142
  )
143
143
  import_cols = hdrs
144
144
  try:
145
- db.populate_table(schemaname, tablename, data, import_cols, get_ts)
145
+ db.populate_table(schemaname, tablename, data, import_cols, get_ts, existing_table=not is_new)
146
146
  db.commit()
147
147
  except ErrInfo:
148
148
  raise
execsql/importers/csv.py CHANGED
@@ -91,7 +91,7 @@ def importtable(
91
91
  other_msg=f"Non-existent table name ({tablename}) used with the IMPORT_FILE metacommand",
92
92
  )
93
93
  try:
94
- db.import_tabular_file(schemaname, tablename, inf, skipheader=True)
94
+ db.import_tabular_file(schemaname, tablename, inf, skipheader=True, existing_table=not is_new)
95
95
  db.commit()
96
96
  except ErrInfo:
97
97
  raise
@@ -474,13 +474,29 @@ JSON_VARIANT_FORMATS = ["JSON_TS", "JSON_TABLESCHEMA"]
474
474
  QUERY_EXPORT_FORMATS = (
475
475
  DELIMITED_FORMATS
476
476
  + TEXT_FORMATS
477
- + ["ODS", "XLSX", "JSON", "HTML", "CGI-HTML", "VALUES", "LATEX", "RAW", "B64", "FEATHER", "YAML", "MARKDOWN", "MD"]
477
+ + [
478
+ "ODS",
479
+ "XLSX",
480
+ "JSON",
481
+ "JSONL",
482
+ "HTML",
483
+ "CGI-HTML",
484
+ "VALUES",
485
+ "LATEX",
486
+ "RAW",
487
+ "B64",
488
+ "FEATHER",
489
+ "YAML",
490
+ "MARKDOWN",
491
+ "MD",
492
+ ]
478
493
  )
479
494
  TABLE_EXPORT_FORMATS = (
480
495
  DELIMITED_FORMATS
481
496
  + TEXT_FORMATS
482
497
  + [
483
498
  "JSON",
499
+ "JSONL",
484
500
  "XML",
485
501
  "VALUES",
486
502
  "HTML",
@@ -21,7 +21,7 @@ What's actually here:
21
21
  ``x_set_counter``.
22
22
  - IMPORT / scanning configuration metacommands (the ``CONFIG …`` form):
23
23
  ``x_empty_strings``, ``x_trim_strings``, ``x_replace_newlines``,
24
- ``x_empty_rows``, ``x_only_strings``, ``x_boolean_int``,
24
+ ``x_empty_rows``, ``x_exact_decimals``, ``x_only_strings``, ``x_boolean_int``,
25
25
  ``x_boolean_words``, ``x_fold_col_hdrs``, ``x_trim_col_hdrs``,
26
26
  ``x_clean_col_hdrs``, ``x_del_empty_cols``, ``x_create_col_hdrs``,
27
27
  ``x_dedup_col_hdrs``, ``x_import_common_cols_only``, ``x_max_int``.
@@ -301,6 +301,12 @@ def x_only_strings(**kwargs: Any) -> None:
301
301
  return None
302
302
 
303
303
 
304
+ def x_exact_decimals(**kwargs: Any) -> None:
305
+ flag = kwargs["yesno"].lower()
306
+ _state.conf.exact_decimals = flag in ("yes", "on", "true", "1")
307
+ return None
308
+
309
+
304
310
  def x_boolean_int(**kwargs: Any) -> None:
305
311
  flag = kwargs["yesno"].lower()
306
312
  _state.conf.boolean_int = flag in ("yes", "on", "true", "1")
@@ -64,6 +64,7 @@ from execsql.metacommands.data import (
64
64
  x_dedup_col_hdrs,
65
65
  x_empty_rows,
66
66
  x_empty_strings,
67
+ x_exact_decimals,
67
68
  x_fold_col_hdrs,
68
69
  x_import_common_cols_only,
69
70
  x_max_int,
@@ -768,6 +769,12 @@ def build_dispatch_table() -> MetaCommandList:
768
769
  description="EMPTY_ROWS",
769
770
  category="config_option",
770
771
  )
772
+ mcl.add(
773
+ r"^\s*CONFIG\s+EXACT_DECIMALS\s+(?P<yesno>YES|NO|ON|OFF|TRUE|FALSE|0|1)\s*$",
774
+ x_exact_decimals,
775
+ description="EXACT_DECIMALS",
776
+ category="config_option",
777
+ )
771
778
  mcl.add(
772
779
  r"^\s*CONFIG\s+ONLY_STRINGS\s+(?P<yesno>YES|NO|ON|OFF|TRUE|FALSE|0|1)\s*$",
773
780
  x_only_strings,
@@ -27,7 +27,7 @@ from execsql.exporters.delimited import write_delimited_file
27
27
  from execsql.exporters.duckdb import write_query_to_duckdb
28
28
  from execsql.exporters.feather import write_query_to_feather, write_query_to_hdf5
29
29
  from execsql.exporters.html import write_query_to_cgi_html, write_query_to_html
30
- from execsql.exporters.json import write_query_to_json, write_query_to_json_ts
30
+ from execsql.exporters.json import write_query_to_json, write_query_to_json_ts, write_query_to_jsonl
31
31
  from execsql.exporters.latex import write_query_to_latex
32
32
  from execsql.exporters.ods import write_queries_to_ods, write_query_to_ods
33
33
  from execsql.exporters.xlsx import write_queries_to_xlsx, write_query_to_xlsx
@@ -123,6 +123,8 @@ def _dispatch_format(
123
123
  write_query_to_xml(select_stmt, xml_table, db, outfile, append, desc=description, zipfile=zipfilename)
124
124
  elif filefmt == "json":
125
125
  write_query_to_json(select_stmt, db, outfile, append, desc=description, zipfile=zipfilename)
126
+ elif filefmt == "jsonl":
127
+ write_query_to_jsonl(select_stmt, db, outfile, append, desc=description, zipfile=zipfilename)
126
128
  elif filefmt in ("json_ts", "json_tableschema"):
127
129
  write_query_to_json_ts(select_stmt, db, outfile, append, not notype, desc=description, zipfile=zipfilename)
128
130
  elif filefmt == "values":
@@ -158,7 +158,7 @@ def x_copy(**kwargs: Any) -> None:
158
158
  return get_ts_tablespec
159
159
 
160
160
  try:
161
- db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts)
161
+ db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
162
162
  db2.commit()
163
163
  except BaseException:
164
164
  _close_rowsource(rows_to_close)
@@ -245,7 +245,7 @@ def x_copy_query(**kwargs: Any) -> None:
245
245
  return get_ts_tablespec
246
246
 
247
247
  try:
248
- db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts)
248
+ db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
249
249
  db2.commit()
250
250
  except BaseException:
251
251
  _close_rowsource(rows_to_close)
@@ -254,6 +254,7 @@ _METACOMMANDS: tuple[Metacommand, ...] = (
254
254
  "CONFIG DELETE_EMPTY_COLUMNS YES|NO",
255
255
  "CONFIG EMPTY_ROWS YES|NO",
256
256
  "CONFIG EMPTY_STRINGS YES|NO",
257
+ "CONFIG EXACT_DECIMALS YES|NO",
257
258
  "CONFIG EXPORT_ROW_BUFFER <n>",
258
259
  "CONFIG FOLD_COLUMN_HEADERS NO|LOWER|UPPER",
259
260
  "CONFIG GUI_LEVEL <n>",
execsql/models.py CHANGED
@@ -49,6 +49,10 @@ __all__ = [
49
49
 
50
50
  ColumnType = tuple[str, type[DataType], int | None, bool, int | None, int | None]
51
51
 
52
+ # Every decimal of up to this many significant digits reads back unchanged
53
+ # from an IEEE double (DBL_DIG).
54
+ _DOUBLE_EXACT_DIGITS = 15
55
+
52
56
 
53
57
  def _text_length(value: Any) -> int:
54
58
  """Return the length of *value* written as text (a column may turn out to be text)."""
@@ -71,9 +75,14 @@ class Column:
71
75
  """Accumulate match counts and length statistics for a single data type."""
72
76
 
73
77
  # Accumulates the count of matches for each data type, plus the maximum length if appropriate.
74
- def __init__(self, data_type_obj: DataType) -> None:
75
- """Initialise the accumulator for the given data type."""
78
+ def __init__(self, data_type_obj: DataType, *, exact_decimals: bool = False) -> None:
79
+ """Initialise the accumulator for the given data type.
80
+
81
+ *exact_decimals*: the ``exact_decimals`` setting; see
82
+ :meth:`chooses_float`.
83
+ """
76
84
  self.dt = data_type_obj
85
+ self.exact_decimals = exact_decimals
77
86
  self.failed = False
78
87
  self.count = 0
79
88
  self.maxlen = 0
@@ -127,6 +136,23 @@ class Column:
127
136
  self.maxscale = max(self.dt.scale, self.maxscale)
128
137
  self.max_int_digits = max(int_digits, self.max_int_digits)
129
138
 
139
+ def chooses_float(self) -> bool:
140
+ """Whether a decimal column gives way to a float column.
141
+
142
+ A column of decimals with different scales (``1.50``, ``2``,
143
+ ``-3.25``) is a float unless the ``exact_decimals`` setting is on,
144
+ or a value has more digits than a double holds exactly: every
145
+ decimal of up to 15 significant digits reads back unchanged from
146
+ a double, so such a column loses nothing, and a longer one
147
+ (``1234567890.1234567891``) stays ``DECIMAL(p, s)``.
148
+ """
149
+ return (
150
+ self.varscale
151
+ and not self.exact_decimals
152
+ and self.maxprecision is not None
153
+ and self.maxprecision <= _DOUBLE_EXACT_DIGITS
154
+ )
155
+
130
156
  def __init__(self, colname: str, infer_strings: bool = True) -> None:
131
157
  """Create a column characteriser for the named column.
132
158
 
@@ -169,7 +195,7 @@ class Column:
169
195
  self.Accum(DT_Boolean()),
170
196
  self.Accum(DT_Integer()),
171
197
  self.Accum(DT_Long()),
172
- self.Accum(DT_Decimal()),
198
+ self.Accum(DT_Decimal(), exact_decimals=conf.exact_decimals),
173
199
  self.Accum(DT_Float()),
174
200
  self.Accum(DT_Character()),
175
201
  self.Accum(DT_Varchar()),
@@ -262,7 +288,7 @@ class Column:
262
288
  break
263
289
  else:
264
290
  if ac.dt.precspec:
265
- if ac.dt.precision is not None and ac.dt.scale is not None:
291
+ if ac.dt.precision is not None and ac.dt.scale is not None and not ac.chooses_float():
266
292
  sel_type = ac
267
293
  break
268
294
  else:
@@ -85,6 +85,14 @@ _DEFER_RX = re.compile(r"!\{([$@&~#+]?\w+)\}!")
85
85
  _VARLIKE = re.compile(r"!![$@&~#]?\w+!!", re.I)
86
86
 
87
87
 
88
+ def _warn_unsubstituted(cmd: str) -> None:
89
+ """Log a command that still looks like it holds a variable; print it too if ``write_warnings`` is on."""
90
+ from execsql.utils.errors import write_warning
91
+ from execsql.utils.fileio import redact
92
+
93
+ write_warning(f"There is a potential un-substituted variable in the command\n {redact(cmd)}")
94
+
95
+
88
96
  def _stack_localvars(ctx: RuntimeContext) -> SubVarSet | None:
89
97
  """Build the merged ``~`` local + ``#`` param overlay for the current scope.
90
98
 
@@ -254,11 +262,7 @@ def _exec_sql(
254
262
  effective_locals = _stack_localvars(ctx) or localvars
255
263
  cmd = substitute_vars(text, effective_locals, ctx=ctx)
256
264
  if _VARLIKE.search(cmd):
257
- from execsql.utils.fileio import redact
258
-
259
- output.write(
260
- f"Warning: There is a potential un-substituted variable in the command\n {redact(cmd)}\n",
261
- )
265
+ _warn_unsubstituted(cmd)
262
266
  e = None
263
267
  result = None
264
268
  try:
@@ -323,11 +327,7 @@ def _exec_metacommand(
323
327
  assert subvars is not None
324
328
 
325
329
  if _VARLIKE.search(cmd):
326
- from execsql.utils.fileio import redact
327
-
328
- output.write(
329
- f"Warning: There is a potential un-substituted variable in the command\n {redact(cmd)}\n",
330
- )
330
+ _warn_unsubstituted(cmd)
331
331
  e = None
332
332
  try:
333
333
  applies, result = metacommandlist.eval(cmd)
@@ -317,6 +317,10 @@ class SubVarSet:
317
317
  Uses case-insensitive string search instead of per-variable regex
318
318
  compilation to avoid O(N) ``re.compile`` calls on every invocation.
319
319
  """
320
+ # Every token below starts with one of these, so text without them
321
+ # can't match; skip the scan over every defined variable.
322
+ if "!!" not in command_str and "!'!" not in command_str and '!"!' not in command_str:
323
+ return command_str, False
320
324
  cmd_lower = command_str.lower()
321
325
 
322
326
  def _check_nul(name: str, value: str) -> None:
execsql/types.py CHANGED
@@ -855,8 +855,8 @@ dbt_dsn.name_datatype(DT_Binary, "LONGBINARY")
855
855
 
856
856
  dbt_mysql = DbType("MySQL")
857
857
  dbt_mysql.name_datatype(DT_TimestampTZ, "varchar", True, "char")
858
- dbt_mysql.name_datatype(DT_Timestamp, "datetime", conv_mod_fn=lambda x: x if x is not None else "")
859
- dbt_mysql.name_datatype(DT_Date, "date", conv_mod_fn=lambda x: x if x is not None else "")
858
+ dbt_mysql.name_datatype(DT_Timestamp, "datetime")
859
+ dbt_mysql.name_datatype(DT_Date, "date")
860
860
  dbt_mysql.name_datatype(DT_Time, "time")
861
861
  dbt_mysql.name_datatype(DT_Integer, "integer", False, "signed integer")
862
862
  dbt_mysql.name_datatype(DT_Long, "bigint", False, "signed integer")
execsql/utils/errors.py CHANGED
@@ -274,8 +274,8 @@ def write_warning(warning_msg: str, *, always: bool = False) -> None:
274
274
  warning_msg: The warning text to emit.
275
275
  always: When ``True``, always write to stderr regardless of the
276
276
  ``conf.write_warnings`` setting. Use this for structural warnings
277
- (e.g. IF-level mismatch, unsubstituted variables) that should always
278
- be visible. When ``False`` (default), stderr output is gated by
277
+ (e.g. IF-level mismatch) that should always be visible. When
278
+ ``False`` (default), stderr output is gated by
279
279
  ``conf.write_warnings``.
280
280
  """
281
281
  if _state.exec_log is not None: