execsql2 2.34.0__tar.gz → 2.35.0__tar.gz

{execsql2-2.34.0 → execsql2-2.35.0}/CHANGELOG.md

@@ -13,6 +13,22 @@ ______________________________________________________________________
 
 ______________________________________________________________________
 
+## [2.35.0] - 2026-10-18
+
+### Added
+
+- **`exact_decimals`** (`[input]`) and **`CONFIG EXACT_DECIMALS YES|NO`**: with Yes, `IMPORT` and `COPY … TO NEW` give a column of decimals with different numbers of places an exact `DECIMAL(p,s)` type instead of a floating-point one.
+
+### Changed
+
+- **A column of decimals with different numbers of places** (`1.50`, `2`, `-3.25`) gets a floating-point type again when `IMPORT` or `COPY … TO NEW` creates a table (it was `DECIMAL(p,s)` from 2.22.0). A column with a value of more than 15 significant digits stays `DECIMAL(p,s)`, so no digit is lost. Set `exact_decimals` to keep every such column exact.
+
+### Fixed
+
+- **The "potential un-substituted variable" warning** is printed only when `write_warnings` is on, to stderr as `**** Warning …`, as documented; it was printed to stdout for every command with an undefined `!!var!!`. It is always written to `execsql.log`.
+
+______________________________________________________________________
+
 ## [2.34.0] - 2026-10-18
 
 ### Added

{execsql2-2.34.0 → execsql2-2.35.0}/CONTRIBUTING.md

@@ -64,6 +64,19 @@ just test-templates-down   # removes the containers
 
 SQL Server is included only when an ODBC driver for it is installed. A template bug the tests have found but not yet fixed is marked with `known_bug(...)`, a strict expected failure: the fix makes the test pass, which fails the run until the marker is removed with it.
 
+### Script behavior corpus
+
+`tests/corpus/` checks that whole scripts do under execsql2 what they did under upstream execsql 1.130.1: the same exit status, console output, files written and table contents. Each case is a directory in `tests/corpus/cases/` holding `script.sql` and the files it reads. Upstream's results are recorded in `tests/corpus/results/`, so the normal test run compares against them without upstream installed. A variable a case checks is `WRITE`n to a file, because upstream has no way to read variables after a run.
+
+To add a case, create its directory, then record what upstream does (this installs execsql 1.130.1 from PyPI under Python 3.12 into `~/.cache/execsql-corpus/` on first use):
+
+```bash
+uv run python -m tests.corpus.record my_new_case
+uv run pytest tests/corpus
+```
+
+Use only syntax upstream accepts; read `_execsql/execsql.py` when unsure. A case must not open a GUI dialog (`HALT "message"`, `PROMPT`) or loop forever under upstream: both hang the recording. A difference from upstream fails the test unless `tests/corpus/changes.json` lists it with both values, a `status` (`accepted` for an upstream bug execsql2 fixed on purpose, `open` for one not yet decided) and the reason.
+
 ## Building the Docs
 
 ```bash

{execsql2-2.34.0 → execsql2-2.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: execsql2
-Version: 2.34.0
+Version: 2.35.0
 Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
 Project-URL: Homepage, https://execsql2.readthedocs.io/en/stable/
 Project-URL: Repository, https://github.com/geocoug/execsql
@@ -441,7 +441,7 @@ execsql format --no-sql --in-place scripts/
 ```yaml
 repos:
   - repo: https://github.com/geocoug/execsql
-    rev: v2.34.0
+    rev: v2.35.0
     hooks:
       - id: execsql-format
       - id: execsql-lint

{execsql2-2.34.0 → execsql2-2.35.0}/README.md

@@ -321,7 +321,7 @@ execsql format --no-sql --in-place scripts/
 ```yaml
 repos:
   - repo: https://github.com/geocoug/execsql
-    rev: v2.34.0
+    rev: v2.35.0
     hooks:
       - id: execsql-format
       - id: execsql-lint

{execsql2-2.34.0 → execsql2-2.35.0}/docs/about/divergence.md

@@ -92,6 +92,7 @@ New options in `execsql.conf`:
 | -------------------------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `use_keyring`                    | `[connect]`   | Use the OS keyring for credential storage (default: `yes`).                                                                                                                                                      |
 | `trust_server_certificate`       | `[connect]`   | SQL Server: accept a self-signed server certificate (ODBC Driver 18 verifies it by default); also `?trust_server_certificate=yes` on an `mssql://` URL. Default: `no`.                                           |
+| `exact_decimals`                 | `[input]`     | Give a column of decimals with different numbers of places an exact `DECIMAL(p,s)` type instead of a float (default: `no`); also `CONFIG EXACT_DECIMALS`.                                                        |
 | `show_progress`                  | `[input]`     | Enable Rich progress bar for IMPORT (default: `no`).                                                                                                                                                             |
 | `import_progress_interval`       | `[input]`     | Log a status line every N rows during IMPORT (default: `0`).                                                                                                                                                     |
 | `gui_framework`                  | `[interface]` | GUI backend: `tkinter` (default) or `textual` (terminal UI).                                                                                                                                                     |
@@ -257,6 +258,7 @@ All 33 mutable runtime globals in `state.py` have been consolidated into a `Runt
 
 - **`$HOSTNAME`** — Network name of the machine running execsql (`platform.node()`). Useful for log messages and environment detection.
 - **`$SYSTEM_CMD_PID`** — New system variable set to the PID of the background process when `SHELL … CONTINUE` is used.
+- **`!'!var!'!` adds the quotes** — `!'!name!'!` becomes a complete string literal: the value is wrapped in single quotes and its apostrophes doubled (`O'Brien` → `'O''Brien'`), so a script writes `where author = !'!name!'!`. Upstream only doubled the apostrophes, so its scripts wrote the quotes themselves (`'!'!name!'!'`); in execsql2 that form has two sets of quotes and fails. Remove the outer quotes when moving such a script.
 - **Cycle detection** — `substitute_vars()` raises an error after 100 iterations to prevent infinite loops when variables reference each other cyclically. Upstream had no protection.
 - **O(1) substitution** — Variable substitution uses a single combined regex and dict lookup instead of O(V) per-variable regex passes. Behavior is identical; performance is improved.
 - **Lazy `$RANDOM`/`$UUID`** — These system variables are now computed on first access rather than generated unconditionally for every statement. Behavior is identical when referenced; scripts that never reference them skip the computation entirely.
@@ -288,6 +290,7 @@ The legacy flat command-list engine (`_parse_script_lines` / `runscripts` / `Com
 
 - **Native INCLUDE handling** — The AST executor parses INCLUDE'd files with the AST parser and executes them through the tree-walking executor. Control flow structures (IF/LOOP/BATCH/SCRIPT) in included files are fully tree-driven, with correct deferred variable handling and source-span error reporting.
 - **Circular INCLUDE detection** — The executor tracks the full include chain and detects circular references (e.g. A includes B includes A), reporting the full chain in the error message. Upstream has no such detection.
+- **LOOP conditions are read on every pass** — The variables in a `LOOP WHILE` / `LOOP UNTIL` condition are substituted afresh before each pass, so `LOOP WHILE (IS_GT(10, !!n!!))` with `SUB_ADD n 1` in the body stops when `n` reaches 10. Upstream substituted them once, when the `LOOP` line ran, so a condition on a variable already defined never changed: the loop ran until a `BREAK`, or forever. Upstream scripts got per-pass conditions only from variables not yet defined when the loop started ("phantom variables") or from deferred `!{var}!` references, and both still work.
 - **BREAK outside LOOP is an error** — `BREAK` outside a loop block now raises an error (exit 1) instead of being silently ignored. This catches script bugs that upstream would not report.
 - **File writer is a thread, one per run** — `WRITE`, `TEE` and the other text-file metacommands hand their output to a background thread owned by the run's `RuntimeContext`, which `execsql.run()` stops before returning. Upstream uses one `multiprocessing` subprocess for the whole program; under the `spawn` start method (macOS, Windows) that re-imported the calling program, so a library caller without an `if __name__ == "__main__":` guard ran its script twice.
 - **Instance-scoped script registry** — Named SCRIPT blocks are stored on the `RuntimeContext` instance instead of a module-level dict, preventing cross-execution contamination.
@@ -435,6 +438,8 @@ These are behavioral changes driven by security or correctness issues in the ups
 | DuckDB float columns were single precision                         | `dbt_duckdb` mapped `DT_Float` to `REAL`. `DT_Float` is documented as IEEE double precision and a Python float always is one, so every float written to DuckDB was narrowed: `1234.567` was stored and returned as `1234.5670166015625`. PostgreSQL, SQLite, SQL Server, MySQL, Access, Firebird and DSN all map `DT_Float` to a double; DuckDB was the only backend that did not, and is now `DOUBLE`.                                                                                                                                                                                                                                                                                                                                                                                                            |
 | VALUES export wrote dates and timestamps unquoted                  | The literal renderer quoted `str` and emitted `NULL` for `None`, but sent everything else through `str()` bare. A `date` became `2026-09-24` in the generated `INSERT`, which is not a syntax error — it is a valid SQL expression evaluating to 1993, so PostgreSQL inserted that and reported nothing. A `timestamp` contains a space and failed outright. Quoting is now the default and only numbers are emitted bare, so a type the renderer has not been taught about is safe rather than silently wrong.                                                                                                                                                                                                                                                                                                    |
 | LaTeX export escaped only the underscore                           | Cells were written as `str(c).replace("_", r"\_")`, leaving every other LaTeX special raw. `&` is the column separator this exporter itself emits, so an ampersand in a value added a column and the `.tex` no longer compiled; `%` commented out the rest of the row. All of `\ & % $ # _ { } ~ ^` are now escaped, in both the header and the body.                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| `ON ERROR_HALT WRITE … TO` a file wrote nothing                    | Upstream passed the `(text, flag)` pair that variable substitution returns on as the text, so writing the message failed after the file was opened and the file stayed empty. The message is written.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| Mixed-scale decimals keep every digit                              | `IMPORT` / `COPY … TO NEW` gave a column of decimals with different numbers of places (`1.50`, `2`, `-3.25`) a floating-point type, which also held values a double cannot store exactly: `1234567890.1234567891` was stored as `1234567890.1234567`. Such a column is still a float, unless a value has more than 15 significant digits; then it is `DECIMAL(p,s)`. The `exact_decimals` setting makes every such column exact.                                                                                                                                                                                                                                                                                                                                                                                   |
 | LaTeX export wrote NULL as the literal "None"                      | `str(c)` was applied to every value including `None`, so a NULL appeared in the typeset table as the word None. It is now an empty cell, matching every other exporter.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
 | HTML export rendered zero as an empty cell                         | Cells were written as `html.escape(str(v)) if v else ''`. `0`, `False`, and `""` are all falsy, so a zero measurement rendered exactly like a NULL and the two could not be told apart in the output. The test is now `if v is not None`. Both `export_html` and `export_cgi_html` were affected.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | Markdown export broke the table on an embedded newline             | `_cell()` escaped pipes but not newlines, and a GFM table row is a single line: a value containing one ended the row mid-cell and left the rest of that row's values on a stray line outside the table entirely. Newlines are now written as `<br>`, which is how GFM expresses a line break inside a cell.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |

{execsql2-2.34.0 → execsql2-2.35.0}/docs/guides/formatter.md

@@ -272,7 +272,7 @@ select id,name,created_at from users where active = true order by name;
 ```yaml
 repos:
   - repo: https://github.com/geocoug/execsql
-    rev: v2.34.0
+    rev: v2.35.0
     hooks:
       - id: execsql-format
       - id: execsql-lint # optional: fail the commit on lint errors

{execsql2-2.34.0 → execsql2-2.35.0}/docs/reference/configuration.md

@@ -141,6 +141,10 @@ The section and property names that may be used in a configuration file are list
 
 :   Determines whether empty strings in the input are preserved or, alternatively, will be replaced by NULL. The property value should be either "Yes" or "No". The default, "Yes", indicates that empty strings are allowed. A value of "No" will cause all empty strings to be replaced by NULL. When this is set to "No", a string value consisting of a sequence of zero or more space characters will be considered to be an empty string. There is no command-line option corresponding to this configuration parameter, but the metacommand [CONFIG EMPTY_STRINGS](metacommands.md#empty_strings) can also be used to change this configuration item.
 
+`exact_decimals`
+
+:   Determines the data type of a column of decimal numbers with different numbers of decimal places, such as `1.50`, `2` and `-3.25`, when the [IMPORT](metacommands.md#import) or [COPY](metacommands.md#copy) metacommand creates a table. The default, "No", gives such a column a floating-point type (`double precision` on PostgreSQL, `double` on MySQL and DuckDB, `float` on SQL Server), unless a value has more than 15 significant digits, which a floating-point column cannot hold exactly: then the column is an exact decimal, so no digit is lost. "Yes" always gives such a column an exact decimal type wide enough for every value (`numeric(3,2)` for those values), for values such as money amounts that should be stored and compared exactly. A column whose values all have the same number of decimal places is an exact decimal either way. The metacommand [CONFIG EXACT_DECIMALS](metacommands.md#exact_decimals) can also be used to change this configuration item.
+
 `fold_column_headers` { #setting_fold_column_headers }
 
 :   Whether or not to fold (convert) the case of all column headers to lowercase or uppercase, or to leave them unchanged when data are [IMPORTed](metacommands.md#import). Valid values are "No" (the default), "Lower", and "Upper". Case does not matter in the specification.

{execsql2-2.34.0 → execsql2-2.35.0}/docs/reference/lint.md

@@ -158,7 +158,7 @@ The `execsql-lint` [pre-commit](https://pre-commit.com/) hook runs `execsql lint
 ```yaml
 repos:
   - repo: https://github.com/geocoug/execsql
-    rev: v2.34.0
+    rev: v2.35.0
     hooks:
       - id: execsql-lint
         args: [--ignore, V002] # optional: any execsql lint option

{execsql2-2.34.0 → execsql2-2.35.0}/docs/reference/metacommands.md

@@ -597,6 +597,21 @@ CONFIG MAX_INT <integer_value>
 Specifies the threshold between integer and bigint data types that is used by the [IMPORT](#import) and [COPY](#copy) metacommands when creating a new table. Any column with integer values less than or equal to this value (max_int) and greater than or equal to -1 × max_int - 1 will be considered to have an integer type. Any column with values outside this range will be considered to have a bigint type. The default value for max_int is 2147483647. The max_int value can also be altered using a configuration option. The bare form `MAX_INT <integer_value>`, without `CONFIG`, does the same.
 
 
+<a id="exact_decimals"></a>
+```
+CONFIG EXACT_DECIMALS YES|NO
+```
+
+Controls the data type the [IMPORT](#import) and [COPY](#copy) metacommands give a column of decimal numbers with different numbers of decimal places when they create a table. With "No" (the default), `1.50`, `2` and `-3.25` make a floating-point column; a column with a value of more than 15 significant digits, such as `1234567890.1234567891`, is an exact decimal even then, because a floating-point column would cut it. With "Yes" every such column is an exact decimal wide enough for every value, `numeric(3,2)` for the first example:
+
+```
+-- !x! CONFIG EXACT_DECIMALS YES
+-- !x! IMPORT TO NEW payments FROM payments.csv
+```
+
+A column whose values all have the same number of decimal places is an exact decimal either way. This can also be set with the `exact_decimals` [configuration setting](configuration.md#config_input).
+
+
 ```
 CONFIG ONLY_STRINGS YES|NO
 ```

{execsql2-2.34.0 → execsql2-2.35.0}/extras/vscode-execsql/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "execsql-syntax",
-  "version": "2.34.0",
+  "version": "2.35.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "execsql-syntax",
-      "version": "2.34.0",
+      "version": "2.35.0",
       "license": "GPL-3.0-or-later",
       "dependencies": {
         "vscode-languageclient": "10.1.2"

{execsql2-2.34.0 → execsql2-2.35.0}/extras/vscode-execsql/package.json

@@ -2,7 +2,7 @@
   "name": "execsql-syntax",
   "displayName": "execsql",
   "description": "Syntax highlighting, lint findings, completion, hover, navigation, quick fixes and formatting for execsql scripts.",
-  "version": "2.34.0",
+  "version": "2.35.0",
   "publisher": "geocoug",
   "license": "GPL-3.0-or-later",
   "icon": "icon.png",

{execsql2-2.34.0 → execsql2-2.35.0}/extras/vscode-execsql/syntaxes/execsql.tmLanguage.json

@@ -113,7 +113,7 @@
     },
     "config-option-names": {
       "comment": "Config option names used after the config keyword",
-      "match": "(?i)\\b(import_only_common_columns|console_wait_when_error|console_wait_when_done|create_column_headers|clean_column_headers|dao_flush_delay_secs|dedup_column_headers|delete_empty_columns|fold_column_headers|trim_column_headers|log_write_messages|export_row_buffer|import_row_buffer|make_export_dirs|replace_newlines|quote_all_text|write_warnings|boolean_words|empty_strings|hdf5_text_len|show_progress|zip_buffer_mb|log_datavars|only_strings|trim_strings|write_prefix|write_suffix|boolean_int|empty_rows|scan_lines|gui_level|log_sql|max_int)\\b",
+      "match": "(?i)\\b(import_only_common_columns|console_wait_when_error|console_wait_when_done|create_column_headers|clean_column_headers|dao_flush_delay_secs|dedup_column_headers|delete_empty_columns|fold_column_headers|trim_column_headers|log_write_messages|export_row_buffer|import_row_buffer|make_export_dirs|replace_newlines|exact_decimals|quote_all_text|write_warnings|boolean_words|empty_strings|hdf5_text_len|show_progress|zip_buffer_mb|log_datavars|only_strings|trim_strings|write_prefix|write_suffix|boolean_int|empty_rows|scan_lines|gui_level|log_sql|max_int)\\b",
       "name": "support.constant.config.execsql"
     },
     "export-formats": {

{execsql2-2.34.0 → execsql2-2.35.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "execsql2"
-version = "2.34.0"
+version = "2.35.0"
 description = "Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables."
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE.txt" }
@@ -188,7 +188,7 @@ line-ending = "auto"
 exclude = ["*.md"]
 
 [tool.bumpversion]
-current_version = "2.34.0"
+current_version = "2.35.0"
 commit = true
 tag = true
 tag_name = "v{new_version}"

{execsql2-2.34.0 → execsql2-2.35.0}/src/execsql/api.py

@@ -178,6 +178,7 @@ class _LibraryConfig:
         self.boolean_int = True
         self.boolean_words = False
         self.empty_strings = True
+        self.exact_decimals = False
         self.only_strings = False
         self.empty_rows = True
         self.del_empty_cols = False

{execsql2-2.34.0 → execsql2-2.35.0}/src/execsql/config.py

@@ -300,6 +300,7 @@ class ConfigData:
         self._get_bool(cp, self._INPUT_SECTION, "boolean_int", "boolean_int")
         self._get_bool(cp, self._INPUT_SECTION, "boolean_words", "boolean_words")
         self._get_bool(cp, self._INPUT_SECTION, "empty_strings", "empty_strings")
+        self._get_bool(cp, self._INPUT_SECTION, "exact_decimals", "exact_decimals")
         self._get_bool(cp, self._INPUT_SECTION, "only_strings", "only_strings")
         self._get_bool(cp, self._INPUT_SECTION, "empty_rows", "empty_rows")
         self._get_bool(cp, self._INPUT_SECTION, "delete_empty_columns", "del_empty_cols")
@@ -503,6 +504,7 @@ class ConfigData:
         self.boolean_int = True
         self.boolean_words = False
         self.empty_strings = True
+        self.exact_decimals = False
         self.only_strings = False
         self.empty_rows = True
         self.del_empty_cols = False

{execsql2-2.34.0 → execsql2-2.35.0}/src/execsql/data/execsql.conf.template

@@ -115,6 +115,13 @@
 # Values: Yes or No.  Default: Yes.
 #empty_strings=Yes
 
+# Whether a column of decimal numbers with different numbers of decimal places
+# (1.5, 2.25) gets an exact DECIMAL(precision, scale) type when IMPORT or COPY
+# creates a table.  With No it gets a floating-point type, unless a value has
+# more than 15 significant digits (then it is DECIMAL, so no digit is lost).
+# Values: Yes or No.  Default: No.
+#exact_decimals=No
+
 # Whether to fold column headers to lowercase, uppercase, or leave unchanged.
 # Values: No, Lower, or Upper.  Default: No.
 #fold_column_headers=No

{execsql2-2.34.0 → execsql2-2.35.0}/src/execsql/metacommands/data.py

@@ -21,7 +21,7 @@ What's actually here:
   ``x_set_counter``.
 - IMPORT / scanning configuration metacommands (the ``CONFIG …`` form):
   ``x_empty_strings``, ``x_trim_strings``, ``x_replace_newlines``,
-  ``x_empty_rows``, ``x_only_strings``, ``x_boolean_int``,
+  ``x_empty_rows``, ``x_exact_decimals``, ``x_only_strings``, ``x_boolean_int``,
   ``x_boolean_words``, ``x_fold_col_hdrs``, ``x_trim_col_hdrs``,
   ``x_clean_col_hdrs``, ``x_del_empty_cols``, ``x_create_col_hdrs``,
   ``x_dedup_col_hdrs``, ``x_import_common_cols_only``, ``x_max_int``.
@@ -301,6 +301,12 @@ def x_only_strings(**kwargs: Any) -> None:
     return None
 
 
+def x_exact_decimals(**kwargs: Any) -> None:
+    flag = kwargs["yesno"].lower()
+    _state.conf.exact_decimals = flag in ("yes", "on", "true", "1")
+    return None
+
+
 def x_boolean_int(**kwargs: Any) -> None:
     flag = kwargs["yesno"].lower()
     _state.conf.boolean_int = flag in ("yes", "on", "true", "1")

{execsql2-2.34.0 → execsql2-2.35.0}/src/execsql/metacommands/dispatch.py

@@ -64,6 +64,7 @@ from execsql.metacommands.data import (
     x_dedup_col_hdrs,
     x_empty_rows,
     x_empty_strings,
+    x_exact_decimals,
     x_fold_col_hdrs,
     x_import_common_cols_only,
     x_max_int,
@@ -768,6 +769,12 @@ def build_dispatch_table() -> MetaCommandList:
         description="EMPTY_ROWS",
         category="config_option",
     )
+    mcl.add(
+        r"^\s*CONFIG\s+EXACT_DECIMALS\s+(?P<yesno>YES|NO|ON|OFF|TRUE|FALSE|0|1)\s*$",
+        x_exact_decimals,
+        description="EXACT_DECIMALS",
+        category="config_option",
+    )
     mcl.add(
         r"^\s*CONFIG\s+ONLY_STRINGS\s+(?P<yesno>YES|NO|ON|OFF|TRUE|FALSE|0|1)\s*$",
         x_only_strings,

{execsql2-2.34.0 → execsql2-2.35.0}/src/execsql/metacommands/reference.py

@@ -254,6 +254,7 @@ _METACOMMANDS: tuple[Metacommand, ...] = (
             "CONFIG DELETE_EMPTY_COLUMNS YES|NO",
             "CONFIG EMPTY_ROWS YES|NO",
             "CONFIG EMPTY_STRINGS YES|NO",
+            "CONFIG EXACT_DECIMALS YES|NO",
             "CONFIG EXPORT_ROW_BUFFER <n>",
             "CONFIG FOLD_COLUMN_HEADERS NO|LOWER|UPPER",
             "CONFIG GUI_LEVEL <n>",

{execsql2-2.34.0 → execsql2-2.35.0}/src/execsql/models.py

@@ -49,6 +49,10 @@ __all__ = [
 
 ColumnType = tuple[str, type[DataType], int | None, bool, int | None, int | None]
 
+# Every decimal of up to this many significant digits reads back unchanged
+# from an IEEE double (DBL_DIG).
+_DOUBLE_EXACT_DIGITS = 15
+
 
 def _text_length(value: Any) -> int:
     """Return the length of *value* written as text (a column may turn out to be text)."""
@@ -71,9 +75,14 @@ class Column:
         """Accumulate match counts and length statistics for a single data type."""
 
         # Accumulates the count of matches for each data type, plus the maximum length if appropriate.
-        def __init__(self, data_type_obj: DataType) -> None:
-            """Initialise the accumulator for the given data type."""
+        def __init__(self, data_type_obj: DataType, *, exact_decimals: bool = False) -> None:
+            """Initialise the accumulator for the given data type.
+
+            *exact_decimals*: the ``exact_decimals`` setting; see
+            :meth:`chooses_float`.
+            """
             self.dt = data_type_obj
+            self.exact_decimals = exact_decimals
             self.failed = False
             self.count = 0
             self.maxlen = 0
@@ -127,6 +136,23 @@ class Column:
                     self.maxscale = max(self.dt.scale, self.maxscale)
                     self.max_int_digits = max(int_digits, self.max_int_digits)
 
+        def chooses_float(self) -> bool:
+            """Whether a decimal column gives way to a float column.
+
+            A column of decimals with different scales (``1.50``, ``2``,
+            ``-3.25``) is a float unless the ``exact_decimals`` setting is on,
+            or a value has more digits than a double holds exactly: every
+            decimal of up to 15 significant digits reads back unchanged from
+            a double, so such a column loses nothing, and a longer one
+            (``1234567890.1234567891``) stays ``DECIMAL(p, s)``.
+            """
+            return (
+                self.varscale
+                and not self.exact_decimals
+                and self.maxprecision is not None
+                and self.maxprecision <= _DOUBLE_EXACT_DIGITS
+            )
+
     def __init__(self, colname: str, infer_strings: bool = True) -> None:
         """Create a column characteriser for the named column.
 
@@ -169,7 +195,7 @@ class Column:
                 self.Accum(DT_Boolean()),
                 self.Accum(DT_Integer()),
                 self.Accum(DT_Long()),
-                self.Accum(DT_Decimal()),
+                self.Accum(DT_Decimal(), exact_decimals=conf.exact_decimals),
                 self.Accum(DT_Float()),
                 self.Accum(DT_Character()),
                 self.Accum(DT_Varchar()),
@@ -262,7 +288,7 @@ class Column:
                                 break
                     else:
                         if ac.dt.precspec:
-                            if ac.dt.precision is not None and ac.dt.scale is not None:
+                            if ac.dt.precision is not None and ac.dt.scale is not None and not ac.chooses_float():
                                 sel_type = ac
                                 break
                         else:

{execsql2-2.34.0 → execsql2-2.35.0}/src/execsql/script/executor.py

@@ -85,6 +85,14 @@ _DEFER_RX = re.compile(r"!\{([$@&~#+]?\w+)\}!")
 _VARLIKE = re.compile(r"!![$@&~#]?\w+!!", re.I)
 
 
+def _warn_unsubstituted(cmd: str) -> None:
+    """Log a command that still looks like it holds a variable; print it too if ``write_warnings`` is on."""
+    from execsql.utils.errors import write_warning
+    from execsql.utils.fileio import redact
+
+    write_warning(f"There is a potential un-substituted variable in the command\n     {redact(cmd)}")
+
+
 def _stack_localvars(ctx: RuntimeContext) -> SubVarSet | None:
     """Build the merged ``~`` local + ``#`` param overlay for the current scope.
 
@@ -254,11 +262,7 @@ def _exec_sql(
     effective_locals = _stack_localvars(ctx) or localvars
     cmd = substitute_vars(text, effective_locals, ctx=ctx)
     if _VARLIKE.search(cmd):
-        from execsql.utils.fileio import redact
-
-        output.write(
-            f"Warning: There is a potential un-substituted variable in the command\n     {redact(cmd)}\n",
-        )
+        _warn_unsubstituted(cmd)
     e = None
     result = None
     try:
@@ -323,11 +327,7 @@ def _exec_metacommand(
     assert subvars is not None
 
     if _VARLIKE.search(cmd):
-        from execsql.utils.fileio import redact
-
-        output.write(
-            f"Warning: There is a potential un-substituted variable in the command\n     {redact(cmd)}\n",
-        )
+        _warn_unsubstituted(cmd)
     e = None
     try:
         applies, result = metacommandlist.eval(cmd)

{execsql2-2.34.0 → execsql2-2.35.0}/src/execsql/utils/errors.py

@@ -274,8 +274,8 @@ def write_warning(warning_msg: str, *, always: bool = False) -> None:
         warning_msg: The warning text to emit.
         always: When ``True``, always write to stderr regardless of the
             ``conf.write_warnings`` setting.  Use this for structural warnings
-            (e.g. IF-level mismatch, unsubstituted variables) that should always
-            be visible.  When ``False`` (default), stderr output is gated by
+            (e.g. IF-level mismatch) that should always be visible.  When
+            ``False`` (default), stderr output is gated by
             ``conf.write_warnings``.
     """
     if _state.exec_log is not None:

{execsql2-2.34.0 → execsql2-2.35.0}/tests/conftest.py

@@ -38,6 +38,7 @@ def minimal_conf():
         max_int=2_147_483_647,
         # Column / DataTable
         only_strings=False,
+        exact_decimals=False,
         trim_strings=False,
         replace_newlines=False,
         empty_strings=True,

execsql2-2.35.0/tests/corpus/cases/batch_and_autocommit/script.sql

@@ -0,0 +1,16 @@
+-- BEGIN BATCH / ROLLBACK, BEGIN BATCH / END BATCH, and AUTOCOMMIT OFF.
+create table t (v text);
+-- !x! BEGIN BATCH
+insert into t values ('batch rolled back');
+-- !x! ROLLBACK
+-- !x! END BATCH
+-- !x! BEGIN BATCH
+insert into t values ('batch kept');
+-- !x! END BATCH
+-- !x! AUTOCOMMIT OFF
+insert into t values ('autocommit off, committed');
+commit;
+insert into t values ('autocommit off, rolled back');
+rollback;
+-- !x! AUTOCOMMIT ON
+insert into t values ('autocommit on');

execsql2-2.35.0/tests/corpus/cases/boolean_and_empty_config/data.csv

@@ -0,0 +1,4 @@
+zero_one,words,padded,maybe_empty
+1,yes,  a  ,x
+0,no, b,
+1,yes,"c "," "

execsql2-2.35.0/tests/corpus/cases/boolean_and_empty_config/script.sql

@@ -0,0 +1,13 @@
+-- Settings that change how imported values are read.
+-- !x! IMPORT TO NEW default_read FROM data.csv
+-- !x! CONFIG BOOLEAN_INT NO
+-- !x! IMPORT TO NEW boolean_int_off FROM data.csv
+-- !x! CONFIG BOOLEAN_INT YES
+-- !x! BOOLEAN_WORDS ON
+-- !x! IMPORT TO NEW boolean_words_on FROM data.csv
+-- !x! BOOLEAN_WORDS OFF
+-- !x! EMPTY_STRINGS NO
+-- !x! IMPORT TO NEW empty_strings_no FROM data.csv
+-- !x! EMPTY_STRINGS YES
+-- !x! TRIM_STRINGS YES
+-- !x! IMPORT TO NEW trimmed FROM data.csv

execsql2-2.35.0/tests/corpus/cases/copy_between_databases/script.sql

@@ -0,0 +1,17 @@
+-- CONNECT a second SQLite database and COPY between them.
+create table src (id integer, name text, amt real, flag integer);
+insert into src values (1, 'a', 1.5, 1), (2, 'b', null, 0), (3, '00701', 3, 1);
+-- !x! CONNECT TO SQLITE(FILE=other.db, NEW) AS other
+-- !x! COPY src FROM initial TO NEW copied IN other
+-- !x! COPY QUERY <<select id, name from src where id > 1;>> FROM initial TO NEW q IN other
+-- !x! USE other
+create table existing (id integer, name text, amt real, flag integer);
+-- !x! USE initial
+-- !x! COPY src FROM initial TO existing IN other
+-- !x! COPY copied FROM other TO NEW back IN initial
+-- !x! USE other
+-- !x! EXPORT copied TO copied.csv AS CSV
+-- !x! EXPORT q TO q.csv AS CSV
+-- !x! EXPORT existing TO existing.csv AS CSV
+-- !x! USE initial
+-- !x! DISCONNECT other

execsql2-2.35.0/tests/corpus/cases/counters/script.sql

@@ -0,0 +1,11 @@
+-- Counter variables advance on every reference.
+create table t (n integer);
+insert into t values (!!$COUNTER_1!!);
+insert into t values (!!$COUNTER_1!!);
+insert into t values (!!$COUNTER_2!!);
+-- !x! SET COUNTER 1 TO 10
+insert into t values (!!$COUNTER_1!!);
+-- !x! RESET COUNTER 1
+insert into t values (!!$COUNTER_1!!);
+-- !x! RESET COUNTERS
+insert into t values (!!$COUNTER_2!!);

execsql2-2.35.0/tests/corpus/cases/decimal_digits/long.csv

@@ -0,0 +1,3 @@
+v
+1234567890.1234567891
+1.5

execsql2-2.35.0/tests/corpus/cases/decimal_digits/script.sql

@@ -0,0 +1,3 @@
+-- The column type IMPORT gives decimals with different numbers of places.
+-- !x! WRITE CREATE_TABLE short FROM short.csv TO ddl.sql
+-- !x! WRITE CREATE_TABLE long FROM long.csv TO ddl.sql

execsql2-2.35.0/tests/corpus/cases/decimal_digits/short.csv

@@ -0,0 +1,4 @@
+v
+1.50
+2
+-3.25

execsql2-2.35.0/tests/corpus/cases/error_halt_off/script.sql

@@ -0,0 +1,13 @@
+-- With ERROR_HALT OFF a failed statement does not stop the script.
+create table t (id integer primary key);
+insert into t values (1);
+-- !x! ERROR_HALT OFF
+insert into t values (1);
+insert into no_such_table values (1);
+-- !x! ERROR_HALT ON
+insert into t values (2);
+-- !x! METACOMMAND_ERROR_HALT OFF
+-- !x! IMPORT TO t FROM missing.csv
+-- !x! METACOMMAND_ERROR_HALT ON
+insert into t values (3);
+-- !x! WRITE "reached the end" TO out.txt

execsql2-2.35.0/tests/corpus/cases/error_stops_script/script.sql

@@ -0,0 +1,7 @@
+-- A failed statement stops the script: what ran before stays, nothing after runs.
+create table t (id integer primary key);
+insert into t values (1);
+-- !x! WRITE "before" TO out.txt
+insert into t values (1);
+insert into t values (2);
+-- !x! WRITE "after" TO out.txt

execsql2-2.35.0/tests/corpus/cases/execute_script_args/script.sql

@@ -0,0 +1,21 @@
+-- BEGIN SCRIPT / EXECUTE SCRIPT with arguments, local variables and nesting.
+create table log (msg text);
+-- !x! BEGIN SCRIPT add_row (what, times)
+-- !x! SUB_LOCAL ~note from add_row
+insert into log values ('!!#what!! x !!#times!! (!!~note!!)');
+-- !x! END SCRIPT add_row
+-- !x! BEGIN SCRIPT outer
+-- !x! EXECUTE SCRIPT add_row WITH ARGS (what=inner, times=2)
+-- !x! END SCRIPT outer
+-- !x! EXECUTE SCRIPT add_row WITH ARGS (what=plain, times=1)
+-- !x! EXECUTE SCRIPT outer
+-- !x! SUB k 0
+-- !x! BEGIN SCRIPT bump
+-- !x! SUB_ADD k 1
+insert into log values ('bump !!k!!');
+-- !x! END SCRIPT bump
+-- !x! EXECUTE SCRIPT bump
+-- !x! EXECUTE SCRIPT bump
+-- !x! EXECUTE SCRIPT bump
+-- !x! WRITE "note outside=[!!~note!!] k=!!k!!" TO out.txt
+-- !x! IF (SCRIPT_EXISTS(bump)) { WRITE "bump exists" TO out.txt }

execsql2-2.35.0/tests/corpus/cases/export_query_append/script.sql

@@ -0,0 +1,8 @@
+-- EXPORT QUERY, APPEND, TEE and DESCRIPTION.
+create table t (id integer, name text);
+insert into t values (1, 'a'), (2, 'b'), (3, 'c');
+-- !x! EXPORT QUERY <<select id, name from t where id < 3 order by id;>> TO q.csv AS CSV
+-- !x! EXPORT QUERY <<select id, name from t where id = 3;>> APPEND TO q.csv AS CSV
+-- !x! EXPORT t TO stdout AS CSV
+-- !x! EXPORT QUERY <<select count(*) as n from t;>> TEE TO n.txt AS TXT
+-- !x! EXPORT t TO d.txt AS TXT DESCRIPTION "the t table"

execsql2-2.35.0/tests/corpus/cases/export_structured_formats/script.sql

@@ -0,0 +1,7 @@
+-- EXPORT to JSON, JSON table schema, VALUES and LaTeX.
+create table t (id integer, name text, amt real, day text);
+insert into t values (1, 'Ann', 1.5, '2024-01-05'), (2, 'O''Neil', null, null);
+-- !x! EXPORT t TO t.json AS JSON
+-- !x! EXPORT t TO t_schema.json AS JSON_TS
+-- !x! EXPORT t TO t_values.sql AS VALUES
+-- !x! EXPORT t TO t.tex AS LATEX

execsql2-2.35.0/tests/corpus/cases/export_text_formats/script.sql

@@ -0,0 +1,9 @@
+-- EXPORT to the delimited and text formats.
+create table t (id integer, name text, amt real, note text);
+insert into t values (1, 'Ann', 1.5, 'has, comma'), (2, 'Bob', null, 'say "hi"'), (3, 'Cé', -2, null);
+-- !x! EXPORT t TO t.csv AS CSV
+-- !x! EXPORT t TO t.tsv AS TSV
+-- !x! EXPORT t TO t.tab AS TAB
+-- !x! EXPORT t TO t.txt AS TXT
+-- !x! EXPORT t TO t.plain AS PLAIN
+-- !x! EXPORT t TO t.us AS US