execsql2 2.33.0__py3-none-any.whl → 2.35.0__py3-none-any.whl

execsql/api.py

@@ -178,6 +178,7 @@ class _LibraryConfig:
         self.boolean_int = True
         self.boolean_words = False
         self.empty_strings = True
+        self.exact_decimals = False
         self.only_strings = False
         self.empty_rows = True
         self.del_empty_cols = False

execsql/config.py

@@ -300,6 +300,7 @@ class ConfigData:
         self._get_bool(cp, self._INPUT_SECTION, "boolean_int", "boolean_int")
         self._get_bool(cp, self._INPUT_SECTION, "boolean_words", "boolean_words")
         self._get_bool(cp, self._INPUT_SECTION, "empty_strings", "empty_strings")
+        self._get_bool(cp, self._INPUT_SECTION, "exact_decimals", "exact_decimals")
         self._get_bool(cp, self._INPUT_SECTION, "only_strings", "only_strings")
         self._get_bool(cp, self._INPUT_SECTION, "empty_rows", "empty_rows")
         self._get_bool(cp, self._INPUT_SECTION, "delete_empty_columns", "del_empty_cols")
@@ -503,6 +504,7 @@ class ConfigData:
         self.boolean_int = True
         self.boolean_words = False
         self.empty_strings = True
+        self.exact_decimals = False
         self.only_strings = False
         self.empty_rows = True
         self.del_empty_cols = False

execsql/data/execsql.conf.template

@@ -115,6 +115,13 @@
 # Values: Yes or No.  Default: Yes.
 #empty_strings=Yes
 
+# Whether a column of decimal numbers with different numbers of decimal places
+# (1.5, 2.25) gets an exact DECIMAL(precision, scale) type when IMPORT or COPY
+# creates a table.  With No it gets a floating-point type, unless a value has
+# more than 15 significant digits (then it is DECIMAL, so no digit is lost).
+# Values: Yes or No.  Default: No.
+#exact_decimals=No
+
 # Whether to fold column headers to lowercase, uppercase, or leave unchanged.
 # Values: No, Lower, or Upper.  Default: No.
 #fold_column_headers=No

execsql/db/base.py

@@ -24,7 +24,9 @@ from collections.abc import Callable, Generator, Iterator
 
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc
+from execsql.db.target_types import ColumnKinds, existing_table_converter, kind_for_python_type
 from execsql.db.tiers import SupportTier, announce_tier
+from execsql.types import DataType
 import execsql.state as _state
 
 __all__ = ["Database", "DatabasePool"]
@@ -604,6 +606,42 @@ class Database(ABC):
             rows = curs.fetchall()
         return [row[0] for row in rows]
 
+    def column_kinds(self, sq_name: str) -> ColumnKinds:
+        """The execsql data type each column of table *sq_name* converts values to (see ``db.target_types``).
+
+        *sq_name* is schema-qualified and quoted, as for an ``INSERT``, and
+        is looked up the way the ``INSERT`` finds it.  Keys are lower-cased
+        column names; a column with no fitting execsql type maps to ``None``.
+        This default reads the driver's description of an empty query on the
+        table; adapters whose catalog names types more exactly override it.
+        """
+        description, _rows = self._column_type_query(sq_name, f"select * from {sq_name} where 1=0;")
+        return {d[0].lower(): self._described_kind(d) for d in description}
+
+    def _described_kind(self, description: Any) -> type[DataType] | None:
+        """The execsql type for one column of a cursor description (a DB-API 7-item sequence)."""
+        return kind_for_python_type(description[1], description[5])
+
+    def _column_type_query(self, sq_name: str, sql: str, params: list | None = None) -> tuple[Any, list]:
+        """Run *sql* for :meth:`column_kinds` of *sq_name*; return the cursor description and the rows."""
+        with self._cursor() as curs:
+            try:
+                if params is None:
+                    curs.execute(sql)
+                else:
+                    curs.execute(sql, params)
+            except ErrInfo:
+                raise
+            except Exception as e:
+                self.rollback()
+                raise ErrInfo(
+                    type="db",
+                    command_text=sql,
+                    exception_msg=exception_desc(),
+                    other_msg=f"Failed to get the column types of table {sq_name} of {self.name()}",
+                ) from e
+            return curs.description, curs.fetchall()
+
     def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
         """Return ``True`` if *view_name* (optionally in *schema_name*) exists."""
         with self._cursor() as curs:
@@ -689,11 +727,17 @@ class Database(ABC):
         rowsource: Any,
         column_list: list[str],
         tablespec_src: Callable,
+        *,
+        existing_table: bool = False,
     ) -> None:
         """Bulk-insert rows from *rowsource* into *table_name* using the columns in *column_list*.
 
         *rowsource* must be a generator yielding lists of values in column order.
         *tablespec_src* is a zero-argument callable that returns the table's type specification.
+        Each value is converted to the type scanned for its column; with
+        *existing_table* (the table was there before the IMPORT or COPY), the
+        type of the column it goes into decides instead (see
+        ``db.target_types.existing_table_converter``).
         """
         # The rowsource argument must be a generator yielding a list of values for the columns of the table.
         # The column_list argument must an iterable containing column names.  This may be a subset of
@@ -713,8 +757,13 @@ class Database(ABC):
         sel_cols = [col for col in ts_colnames if col in column_list]
         incl_col = [col in column_list for col in ts_colnames]
         # Type conversion functions for the rowsource.
-        type_objs = [col.column_type()[1]() for col in tablespec.cols]
-        type_mod_fn = [self.type.dialect[col.column_type()[1]][3] for col in tablespec.cols]
+        converters = [self._scanned_type_converter(col.column_type()[1]) for col in tablespec.cols]
+        if existing_table:
+            kinds = self.column_kinds(sq_name)
+            converters = [
+                existing_table_converter(kinds.get(col.name.lower()), col.column_type()[1], convert, col.name, sq_name)
+                for col, convert in zip(tablespec.cols, converters)
+            ]
         # Construct INSERT statement.
         columns = [self.type.quoted(col) for col in sel_cols]
         colspec = ",".join(columns)
@@ -812,10 +861,7 @@ class Database(ABC):
                                             line[i] = None
                             # Trailing values left off the line are NULL.
                             line = list(line) + [None] * (len(ts_colnames) - len(line))
-                            lt = [
-                                type_objs[i].from_data(val) if val is not None else None for i, val in enumerate(line)
-                            ]
-                            lt = [type_mod_fn[i](v) if type_mod_fn[i] else v for i, v in enumerate(lt)]
+                            lt = [converters[i](val) for i, val in enumerate(line)]
                             row = []
                             for i, v in enumerate(lt):
                                 if incl_col[i]:
@@ -869,14 +915,31 @@ class Database(ABC):
                 f"IMPORT into {sq_name} complete: {total_rows} rows imported.",
             )
 
+    def _scanned_type_converter(self, data_type: type[DataType]) -> Callable[[Any], Any]:
+        """Convert a value to *data_type*, then apply this DBMS's adjustment for that type, if any.
+
+        NULL is not converted, but it is adjusted: MySQL's adjustment for
+        dates turns it into ``""``.
+        """
+        from_data = data_type().from_data
+        adjust = self.type.dialect[data_type][3]
+        if adjust is None:
+            return lambda value: None if value is None else from_data(value)
+        return lambda value: adjust(None if value is None else from_data(value))
+
     def import_tabular_file(
         self,
         schema_name: str | None,
         table_name: str,
         csv_file_obj: Any,
         skipheader: bool,
+        *,
+        existing_table: bool = False,
     ) -> None:
-        """Import a CSV/tabular file into *table_name*; column names must be compatible."""
+        """Import a CSV/tabular file into *table_name*; column names must be compatible.
+
+        *existing_table*: the table was there before the IMPORT (see :meth:`populate_table`).
+        """
         # Import a text (CSV) file containing tabular data to a table.  Columns must be compatible.
         if not self.table_exists(table_name, schema_name):
             raise ErrInfo(
@@ -912,7 +975,7 @@ class Database(ABC):
 
         f = csv_file_obj.reader()
         next(f)
-        self.populate_table(schema_name, table_name, f, import_cols, get_ts)
+        self.populate_table(schema_name, table_name, f, import_cols, get_ts, existing_table=existing_table)
 
     def import_entire_file(
         self,

execsql/db/duckdb.py

@@ -18,6 +18,8 @@ from pathlib import Path
 from typing import Any
 
 from execsql.db.base import Database, statement_start
+from execsql.db.target_types import kind_for_type_name
+from execsql.types import DataType
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error, install_hint
 import execsql.state as _state
@@ -320,6 +322,10 @@ class DuckDBDatabase(Database):
 
         return insert
 
+    def _described_kind(self, description: Any) -> type[DataType] | None:
+        """DuckDB describes a column by its type's name (``DECIMAL(18,3)``, ``TIMESTAMP WITH TIME ZONE``)."""
+        return kind_for_type_name(str(description[1]))
+
     def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
         """Return True if the named view exists in the DuckDB database."""
         # DuckDB information_schema has no 'views' table; views are listed in 'tables'

execsql/db/mysql.py

@@ -13,6 +13,7 @@ import tempfile
 from typing import Any, cast
 
 from execsql.db.base import Database
+from execsql.db.target_types import ColumnKinds, kind_for_type_name
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error, install_hint
 from execsql.utils.auth import (
@@ -41,6 +42,11 @@ _PYTHON_TO_MYSQL_CHARSET: dict[str, str] = {
 }
 
 
+def _text(value: Any) -> str:
+    """A catalog value as text: some MySQL versions return ``SHOW COLUMNS`` types as bytes."""
+    return value.decode() if isinstance(value, (bytes, bytearray)) else str(value)
+
+
 def _crlf_terminated(filename: str) -> bool:
     """True if the file's first line ends with CRLF.
 
@@ -239,6 +245,11 @@ class MySQLDatabase(Database):
             self._fold_identifier(schema_name) if schema_name is not None else None,
         )
 
+    def column_kinds(self, sq_name: str) -> ColumnKinds:
+        """The execsql type of each column of *sq_name*, from ``SHOW COLUMNS`` (which tells BOOLEAN from TINYINT)."""
+        _description, rows = self._column_type_query(sq_name, f"show columns from {sq_name};")
+        return {str(row[0]).lower(): kind_for_type_name(_text(row[1])) for row in rows}
+
     def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
         return super().view_exists(
             self._fold_identifier(view_name),
@@ -305,7 +316,11 @@ class MySQLDatabase(Database):
                     )
                     self.conn = db_conn()
                 remember_password("MySQL", self.db_name, self.user, self.server_name, port=self.port)
-                self.execute("set session sql_mode='ANSI';")
+                # A fixed mode, not the server's: quote_literal relies on
+                # backslash escapes being on (no NO_BACKSLASH_ESCAPES).
+                # STRICT_TRANS_TABLES makes a value MySQL can't store as
+                # given an error instead of a clamped or truncated one (#101).
+                self.execute("set session sql_mode='ANSI,STRICT_TRANS_TABLES';")
             except SystemExit:
                 # If the user canceled the password prompt.
                 raise
@@ -343,14 +358,50 @@ class MySQLDatabase(Database):
             rows = curs.fetchall()
         return len(rows) > 0
 
+    def _fail_on_load_warnings(self, file_name: str, sq_name: str) -> None:
+        """Roll back and raise if the LOAD DATA just run changed any value while the session is in strict mode.
+
+        ``LOAD DATA LOCAL`` ignores strict mode: a value MySQL cannot store as
+        given is clamped or zeroed with a warning, not rejected (#101).  This
+        makes the load fail as the same rows inserted one by one would.  Notes
+        (a decimal rounded to the column's scale) are not warnings and pass, as
+        they do for an ``INSERT``.  A script that turned strict mode off
+        (``SET SESSION sql_mode = 'ANSI'``) keeps MySQL's changed values.
+        """
+        with self._cursor() as curs:
+            curs.execute("show warnings;")
+            warnings = curs.fetchall()
+            curs.execute("select @@session.sql_mode;")
+            mode = _text(curs.fetchone()[0]).upper()
+        if "STRICT_TRANS_TABLES" not in mode and "STRICT_ALL_TABLES" not in mode:
+            return
+        problems = [_text(row[2]) for row in warnings if _text(row[0]) in ("Warning", "Error")]
+        if not problems:
+            return
+        self.rollback()
+        shown = "; ".join(problems[:3])
+        more = f" (and {len(problems) - 3} more)" if len(problems) > 3 else ""
+        raise ErrInfo(
+            type="error",
+            other_msg=(
+                f"Can't import {file_name} into {sq_name}: MySQL would have stored changed values: {shown}{more}. "
+                "Nothing was loaded."
+            ),
+        )
+
     def import_tabular_file(
         self,
         schema_name: str | None,
         table_name: str,
         csv_file_obj: Any,
         skipheader: bool,
+        *,
+        existing_table: bool = False,
     ) -> None:
-        """Import a delimited file into a MySQL table."""
+        """Import a delimited file into a MySQL table.
+
+        The file's values go to the server as text, unconverted, so *existing_table* changes nothing here.
+        """
         # Import a file to a table.  Columns must be compatible.
         sq_name = self.schema_qualified_table_name(schema_name, table_name)
         if not self.table_exists(table_name, schema_name):
@@ -436,6 +487,7 @@ class MySQLDatabase(Database):
                 )
             try:
                 self.execute(import_sql)
+                self._fail_on_load_warnings(csv_file_obj.csvfname, sq_name)
             finally:
                 if load_file != csv_file_obj.csvfname:
                     os.unlink(load_file)

execsql/db/oracle.py

@@ -12,6 +12,8 @@ import re
 from typing import Any
 
 from execsql.db.base import Database
+from execsql.db.target_types import kind_for_type_name
+from execsql.types import DataType
 from execsql.db.tiers import SupportTier
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error, install_hint
@@ -49,6 +51,29 @@ def _oracle_driver() -> Any:
         return None
 
 
+# oracledb DbType names -> the type names db.target_types reads.  NUMBER is
+# read from its precision and scale; a type not listed has no execsql type.
+_ORACLE_TYPE_NAMES = {
+    "DB_TYPE_VARCHAR": "varchar2",
+    "DB_TYPE_NVARCHAR": "nvarchar2",
+    "DB_TYPE_CHAR": "char",
+    "DB_TYPE_NCHAR": "nchar",
+    "DB_TYPE_LONG": "clob",
+    "DB_TYPE_CLOB": "clob",
+    "DB_TYPE_NCLOB": "nclob",
+    "DB_TYPE_BINARY_FLOAT": "binary_float",
+    "DB_TYPE_BINARY_DOUBLE": "binary_double",
+    "DB_TYPE_BOOLEAN": "boolean",
+    "DB_TYPE_DATE": "date",
+    "DB_TYPE_TIMESTAMP": "timestamp",
+    "DB_TYPE_TIMESTAMP_TZ": "timestamp with time zone",
+    "DB_TYPE_TIMESTAMP_LTZ": "timestamp with local time zone",
+    "DB_TYPE_RAW": "raw",
+    "DB_TYPE_LONG_RAW": "long raw",
+    "DB_TYPE_BLOB": "blob",
+}
+
+
 class OracleDatabase(Database):
     """Oracle adapter using the python-oracledb driver."""
 
@@ -268,6 +293,17 @@ class OracleDatabase(Database):
             rows = curs.fetchall()
         return [row[0] for row in rows]
 
+    def _described_kind(self, description: Any) -> type[DataType] | None:
+        """oracledb describes a column by a ``DbType`` (``DB_TYPE_NUMBER``), with its precision and scale."""
+        name = getattr(description[1], "name", "")
+        if name == "DB_TYPE_NUMBER":
+            precision, scale = description[4], description[5]
+            # NUMBER with no precision, and FLOAT, report scale -127.
+            type_name = f"number({precision},{scale})" if precision and scale is not None and scale >= 0 else "number"
+        else:
+            type_name = _ORACLE_TYPE_NAMES.get(name, "")
+        return kind_for_type_name(type_name, oracle=True)
+
     def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
         """Return True if the named view exists in the Oracle database."""
         # Oracle folds unquoted identifiers to uppercase.

execsql/db/postgres.py

@@ -14,6 +14,7 @@ import re
 from typing import Any
 
 from execsql.db.base import Database
+from execsql.db.target_types import ColumnKinds, kind_for_type_name
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error, install_hint
 from execsql.utils.auth import (
@@ -224,6 +225,16 @@ class PostgresDatabase(Database):
             rows = curs.fetchall()
         return len(rows) > 0
 
+    def column_kinds(self, sq_name: str) -> ColumnKinds:
+        """The execsql type of each column of *sq_name*, from the catalog (``regclass`` follows the search path)."""
+        _description, rows = self._column_type_query(
+            sq_name,
+            "select attname, format_type(atttypid, atttypmod) from pg_attribute "
+            "where attrelid = %s::regclass and attnum > 0 and not attisdropped order by attnum;",
+            [sq_name],
+        )
+        return {name.lower(): kind_for_type_name(type_name) for name, type_name in rows}
+
     def view_exists(self, view_name: str, schema_name: str | None = None) -> bool:
         """Return True if the named view exists in the PostgreSQL database."""
         if schema_name is not None:
@@ -279,8 +290,13 @@ class PostgresDatabase(Database):
         table_name: str,
         csv_file_obj: Any,
         skipheader: bool,
+        *,
+        existing_table: bool = False,
     ) -> None:
-        """Import a delimited file into a PostgreSQL table, using COPY when possible."""
+        """Import a delimited file into a PostgreSQL table, using COPY when possible.
+
+        The file's values go to the server as text, unconverted, so *existing_table* changes nothing here.
+        """
         # Import a file to a table.  Columns must be compatible.
         sq_name = self.schema_qualified_table_name(schema_name, table_name)
         if not self.table_exists(table_name, schema_name):

execsql/db/sqlite.py

@@ -182,8 +182,14 @@ class SQLiteDatabase(Database):
         rowsource: Any,
         column_list: list[str],
         tablespec_src: Any,
+        *,
+        existing_table: bool = False,
     ) -> None:
-        """Populate a SQLite table from a row source generator."""
+        """Populate a SQLite table from a row source generator.
+
+        Values are inserted as the source gives them (dates, times and
+        decimals as text), so *existing_table* changes nothing here.
+        """
         # The rowsource argument must be a generator yielding a list of values for the columns of the table.
         # The column_list argument must an iterable containing column names in the same order as produced by the rowsource.
         sq_name = self.schema_qualified_table_name(None, table_name)

execsql/db/sqlserver.py

@@ -8,6 +8,7 @@ Server via ``pyodbc``.  Corresponds to ``-t s`` on the CLI.
 """
 
 from execsql.db.base import Database
+from execsql.db.target_types import ColumnKinds, kind_for_type_name
 from execsql.db.odbc import connection_string, fresh_sessions, loggable
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import fatal_error, install_hint
@@ -228,6 +229,29 @@ class SqlServerDatabase(Database):
             rows = curs.fetchall()
         return len(rows) > 0
 
+    def column_kinds(self, sq_name: str) -> ColumnKinds:
+        """The execsql type of each column of *sq_name*, from ``sys.columns``.
+
+        pyodbc describes ``datetimeoffset`` and ``varbinary`` alike, so the
+        catalog is read instead.  A table it does not find through
+        ``object_id`` (a ``#temp`` table, which lives in tempdb) falls back
+        to the driver's description.
+        """
+        _description, rows = self._column_type_query(
+            sq_name,
+            "select c.name, type_name(c.system_type_id), c.precision, c.scale from sys.columns c "
+            "where c.object_id = object_id(?) order by c.column_id;",
+            [sq_name],
+        )
+        if not rows:
+            return super().column_kinds(sq_name)
+        return {
+            name.lower(): kind_for_type_name(
+                f"{type_name}({precision},{scale})" if type_name in ("decimal", "numeric") else type_name,
+            )
+            for name, type_name, precision, scale in rows
+        }
+
     def drop_table(self, tablename: str) -> None:
         """Drop *tablename*, schema-qualified and quoted by the caller."""
         # SQL Server and Firebird will throw an error if there are foreign keys to the table.