execsql2 2.29.0__py3-none-any.whl → 2.31.1__py3-none-any.whl

execsql/db/access.py

@@ -15,6 +15,7 @@ from pathlib import Path
 from typing import Any, cast
 
 from execsql.db.base import Database, _fetched
+from execsql.db.odbc import odbc_value
 from execsql.db.tiers import SupportTier
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error, install_hint
@@ -119,10 +120,11 @@ class AccessDatabase(Database):
         def _try_odbc_drivers():
             db_name = str(Path(self.db_name).resolve())
             for cs, jet4flag in self.connection_strings:
+                connstr = cs % odbc_value(db_name)
+                logstr = connstr
                 if self.need_passwd:
-                    connstr = f"{cs % db_name} Uid={self.user}; Pwd={self.password};"
-                else:
-                    connstr = cs % db_name
+                    connstr += f" Uid={odbc_value(self.user)}; Pwd={odbc_value(self.password)};"
+                    logstr += f" Uid={odbc_value(self.user)}; Pwd=***;"
                 try:
                     # Access is file-based but a UNC path on an
                     # unreachable share can still block forever; cap
@@ -133,12 +135,12 @@ class AccessDatabase(Database):
                     errors.append(e)
                     if _state.exec_log is not None:
                         _state.exec_log.log_status_info(
-                            f"Could not connect via ODBC using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
+                            f"Could not connect via ODBC using: {logstr}",
                         )
                 else:
                     if _state.exec_log is not None:
                         _state.exec_log.log_status_info(
-                            f"Connected via ODBC using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
+                            f"Connected via ODBC using: {logstr}",
                         )
                     self.jet4 = jet4flag
                     return True

execsql/db/base.py

@@ -641,6 +641,24 @@ class Database(ABC):
 
         raise DatabaseNotImplementedError(self.name(), "role_exists")
 
+    def drop_table_for_replacement(self, schema_name: str | None, table_name: str) -> None:
+        """Drop the table a ``TO REPLACEMENT`` import or copy is about to recreate, if it is there.
+
+        Not every DBMS has ``DROP TABLE IF EXISTS``: SQL Server, Oracle,
+        Firebird and Access refuse to drop a table that does not exist.  The
+        drop is tried first, and its error is ignored only when the table is
+        confirmed absent, so a drop that would have succeeded always runs.
+        """
+        try:
+            self.drop_table(self.schema_qualified_table_name(schema_name, table_name))
+        except Exception:
+            try:
+                there = self.table_exists_where_created(table_name, schema_name)
+            except Exception:
+                there = True
+            if there:
+                raise
+
     def drop_table(self, tablename: str) -> None:
         """Drop *tablename* if it exists; *tablename* must already be schema-qualified and quoted."""
         # The 'tablename' argument should be schema-qualified and quoted as necessary.

execsql/db/duckdb.py

@@ -9,6 +9,7 @@ on the CLI.
 """
 
 import re
+from collections import deque
 from pathlib import Path
 from typing import Any
 
@@ -28,21 +29,83 @@ _TRANSACTION_RX = re.compile(r"(?P<begin>BEGIN|START\s+TRANSACTION)\b|(?P<end>CO
 
 
 class _SharedCursor:
-    """The connection, used as a cursor so a statement joins its open transaction.
-
-    A DuckDB cursor is a separate connection with a transaction of its own,
-    and closing it rolls that transaction back.  ``close()`` therefore leaves
-    the connection open.
+    """A cursor that runs on the database's one connection.
+
+    A DuckDB cursor is a separate connection: a temporary table created on
+    one is invisible to the next, and a cursor's transaction is its own.  So
+    every statement runs on the connection itself.  A connection holds one
+    pending result, though, and the next statement discards it; COPY reads
+    its source twice at once (the rows, and a scan for their types).  So
+    before another statement runs, the result this cursor still has to
+    deliver is read into memory (:meth:`DuckDBDatabase._take_connection`).
+    An EXPORT, which runs nothing else while it reads, still streams.
     """
 
-    def __init__(self, conn: Any) -> None:
-        self._conn = conn
+    def __init__(self, db: DuckDBDatabase) -> None:
+        self._db = db
+        self._conn = db.conn
+        self._buffer: deque[Any] | None = None  # the rest of the result, once read ahead
+        self._pending = False  # a result the connection still holds for this cursor
+        self.description: Any = None
+        self.arraysize = 1000
 
     def __getattr__(self, name: str) -> Any:
         return getattr(self._conn, name)
 
+    def execute(self, sql: Any, params: Any = None) -> _SharedCursor:
+        self._db._take_connection(self)
+        if params is None:
+            self._conn.execute(sql)
+        else:
+            self._conn.execute(sql, params)
+        self.description = self._conn.description
+        self._buffer = None
+        self._pending = self.description is not None
+        return self
+
+    def executemany(self, sql: Any, params: Any) -> _SharedCursor:
+        self._db._take_connection(self)
+        self._conn.executemany(sql, params)
+        self.description = self._conn.description
+        self._buffer = None
+        self._pending = False
+        return self
+
+    def read_ahead(self) -> None:
+        """Move the rest of the pending result into memory, freeing the connection."""
+        if self._pending:
+            self._buffer = deque(self._conn.fetchall())
+            self._pending = False
+
+    def fetchmany(self, size: int | None = None) -> list[Any]:
+        size = size or self.arraysize
+        if self._buffer is not None:
+            return [self._buffer.popleft() for _ in range(min(size, len(self._buffer)))]
+        if not self._pending:
+            return []
+        rows: list[Any] = self._conn.fetchmany(size)
+        if not rows:
+            self._pending = False
+        return rows
+
+    def fetchone(self) -> Any:
+        rows = self.fetchmany(1)
+        return rows[0] if rows else None
+
+    def fetchall(self) -> list[Any]:
+        if self._buffer is not None:
+            rows, self._buffer = list(self._buffer), deque()
+            return rows
+        if not self._pending:
+            return []
+        self._pending = False
+        return list(self._conn.fetchall())
+
     def close(self) -> None:
-        pass
+        """Leave the connection, and any transaction on it, open."""
+        self._buffer = None
+        self._pending = False
+        self._db._release_connection(self)
 
 
 class DuckDBDatabase(Database):
@@ -51,12 +114,15 @@ class DuckDBDatabase(Database):
     The duckdb driver commits each statement unless a transaction was begun,
     and has no way to ask whether one is open, so this adapter begins one
     itself while statements are held (AUTOCOMMIT OFF, BEGIN BATCH) and keeps
-    track of it.  While it is open, every statement runs on the connection
-    itself rather than on a cursor (see :class:`_SharedCursor`).
+    track of it.  Every statement runs on the connection itself rather than
+    on a cursor (see :class:`_SharedCursor`), so it joins that transaction
+    and sees the run's temporary tables.
     """
 
     #: A transaction this adapter or a script's own BEGIN opened, not yet ended.
     in_transaction: bool = False
+    #: The cursor whose result the connection holds, if any.
+    _holder: _SharedCursor | None = None
 
     def __init__(self, DuckDB_fn: str) -> None:
         try:
@@ -83,12 +149,21 @@ class DuckDBDatabase(Database):
         return f"DuckDBDatabase({self.db_name!r})"
 
     def cursor(self) -> Any:
-        """A cursor, or while a transaction is open, the connection itself."""
+        """A cursor on the database's one connection (see :class:`_SharedCursor`)."""
         if self.conn is None:
             self.open_db()
-        if self.in_transaction:
-            return _SharedCursor(self.conn)
-        return self.conn.cursor()
+        return _SharedCursor(self)
+
+    def _take_connection(self, cursor: _SharedCursor | None = None) -> None:
+        """Before a statement runs, read ahead the result another cursor has not finished with."""
+        holder = self._holder
+        if holder is not None and holder is not cursor:
+            holder.read_ahead()
+        self._holder = cursor
+
+    def _release_connection(self, cursor: _SharedCursor) -> None:
+        if self._holder is cursor:
+            self._holder = None
 
     def begin_transaction(self) -> None:
         """Open a transaction if none is open."""
@@ -96,6 +171,7 @@ class DuckDBDatabase(Database):
             self.open_db()
         if not self.in_transaction:
             # A second BEGIN would abort the open transaction, hence the tracking.
+            self._take_connection()
             self.conn.execute("BEGIN TRANSACTION")
             self.in_transaction = True
 
@@ -124,17 +200,23 @@ class DuckDBDatabase(Database):
 
     def commit(self) -> None:
         """Commit the open transaction if autocommit is enabled."""
+        self._take_connection()
         super().commit()
         if self.autocommit:
             self.in_transaction = False
 
     def rollback(self) -> None:
         """Roll back the open transaction."""
+        try:
+            self._take_connection()
+        except Exception:  # best effort, as the rollback is: a failed transaction may not give up its rows
+            self._holder = None
         super().rollback()
         self.in_transaction = False
 
     def close(self) -> None:
         """Close the connection; an open transaction is rolled back."""
+        self._holder = None
         super().close()
         self.in_transaction = False
 

execsql/db/factory.py

@@ -106,9 +106,19 @@ def db_SqlServer(
     port: int | None = None,
     encoding: str | None = None,
     password: str | None = None,
+    trust_server_certificate: bool | None = None,
 ) -> SqlServerDatabase:
     """Open a Microsoft SQL Server connection via pyodbc."""
-    return SqlServerDatabase(server_name, database_name, user, pw_needed, port, encoding, password=password)
+    return SqlServerDatabase(
+        server_name,
+        database_name,
+        user,
+        pw_needed,
+        port,
+        encoding,
+        password=password,
+        trust_server_certificate=trust_server_certificate,
+    )
 
 
 def db_MySQL(

execsql/db/mysql.py

@@ -7,7 +7,9 @@ Implements :class:`MySQLDatabase`, which connects to MySQL and MariaDB
 servers via ``pymysql``.  Corresponds to ``-t m`` on the CLI.
 """
 
+import os
 import re
+import tempfile
 from typing import Any, cast
 
 from execsql.db.base import Database
@@ -53,6 +55,49 @@ def _crlf_terminated(filename: str) -> bool:
     return i > 0 and chunk[i - 1 : i] == b"\r"
 
 
+def _data_length(filename: str) -> int | None:
+    """Return the file's length without its trailing blank lines, or None if it has none.
+
+    LOAD DATA reads each blank line at the end of a file as a row of NULLs;
+    every other import path drops them.  The file is read backwards from the
+    end, as bytes, so no decoding can hide a line ending.
+    """
+    try:
+        with open(filename, "rb") as f:
+            size = f.seek(0, 2)
+            pos = size
+            content_end = 0
+            while pos > 0:
+                start = max(0, pos - 65536)
+                f.seek(start)
+                block = f.read(pos - start)
+                stripped = len(block.rstrip(b"\r\n"))
+                if stripped:
+                    content_end = start + stripped
+                    break
+                pos = start
+            f.seek(content_end)
+            ending = f.read(2)
+    except OSError:
+        return None
+    # Keep the line ending of the last line of data.
+    keep = content_end + (2 if ending == b"\r\n" else 1 if ending[:1] in (b"\r", b"\n") else 0)
+    return keep if keep < size else None
+
+
+def _copy_head(filename: str, length: int) -> str:
+    """Copy the first *length* bytes of *filename* to a temporary file; return its path."""
+    with open(filename, "rb") as src, tempfile.NamedTemporaryFile("wb", suffix=".csv", delete=False) as dst:
+        remaining = length
+        while remaining > 0:
+            chunk = src.read(min(remaining, 1 << 20))
+            if not chunk:
+                break
+            dst.write(chunk)
+            remaining -= len(chunk)
+    return dst.name
+
+
 class MySQLDatabase(Database):
     """MySQL and MariaDB adapter using the pymysql package."""
 
@@ -353,7 +398,11 @@ class MySQLDatabase(Database):
             # String literals in this session treat a backslash as an escape,
             # so the file name, delimiter and quote character are quoted with
             # quote_literal, which doubles it.
-            import_sql = f"load data local infile {self.quote_literal(csv_file_obj.csvfname)} into table {sq_name}"
+            load_file = csv_file_obj.csvfname
+            data_length = _data_length(load_file)
+            if data_length is not None:
+                load_file = _copy_head(load_file, data_length)
+            import_sql = f"load data local infile {self.quote_literal(load_file)} into table {sq_name}"
             if csv_file_obj.encoding:
                 charset = _PYTHON_TO_MYSQL_CHARSET.get(csv_file_obj.encoding.lower(), csv_file_obj.encoding)
                 import_sql = f"{import_sql} character set {charset}"
@@ -385,7 +434,11 @@ class MySQLDatabase(Database):
                 _state.exec_log.log_status_info(
                     f"IMPORTing {csv_file_obj.csvfname} using the DBMS' fast file reading routine",
                 )
-            self.execute(import_sql)
+            try:
+                self.execute(import_sql)
+            finally:
+                if load_file != csv_file_obj.csvfname:
+                    os.unlink(load_file)
         else:
             data_indexes = [csv_file_cols.index(col) for col in import_cols]
             paramspec = ",".join(["%s"] * len(import_cols))
@@ -450,7 +503,7 @@ class MySQLDatabase(Database):
                                             if not _state.conf.empty_strings and line[i].strip() == "":
                                                 line[i] = None
                                 # Pad short line with nulls
-                                line.extend([None] * (len(import_cols) - len(line)))
+                                line.extend([None] * (len(csv_file_cols) - len(line)))
                                 linedata = [line[ix] for ix in data_indexes]
                                 add_line = True
                                 if not _state.conf.empty_rows:

execsql/db/odbc.py

@@ -0,0 +1,45 @@
+"""
+ODBC connection strings for the SQL Server and Access adapters.
+
+An ODBC connection string is a ``;``-separated list of ``KEY=value``
+attributes, so a value containing ``;`` (a password, a file path, a name
+taken from a substitution variable) would end its attribute early and
+start another.  :func:`odbc_value` brace-quotes such values;
+:func:`connection_string` joins the attributes, and :func:`loggable`
+gives the same string with the password masked for the log.
+"""
+
+from __future__ import annotations
+
+__all__ = ["connection_string", "loggable", "odbc_value"]
+
+_SECRET_KEYS = frozenset({"pwd", "password"})
+
+
+def odbc_value(value: object) -> str:
+    """Return *value* as an ODBC attribute value, brace-quoted when it needs it.
+
+    A ``;`` ends an attribute and a leading ``{`` opens a quoted value, so a
+    value with either is wrapped in ``{…}`` with each ``}`` doubled.  Other
+    values are written as they are, because older drivers do not all accept
+    braces around every attribute.
+    """
+    text = str(value)
+    if ";" in text or text.startswith("{"):
+        return "{" + text.replace("}", "}}") + "}"
+    return text
+
+
+def connection_string(driver: str, attributes: list[tuple[str, object]]) -> str:
+    """Return ``DRIVER={driver};KEY=value;…`` with every value quoted as needed."""
+    parts = [f"DRIVER={{{driver}}}"]
+    parts.extend(f"{key}={odbc_value(value)}" for key, value in attributes)
+    return ";".join(parts) + ";"
+
+
+def loggable(driver: str, attributes: list[tuple[str, object]]) -> str:
+    """Return the connection string with the password replaced by ``***``."""
+    return connection_string(
+        driver,
+        [(key, "***" if key.lower() in _SECRET_KEYS else value) for key, value in attributes],
+    )

execsql/db/postgres.py

@@ -462,7 +462,7 @@ class PostgresDatabase(Database):
                                             if not _state.conf.empty_strings and line[i].strip() == "":
                                                 line[i] = None
                                 # Pad short line with nulls
-                                line.extend([None] * (len(import_cols) - len(line)))
+                                line.extend([None] * (len(csv_file_cols) - len(line)))
                                 linedata = [line[ix] for ix in data_indexes]
                                 add_line = True
                                 if not _state.conf.empty_rows:

execsql/db/sqlserver.py

@@ -7,9 +7,8 @@ Implements :class:`SqlServerDatabase`, which connects to Microsoft SQL
 Server via ``pyodbc``.  Corresponds to ``-t s`` on the CLI.
 """
 
-import re
-
 from execsql.db.base import Database
+from execsql.db.odbc import connection_string, loggable
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import fatal_error, install_hint
 from execsql.utils.auth import (
@@ -34,9 +33,10 @@ class SqlServerDatabase(Database):
         db_name: str,
         user_name: str | None,
         need_passwd: bool = False,
-        port: int | None = 1433,
+        port: int | None = None,
         encoding: str | None = "latin1",
         password: str | None = None,
+        trust_server_certificate: bool | None = None,
     ) -> None:
         try:
             import pyodbc  # noqa: F401
@@ -51,6 +51,15 @@ class SqlServerDatabase(Database):
             need_passwd=need_passwd,
             port=port if port else 1433,
         )
+        # Only a port that was asked for goes into the connection string: a
+        # named instance (``host\instance``) is found through SQL Server
+        # Browser, which an explicit port bypasses.
+        self.explicit_port = port or None
+        # None: the [connect] trust_server_certificate setting, which also
+        # covers connections a script opens with CONNECT.
+        if trust_server_certificate is None:
+            trust_server_certificate = bool(getattr(_state.conf, "trust_server_certificate", False))
+        self.trust_server_certificate = trust_server_certificate
         self.type = dbt_sqlserver
         self.password = password
         self.encoding = encoding or "latin1"  # Default on installation of SQL Server
@@ -79,6 +88,12 @@ class SqlServerDatabase(Database):
         """
         return "[" + identifier.replace("]", "]]") + "]"
 
+    def _server_attribute(self) -> str:
+        """Return the ``SERVER`` value: ``host,port`` when a port was given."""
+        if self.explicit_port and "," not in self.server_name:
+            return f"{self.server_name},{self.explicit_port}"
+        return str(self.server_name)
+
     def open_db(self) -> None:
         """Open a connection to the SQL Server database via pyodbc."""
         import pyodbc
@@ -95,6 +110,7 @@ class SqlServerDatabase(Database):
                 )
             # Use pyodbc to connect.  Try different driver versions from newest to oldest.
             ssdrivers = (
+                "ODBC Driver 18 for SQL Server",
                 "ODBC Driver 17 for SQL Server",
                 "ODBC Driver 13.1 for SQL Server",
                 "ODBC Driver 13 for SQL Server",
@@ -108,23 +124,23 @@ class SqlServerDatabase(Database):
             errors: list[BaseException] = []
 
             def _try_drivers():
+                attributes: list[tuple[str, object]] = [
+                    ("SERVER", self._server_attribute()),
+                    ("MARS_Connection", "Yes"),
+                    ("DATABASE", self.db_name),
+                ]
+                if self.user:
+                    attributes.append(("Uid", self.user))
+                    if self.password:
+                        attributes.append(("Pwd", self.password))
+                else:
+                    attributes.append(("Trusted_Connection", "yes"))
+                if self.trust_server_certificate:
+                    # Driver 18 encrypts and verifies the server's certificate
+                    # by default; this accepts a self-signed one.
+                    attributes.append(("TrustServerCertificate", "yes"))
                 for drv in ssdrivers:
-                    if self.user:
-                        if self.password:
-                            connstr = (
-                                f"DRIVER={{{drv}}};SERVER={self.server_name};MARS_Connection=Yes; "
-                                f"DATABASE={self.db_name};Uid={self.user};Pwd={self.password}"
-                            )
-                        else:
-                            connstr = (
-                                f"DRIVER={{{drv}}};SERVER={self.server_name};MARS_Connection=Yes; "
-                                f"DATABASE={self.db_name};Uid={self.user}"
-                            )
-                    else:
-                        connstr = (
-                            f"DRIVER={{{drv}}};SERVER={self.server_name};MARS_Connection=Yes; "
-                            f"DATABASE={self.db_name};Trusted_Connection=yes"
-                        )
+                    connstr = connection_string(drv, attributes)
                     try:
                         # 30 s connect timeout matches the Postgres adapter
                         # default; pyodbc treats `timeout` as the login-and-
@@ -134,12 +150,12 @@ class SqlServerDatabase(Database):
                         errors.append(e)
                         if _state.exec_log is not None:
                             _state.exec_log.log_status_info(
-                                f"Could not connect using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
+                                f"Could not connect using: {loggable(drv, attributes)}",
                             )
                     else:
                         if _state.exec_log is not None:
                             _state.exec_log.log_status_info(
-                                f"Connected using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
+                                f"Connected using: {loggable(drv, attributes)}",
                             )
                         return True
                 return False