execsql2 2.24.0__py3-none-any.whl → 2.25.0__py3-none-any.whl

execsql/data/execsql.conf.template

@@ -46,7 +46,7 @@
 #password_prompt=Yes
 
 # Whether to use the OS credential store (keyring) for password storage.
-# Requires: pip install execsql2[auth]
+# Requires the auth extra: uv tool install "execsql2[auth]"
 # Values: Yes or No.  Default: Yes.
 #use_keyring=Yes
 
@@ -71,7 +71,8 @@
 [input]
 # Settings for handling imported data.
 
-# Whether to convert numeric values to double precision when using MS Access.
+# MS Access: create columns for decimal data as NUMERIC(precision,scale)
+# (Yes) or as DOUBLE (No) when IMPORT or COPY creates a table.
 # Values: Yes or No.  Default: No.
 #access_use_numeric=No
 
@@ -261,6 +262,15 @@
 # Values: Yes or No.  Default: No.
 #use_tls=No
 
+# Whether to verify the SMTP server's certificate when use_ssl or use_tls is on.
+# Turn off only for a test relay; prefer ca_file for an internal certificate.
+# Values: Yes or No.  Default: Yes.
+#verify_certificate=Yes
+
+# CA certificate file (PEM) to trust, for a server with an internal or
+# self-signed certificate.  Default: the system's trusted CAs.
+#ca_file=
+
 # Email body format.
 # Values: plain or html.  Default: plain.
 #email_format=plain
@@ -297,8 +307,8 @@
 # Values: Yes or No.  Default: Yes.
 #allow_rm_file=Yes
 
-# Whether to allow the SERVE metacommand (which opens an HTTP server on
-# a local port to serve a single file). Set to No to disable.
+# Whether to allow the SERVE metacommand (which copies a file to stdout
+# with HTTP headers, for CGI use). Set to No to disable.
 # Also controllable via --no-serve CLI flag.
 # Values: Yes or No.  Default: Yes.
 #allow_serve=Yes

execsql/db/access.py

@@ -18,7 +18,14 @@ from execsql.db.base import Database, _fetched
 from execsql.db.tiers import SupportTier
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error
-from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
+from execsql.utils.auth import (
+    begin_login,
+    clear_stored_password,
+    get_password,
+    is_login_failure,
+    password_from_keyring,
+    remember_password,
+)
 import execsql.state as _state
 
 __all__ = ["AccessDatabase"]
@@ -76,7 +83,6 @@ class AccessDatabase(Database):
         self.password = password
         # Encoding is only applicable to Jet < 4.0: non-accdb databases.
         self.encoding = encoding or "windows-1252"
-        self.encode_commands = True
         self.dao_conn: Any = None
         self.conn = None  # ODBC connection
         self.paramstr = "?"
@@ -104,9 +110,12 @@ class AccessDatabase(Database):
         if self.conn is not None:
             self.conn.close()
             self.conn = None
+        begin_login()
         if self.need_passwd and self.user and self.password is None:
             self.password = get_password("MS-Access", self.db_name, self.user)
 
+        errors: list[BaseException] = []
+
         def _try_odbc_drivers():
             db_name = str(Path(self.db_name).resolve())
             for cs, jet4flag in self.connection_strings:
@@ -120,7 +129,8 @@ class AccessDatabase(Database):
                     # the connect attempt at 30 s to match the other
                     # adapters.
                     self.conn = pyodbc.connect(connstr, timeout=30)
-                except Exception:
+                except Exception as e:
+                    errors.append(e)
                     if _state.exec_log is not None:
                         _state.exec_log.log_status_info(
                             f"Could not connect via ODBC using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
@@ -134,7 +144,8 @@ class AccessDatabase(Database):
                     return True
             return False
 
-        if not _try_odbc_drivers() and password_from_keyring():
+        # Only a rejected password means the stored one is stale.
+        if not _try_odbc_drivers() and password_from_keyring() and any(is_login_failure(e) for e in errors):
             clear_stored_password("MS-Access", self.db_name, self.user)
             self.password = get_password(
                 "MS-Access",
@@ -144,6 +155,8 @@ class AccessDatabase(Database):
                 other_msg="(stored credential failed — enter current password)",
             )
             _try_odbc_drivers()
+        if self.conn and self.user:
+            remember_password("MS-Access", self.db_name, self.user)
 
         if not self.conn:
             raise ErrInfo(
@@ -158,10 +171,13 @@ class AccessDatabase(Database):
         if self.dao_conn is not None:
             self.dao_conn.Close()
             self.dao_conn = None
+        begin_login()
         if self.need_passwd and self.user and self.password is None:
             self.password = get_password("MS-Access", self.db_name, self.user)
         dao_engines = ("DAO.DBEngine.120", "DAO.DBEngine.36")
 
+        dao_errors: list[BaseException] = []
+
         def _try_dao_engines():
             for engine in dao_engines:
                 try:
@@ -175,7 +191,8 @@ class AccessDatabase(Database):
                         )
                     else:
                         self.dao_conn = daoEngine.OpenDatabase(self.db_name)
-                except Exception:
+                except Exception as e:
+                    dao_errors.append(e)
                     if _state.exec_log is not None:
                         _state.exec_log.log_status_info(f"Could not connect via DAO using: {engine}")
                 else:
@@ -184,7 +201,7 @@ class AccessDatabase(Database):
                     return True
             return False
 
-        if not _try_dao_engines() and password_from_keyring():
+        if not _try_dao_engines() and password_from_keyring() and any(is_login_failure(e) for e in dao_errors):
             clear_stored_password("MS-Access", self.db_name, self.user)
             self.password = get_password(
                 "MS-Access",
@@ -194,6 +211,8 @@ class AccessDatabase(Database):
                 other_msg="(stored credential failed — enter current password)",
             )
             _try_dao_engines()
+        if self.dao_conn and self.user:
+            remember_password("MS-Access", self.db_name, self.user)
 
         if not self.dao_conn:
             raise ErrInfo(
@@ -339,6 +358,9 @@ class AccessDatabase(Database):
 
         return headers, iter(dict_row, None)
 
+    # table_exists() without a schema looks only where CREATE TABLE puts a table: one namespace.
+    unqualified_lookup_is_scoped = True
+
     def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
         """Return True if the named table exists in the Access database.
 
@@ -418,9 +440,8 @@ class AccessDatabase(Database):
         return False
 
     def drop_table(self, tablename: str) -> None:
-        """Drop the named table from the Access database."""
+        """Drop *tablename*, quoted by the caller."""
         self.dao_flush_check()
-        tablename = self.type.quoted(tablename)
         self.execute(f"drop table {tablename};")
 
     def as_datetime(self, val: Any) -> datetime.datetime | datetime.date | datetime.time | None:

execsql/db/base.py

@@ -98,7 +98,6 @@ class Database(ABC):
     password: str | None
     port: int | None
     encoding: Any
-    encode_commands: bool
     paramstr: str
     conn: Any
     autocommit: bool
@@ -143,7 +142,6 @@ class Database(ABC):
         self.password: str | None = None
         self.port = port
         self.encoding = encoding
-        self.encode_commands = True
         self.paramstr = "?"
         self.conn = None
         self.autocommit = True
@@ -494,6 +492,21 @@ class Database(ABC):
             rows = curs.fetchall()
         return len(rows) > 0
 
+    #: Whether :meth:`table_exists` without a schema already looks only where
+    #: an unqualified ``CREATE TABLE`` puts a table (one namespace, or the
+    #: search path).  When not, the check is narrowed to :meth:`current_schema`.
+    unqualified_lookup_is_scoped = False
+
+    def current_schema(self) -> str | None:
+        """The schema (MySQL: database) an unqualified table name is created in, or ``None`` if unknown."""
+        return None
+
+    def _query_current_schema(self, sql: str) -> str | None:
+        with self._cursor() as curs:
+            curs.execute(sql)
+            row = curs.fetchone()
+        return str(row[0]) if row and row[0] else None
+
     def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
         """Return ``True`` if *table_name* (optionally in *schema_name*) exists."""
         with self._cursor() as curs:
@@ -665,6 +678,17 @@ class Database(ABC):
         eof = False
         total_rows = 0
         self.before_statement(sql)
+        # The rows go in as several batches and are committed once, by the
+        # caller.  Most drivers hold every batch in the transaction they open
+        # themselves; DuckDB's commits each statement unless one is open, so a
+        # failure would leave the batches before it behind.
+        self.begin_transaction()
+        if self.driver_autocommits:
+            self.warn_once(
+                "import",
+                f"{self.name()} commits each statement as it runs, so an IMPORT or COPY that fails part-way "
+                "leaves the rows already loaded.",
+            )
 
         # Optional rich progress bar for long-running imports.
         use_progress = getattr(_state.conf, "show_progress", False)
@@ -739,6 +763,8 @@ class Database(ABC):
                                             )
                                         if not _state.conf.empty_strings and line[i].strip() == "":
                                             line[i] = None
+                            # Trailing values left off the line are NULL.
+                            line = list(line) + [None] * (len(ts_colnames) - len(line))
                             lt = [
                                 type_objs[i].from_data(val) if val is not None else None for i, val in enumerate(line)
                             ]
@@ -777,13 +803,19 @@ class Database(ABC):
                     break
             return total_rows
 
-        with self._cursor() as curs:
-            if use_progress and progress_ctx is not None:
-                with progress_ctx:
-                    task_id = progress_ctx.add_task(sq_name, total=None)
+        try:
+            with self._cursor() as curs:
+                if use_progress and progress_ctx is not None:
+                    with progress_ctx:
+                        task_id = progress_ctx.add_task(sq_name, total=None)
+                        _import_loop(curs)
+                else:
                     _import_loop(curs)
-            else:
-                _import_loop(curs)
+        except BaseException:
+            # A bad value or short row stops the load as surely as a failed
+            # insert does; either way none of it is kept.
+            self.rollback()
+            raise
 
         if _state.exec_log:
             _state.exec_log.log_status_info(

execsql/db/dsn.py

@@ -13,7 +13,14 @@ from execsql.db.base import Database
 from execsql.db.tiers import SupportTier
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error
-from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
+from execsql.utils.auth import (
+    begin_login,
+    clear_stored_password,
+    get_password,
+    is_login_failure,
+    password_from_keyring,
+    remember_password,
+)
 import execsql.state as _state
 
 __all__ = ["DsnDatabase"]
@@ -56,7 +63,6 @@ class DsnDatabase(Database):
         self.type = dbt_dsn
         self.password = password
         self.encoding = encoding
-        self.encode_commands = True
         self.paramstr = "?"
         self.open_db()
         self.password = None  # Clear cleartext password after successful connection
@@ -72,6 +78,7 @@ class DsnDatabase(Database):
         if self.conn is not None:
             self.conn.close()
             self.conn = None
+        begin_login()
         if self.need_passwd and self.user and self.password is None:
             self.password = get_password("DSN", self.db_name, self.user)
 
@@ -126,8 +133,9 @@ class DsnDatabase(Database):
 
         try:
             _try_connect()
-        except ErrInfo:
-            if not password_from_keyring():
+        except ErrInfo as e:
+            # Only a rejected password means the stored one is stale.
+            if not (password_from_keyring() and is_login_failure(e)):
                 raise
             clear_stored_password("DSN", self.db_name, self.user)
             self.password = get_password(
@@ -139,6 +147,7 @@ class DsnDatabase(Database):
             )
             self.conn = None
             _try_connect()
+        remember_password("DSN", self.db_name, self.user or "")
 
     def exec_cmd(self, querycommand: str) -> None:
         """Execute a stored procedure by name."""

execsql/db/duckdb.py

@@ -74,9 +74,11 @@ class DuckDBDatabase(Database):
         )
         self.type = dbt_duckdb
         self.catalog_name = Path(DuckDB_fn).stem
-        self.encode_commands = False
         self.open_db()
 
+    def current_schema(self) -> str | None:
+        return self._query_current_schema("select current_schema()")
+
     def __repr__(self) -> str:
         return f"DuckDBDatabase({self.db_name!r})"
 

execsql/db/factory.py

@@ -64,7 +64,16 @@ def db_Postgres(
     password: str | None = None,
 ) -> PostgresDatabase:
     """Open a new PostgreSQL connection via psycopg (psycopg3)."""
-    return PostgresDatabase(server_name, database_name, user, pw_needed, port, new_db=new_db, password=password)
+    return PostgresDatabase(
+        server_name,
+        database_name,
+        user,
+        pw_needed,
+        port,
+        new_db=new_db,
+        encoding=encoding,
+        password=password,
+    )
 
 
 def db_SQLite(
@@ -143,7 +152,7 @@ def db_Oracle(
     encoding: str | None = None,
     password: str | None = None,
 ) -> OracleDatabase:
-    """Open an Oracle database connection via cx_Oracle (python-oracledb)."""
+    """Open an Oracle database connection via python-oracledb."""
     return OracleDatabase(server_name, database_name, user, pw_needed, port, encoding, password=password)
 
 

execsql/db/firebird.py

@@ -12,7 +12,14 @@ from execsql.db.base import Database
 from execsql.db.tiers import SupportTier
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error
-from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
+from execsql.utils.auth import (
+    begin_login,
+    clear_stored_password,
+    get_password,
+    is_login_failure,
+    password_from_keyring,
+    remember_password,
+)
 import execsql.state as _state
 
 __all__ = ["FirebirdDatabase"]
@@ -54,7 +61,6 @@ class FirebirdDatabase(Database):
         self.type = dbt_firebird
         self.password = password
         self.encoding = encoding or "latin1"
-        self.encode_commands = True
         self.paramstr = "?"
         self.open_db()
         self.password = None  # Clear cleartext password after successful connection
@@ -100,28 +106,34 @@ class FirebirdDatabase(Database):
 
         if self.conn is None:
             try:
+                begin_login()
                 if self.user and self.need_passwd and not self.password:
                     self.password = get_password(
                         "Firebird",
                         self.db_name,
                         self.user,
                         server_name=self.server_name,
+                        port=self.port,
                     )
                 try:
                     self.conn = db_conn()
-                except Exception:
-                    if not password_from_keyring():
+                except Exception as e:
+                    # Only a rejected password means the stored one is stale;
+                    # a timeout or a missing database leaves it alone.
+                    if not (password_from_keyring() and is_login_failure(e)):
                         raise
-                    clear_stored_password("Firebird", self.db_name, self.user, self.server_name)
+                    clear_stored_password("Firebird", self.db_name, self.user, self.server_name, port=self.port)
                     self.password = get_password(
                         "Firebird",
                         self.db_name,
                         self.user,
                         server_name=self.server_name,
+                        port=self.port,
                         skip_keyring=True,
                         other_msg="(stored credential failed — enter current password)",
                     )
                     self.conn = db_conn()
+                remember_password("Firebird", self.db_name, self.user, self.server_name, port=self.port)
             except SystemExit:
                 # If the user canceled the password prompt.
                 raise
@@ -143,6 +155,9 @@ class FirebirdDatabase(Database):
                 raise
             _state.subvars.add_substitution("$LAST_ROWCOUNT", curs.rowcount)
 
+    # table_exists() without a schema looks only where CREATE TABLE puts a table: one namespace.
+    unqualified_lookup_is_scoped = True
+
     def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
         """Return True if the named table exists in the Firebird database."""
         sql = (
@@ -243,8 +258,7 @@ class FirebirdDatabase(Database):
         return len(rows) > 0
 
     def drop_table(self, tablename: str) -> None:
-        """Drop the named table from the Firebird database."""
+        """Drop *tablename*, schema-qualified and quoted by the caller."""
         # Firebird will thrown an error if there are foreign keys into the table.
-        tablename = self.type.quoted(tablename)
         self.execute(f"DROP TABLE {tablename};")
         self.conn.commit()

execsql/db/mysql.py

@@ -13,7 +13,14 @@ from typing import Any, cast
 from execsql.db.base import Database
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error
-from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
+from execsql.utils.auth import (
+    begin_login,
+    clear_stored_password,
+    get_password,
+    is_login_failure,
+    password_from_keyring,
+    remember_password,
+)
 import execsql.state as _state
 
 __all__ = ["MySQLDatabase"]
@@ -93,7 +100,6 @@ class MySQLDatabase(Database):
         # Greek, or Cyrillic at all — it raises UnicodeEncodeError before the
         # server is reached — so it silently capped what execsql could move.
         self.encoding = encoding or "utf8mb4"
-        self.encode_commands = True
         self.paramstr = "%s"
         self.open_db()
         self.password = None  # Clear cleartext password after successful connection
@@ -149,6 +155,9 @@ class MySQLDatabase(Database):
         """Lowercase *name* when the server is case-insensitive (LCTN 1/2)."""
         return name.lower() if self._lower_case_table_names() in (1, 2) else name
 
+    def current_schema(self) -> str | None:
+        return self._query_current_schema("select database()")
+
     # NB: schema_exists is overridden below to return False unconditionally
     # — MySQL's pre-existing behavior. The case-folding only matters for
     # adapters where schema lookups are meaningful, which MySQL skips.
@@ -209,28 +218,34 @@ class MySQLDatabase(Database):
 
         if self.conn is None:
             try:
+                begin_login()
                 if self.user and self.need_passwd and not self.password:
                     self.password = get_password(
                         "MySQL",
                         self.db_name,
                         self.user,
                         server_name=self.server_name,
+                        port=self.port,
                     )
                 try:
                     self.conn = db_conn()
-                except Exception:
-                    if not password_from_keyring():
+                except Exception as e:
+                    # Only a rejected password means the stored one is stale;
+                    # a timeout or a missing database leaves it alone.
+                    if not (password_from_keyring() and is_login_failure(e)):
                         raise
-                    clear_stored_password("MySQL", self.db_name, self.user, self.server_name)
+                    clear_stored_password("MySQL", self.db_name, self.user, self.server_name, port=self.port)
                     self.password = get_password(
                         "MySQL",
                         self.db_name,
                         self.user,
                         server_name=self.server_name,
+                        port=self.port,
                         skip_keyring=True,
                         other_msg="(stored credential failed — enter current password)",
                     )
                     self.conn = db_conn()
+                remember_password("MySQL", self.db_name, self.user, self.server_name, port=self.port)
                 self.execute("set session sql_mode='ANSI';")
             except SystemExit:
                 # If the user canceled the password prompt.
@@ -306,8 +321,12 @@ class MySQLDatabase(Database):
                 )
             import_cols = csv_file_cols
         input_col_list = ",".join(import_cols)
+        # LOAD DATA is told the file has no escape character (ESCAPED BY ''),
+        # so a file where one was detected takes the row-by-row path, which
+        # reads the format that was detected.
         if (
-            data_table_cols == csv_file_cols
+            csv_file_obj.escapechar is None
+            and data_table_cols == csv_file_cols
             and _state.conf.empty_strings
             and _state.conf.empty_rows
             and not _state.conf.del_empty_cols
@@ -315,19 +334,21 @@ class MySQLDatabase(Database):
             and not _state.conf.trim_strings
             and not _state.conf.replace_newlines
         ):
-            safe_fname = csv_file_obj.csvfname.replace("'", "''")
-            import_sql = f"load data local infile '{safe_fname}' into table {sq_name}"
+            # String literals in this session treat a backslash as an escape,
+            # so the file name, delimiter and quote character are quoted with
+            # quote_literal, which doubles it.
+            import_sql = f"load data local infile {self.quote_literal(csv_file_obj.csvfname)} into table {sq_name}"
             if csv_file_obj.encoding:
                 charset = _PYTHON_TO_MYSQL_CHARSET.get(csv_file_obj.encoding.lower(), csv_file_obj.encoding)
                 import_sql = f"{import_sql} character set {charset}"
-            if csv_file_obj.delimiter or csv_file_obj.quotechar:
-                import_sql = import_sql + " columns"
-                if csv_file_obj.delimiter:
-                    safe_delim = csv_file_obj.delimiter.replace("'", "''")
-                    import_sql = f"{import_sql} terminated by '{safe_delim}'"
-                if csv_file_obj.quotechar:
-                    safe_quote = csv_file_obj.quotechar.replace("'", "''")
-                    import_sql = f"{import_sql} optionally enclosed by '{safe_quote}'"
+            import_sql = import_sql + " columns"
+            if csv_file_obj.delimiter:
+                import_sql = f"{import_sql} terminated by {self.quote_literal(csv_file_obj.delimiter)}"
+            if csv_file_obj.quotechar:
+                import_sql = f"{import_sql} optionally enclosed by {self.quote_literal(csv_file_obj.quotechar)}"
+            # MySQL's default ESCAPED BY '\\' turned C:\new into a newline and
+            # \N into NULL; every other backend keeps the text as written.
+            import_sql = f"{import_sql} escaped by ''"
             # Tell LOAD DATA what ends a line.  Its default is "\n", so a CRLF
             # file left a carriage return on the last field of every row: the
             # value was not empty, NULLIF below could not see it, and a date