execsql2 2.23.1__py3-none-any.whl → 2.25.0__py3-none-any.whl

execsql/config_validation.py

@@ -0,0 +1,282 @@
+"""Check execsql.conf files and report every problem with its file and line.
+
+A run reads config files with :class:`~execsql.config.ConfigData`, which stops
+at the first invalid value and silently ignores anything it does not
+recognize — a misspelled key or section simply does nothing. This module
+backs ``execsql config --validate``: it reads the same files in the same order
+and reports all of it.
+
+Values are checked by :meth:`ConfigData._apply_file`, the method a run uses,
+applied to one option at a time on a fresh instance. Validation therefore
+cannot accept a value a run rejects, or reject one a run accepts.
+"""
+
+from __future__ import annotations
+
+import configparser
+import difflib
+import os
+import re
+import sys
+import warnings
+from collections import deque
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from execsql.config import ConfigData, describe_parse_error
+from execsql.exceptions import ConfigError
+
+if TYPE_CHECKING:
+    from execsql.config import _VariablePool
+
+__all__ = ["ConfigProblem", "known_options", "validate_config"]
+
+#: Sections whose keys are free-form: variable names, and numbered include files.
+_FREE_SECTIONS = frozenset({"variables", "include_required", "include_optional", configparser.DEFAULTSECT})
+
+#: Keys that chain to another config file, and the platform each applies on.
+_CHAIN_KEYS = {
+    "config_file": None,
+    "linux_config_file": "linux",
+    "macos_config_file": "darwin",
+    "win_config_file": "win32",
+}
+
+
+@dataclass(frozen=True)
+class ConfigProblem:
+    """One problem in one config file.
+
+    Attributes:
+        file: The config file.
+        line: 1-based line number, or ``0`` when the problem has no single line.
+        severity: ``"error"`` — a run fails on it — or ``"warning"`` — a run
+            silently ignores it.
+        message: What is wrong, and the likely fix when there is one.
+    """
+
+    file: str
+    line: int
+    severity: str
+    message: str
+
+
+def known_options() -> dict[str, set[str]]:
+    """Every section execsql reads, and the keys it reads in each.
+
+    Free-form sections (``[variables]`` and the include sections) map to an
+    empty set: any key is accepted there.
+    """
+    probe = ConfigData.__new__(ConfigData)
+    probe._set_defaults()
+    probe._read_known_options(configparser.ConfigParser())  # fills ConfigData._option_keys
+    known: dict[str, set[str]] = {}
+    for section, key, _attr in ConfigData._option_keys:
+        known.setdefault(section, set()).add(key)
+    known.setdefault(ConfigData._CONFIG_SECTION, set()).update(_CHAIN_KEYS)
+    for section in _FREE_SECTIONS - {configparser.DEFAULTSECT}:
+        known[section] = set()
+    return known
+
+
+def _line_numbers(text: str) -> tuple[dict[str, int], dict[tuple[str, str], int]]:
+    """Where each section header and each key is, by 1-based line number.
+
+    Keys are lower-cased, as :class:`configparser.ConfigParser` does.
+    Continuation lines (indented) and comments are skipped.
+    """
+    sections: dict[str, int] = {}
+    keys: dict[tuple[str, str], int] = {}
+    current: str | None = None
+    for number, raw in enumerate(text.splitlines(), 1):
+        line = raw.strip()
+        if not line or line[0] in "#;":
+            continue
+        header = re.fullmatch(r"\[(.+)\]", line)
+        if header:
+            current = header.group(1)
+            sections.setdefault(current, number)
+            continue
+        if raw[0].isspace() or current is None:
+            continue
+        option = re.match(r"([^=:]+?)\s*[=:]", line)
+        if option:
+            keys[(current, option.group(1).strip().lower())] = number
+    return sections, keys
+
+
+def _suggest(word: str, candidates: Iterable[str]) -> str | None:
+    match = difflib.get_close_matches(word, sorted(candidates), n=1, cutoff=0.7)
+    return match[0] if match else None
+
+
+def _check_value(
+    section: str,
+    key: str,
+    value: str,
+    variable_pool: _VariablePool,
+    current_script: str,
+) -> str | None:
+    """Apply one option the way a run would; return why it fails, or ``None``."""
+    single = configparser.ConfigParser()
+    single.read_dict({section: {key: value.replace("%", "%%")}})
+    probe = ConfigData.__new__(ConfigData)
+    probe._set_defaults()
+    try:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")  # enc_password's deprecation notice
+            probe._apply_file(single, variable_pool, current_script)
+    except ConfigError as exc:
+        return str(exc)
+    except ValueError as exc:
+        if section in ("include_required", "include_optional"):
+            return f"keys in [{section}] must be numbers, not {key!r}"
+        return str(exc)
+    except Exception as exc:
+        return f"{type(exc).__name__}: {exc}"
+    return None
+
+
+def _check_rule_codes(value: str) -> str | None:
+    """Rule codes in ``[lint] select`` / ``ignore``, checked as ``execsql lint`` checks them."""
+    from execsql.cli.lint import resolve_selectors
+
+    try:
+        resolve_selectors([value])
+    except ValueError as exc:
+        return str(exc)
+    return None
+
+
+def _is_secret(key: str) -> bool:
+    """Whether *key* holds a password (``password``, ``enc_password``, ``passwd``)."""
+    return "passw" in key.lower()
+
+
+def _check_file(
+    path: str,
+    known: dict[str, set[str]],
+    variable_pool: _VariablePool,
+    current_script: str,
+) -> tuple[list[ConfigProblem], list[str]]:
+    """Every problem in one file, and the config files it chains to (in queue order)."""
+    problems: list[ConfigProblem] = []
+
+    def problem(line: int, severity: str, message: str) -> None:
+        problems.append(ConfigProblem(path, line, severity, message))
+
+    try:
+        with open(path) as f:  # the encoding ConfigParser.read uses
+            text = f.read()
+    except (OSError, UnicodeDecodeError) as exc:
+        problem(0, "error", f"cannot read: {exc}")
+        return problems, []
+
+    cp = configparser.ConfigParser()
+    try:
+        cp.read_string(text, source=path)
+    except configparser.ParsingError as exc:
+        if isinstance(exc, configparser.MissingSectionHeaderError):
+            line, text = describe_parse_error(exc)
+            problem(line, "error", text)
+        else:  # every unparsable line, not only the first
+            for err_line, err_text in exc.errors:
+                problem(err_line, "error", f"cannot parse line: {err_text.strip()}")
+        return problems, []
+    except configparser.Error as exc:
+        line, text = describe_parse_error(exc)
+        problem(line, "error", text)
+        return problems, []
+
+    section_lines, key_lines = _line_numbers(text)
+
+    for section in cp.sections():
+        if section not in known:
+            hint = _suggest(section, known)
+            problem(
+                section_lines.get(section, 0),
+                "warning",
+                f"unknown section [{section}]" + (f"; did you mean [{hint}]?" if hint else "; a run ignores it"),
+            )
+
+    chained: list[str] = []
+    for (section, key), line in key_lines.items():
+        if section not in known and section != configparser.DEFAULTSECT:
+            continue  # reported once, as an unknown section
+        if section not in _FREE_SECTIONS and key not in known[section]:
+            elsewhere = sorted(s for s, keys in known.items() if key in keys)
+            if elsewhere:
+                message = f"{key} belongs in [{elsewhere[0]}], not [{section}]"
+            else:
+                hint = _suggest(key, known[section])
+                message = f"unknown key {key!r} in [{section}]" + (
+                    f"; did you mean {hint!r}?" if hint else "; a run ignores it"
+                )
+            problem(line, "warning", message)
+            continue
+        if section == configparser.DEFAULTSECT:
+            continue
+        try:
+            value = cp.get(section, key)
+        except configparser.Error as exc:
+            problem(line, "error", describe_parse_error(exc)[1])
+            continue
+        if key in _CHAIN_KEYS:
+            platform = _CHAIN_KEYS[key]
+            if platform and not sys.platform.startswith(platform):
+                continue  # another platform's file; a run here never reads it
+            expand_home = os.name == "posix" if key == "config_file" else True
+            target = ConfigData._chain_target(value, variable_pool, expand_home=expand_home)
+            if Path(target).is_file():
+                chained.append(target)
+            else:
+                problem(line, "warning", f"{key} = {value}: no such file; a run skips it")
+            continue
+        failure = _check_value(section, key, value, variable_pool, current_script)
+        if failure is None and section == ConfigData._LINT_SECTION:
+            failure = _check_rule_codes(value)
+        if failure:
+            if _is_secret(key):
+                # Never echo a password, even an obfuscated one, into logs.
+                failure = failure.replace(value, "***") if value else failure
+                value = "***"
+            problem(line, "error", f"{key} = {value}: {failure}")
+
+    return problems, chained
+
+
+def validate_config(
+    script_path: str,
+    variable_pool: _VariablePool,
+    config_file: str | None = None,
+) -> tuple[list[str], list[ConfigProblem]]:
+    """Check every config file a run from *script_path* would read.
+
+    Args:
+        script_path: Directory of the script a run would execute; its
+            ``execsql.conf`` is included, as for a run.
+        variable_pool: Substitution variables for expanding chained file paths
+            and checking ``[variables]`` names.
+        config_file: The file ``--config`` names, read last.
+
+    Returns:
+        The files checked, in the order a run reads them, and every problem
+        found, in file order and then line order.
+    """
+    known = known_options()
+    current_script = str(Path(sys.argv[0]).resolve())
+    queue: deque[str] = deque(ConfigData._search_paths(script_path, config_file))
+    checked: list[str] = []
+    problems: list[ConfigProblem] = []
+    while queue and len(checked) < ConfigData._MAX_CONFIG_CHAIN:
+        path = queue.popleft()
+        if path in checked or not Path(path).is_file():
+            continue
+        checked.append(path)
+        found, chained = _check_file(path, known, variable_pool, current_script)
+        problems.extend(sorted(found, key=lambda p: p.line))
+        for target in chained:
+            queue.appendleft(target)
+    return checked, problems

execsql/data/execsql.conf.template

@@ -12,7 +12,7 @@
 #   4. Working directory: current directory
 #
 # Later files override earlier ones. CLI arguments override everything.
-# Generate this template with: execsql --init-config > execsql.conf
+# Generate this template with: execsql config --init > execsql.conf
 
 
 [connect]
@@ -46,7 +46,7 @@
 #password_prompt=Yes
 
 # Whether to use the OS credential store (keyring) for password storage.
-# Requires: pip install execsql2[auth]
+# Requires the auth extra: uv tool install "execsql2[auth]"
 # Values: Yes or No.  Default: Yes.
 #use_keyring=Yes
 
@@ -71,7 +71,8 @@
 [input]
 # Settings for handling imported data.
 
-# Whether to convert numeric values to double precision when using MS Access.
+# MS Access: create columns for decimal data as NUMERIC(precision,scale)
+# (Yes) or as DOUBLE (No) when IMPORT or COPY creates a table.
 # Values: Yes or No.  Default: No.
 #access_use_numeric=No
 
@@ -261,6 +262,15 @@
 # Values: Yes or No.  Default: No.
 #use_tls=No
 
+# Whether to verify the SMTP server's certificate when use_ssl or use_tls is on.
+# Turn off only for a test relay; prefer ca_file for an internal certificate.
+# Values: Yes or No.  Default: Yes.
+#verify_certificate=Yes
+
+# CA certificate file (PEM) to trust, for a server with an internal or
+# self-signed certificate.  Default: the system's trusted CAs.
+#ca_file=
+
 # Email body format.
 # Values: plain or html.  Default: plain.
 #email_format=plain
@@ -297,8 +307,8 @@
 # Values: Yes or No.  Default: Yes.
 #allow_rm_file=Yes
 
-# Whether to allow the SERVE metacommand (which opens an HTTP server on
-# a local port to serve a single file). Set to No to disable.
+# Whether to allow the SERVE metacommand (which copies a file to stdout
+# with HTTP headers, for CGI use). Set to No to disable.
 # Also controllable via --no-serve CLI flag.
 # Values: Yes or No.  Default: Yes.
 #allow_serve=Yes
@@ -349,6 +359,29 @@
 #win_config_file=
 
 
+[format]
+# Layout for `execsql format`. A flag on the command line wins over these.
+# A run ignores this section.
+
+# Spaces per indent level, for metacommand blocks and SQL.
+#indent=4
+# Put commas at the start of lines instead of the end.
+#leading_comma=No
+# Reformat SQL with sqlglot (needs the [formatter] extra). No = metacommands only.
+#sql=Yes
+
+
+[lint]
+# Rules for `execsql lint`: codes or prefixes, comma-separated (e.g. V001,F).
+# --select or --ignore on the command line replaces the value here.
+# A run ignores this section.
+
+#select=
+#ignore=
+# Yes = exit 1 on warnings too, not only on errors (--strict / --no-strict).
+#strict=No
+
+
 [variables]
 # Substitution variables defined at startup. No pre-defined keys.
 # Example:

execsql/data/script_header.sql

@@ -0,0 +1,23 @@
+-- {name}
+--
+-- PURPOSE
+--
+--
+-- NOTES
+--  1.
+--
+-- PROJECT
+--
+--
+-- COPYRIGHT
+--  Copyright (c) {year},
+--
+-- AUTHORS
+--
+--
+-- HISTORY
+--   Date        Remarks
+--  ----------  -----------------------------------------------------------------
+--  {date}  Created.
+--
+-- ==============================================================================

execsql/db/access.py

@@ -18,7 +18,14 @@ from execsql.db.base import Database, _fetched
 from execsql.db.tiers import SupportTier
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error
-from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
+from execsql.utils.auth import (
+    begin_login,
+    clear_stored_password,
+    get_password,
+    is_login_failure,
+    password_from_keyring,
+    remember_password,
+)
 import execsql.state as _state
 
 __all__ = ["AccessDatabase"]
@@ -76,7 +83,6 @@ class AccessDatabase(Database):
         self.password = password
         # Encoding is only applicable to Jet < 4.0: non-accdb databases.
         self.encoding = encoding or "windows-1252"
-        self.encode_commands = True
         self.dao_conn: Any = None
         self.conn = None  # ODBC connection
         self.paramstr = "?"
@@ -104,9 +110,12 @@ class AccessDatabase(Database):
         if self.conn is not None:
             self.conn.close()
             self.conn = None
+        begin_login()
         if self.need_passwd and self.user and self.password is None:
             self.password = get_password("MS-Access", self.db_name, self.user)
 
+        errors: list[BaseException] = []
+
         def _try_odbc_drivers():
             db_name = str(Path(self.db_name).resolve())
             for cs, jet4flag in self.connection_strings:
@@ -120,7 +129,8 @@ class AccessDatabase(Database):
                     # the connect attempt at 30 s to match the other
                     # adapters.
                     self.conn = pyodbc.connect(connstr, timeout=30)
-                except Exception:
+                except Exception as e:
+                    errors.append(e)
                     if _state.exec_log is not None:
                         _state.exec_log.log_status_info(
                             f"Could not connect via ODBC using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
@@ -134,7 +144,8 @@ class AccessDatabase(Database):
                     return True
             return False
 
-        if not _try_odbc_drivers() and password_from_keyring():
+        # Only a rejected password means the stored one is stale.
+        if not _try_odbc_drivers() and password_from_keyring() and any(is_login_failure(e) for e in errors):
             clear_stored_password("MS-Access", self.db_name, self.user)
             self.password = get_password(
                 "MS-Access",
@@ -144,6 +155,8 @@ class AccessDatabase(Database):
                 other_msg="(stored credential failed — enter current password)",
             )
             _try_odbc_drivers()
+        if self.conn and self.user:
+            remember_password("MS-Access", self.db_name, self.user)
 
         if not self.conn:
             raise ErrInfo(
@@ -158,10 +171,13 @@ class AccessDatabase(Database):
         if self.dao_conn is not None:
             self.dao_conn.Close()
             self.dao_conn = None
+        begin_login()
         if self.need_passwd and self.user and self.password is None:
             self.password = get_password("MS-Access", self.db_name, self.user)
         dao_engines = ("DAO.DBEngine.120", "DAO.DBEngine.36")
 
+        dao_errors: list[BaseException] = []
+
         def _try_dao_engines():
             for engine in dao_engines:
                 try:
@@ -175,7 +191,8 @@ class AccessDatabase(Database):
                         )
                     else:
                         self.dao_conn = daoEngine.OpenDatabase(self.db_name)
-                except Exception:
+                except Exception as e:
+                    dao_errors.append(e)
                     if _state.exec_log is not None:
                         _state.exec_log.log_status_info(f"Could not connect via DAO using: {engine}")
                 else:
@@ -184,7 +201,7 @@ class AccessDatabase(Database):
                     return True
             return False
 
-        if not _try_dao_engines() and password_from_keyring():
+        if not _try_dao_engines() and password_from_keyring() and any(is_login_failure(e) for e in dao_errors):
             clear_stored_password("MS-Access", self.db_name, self.user)
             self.password = get_password(
                 "MS-Access",
@@ -194,6 +211,8 @@ class AccessDatabase(Database):
                 other_msg="(stored credential failed — enter current password)",
             )
             _try_dao_engines()
+        if self.dao_conn and self.user:
+            remember_password("MS-Access", self.db_name, self.user)
 
         if not self.dao_conn:
             raise ErrInfo(
@@ -339,6 +358,9 @@ class AccessDatabase(Database):
 
         return headers, iter(dict_row, None)
 
+    # table_exists() without a schema looks only where CREATE TABLE puts a table: one namespace.
+    unqualified_lookup_is_scoped = True
+
     def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
         """Return True if the named table exists in the Access database.
 
@@ -418,9 +440,8 @@ class AccessDatabase(Database):
         return False
 
     def drop_table(self, tablename: str) -> None:
-        """Drop the named table from the Access database."""
+        """Drop *tablename*, quoted by the caller."""
         self.dao_flush_check()
-        tablename = self.type.quoted(tablename)
         self.execute(f"drop table {tablename};")
 
     def as_datetime(self, val: Any) -> datetime.datetime | datetime.date | datetime.time | None:

execsql/db/base.py

@@ -98,7 +98,6 @@ class Database(ABC):
     password: str | None
     port: int | None
     encoding: Any
-    encode_commands: bool
     paramstr: str
     conn: Any
     autocommit: bool
@@ -143,7 +142,6 @@ class Database(ABC):
         self.password: str | None = None
         self.port = port
         self.encoding = encoding
-        self.encode_commands = True
         self.paramstr = "?"
         self.conn = None
         self.autocommit = True
@@ -494,6 +492,21 @@ class Database(ABC):
             rows = curs.fetchall()
         return len(rows) > 0
 
+    #: Whether :meth:`table_exists` without a schema already looks only where
+    #: an unqualified ``CREATE TABLE`` puts a table (one namespace, or the
+    #: search path).  When not, the check is narrowed to :meth:`current_schema`.
+    unqualified_lookup_is_scoped = False
+
+    def current_schema(self) -> str | None:
+        """The schema (MySQL: database) an unqualified table name is created in, or ``None`` if unknown."""
+        return None
+
+    def _query_current_schema(self, sql: str) -> str | None:
+        with self._cursor() as curs:
+            curs.execute(sql)
+            row = curs.fetchone()
+        return str(row[0]) if row and row[0] else None
+
     def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
         """Return ``True`` if *table_name* (optionally in *schema_name*) exists."""
         with self._cursor() as curs:
@@ -665,6 +678,17 @@ class Database(ABC):
         eof = False
         total_rows = 0
         self.before_statement(sql)
+        # The rows go in as several batches and are committed once, by the
+        # caller.  Most drivers hold every batch in the transaction they open
+        # themselves; DuckDB's commits each statement unless one is open, so a
+        # failure would leave the batches before it behind.
+        self.begin_transaction()
+        if self.driver_autocommits:
+            self.warn_once(
+                "import",
+                f"{self.name()} commits each statement as it runs, so an IMPORT or COPY that fails part-way "
+                "leaves the rows already loaded.",
+            )
 
         # Optional rich progress bar for long-running imports.
         use_progress = getattr(_state.conf, "show_progress", False)
@@ -739,6 +763,8 @@ class Database(ABC):
                                             )
                                         if not _state.conf.empty_strings and line[i].strip() == "":
                                             line[i] = None
+                            # Trailing values left off the line are NULL.
+                            line = list(line) + [None] * (len(ts_colnames) - len(line))
                             lt = [
                                 type_objs[i].from_data(val) if val is not None else None for i, val in enumerate(line)
                             ]
@@ -777,13 +803,19 @@ class Database(ABC):
                     break
             return total_rows
 
-        with self._cursor() as curs:
-            if use_progress and progress_ctx is not None:
-                with progress_ctx:
-                    task_id = progress_ctx.add_task(sq_name, total=None)
+        try:
+            with self._cursor() as curs:
+                if use_progress and progress_ctx is not None:
+                    with progress_ctx:
+                        task_id = progress_ctx.add_task(sq_name, total=None)
+                        _import_loop(curs)
+                else:
                     _import_loop(curs)
-            else:
-                _import_loop(curs)
+        except BaseException:
+            # A bad value or short row stops the load as surely as a failed
+            # insert does; either way none of it is kept.
+            self.rollback()
+            raise
 
         if _state.exec_log:
             _state.exec_log.log_status_info(

execsql/db/dsn.py

@@ -13,7 +13,14 @@ from execsql.db.base import Database
 from execsql.db.tiers import SupportTier
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error
-from execsql.utils.auth import clear_stored_password, get_password, password_from_keyring
+from execsql.utils.auth import (
+    begin_login,
+    clear_stored_password,
+    get_password,
+    is_login_failure,
+    password_from_keyring,
+    remember_password,
+)
 import execsql.state as _state
 
 __all__ = ["DsnDatabase"]
@@ -56,7 +63,6 @@ class DsnDatabase(Database):
         self.type = dbt_dsn
         self.password = password
         self.encoding = encoding
-        self.encode_commands = True
         self.paramstr = "?"
         self.open_db()
         self.password = None  # Clear cleartext password after successful connection
@@ -72,6 +78,7 @@ class DsnDatabase(Database):
         if self.conn is not None:
             self.conn.close()
             self.conn = None
+        begin_login()
         if self.need_passwd and self.user and self.password is None:
             self.password = get_password("DSN", self.db_name, self.user)
 
@@ -126,8 +133,9 @@ class DsnDatabase(Database):
 
         try:
             _try_connect()
-        except ErrInfo:
-            if not password_from_keyring():
+        except ErrInfo as e:
+            # Only a rejected password means the stored one is stale.
+            if not (password_from_keyring() and is_login_failure(e)):
                 raise
             clear_stored_password("DSN", self.db_name, self.user)
             self.password = get_password(
@@ -139,6 +147,7 @@ class DsnDatabase(Database):
             )
             self.conn = None
             _try_connect()
+        remember_password("DSN", self.db_name, self.user or "")
 
     def exec_cmd(self, querycommand: str) -> None:
         """Execute a stored procedure by name."""

execsql/db/duckdb.py

@@ -74,9 +74,11 @@ class DuckDBDatabase(Database):
         )
         self.type = dbt_duckdb
         self.catalog_name = Path(DuckDB_fn).stem
-        self.encode_commands = False
         self.open_db()
 
+    def current_schema(self) -> str | None:
+        return self._query_current_schema("select current_schema()")
+
     def __repr__(self) -> str:
         return f"DuckDBDatabase({self.db_name!r})"