execsql2 2.23.0__py3-none-any.whl → 2.24.0__py3-none-any.whl

execsql/config_validation.py

@@ -0,0 +1,282 @@
+"""Check execsql.conf files and report every problem with its file and line.
+
+A run reads config files with :class:`~execsql.config.ConfigData`, which stops
+at the first invalid value and silently ignores anything it does not
+recognize — a misspelled key or section simply does nothing. This module
+backs ``execsql config --validate``: it reads the same files in the same order
+and reports all of it.
+
+Values are checked by :meth:`ConfigData._apply_file`, the method a run uses,
+applied to one option at a time on a fresh instance. Validation therefore
+cannot accept a value a run rejects, or reject one a run accepts.
+"""
+
+from __future__ import annotations
+
+import configparser
+import difflib
+import os
+import re
+import sys
+import warnings
+from collections import deque
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from execsql.config import ConfigData, describe_parse_error
+from execsql.exceptions import ConfigError
+
+if TYPE_CHECKING:
+    from execsql.config import _VariablePool
+
+__all__ = ["ConfigProblem", "known_options", "validate_config"]
+
+#: Sections whose keys are free-form: variable names, and numbered include files.
+_FREE_SECTIONS = frozenset({"variables", "include_required", "include_optional", configparser.DEFAULTSECT})
+
+#: Keys that chain to another config file, and the platform each applies on.
+_CHAIN_KEYS = {
+    "config_file": None,
+    "linux_config_file": "linux",
+    "macos_config_file": "darwin",
+    "win_config_file": "win32",
+}
+
+
+@dataclass(frozen=True)
+class ConfigProblem:
+    """One problem in one config file.
+
+    Attributes:
+        file: The config file.
+        line: 1-based line number, or ``0`` when the problem has no single line.
+        severity: ``"error"`` — a run fails on it — or ``"warning"`` — a run
+            silently ignores it.
+        message: What is wrong, and the likely fix when there is one.
+    """
+
+    file: str
+    line: int
+    severity: str
+    message: str
+
+
+def known_options() -> dict[str, set[str]]:
+    """Every section execsql reads, and the keys it reads in each.
+
+    Free-form sections (``[variables]`` and the include sections) map to an
+    empty set: any key is accepted there.
+    """
+    probe = ConfigData.__new__(ConfigData)
+    probe._set_defaults()
+    probe._read_known_options(configparser.ConfigParser())  # fills ConfigData._option_keys
+    known: dict[str, set[str]] = {}
+    for section, key, _attr in ConfigData._option_keys:
+        known.setdefault(section, set()).add(key)
+    known.setdefault(ConfigData._CONFIG_SECTION, set()).update(_CHAIN_KEYS)
+    for section in _FREE_SECTIONS - {configparser.DEFAULTSECT}:
+        known[section] = set()
+    return known
+
+
+def _line_numbers(text: str) -> tuple[dict[str, int], dict[tuple[str, str], int]]:
+    """Where each section header and each key is, by 1-based line number.
+
+    Keys are lower-cased, as :class:`configparser.ConfigParser` does.
+    Continuation lines (indented) and comments are skipped.
+    """
+    sections: dict[str, int] = {}
+    keys: dict[tuple[str, str], int] = {}
+    current: str | None = None
+    for number, raw in enumerate(text.splitlines(), 1):
+        line = raw.strip()
+        if not line or line[0] in "#;":
+            continue
+        header = re.fullmatch(r"\[(.+)\]", line)
+        if header:
+            current = header.group(1)
+            sections.setdefault(current, number)
+            continue
+        if raw[0].isspace() or current is None:
+            continue
+        option = re.match(r"([^=:]+?)\s*[=:]", line)
+        if option:
+            keys[(current, option.group(1).strip().lower())] = number
+    return sections, keys
+
+
+def _suggest(word: str, candidates: Iterable[str]) -> str | None:
+    match = difflib.get_close_matches(word, sorted(candidates), n=1, cutoff=0.7)
+    return match[0] if match else None
+
+
+def _check_value(
+    section: str,
+    key: str,
+    value: str,
+    variable_pool: _VariablePool,
+    current_script: str,
+) -> str | None:
+    """Apply one option the way a run would; return why it fails, or ``None``."""
+    single = configparser.ConfigParser()
+    single.read_dict({section: {key: value.replace("%", "%%")}})
+    probe = ConfigData.__new__(ConfigData)
+    probe._set_defaults()
+    try:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")  # enc_password's deprecation notice
+            probe._apply_file(single, variable_pool, current_script)
+    except ConfigError as exc:
+        return str(exc)
+    except ValueError as exc:
+        if section in ("include_required", "include_optional"):
+            return f"keys in [{section}] must be numbers, not {key!r}"
+        return str(exc)
+    except Exception as exc:
+        return f"{type(exc).__name__}: {exc}"
+    return None
+
+
+def _check_rule_codes(value: str) -> str | None:
+    """Rule codes in ``[lint] select`` / ``ignore``, checked as ``execsql lint`` checks them."""
+    from execsql.cli.lint import resolve_selectors
+
+    try:
+        resolve_selectors([value])
+    except ValueError as exc:
+        return str(exc)
+    return None
+
+
+def _is_secret(key: str) -> bool:
+    """Whether *key* holds a password (``password``, ``enc_password``, ``passwd``)."""
+    return "passw" in key.lower()
+
+
+def _check_file(
+    path: str,
+    known: dict[str, set[str]],
+    variable_pool: _VariablePool,
+    current_script: str,
+) -> tuple[list[ConfigProblem], list[str]]:
+    """Every problem in one file, and the config files it chains to (in queue order)."""
+    problems: list[ConfigProblem] = []
+
+    def problem(line: int, severity: str, message: str) -> None:
+        problems.append(ConfigProblem(path, line, severity, message))
+
+    try:
+        with open(path) as f:  # the encoding ConfigParser.read uses
+            text = f.read()
+    except (OSError, UnicodeDecodeError) as exc:
+        problem(0, "error", f"cannot read: {exc}")
+        return problems, []
+
+    cp = configparser.ConfigParser()
+    try:
+        cp.read_string(text, source=path)
+    except configparser.ParsingError as exc:
+        if isinstance(exc, configparser.MissingSectionHeaderError):
+            line, text = describe_parse_error(exc)
+            problem(line, "error", text)
+        else:  # every unparsable line, not only the first
+            for err_line, err_text in exc.errors:
+                problem(err_line, "error", f"cannot parse line: {err_text.strip()}")
+        return problems, []
+    except configparser.Error as exc:
+        line, text = describe_parse_error(exc)
+        problem(line, "error", text)
+        return problems, []
+
+    section_lines, key_lines = _line_numbers(text)
+
+    for section in cp.sections():
+        if section not in known:
+            hint = _suggest(section, known)
+            problem(
+                section_lines.get(section, 0),
+                "warning",
+                f"unknown section [{section}]" + (f"; did you mean [{hint}]?" if hint else "; a run ignores it"),
+            )
+
+    chained: list[str] = []
+    for (section, key), line in key_lines.items():
+        if section not in known and section != configparser.DEFAULTSECT:
+            continue  # reported once, as an unknown section
+        if section not in _FREE_SECTIONS and key not in known[section]:
+            elsewhere = sorted(s for s, keys in known.items() if key in keys)
+            if elsewhere:
+                message = f"{key} belongs in [{elsewhere[0]}], not [{section}]"
+            else:
+                hint = _suggest(key, known[section])
+                message = f"unknown key {key!r} in [{section}]" + (
+                    f"; did you mean {hint!r}?" if hint else "; a run ignores it"
+                )
+            problem(line, "warning", message)
+            continue
+        if section == configparser.DEFAULTSECT:
+            continue
+        try:
+            value = cp.get(section, key)
+        except configparser.Error as exc:
+            problem(line, "error", describe_parse_error(exc)[1])
+            continue
+        if key in _CHAIN_KEYS:
+            platform = _CHAIN_KEYS[key]
+            if platform and not sys.platform.startswith(platform):
+                continue  # another platform's file; a run here never reads it
+            expand_home = os.name == "posix" if key == "config_file" else True
+            target = ConfigData._chain_target(value, variable_pool, expand_home=expand_home)
+            if Path(target).is_file():
+                chained.append(target)
+            else:
+                problem(line, "warning", f"{key} = {value}: no such file; a run skips it")
+            continue
+        failure = _check_value(section, key, value, variable_pool, current_script)
+        if failure is None and section == ConfigData._LINT_SECTION:
+            failure = _check_rule_codes(value)
+        if failure:
+            if _is_secret(key):
+                # Never echo a password, even an obfuscated one, into logs.
+                failure = failure.replace(value, "***") if value else failure
+                value = "***"
+            problem(line, "error", f"{key} = {value}: {failure}")
+
+    return problems, chained
+
+
+def validate_config(
+    script_path: str,
+    variable_pool: _VariablePool,
+    config_file: str | None = None,
+) -> tuple[list[str], list[ConfigProblem]]:
+    """Check every config file a run from *script_path* would read.
+
+    Args:
+        script_path: Directory of the script a run would execute; its
+            ``execsql.conf`` is included, as for a run.
+        variable_pool: Substitution variables for expanding chained file paths
+            and checking ``[variables]`` names.
+        config_file: The file ``--config`` names, read last.
+
+    Returns:
+        The files checked, in the order a run reads them, and every problem
+        found, in file order and then line order.
+    """
+    known = known_options()
+    current_script = str(Path(sys.argv[0]).resolve())
+    queue: deque[str] = deque(ConfigData._search_paths(script_path, config_file))
+    checked: list[str] = []
+    problems: list[ConfigProblem] = []
+    while queue and len(checked) < ConfigData._MAX_CONFIG_CHAIN:
+        path = queue.popleft()
+        if path in checked or not Path(path).is_file():
+            continue
+        checked.append(path)
+        found, chained = _check_file(path, known, variable_pool, current_script)
+        problems.extend(sorted(found, key=lambda p: p.line))
+        for target in chained:
+            queue.appendleft(target)
+    return checked, problems

execsql/data/execsql.conf.template

@@ -12,7 +12,7 @@
 #   4. Working directory: current directory
 #
 # Later files override earlier ones. CLI arguments override everything.
-# Generate this template with: execsql --init-config > execsql.conf
+# Generate this template with: execsql config --init > execsql.conf
 
 
 [connect]
@@ -349,6 +349,29 @@
 #win_config_file=
 
 
+[format]
+# Layout for `execsql format`. A flag on the command line wins over these.
+# A run ignores this section.
+
+# Spaces per indent level, for metacommand blocks and SQL.
+#indent=4
+# Put commas at the start of lines instead of the end.
+#leading_comma=No
+# Reformat SQL with sqlglot (needs the [formatter] extra). No = metacommands only.
+#sql=Yes
+
+
+[lint]
+# Rules for `execsql lint`: codes or prefixes, comma-separated (e.g. V001,F).
+# --select or --ignore on the command line replaces the value here.
+# A run ignores this section.
+
+#select=
+#ignore=
+# Yes = exit 1 on warnings too, not only on errors (--strict / --no-strict).
+#strict=No
+
+
 [variables]
 # Substitution variables defined at startup. No pre-defined keys.
 # Example:

execsql/data/script_header.sql

@@ -0,0 +1,23 @@
+-- {name}
+--
+-- PURPOSE
+--
+--
+-- NOTES
+--  1.
+--
+-- PROJECT
+--
+--
+-- COPYRIGHT
+--  Copyright (c) {year},
+--
+-- AUTHORS
+--
+--
+-- HISTORY
+--   Date        Remarks
+--  ----------  -----------------------------------------------------------------
+--  {date}  Created.
+--
+-- ==============================================================================

execsql/db/access.py

@@ -14,7 +14,7 @@ import time
 from pathlib import Path
 from typing import Any, cast
 
-from execsql.db.base import Database
+from execsql.db.base import Database, _fetched
 from execsql.db.tiers import SupportTier
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error
@@ -231,7 +231,13 @@ class AccessDatabase(Database):
         if time.time() - self.last_dao_time < 5.0:
             time.sleep(5 - (time.time() - self.last_dao_time))
 
-    def execute(self, sqlcmd: Any, paramlist: list | None = None) -> None:
+    def execute(
+        self,
+        sqlcmd: Any,
+        paramlist: list | None = None,
+        *,
+        fetch: bool = False,
+    ) -> tuple[list[str], list] | None:
         """Execute a SQL command, handling encoding, DAO flush, and temporary queries."""
 
         # A shortcut to self.cursor().execute() that handles encoding and that
@@ -239,9 +245,15 @@ class AccessDatabase(Database):
         # to allow Jet's read buffer to be flushed (see https://support.microsoft.com/en-us/kb/225048).
         # This also handles the 'CREATE TEMPORARY QUERY' extension to Access.
         # For Access, commands in a tuple (batch) are executed singly.
-        def exec1(sql: str, paramlist: list | None) -> None:
+        def exec1(sql: str, paramlist: list | None) -> tuple[list[str], list] | None:
             tqd = self.temp_rx.match(sql)
             if tqd:
+                if self.conn is not None and self.holding():
+                    self.warn_once(
+                        "temporary query",
+                        "CREATE TEMPORARY QUERY reopens the Access connection, which rolls back statements "
+                        "not yet committed under AUTOCOMMIT OFF or BEGIN BATCH.",
+                    )
                 qn = tqd.group(3)
                 qsql = sql[tqd.end() :]
                 if self.dao_conn is None:
@@ -258,7 +270,9 @@ class AccessDatabase(Database):
                     self.conn = None
                 if tqd.group(1) and tqd.group(1).strip().lower()[:4] == "temp" and qn not in self.temp_query_names:
                     self.temp_query_names.append(qn)
+                return None
             else:
+                self.before_statement(sql)
                 self.dao_flush_check()
                 with self._cursor() as curs:
                     encoded_sql: Any
@@ -271,12 +285,14 @@ class AccessDatabase(Database):
                     else:
                         curs.execute(encoded_sql, paramlist)
                     _state.subvars.add_substitution("$LAST_ROWCOUNT", curs.rowcount)
+                    return _fetched(curs) if fetch else None
 
         if type(sqlcmd) in (list, tuple):
+            result = None
             for sql in sqlcmd:
-                exec1(sql, paramlist)
-        else:
-            exec1(sqlcmd, paramlist)
+                result = exec1(sql, paramlist)
+            return result
+        return exec1(sqlcmd, paramlist)
 
     def exec_cmd(self, querycommand: str) -> None:
         """Execute a stored query command via DAO."""

execsql/db/base.py

@@ -29,6 +29,21 @@ import execsql.state as _state
 
 __all__ = ["Database", "DatabasePool"]
 
+# Leading whitespace and comments, skipped to find a statement's first words.
+_LEADING_NOISE_RX = re.compile(r"(?:\s+|--[^\n]*(?:\n|$)|/\*.*?\*/)*", re.S)
+
+# Statements that manage a transaction themselves; one is never opened in front of them.
+_TRANSACTION_CONTROL_RX = re.compile(
+    r"(?:BEGIN|START\s+TRANSACTION|COMMIT|END|ROLLBACK|ABORT|SAVEPOINT|RELEASE)\b",
+    re.I,
+)
+
+
+def statement_start(sql: str) -> str:
+    """*sql* from its first keyword on: leading whitespace and comments removed."""
+    match = _LEADING_NOISE_RX.match(sql)
+    return sql[match.end() :] if match else sql
+
 
 def _default_dt_cast() -> dict[type, Callable]:
     """Build the default type-cast mapping used by all database backends."""
@@ -46,6 +61,13 @@ def _default_dt_cast() -> dict[type, Callable]:
     }
 
 
+def _fetched(curs: Any) -> tuple[list[str], list] | None:
+    """The rows a just-executed cursor produced, as ``(column_names, rows)``; ``None`` if none."""
+    if curs.description is None:
+        return None
+    return [d[0] for d in curs.description], curs.fetchall()
+
+
 class Database(ABC):
     """Abstract base class for every DBMS adapter.
 
@@ -215,13 +237,17 @@ class Database(ABC):
         """Return a comma-separated string of *paramcount* parameter placeholders."""
         return ",".join((self.paramstr,) * paramcount)
 
-    def execute(self, sql: Any, paramlist: list | None = None) -> None:
+    def execute(self, sql: Any, paramlist: list | None = None, *, fetch: bool = False) -> tuple[list[str], list] | None:
         """Execute *sql* (optionally with *paramlist*), updating ``$LAST_ROWCOUNT``.
 
+        With *fetch*, return ``(column_names, rows)`` when the statement
+        produces rows (a prompt shows them), else ``None``.
+
         Rolls back the current transaction and re-raises on any driver error.
         """
         if type(sql) in (tuple, list):
             sql = " ".join(sql)
+        self.before_statement(sql)
         try:
             with self._cursor() as curs:
                 if paramlist is None:
@@ -232,6 +258,7 @@ class Database(ABC):
                     _state.subvars.add_substitution("$LAST_ROWCOUNT", curs.rowcount)
                 except Exception:
                     pass  # Non-critical: some drivers lack rowcount support.
+                return _fetched(curs) if fetch else None
         except Exception:
             try:
                 self.rollback()
@@ -244,6 +271,81 @@ class Database(ABC):
         """Execute a stored procedure or function by name."""
         ...
 
+    # ------------------------------------------------------------------
+    # Holding statements until commit (AUTOCOMMIT OFF, BEGIN BATCH)
+    # ------------------------------------------------------------------
+
+    #: Statements the DBMS commits on its own, with everything run before
+    #: them, even inside a transaction (MySQL and Oracle DDL).  Matched against
+    #: the start of the statement; a warning names each one run while holding.
+    implicit_commit_rx: ClassVar[re.Pattern[str] | None] = None
+
+    #: Statements that cannot run inside a transaction on this DBMS; one is
+    #: not opened in front of them.
+    no_transaction_rx: ClassVar[re.Pattern[str] | None] = None
+
+    #: Set on a connection whose driver could only connect in autocommit mode
+    #: (an ODBC DSN without transactions): nothing can be held back.
+    driver_autocommits: bool = False
+
+    def holding(self) -> bool:
+        """Whether statements on this connection wait for a commit.
+
+        True while AUTOCOMMIT is OFF for this database, or inside BEGIN BATCH
+        (which holds every database).
+        """
+        if not self.autocommit:
+            return True
+        status = _state.status
+        return status is not None and status.batch.in_batch()
+
+    def before_statement(self, sql: str) -> None:
+        """Make sure *sql* is held until commit when execsql is holding statements.
+
+        Most drivers already hold every statement: they open a transaction
+        before the first one and keep it open until commit or rollback.  An
+        adapter whose driver does not (SQLite for DDL, DuckDB for everything)
+        overrides :meth:`begin_transaction`.  Where the DBMS itself commits a
+        statement regardless, a warning says so, since the work cannot be
+        rolled back.
+        """
+        if not self.holding():
+            return
+        if self.driver_autocommits:
+            self.warn_once(
+                "driver",
+                f"{self.name()} cannot hold statements until COMMIT: its driver commits each one as it runs. "
+                "AUTOCOMMIT OFF and BEGIN BATCH have no effect on it.",
+            )
+            return
+        start = statement_start(sql)
+        if self.implicit_commit_rx is not None and (m := self.implicit_commit_rx.match(start)):
+            from execsql.utils.errors import write_warning
+
+            write_warning(
+                f"{self.type.dbms_id} commits {' '.join(m.group(0).upper().split())} immediately, together with "
+                "everything run before it; it cannot be rolled back.",
+                always=True,
+            )
+        if _TRANSACTION_CONTROL_RX.match(start):
+            return
+        if self.no_transaction_rx is not None and self.no_transaction_rx.match(start):
+            return
+        self.begin_transaction()
+
+    def begin_transaction(self) -> None:
+        """Open a transaction if none is open.  Most drivers do this themselves; a no-op here."""
+
+    def warn_once(self, key: str, message: str) -> None:
+        """Write *message* as a warning the first time *key* comes up on this connection."""
+        warned: set[str] = self.__dict__.setdefault("_warned", set())
+        if key in warned:
+            return
+        warned.add(key)
+        from execsql.utils.errors import write_warning
+
+        write_warning(message, always=True)
+
     def autocommit_on(self) -> None:
         """Enable autocommit mode so each statement is committed immediately."""
         self.autocommit = True
@@ -562,6 +664,7 @@ class Database(ABC):
         rows = iter(rowsource)
         eof = False
         total_rows = 0
+        self.before_statement(sql)
 
         # Optional rich progress bar for long-running imports.
         use_progress = getattr(_state.conf, "show_progress", False)
@@ -745,6 +848,7 @@ class Database(ABC):
         sq_name = self.schema_qualified_table_name(schema_name, table_name)
         quoted_col = self.quote_identifier(column_name)
         sql = f"insert into {sq_name} ({quoted_col}) values ({self.paramsubs(1)});"
+        self.before_statement(sql)
         with self._cursor() as curs:
             curs.execute(sql, (filedata,))
 

execsql/db/dsn.py

@@ -109,6 +109,8 @@ class DsnDatabase(Database):
                 if "Optional feature not implemented" in excdesc:
                     try:
                         _dsn_connect(autocommit=True)
+                        # This ODBC driver has no transactions: nothing can be held.
+                        self.driver_autocommits = True
                     except Exception as e:
                         raise ErrInfo(
                             type="exception",