execsql2 2.23.1__py3-none-any.whl → 2.24.0__py3-none-any.whl

execsql/config_validation.py

@@ -0,0 +1,282 @@
+"""Check execsql.conf files and report every problem with its file and line.
+
+A run reads config files with :class:`~execsql.config.ConfigData`, which stops
+at the first invalid value and silently ignores anything it does not
+recognize — a misspelled key or section simply does nothing. This module
+backs ``execsql config --validate``: it reads the same files in the same order
+and reports all of it.
+
+Values are checked by :meth:`ConfigData._apply_file`, the method a run uses,
+applied to one option at a time on a fresh instance. Validation therefore
+cannot accept a value a run rejects, or reject one a run accepts.
+"""
+
+from __future__ import annotations
+
+import configparser
+import difflib
+import os
+import re
+import sys
+import warnings
+from collections import deque
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from execsql.config import ConfigData, describe_parse_error
+from execsql.exceptions import ConfigError
+
+if TYPE_CHECKING:
+    from execsql.config import _VariablePool
+
+__all__ = ["ConfigProblem", "known_options", "validate_config"]
+
+#: Sections whose keys are free-form: variable names, and numbered include files.
+_FREE_SECTIONS = frozenset({"variables", "include_required", "include_optional", configparser.DEFAULTSECT})
+
+#: Keys that chain to another config file, and the platform each applies on.
+_CHAIN_KEYS = {
+    "config_file": None,
+    "linux_config_file": "linux",
+    "macos_config_file": "darwin",
+    "win_config_file": "win32",
+}
+
+
+@dataclass(frozen=True)
+class ConfigProblem:
+    """One problem in one config file.
+
+    Attributes:
+        file: The config file.
+        line: 1-based line number, or ``0`` when the problem has no single line.
+        severity: ``"error"`` — a run fails on it — or ``"warning"`` — a run
+            silently ignores it.
+        message: What is wrong, and the likely fix when there is one.
+    """
+
+    file: str
+    line: int
+    severity: str
+    message: str
+
+
+def known_options() -> dict[str, set[str]]:
+    """Every section execsql reads, and the keys it reads in each.
+
+    Free-form sections (``[variables]`` and the include sections) map to an
+    empty set: any key is accepted there.
+    """
+    probe = ConfigData.__new__(ConfigData)
+    probe._set_defaults()
+    probe._read_known_options(configparser.ConfigParser())  # fills ConfigData._option_keys
+    known: dict[str, set[str]] = {}
+    for section, key, _attr in ConfigData._option_keys:
+        known.setdefault(section, set()).add(key)
+    known.setdefault(ConfigData._CONFIG_SECTION, set()).update(_CHAIN_KEYS)
+    for section in _FREE_SECTIONS - {configparser.DEFAULTSECT}:
+        known[section] = set()
+    return known
+
+
+def _line_numbers(text: str) -> tuple[dict[str, int], dict[tuple[str, str], int]]:
+    """Where each section header and each key is, by 1-based line number.
+
+    Keys are lower-cased, as :class:`configparser.ConfigParser` does.
+    Continuation lines (indented) and comments are skipped.
+    """
+    sections: dict[str, int] = {}
+    keys: dict[tuple[str, str], int] = {}
+    current: str | None = None
+    for number, raw in enumerate(text.splitlines(), 1):
+        line = raw.strip()
+        if not line or line[0] in "#;":
+            continue
+        header = re.fullmatch(r"\[(.+)\]", line)
+        if header:
+            current = header.group(1)
+            sections.setdefault(current, number)
+            continue
+        if raw[0].isspace() or current is None:
+            continue
+        option = re.match(r"([^=:]+?)\s*[=:]", line)
+        if option:
+            keys[(current, option.group(1).strip().lower())] = number
+    return sections, keys
+
+
+def _suggest(word: str, candidates: Iterable[str]) -> str | None:
+    match = difflib.get_close_matches(word, sorted(candidates), n=1, cutoff=0.7)
+    return match[0] if match else None
+
+
+def _check_value(
+    section: str,
+    key: str,
+    value: str,
+    variable_pool: _VariablePool,
+    current_script: str,
+) -> str | None:
+    """Apply one option the way a run would; return why it fails, or ``None``."""
+    single = configparser.ConfigParser()
+    single.read_dict({section: {key: value.replace("%", "%%")}})
+    probe = ConfigData.__new__(ConfigData)
+    probe._set_defaults()
+    try:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")  # enc_password's deprecation notice
+            probe._apply_file(single, variable_pool, current_script)
+    except ConfigError as exc:
+        return str(exc)
+    except ValueError as exc:
+        if section in ("include_required", "include_optional"):
+            return f"keys in [{section}] must be numbers, not {key!r}"
+        return str(exc)
+    except Exception as exc:
+        return f"{type(exc).__name__}: {exc}"
+    return None
+
+
+def _check_rule_codes(value: str) -> str | None:
+    """Rule codes in ``[lint] select`` / ``ignore``, checked as ``execsql lint`` checks them."""
+    from execsql.cli.lint import resolve_selectors
+
+    try:
+        resolve_selectors([value])
+    except ValueError as exc:
+        return str(exc)
+    return None
+
+
+def _is_secret(key: str) -> bool:
+    """Whether *key* holds a password (``password``, ``enc_password``, ``passwd``)."""
+    return "passw" in key.lower()
+
+
+def _check_file(
+    path: str,
+    known: dict[str, set[str]],
+    variable_pool: _VariablePool,
+    current_script: str,
+) -> tuple[list[ConfigProblem], list[str]]:
+    """Every problem in one file, and the config files it chains to (in queue order)."""
+    problems: list[ConfigProblem] = []
+
+    def problem(line: int, severity: str, message: str) -> None:
+        problems.append(ConfigProblem(path, line, severity, message))
+
+    try:
+        with open(path) as f:  # the encoding ConfigParser.read uses
+            text = f.read()
+    except (OSError, UnicodeDecodeError) as exc:
+        problem(0, "error", f"cannot read: {exc}")
+        return problems, []
+
+    cp = configparser.ConfigParser()
+    try:
+        cp.read_string(text, source=path)
+    except configparser.ParsingError as exc:
+        if isinstance(exc, configparser.MissingSectionHeaderError):
+            line, text = describe_parse_error(exc)
+            problem(line, "error", text)
+        else:  # every unparsable line, not only the first
+            for err_line, err_text in exc.errors:
+                problem(err_line, "error", f"cannot parse line: {err_text.strip()}")
+        return problems, []
+    except configparser.Error as exc:
+        line, text = describe_parse_error(exc)
+        problem(line, "error", text)
+        return problems, []
+
+    section_lines, key_lines = _line_numbers(text)
+
+    for section in cp.sections():
+        if section not in known:
+            hint = _suggest(section, known)
+            problem(
+                section_lines.get(section, 0),
+                "warning",
+                f"unknown section [{section}]" + (f"; did you mean [{hint}]?" if hint else "; a run ignores it"),
+            )
+
+    chained: list[str] = []
+    for (section, key), line in key_lines.items():
+        if section not in known and section != configparser.DEFAULTSECT:
+            continue  # reported once, as an unknown section
+        if section not in _FREE_SECTIONS and key not in known[section]:
+            elsewhere = sorted(s for s, keys in known.items() if key in keys)
+            if elsewhere:
+                message = f"{key} belongs in [{elsewhere[0]}], not [{section}]"
+            else:
+                hint = _suggest(key, known[section])
+                message = f"unknown key {key!r} in [{section}]" + (
+                    f"; did you mean {hint!r}?" if hint else "; a run ignores it"
+                )
+            problem(line, "warning", message)
+            continue
+        if section == configparser.DEFAULTSECT:
+            continue
+        try:
+            value = cp.get(section, key)
+        except configparser.Error as exc:
+            problem(line, "error", describe_parse_error(exc)[1])
+            continue
+        if key in _CHAIN_KEYS:
+            platform = _CHAIN_KEYS[key]
+            if platform and not sys.platform.startswith(platform):
+                continue  # another platform's file; a run here never reads it
+            expand_home = os.name == "posix" if key == "config_file" else True
+            target = ConfigData._chain_target(value, variable_pool, expand_home=expand_home)
+            if Path(target).is_file():
+                chained.append(target)
+            else:
+                problem(line, "warning", f"{key} = {value}: no such file; a run skips it")
+            continue
+        failure = _check_value(section, key, value, variable_pool, current_script)
+        if failure is None and section == ConfigData._LINT_SECTION:
+            failure = _check_rule_codes(value)
+        if failure:
+            if _is_secret(key):
+                # Never echo a password, even an obfuscated one, into logs.
+                failure = failure.replace(value, "***") if value else failure
+                value = "***"
+            problem(line, "error", f"{key} = {value}: {failure}")
+
+    return problems, chained
+
+
+def validate_config(
+    script_path: str,
+    variable_pool: _VariablePool,
+    config_file: str | None = None,
+) -> tuple[list[str], list[ConfigProblem]]:
+    """Check every config file a run from *script_path* would read.
+
+    Args:
+        script_path: Directory of the script a run would execute; its
+            ``execsql.conf`` is included, as for a run.
+        variable_pool: Substitution variables for expanding chained file paths
+            and checking ``[variables]`` names.
+        config_file: The file ``--config`` names, read last.
+
+    Returns:
+        The files checked, in the order a run reads them, and every problem
+        found, in file order and then line order.
+    """
+    known = known_options()
+    current_script = str(Path(sys.argv[0]).resolve())
+    queue: deque[str] = deque(ConfigData._search_paths(script_path, config_file))
+    checked: list[str] = []
+    problems: list[ConfigProblem] = []
+    while queue and len(checked) < ConfigData._MAX_CONFIG_CHAIN:
+        path = queue.popleft()
+        if path in checked or not Path(path).is_file():
+            continue
+        checked.append(path)
+        found, chained = _check_file(path, known, variable_pool, current_script)
+        problems.extend(sorted(found, key=lambda p: p.line))
+        for target in chained:
+            queue.appendleft(target)
+    return checked, problems

execsql/data/execsql.conf.template

@@ -12,7 +12,7 @@
 #   4. Working directory: current directory
 #
 # Later files override earlier ones. CLI arguments override everything.
-# Generate this template with: execsql --init-config > execsql.conf
+# Generate this template with: execsql config --init > execsql.conf
 
 
 [connect]
@@ -349,6 +349,29 @@
 #win_config_file=
 
 
+[format]
+# Layout for `execsql format`. A flag on the command line wins over these.
+# A run ignores this section.
+
+# Spaces per indent level, for metacommand blocks and SQL.
+#indent=4
+# Put commas at the start of lines instead of the end.
+#leading_comma=No
+# Reformat SQL with sqlglot (needs the [formatter] extra). No = metacommands only.
+#sql=Yes
+
+
+[lint]
+# Rules for `execsql lint`: codes or prefixes, comma-separated (e.g. V001,F).
+# --select or --ignore on the command line replaces the value here.
+# A run ignores this section.
+
+#select=
+#ignore=
+# Yes = exit 1 on warnings too, not only on errors (--strict / --no-strict).
+#strict=No
+
+
 [variables]
 # Substitution variables defined at startup. No pre-defined keys.
 # Example:

execsql/data/script_header.sql

@@ -0,0 +1,23 @@
+-- {name}
+--
+-- PURPOSE
+--
+--
+-- NOTES
+--  1.
+--
+-- PROJECT
+--
+--
+-- COPYRIGHT
+--  Copyright (c) {year},
+--
+-- AUTHORS
+--
+--
+-- HISTORY
+--   Date        Remarks
+--  ----------  -----------------------------------------------------------------
+--  {date}  Created.
+--
+-- ==============================================================================

execsql/db/factory.py

@@ -84,7 +84,7 @@ def db_SQLite(
         if not Path(sqlite_fn).exists():
             raise ErrInfo(
                 type="error",
-                other_msg=f'SQLite database file "{sqlite_fn}" does not exist.',
+                other_msg=f'SQLite database file "{sqlite_fn}" does not exist. Use -n (--new-db) to create it.',
             )
     return SQLiteDatabase(sqlite_fn)
 
@@ -129,7 +129,7 @@ def db_DuckDB(
         if not Path(duckdb_fn).exists():
             raise ErrInfo(
                 type="error",
-                other_msg=f'DuckDB database file "{duckdb_fn}" does not exist.',
+                other_msg=f'DuckDB database file "{duckdb_fn}" does not exist. Use -n (--new-db) to create it.',
             )
     return DuckDBDatabase(duckdb_fn)
 

execsql/debug/repl.py

@@ -1,49 +1,34 @@
 from __future__ import annotations
 
-"""Interactive debug REPL metacommand handler for execsql.
+"""The debug REPL a ``BREAKPOINT`` (or ``run --debug``) opens.
 
-Implements ``x_breakpoint`` — the ``BREAKPOINT`` metacommand — which pauses
-script execution and drops into an interactive read-eval-print loop.
+Input runs as if it were written where the script paused, through
+:mod:`execsql.interactive`: SQL (ending with ``;``, results shown as tables),
+metacommands and blocks, in the paused script's variable scope and
+transaction. A ``SUB`` typed here is still set
+when the script resumes; a ``BREAK`` inside a LOOP leaves the loop; ``HALT``
+ends the run. An error ends only the input that caused it.
 
-The REPL allows the user to:
+Commands to the REPL itself start with ``.``: ``.continue`` and ``.next``
+resume the script, ``.where`` and ``.stack`` show where it paused, ``.quit``
+halts it; ``.vars``, ``.set``, ``.scripts`` and ``.cancel`` inspect and
+change the run.
 
-- Inspect and print substitution variables.
-- Run ad-hoc SQL queries against the current database.
-- Step through the script one statement at a time.
-- Resume or abort execution.
-
-Dispatch is two-way (psql-style): input starting with ``.`` is a REPL
-command (``.continue``, ``.vars [VAR]``, ``.next``, etc.), everything
-else is SQL.  Multi-line SQL is supported: any non-``.`` input opens a
-buffer whose continuation prompt is ``  ...        > ``, accumulating
-until a line ends with ``;``.  ``.cancel`` (or Ctrl-C / EOF) discards
-the partial buffer.
-
-Variable lookup is explicit — ``.vars LOGFILE`` prints one variable;
-``.vars`` lists them all.  There is no bare-identifier lookup, so any
-SQL keyword you type starts a buffer the moment you press Enter.
-
-The trailing ``;`` is the SQL terminator both within one line and across
-multiple lines — the REPL has no read-only mode and DDL on most adapters
-is irreversible, so requiring ``;`` is a small intent gate against
-accidental DML/DDL on mistyped input.  Use SQL ``BEGIN; … ROLLBACK;`` to
-bracket exploratory DML if you need recoverability.
-
-Errors raised by any REPL helper — bad SQL, malformed dot-commands, etc. —
-are caught at the loop level so the session re-prompts instead of escaping
-through ``x_breakpoint`` and being stamped as a "Metacommand error".
+This module also holds the REPL's display helpers (colors, tables, variable
+listings).
 
 In non-interactive environments (CI, piped input, ``sys.stdin.isatty()`` is
 ``False``) the metacommand is silently skipped so automated pipelines are not
 blocked.
 """
 
-import os
 import sys
 from pathlib import Path
 from typing import Any
 
 import execsql.state as _state
+from execsql.interactive import Prompt, common_dot_command, read_eval_loop, unknown_dot_command
+from execsql.utils.color import color_disabled_by_env
 
 __all__ = ["x_breakpoint"]
 
@@ -77,7 +62,7 @@ def _use_color() -> bool:
     global _color_cache  # noqa: PLW0603
     if _color_cache is not None:
         return _color_cache
-    if os.environ.get("NO_COLOR") is not None or os.environ.get("EXECSQL_NO_COLOR") is not None:
+    if color_disabled_by_env():
         _color_cache = False
     else:
         output = _state.output
@@ -123,18 +108,20 @@ _HELP_COMMANDS = [
     (".quit", ".q", "Halt the script (exit 1)"),
     (".vars", ".v", "List all execsql substitution variables"),
     (".vars VAR", ".v VAR", "Print the value of a single variable (e.g. .vars logfile)"),
+    (".vars all", "", "Also list environment (&) variables"),
     (".next", ".n", "Execute the next statement then pause again (step mode)"),
     (".where", ".w", "Show the current script location and upcoming statement"),
     (".stack", "", "Show the command-list stack (script name, line, depth)"),
     (".set VAR VAL", ".s", "Set or update a substitution variable"),
     (".scripts", "", "List all registered SCRIPT definitions"),
     (".scripts NAME", "", "Show detail for a specific SCRIPT"),
-    (".cancel", "", "Discard a partial multi-line SQL buffer"),
+    (".cancel", "", "Discard a statement or block you are typing"),
     (".help", ".h", "Show this help text"),
 ]
 
 _HELP_OTHER = [
-    ("SELECT ...;", "Run SQL — multi-line accepted, terminate with ';' to execute"),
+    ("SELECT ...;", "Run SQL, as the script would here; it may span lines and ends with ';'"),
+    ("-- !x! SUB ...", "Run a metacommand (!x! SUB ... works too); blocks stay open until closed"),
 ]
 
 _HELP_CMD_WIDTH = 13  # width of the command column
@@ -196,8 +183,8 @@ def _write_rule(label: str) -> None:
 def _debug_repl(*, step: bool = False) -> None:
     """Interactive read-eval-print loop for script debugging.
 
-    Reads commands from stdin until the user types ``.continue`` or ``.abort``,
-    or until EOF / KeyboardInterrupt.
+    Reads input until ``.continue``, ``.next`` or ``.quit``, or until Ctrl-D /
+    Ctrl-C on an empty line, which resume the script.
 
     Args:
         step: When ``True``, the entry banner says "Step" instead of
@@ -232,125 +219,37 @@ def _debug_repl(*, step: bool = False) -> None:
     _hint_c = _c(_DIM, "'.c'")
     _write(f"  Type {_hint_help} for commands, {_hint_c} to resume.\n\n")
 
-    sql_buffer: list[str] = []
-
-    while True:
-        prompt = "  ...        > " if sql_buffer else "execsql debug> "
-        try:
-            line = input(prompt).strip()
-        except EOFError:
-            if sql_buffer:
-                sql_buffer.clear()
-                _write("\n  (input discarded)\n")
-                continue
-            _write("\n")
-            return
-        except KeyboardInterrupt:
-            if sql_buffer:
-                sql_buffer.clear()
-                _write("\n  (input discarded)\n")
-                continue
-            _write("\n")
-            return
+    read_eval_loop(_DebugPrompt())
 
-        if not line:
-            continue
-
-        try:
-            if line.startswith("."):
-                cmd = line[1:].strip().lower()
-                if cmd == "cancel":
-                    if sql_buffer:
-                        sql_buffer.clear()
-                        _write("  (input discarded)\n")
-                    continue
-                _handle_dot_command(line)
-                if cmd in ("continue", "c"):
-                    return
-                if cmd in ("abort", "q", "quit"):
-                    return
-                if cmd in ("next", "n"):
-                    return
-                continue
-
-            if sql_buffer:
-                sql_buffer.append(line)
-                joined = " ".join(sql_buffer)
-                if joined.rstrip().endswith(";"):
-                    _run_sql(joined)
-                    sql_buffer.clear()
-                continue
-
-            if line.rstrip().endswith(";"):
-                _run_sql(line)
-                continue
-
-            sql_buffer.append(line)
-        except SystemExit:
-            raise
-        except KeyboardInterrupt:
-            sql_buffer.clear()
-            _write("\n  (interrupted)\n")
-            continue
-        except Exception as exc:
-            sql_buffer.clear()
-            _write(f"  {_c(_RED, 'Error:')} {exc}\n")
-            continue
-
-
-def _handle_dot_command(line: str) -> None:
-    """Dispatch a dot-prefixed REPL command."""
-    # Strip the leading dot and normalize
-    cmd = line[1:].strip().lower()
-
-    if cmd in ("continue", "c"):
-        return  # caller checks and returns from _debug_repl
-    elif cmd in ("abort", "q", "quit"):
+
+class _DebugPrompt(Prompt):
+    name = "execsql debug"
+    source = "<breakpoint>"
+
+    def dot_command(self, command: str, argument: str, line: str) -> bool:
+        return _handle_dot_command(line)
+
+
+def _handle_dot_command(line: str) -> bool:
+    """Dispatch a ``.`` command; return ``False`` when the script should resume."""
+    command, _, argument = line.strip()[1:].partition(" ")
+    command = command.lower()
+    if command in ("continue", "c"):
+        return False
+    if command in ("abort", "q", "quit"):
         raise SystemExit(1)
-    elif cmd in ("help", "h"):
+    if command in ("next", "n"):
+        _enable_step_mode()
+        return False
+    if command in ("help", "h"):
         _write(_format_help())
-    elif cmd in ("vars", "v"):
-        _print_all_vars()
-    elif cmd.startswith("vars ") or cmd.startswith("v "):
-        rest = cmd.split(None, 1)[1].strip() if " " in cmd else ""
-        if rest:
-            _print_var(rest)
-        else:
-            _print_all_vars()
-    elif cmd in ("where", "w"):
+    elif command in ("where", "w"):
         _print_where()
-    elif cmd == "stack":
+    elif command == "stack":
         _print_stack()
-    elif cmd in ("next", "n"):
-        _enable_step_mode()
-    elif cmd.startswith("set ") or cmd == "set":
-        # .set VAR VAL — set or update a substitution variable
-        rest = cmd[4:].strip() if cmd.startswith("set ") else ""
-        if not rest:
-            _write("  Usage: .set VAR VALUE\n")
-        else:
-            parts = rest.split(None, 1)
-            varname = parts[0]
-            value = parts[1] if len(parts) > 1 else ""
-            _set_var(varname, value)
-    elif cmd.startswith("s ") or cmd == "s":
-        # .s VAR VAL — shorthand for .set
-        rest = cmd[2:].strip() if cmd.startswith("s ") else ""
-        if not rest:
-            _write("  Usage: .s VAR VALUE\n")
-        else:
-            parts = rest.split(None, 1)
-            varname = parts[0]
-            value = parts[1] if len(parts) > 1 else ""
-            _set_var(varname, value)
-    elif cmd.startswith("scripts"):
-        rest = cmd[7:].strip()
-        if rest:
-            _print_script_detail(rest)
-        else:
-            _print_scripts()
-    else:
-        _write(f"  {_c(_RED, 'Unknown command:')} {line!r}. Type '.help' for available commands.\n")
+    elif not common_dot_command(command, argument.strip(), line):
+        unknown_dot_command(line)
+    return True
 
 
 # ---------------------------------------------------------------------------
@@ -371,7 +270,7 @@ def _write(text: str) -> None:
 def _print_where() -> None:
     """Print the current script location and the upcoming statement.
 
-    Reads ``_state.last_command`` (a :class:`ScriptCmd`) and displays the
+    Reads ``_state.last_command`` and displays the
     filename, line number, command type, and (truncated) statement text.
     If ``last_command`` is ``None``, reports that the position is unknown.
     """
@@ -531,49 +430,8 @@ def _print_stack() -> None:
     _write("\n")
 
 
-def _run_sql(sql: str) -> None:
-    """Execute ad-hoc SQL and pretty-print results or affected rowcount."""
-    dbs = _state.dbs
-    if dbs is None:
-        _write("  (no database connection is active)\n")
-        return
-    db = dbs.current()
-    if db is None:
-        _write("  (no database connection is active)\n")
-        return
-
-    try:
-        with db._cursor() as curs:
-            try:
-                curs.execute(sql)
-            except Exception as exc:
-                try:
-                    db.rollback()
-                except Exception:
-                    pass
-                _write(f"  {_c(_RED, 'SQL error:')} {exc}\n")
-                return
-
-            try:
-                _state.subvars.add_substitution("$LAST_ROWCOUNT", curs.rowcount)
-            except Exception:
-                pass
-
-            if curs.description is None:
-                rowcount = curs.rowcount if curs.rowcount is not None else -1
-                if rowcount >= 0:
-                    row_word = "row" if rowcount == 1 else "rows"
-                    _write(f"  {_c(_DIM, f'({rowcount} {row_word} affected)')}\n")
-                else:
-                    _write(f"  {_c(_DIM, '(statement executed)')}\n")
-                return
-
-            colnames = [d[0] for d in curs.description]
-            rows = curs.fetchall()
-    except Exception as exc:
-        _write(f"  {_c(_RED, 'SQL error:')} {exc}\n")
-        return
-
+def _print_table(colnames: list[str], rows: list[Any]) -> None:
+    """Print query results as a boxed text table, NULLs dimmed."""
     if not colnames:
         _write("  (query returned no columns)\n")
         return