execsql2 2.23.0__py3-none-any.whl → 2.24.0__py3-none-any.whl

execsql/db/duckdb.py

@@ -8,18 +8,55 @@ analytics databases via the ``duckdb`` package.  Corresponds to ``-t k``
 on the CLI.
 """
 
+import re
 from pathlib import Path
+from typing import Any
 
-from execsql.db.base import Database
+from execsql.db.base import Database, statement_start
 from execsql.exceptions import ErrInfo
 from execsql.utils.errors import exception_desc, fatal_error
 import execsql.state as _state
 
 __all__ = ["DuckDBDatabase"]
 
+# Statements that return rows; any other statement DuckDB answers with a
+# one-column "Count" result (rows inserted, updated or deleted).
+_QUERY_RX = re.compile(r"(?:SELECT|WITH|VALUES|FROM|TABLE|SHOW|DESCRIBE|SUMMARIZE|EXPLAIN|PRAGMA|CALL)\b|\(", re.I)
+
+# A statement that opens a transaction, or one that ends it.
+_TRANSACTION_RX = re.compile(r"(?P<begin>BEGIN|START\s+TRANSACTION)\b|(?P<end>COMMIT|END|ROLLBACK|ABORT)\b", re.I)
+
+
+class _SharedCursor:
+    """The connection, used as a cursor so a statement joins its open transaction.
+
+    A DuckDB cursor is a separate connection with a transaction of its own,
+    and closing it rolls that transaction back.  ``close()`` therefore leaves
+    the connection open.
+    """
+
+    def __init__(self, conn: Any) -> None:
+        self._conn = conn
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._conn, name)
+
+    def close(self) -> None:
+        pass
+
 
 class DuckDBDatabase(Database):
-    """DuckDB in-process analytics adapter using the duckdb package."""
+    """DuckDB in-process analytics adapter using the duckdb package.
+
+    The duckdb driver commits each statement unless a transaction was begun,
+    and has no way to ask whether one is open, so this adapter begins one
+    itself while statements are held (AUTOCOMMIT OFF, BEGIN BATCH) and keeps
+    track of it.  While it is open, every statement runs on the connection
+    itself rather than on a cursor (see :class:`_SharedCursor`).
+    """
+
+    #: A transaction this adapter or a script's own BEGIN opened, not yet ended.
+    in_transaction: bool = False
 
     def __init__(self, DuckDB_fn: str) -> None:
         try:
@@ -43,6 +80,62 @@ class DuckDBDatabase(Database):
     def __repr__(self) -> str:
         return f"DuckDBDatabase({self.db_name!r})"
 
+    def cursor(self) -> Any:
+        """A cursor, or while a transaction is open, the connection itself."""
+        if self.conn is None:
+            self.open_db()
+        if self.in_transaction:
+            return _SharedCursor(self.conn)
+        return self.conn.cursor()
+
+    def begin_transaction(self) -> None:
+        """Open a transaction if none is open."""
+        if self.conn is None:
+            self.open_db()
+        if not self.in_transaction:
+            # A second BEGIN would abort the open transaction, hence the tracking.
+            self.conn.execute("BEGIN TRANSACTION")
+            self.in_transaction = True
+
+    def execute(self, sql: Any, paramlist: list | None = None, *, fetch: bool = False) -> tuple[list[str], list] | None:
+        """Execute *sql*, noting a transaction the statement itself begins or ends.
+
+        The driver's ``rowcount`` is always -1: DuckDB answers an INSERT,
+        UPDATE or DELETE with a one-column ``Count`` result instead, which is
+        read here to set ``$LAST_ROWCOUNT`` as other drivers do.
+        """
+        text = " ".join(sql) if type(sql) in (tuple, list) else sql
+        start = statement_start(text)
+        m = _TRANSACTION_RX.match(start)
+        if m and m.group("begin"):
+            self.in_transaction = True  # so the BEGIN itself runs on the connection
+        query = bool(_QUERY_RX.match(start))
+        result = super().execute(sql, paramlist, fetch=fetch or not query)
+        if m and m.group("end"):
+            self.in_transaction = False
+        if not query and result is not None and result[0] == ["Count"]:
+            rows = result[1]
+            if _state.subvars is not None:
+                _state.subvars.add_substitution("$LAST_ROWCOUNT", rows[0][0] if rows else -1)
+            return None
+        return result if fetch else None
+
+    def commit(self) -> None:
+        """Commit the open transaction if autocommit is enabled."""
+        super().commit()
+        if self.autocommit:
+            self.in_transaction = False
+
+    def rollback(self) -> None:
+        """Roll back the open transaction."""
+        super().rollback()
+        self.in_transaction = False
+
+    def close(self) -> None:
+        """Close the connection; an open transaction is rolled back."""
+        super().close()
+        self.in_transaction = False
+
     def open_db(self) -> None:
         """Open a connection to the DuckDB database file."""
         import duckdb

execsql/db/factory.py

@@ -84,7 +84,7 @@ def db_SQLite(
         if not Path(sqlite_fn).exists():
             raise ErrInfo(
                 type="error",
-                other_msg=f'SQLite database file "{sqlite_fn}" does not exist.',
+                other_msg=f'SQLite database file "{sqlite_fn}" does not exist. Use -n (--new-db) to create it.',
             )
     return SQLiteDatabase(sqlite_fn)
 
@@ -129,7 +129,7 @@ def db_DuckDB(
         if not Path(duckdb_fn).exists():
             raise ErrInfo(
                 type="error",
-                other_msg=f'DuckDB database file "{duckdb_fn}" does not exist.',
+                other_msg=f'DuckDB database file "{duckdb_fn}" does not exist. Use -n (--new-db) to create it.',
             )
     return DuckDBDatabase(duckdb_fn)
 

execsql/db/mysql.py

@@ -49,6 +49,17 @@ def _crlf_terminated(filename: str) -> bool:
 class MySQLDatabase(Database):
     """MySQL and MariaDB adapter using the pymysql package."""
 
+    #: Statements MySQL commits at once, together with everything before them
+    #: ("Statements That Cause an Implicit Commit" in the MySQL manual).
+    #: CREATE and DROP TEMPORARY TABLE are the exception.
+    implicit_commit_rx = re.compile(
+        r"(?:CREATE|ALTER|DROP)\s+(?!TEMPORARY\b)(?:OR\s+REPLACE\s+)?(?:UNIQUE\s+|FULLTEXT\s+|SPATIAL\s+)?\w+"
+        r"|RENAME\s+\w+|TRUNCATE\b|GRANT\b|REVOKE\b|LOCK\s+TABLES?\b|UNLOCK\s+TABLES?\b"
+        r"|BEGIN\b|START\s+TRANSACTION\b|(?:ANALYZE|CHECK|OPTIMIZE|REPAIR)\s+TABLE\b|FLUSH\b"
+        r"|(?:UN)?INSTALL\s+PLUGIN\b|SET\s+PASSWORD\b|SET\s+(?:SESSION\s+|@@)?autocommit\b",
+        re.I,
+    )
+
     def __init__(
         self,
         server_name: str,

execsql/db/oracle.py

@@ -8,6 +8,7 @@ via the ``oracledb`` driver (python-oracledb).  Corresponds to ``-t o``
 on the CLI.
 """
 
+import re
 from typing import Any
 
 from execsql.db.base import Database
@@ -23,6 +24,14 @@ __all__ = ["OracleDatabase"]
 class OracleDatabase(Database):
     """Oracle adapter using the cx_Oracle (python-oracledb) driver."""
 
+    #: Oracle commits every DDL statement at once, together with everything
+    #: before it, including CREATE GLOBAL TEMPORARY TABLE.
+    implicit_commit_rx = re.compile(
+        r"(?:CREATE|ALTER|DROP)\s+(?:OR\s+REPLACE\s+)?(?:GLOBAL\s+TEMPORARY\s+|UNIQUE\s+|BITMAP\s+)?\w+"
+        r"|TRUNCATE\s+\w+|RENAME\b|GRANT\b|REVOKE\b|COMMENT\s+ON\b|(?:NO)?AUDIT\b|ANALYZE\b|PURGE\b|FLASHBACK\b",
+        re.I,
+    )
+
     #: No Oracle server runs in CI; this adapter has no tests.
     support_tier = SupportTier.BEST_EFFORT
     support_tier_name = "Oracle"
@@ -110,13 +119,12 @@ class OracleDatabase(Database):
                 msg = f"Failed to open Oracle database {self.db_name} on {self.server_name}"
                 raise ErrInfo(type="exception", exception_msg=exception_desc(), other_msg=msg) from e
 
-    def execute(self, sql: Any, paramlist: list | None = None) -> None:
+    def execute(self, sql: Any, paramlist: list | None = None, *, fetch: bool = False) -> tuple[list[str], list] | None:
         """Execute a SQL command, stripping any trailing semicolon for Oracle."""
         # Strip any semicolon off the end and pass to the parent method.
         if sql[-1:] == ";":
-            super().execute(sql[:-1], paramlist)
-        else:
-            super().execute(sql, paramlist)
+            return super().execute(sql[:-1], paramlist, fetch=fetch)
+        return super().execute(sql, paramlist, fetch=fetch)
 
     def select_data(self, sql: str) -> tuple[list[str], list]:
         """Return column names and all rows from a SELECT statement."""

execsql/db/sqlite.py

@@ -26,6 +26,10 @@ DEFAULT_CONNECT_TIMEOUT = 30  # seconds
 class SQLiteDatabase(Database):
     """SQLite adapter using the Python standard-library sqlite3 module."""
 
+    #: SQLite refuses VACUUM, ATTACH and DETACH inside a transaction, and
+    #: ignores some PRAGMAs there (``foreign_keys``).
+    no_transaction_rx = re.compile(r"(?:VACUUM|ATTACH|DETACH|PRAGMA)\b", re.I)
+
     def __init__(self, SQLite_fn: str, timeout: float = DEFAULT_CONNECT_TIMEOUT) -> None:
         try:
             import sqlite3  # noqa: F401
@@ -48,6 +52,18 @@ class SQLiteDatabase(Database):
     def __repr__(self) -> str:
         return f"SQLiteDatabase({self.db_name!r})"
 
+    def begin_transaction(self) -> None:
+        """Open a transaction if none is open.
+
+        Python's sqlite3 module opens one on its own only before INSERT,
+        UPDATE, DELETE and REPLACE.  Without this, a CREATE TABLE or DROP run
+        under AUTOCOMMIT OFF or in a batch would be committed at once.
+        """
+        if self.conn is None:
+            self.open_db()
+        if not self.conn.in_transaction:
+            self.conn.execute("BEGIN")
+
     def open_db(self) -> None:
         """Open a connection to the SQLite database file."""
         import sqlite3

execsql/debug/repl.py

@@ -1,49 +1,34 @@
 from __future__ import annotations
 
-"""Interactive debug REPL metacommand handler for execsql.
+"""The debug REPL a ``BREAKPOINT`` (or ``run --debug``) opens.
 
-Implements ``x_breakpoint`` — the ``BREAKPOINT`` metacommand — which pauses
-script execution and drops into an interactive read-eval-print loop.
+Input runs as if it were written where the script paused, through
+:mod:`execsql.interactive`: SQL (ending with ``;``, results shown as tables),
+metacommands and blocks, in the paused script's variable scope and
+transaction. A ``SUB`` typed here is still set
+when the script resumes; a ``BREAK`` inside a LOOP leaves the loop; ``HALT``
+ends the run. An error ends only the input that caused it.
 
-The REPL allows the user to:
+Commands to the REPL itself start with ``.``: ``.continue`` and ``.next``
+resume the script, ``.where`` and ``.stack`` show where it paused, ``.quit``
+halts it; ``.vars``, ``.set``, ``.scripts`` and ``.cancel`` inspect and
+change the run.
 
-- Inspect and print substitution variables.
-- Run ad-hoc SQL queries against the current database.
-- Step through the script one statement at a time.
-- Resume or abort execution.
-
-Dispatch is two-way (psql-style): input starting with ``.`` is a REPL
-command (``.continue``, ``.vars [VAR]``, ``.next``, etc.), everything
-else is SQL.  Multi-line SQL is supported: any non-``.`` input opens a
-buffer whose continuation prompt is ``  ...        > ``, accumulating
-until a line ends with ``;``.  ``.cancel`` (or Ctrl-C / EOF) discards
-the partial buffer.
-
-Variable lookup is explicit — ``.vars LOGFILE`` prints one variable;
-``.vars`` lists them all.  There is no bare-identifier lookup, so any
-SQL keyword you type starts a buffer the moment you press Enter.
-
-The trailing ``;`` is the SQL terminator both within one line and across
-multiple lines — the REPL has no read-only mode and DDL on most adapters
-is irreversible, so requiring ``;`` is a small intent gate against
-accidental DML/DDL on mistyped input.  Use SQL ``BEGIN; … ROLLBACK;`` to
-bracket exploratory DML if you need recoverability.
-
-Errors raised by any REPL helper — bad SQL, malformed dot-commands, etc. —
-are caught at the loop level so the session re-prompts instead of escaping
-through ``x_breakpoint`` and being stamped as a "Metacommand error".
+This module also holds the REPL's display helpers (colors, tables, variable
+listings).
 
 In non-interactive environments (CI, piped input, ``sys.stdin.isatty()`` is
 ``False``) the metacommand is silently skipped so automated pipelines are not
 blocked.
 """
 
-import os
 import sys
 from pathlib import Path
 from typing import Any
 
 import execsql.state as _state
+from execsql.interactive import Prompt, common_dot_command, read_eval_loop, unknown_dot_command
+from execsql.utils.color import color_disabled_by_env
 
 __all__ = ["x_breakpoint"]
 
@@ -77,7 +62,7 @@ def _use_color() -> bool:
     global _color_cache  # noqa: PLW0603
     if _color_cache is not None:
         return _color_cache
-    if os.environ.get("NO_COLOR") is not None or os.environ.get("EXECSQL_NO_COLOR") is not None:
+    if color_disabled_by_env():
         _color_cache = False
     else:
         output = _state.output
@@ -123,18 +108,20 @@ _HELP_COMMANDS = [
     (".quit", ".q", "Halt the script (exit 1)"),
     (".vars", ".v", "List all execsql substitution variables"),
     (".vars VAR", ".v VAR", "Print the value of a single variable (e.g. .vars logfile)"),
+    (".vars all", "", "Also list environment (&) variables"),
     (".next", ".n", "Execute the next statement then pause again (step mode)"),
     (".where", ".w", "Show the current script location and upcoming statement"),
     (".stack", "", "Show the command-list stack (script name, line, depth)"),
     (".set VAR VAL", ".s", "Set or update a substitution variable"),
     (".scripts", "", "List all registered SCRIPT definitions"),
     (".scripts NAME", "", "Show detail for a specific SCRIPT"),
-    (".cancel", "", "Discard a partial multi-line SQL buffer"),
+    (".cancel", "", "Discard a statement or block you are typing"),
     (".help", ".h", "Show this help text"),
 ]
 
 _HELP_OTHER = [
-    ("SELECT ...;", "Run SQL — multi-line accepted, terminate with ';' to execute"),
+    ("SELECT ...;", "Run SQL, as the script would here; it may span lines and ends with ';'"),
+    ("-- !x! SUB ...", "Run a metacommand (!x! SUB ... works too); blocks stay open until closed"),
 ]
 
 _HELP_CMD_WIDTH = 13  # width of the command column
@@ -196,8 +183,8 @@ def _write_rule(label: str) -> None:
 def _debug_repl(*, step: bool = False) -> None:
     """Interactive read-eval-print loop for script debugging.
 
-    Reads commands from stdin until the user types ``.continue`` or ``.abort``,
-    or until EOF / KeyboardInterrupt.
+    Reads input until ``.continue``, ``.next`` or ``.quit``, or until Ctrl-D /
+    Ctrl-C on an empty line, which resume the script.
 
     Args:
         step: When ``True``, the entry banner says "Step" instead of
@@ -232,125 +219,37 @@ def _debug_repl(*, step: bool = False) -> None:
     _hint_c = _c(_DIM, "'.c'")
     _write(f"  Type {_hint_help} for commands, {_hint_c} to resume.\n\n")
 
-    sql_buffer: list[str] = []
-
-    while True:
-        prompt = "  ...        > " if sql_buffer else "execsql debug> "
-        try:
-            line = input(prompt).strip()
-        except EOFError:
-            if sql_buffer:
-                sql_buffer.clear()
-                _write("\n  (input discarded)\n")
-                continue
-            _write("\n")
-            return
-        except KeyboardInterrupt:
-            if sql_buffer:
-                sql_buffer.clear()
-                _write("\n  (input discarded)\n")
-                continue
-            _write("\n")
-            return
+    read_eval_loop(_DebugPrompt())
 
-        if not line:
-            continue
-
-        try:
-            if line.startswith("."):
-                cmd = line[1:].strip().lower()
-                if cmd == "cancel":
-                    if sql_buffer:
-                        sql_buffer.clear()
-                        _write("  (input discarded)\n")
-                    continue
-                _handle_dot_command(line)
-                if cmd in ("continue", "c"):
-                    return
-                if cmd in ("abort", "q", "quit"):
-                    return
-                if cmd in ("next", "n"):
-                    return
-                continue
-
-            if sql_buffer:
-                sql_buffer.append(line)
-                joined = " ".join(sql_buffer)
-                if joined.rstrip().endswith(";"):
-                    _run_sql(joined)
-                    sql_buffer.clear()
-                continue
-
-            if line.rstrip().endswith(";"):
-                _run_sql(line)
-                continue
-
-            sql_buffer.append(line)
-        except SystemExit:
-            raise
-        except KeyboardInterrupt:
-            sql_buffer.clear()
-            _write("\n  (interrupted)\n")
-            continue
-        except Exception as exc:
-            sql_buffer.clear()
-            _write(f"  {_c(_RED, 'Error:')} {exc}\n")
-            continue
-
-
-def _handle_dot_command(line: str) -> None:
-    """Dispatch a dot-prefixed REPL command."""
-    # Strip the leading dot and normalize
-    cmd = line[1:].strip().lower()
-
-    if cmd in ("continue", "c"):
-        return  # caller checks and returns from _debug_repl
-    elif cmd in ("abort", "q", "quit"):
+
+class _DebugPrompt(Prompt):
+    name = "execsql debug"
+    source = "<breakpoint>"
+
+    def dot_command(self, command: str, argument: str, line: str) -> bool:
+        return _handle_dot_command(line)
+
+
+def _handle_dot_command(line: str) -> bool:
+    """Dispatch a ``.`` command; return ``False`` when the script should resume."""
+    command, _, argument = line.strip()[1:].partition(" ")
+    command = command.lower()
+    if command in ("continue", "c"):
+        return False
+    if command in ("abort", "q", "quit"):
         raise SystemExit(1)
-    elif cmd in ("help", "h"):
+    if command in ("next", "n"):
+        _enable_step_mode()
+        return False
+    if command in ("help", "h"):
         _write(_format_help())
-    elif cmd in ("vars", "v"):
-        _print_all_vars()
-    elif cmd.startswith("vars ") or cmd.startswith("v "):
-        rest = cmd.split(None, 1)[1].strip() if " " in cmd else ""
-        if rest:
-            _print_var(rest)
-        else:
-            _print_all_vars()
-    elif cmd in ("where", "w"):
+    elif command in ("where", "w"):
         _print_where()
-    elif cmd == "stack":
+    elif command == "stack":
         _print_stack()
-    elif cmd in ("next", "n"):
-        _enable_step_mode()
-    elif cmd.startswith("set ") or cmd == "set":
-        # .set VAR VAL — set or update a substitution variable
-        rest = cmd[4:].strip() if cmd.startswith("set ") else ""
-        if not rest:
-            _write("  Usage: .set VAR VALUE\n")
-        else:
-            parts = rest.split(None, 1)
-            varname = parts[0]
-            value = parts[1] if len(parts) > 1 else ""
-            _set_var(varname, value)
-    elif cmd.startswith("s ") or cmd == "s":
-        # .s VAR VAL — shorthand for .set
-        rest = cmd[2:].strip() if cmd.startswith("s ") else ""
-        if not rest:
-            _write("  Usage: .s VAR VALUE\n")
-        else:
-            parts = rest.split(None, 1)
-            varname = parts[0]
-            value = parts[1] if len(parts) > 1 else ""
-            _set_var(varname, value)
-    elif cmd.startswith("scripts"):
-        rest = cmd[7:].strip()
-        if rest:
-            _print_script_detail(rest)
-        else:
-            _print_scripts()
-    else:
-        _write(f"  {_c(_RED, 'Unknown command:')} {line!r}. Type '.help' for available commands.\n")
+    elif not common_dot_command(command, argument.strip(), line):
+        unknown_dot_command(line)
+    return True
 
 
 # ---------------------------------------------------------------------------
@@ -371,7 +270,7 @@ def _write(text: str) -> None:
 def _print_where() -> None:
     """Print the current script location and the upcoming statement.
 
-    Reads ``_state.last_command`` (a :class:`ScriptCmd`) and displays the
+    Reads ``_state.last_command`` and displays the
     filename, line number, command type, and (truncated) statement text.
     If ``last_command`` is ``None``, reports that the position is unknown.
     """
@@ -531,49 +430,8 @@ def _print_stack() -> None:
     _write("\n")
 
 
-def _run_sql(sql: str) -> None:
-    """Execute ad-hoc SQL and pretty-print results or affected rowcount."""
-    dbs = _state.dbs
-    if dbs is None:
-        _write("  (no database connection is active)\n")
-        return
-    db = dbs.current()
-    if db is None:
-        _write("  (no database connection is active)\n")
-        return
-
-    try:
-        with db._cursor() as curs:
-            try:
-                curs.execute(sql)
-            except Exception as exc:
-                try:
-                    db.rollback()
-                except Exception:
-                    pass
-                _write(f"  {_c(_RED, 'SQL error:')} {exc}\n")
-                return
-
-            try:
-                _state.subvars.add_substitution("$LAST_ROWCOUNT", curs.rowcount)
-            except Exception:
-                pass
-
-            if curs.description is None:
-                rowcount = curs.rowcount if curs.rowcount is not None else -1
-                if rowcount >= 0:
-                    row_word = "row" if rowcount == 1 else "rows"
-                    _write(f"  {_c(_DIM, f'({rowcount} {row_word} affected)')}\n")
-                else:
-                    _write(f"  {_c(_DIM, '(statement executed)')}\n")
-                return
-
-            colnames = [d[0] for d in curs.description]
-            rows = curs.fetchall()
-    except Exception as exc:
-        _write(f"  {_c(_RED, 'SQL error:')} {exc}\n")
-        return
-
+def _print_table(colnames: list[str], rows: list[Any]) -> None:
+    """Print query results as a boxed text table, NULLs dimmed."""
     if not colnames:
         _write("  (query returned no columns)\n")
         return

execsql/exceptions.py

@@ -35,6 +35,7 @@ Note:
 __all__ = [
     "ExecSqlError",
     "ConfigError",
+    "ConfigFileError",
     "ExecSqlTimeoutError",
     "ErrInfo",
     "DataTypeError",
@@ -74,6 +75,14 @@ class ConfigError(ExecSqlError):
     """Raised for invalid or missing execsql configuration values."""
 
 
+class ConfigFileError(ConfigError):
+    """A :class:`ConfigError` found while reading one config file.
+
+    The message starts with the file's path. ``execsql config --validate``
+    reports every problem in the file, with line numbers.
+    """
+
+
 class ExecSqlTimeoutError(ExecSqlError):
     """Timeout during alarm-timer operations.