execsql2 2.1.2__py3-none-any.whl → 2.4.0__py3-none-any.whl

execsql/{script.py → script/engine.py}

@@ -1,50 +1,29 @@
 from __future__ import annotations
 
-"""
-Core script-execution engine for execsql.
-
-This module contains the data structures and functions that load, parse, and
-drive execution of execsql ``.sql`` script files.  It is the heart of the
-runtime.
-
-Key classes:
-
-- :class:`BatchLevels` — tracks which databases are used in nested BEGIN/END
-  BATCH blocks for commit/rollback handling.
-- :class:`IfItem` / :class:`IfLevels` — stack-based IF/ELSE/ENDIF nesting.
-- :class:`CounterVars` — named integer counters (``@NAME``).
-- :class:`SubVarSet` — global ``!!$VAR!!`` substitution-variable store, plus
-  ``&ENV``, ``@COUNTER``, ``~LOCAL``, and ``#ARG`` prefixes.
-- :class:`LocalSubVarSet` / :class:`ScriptArgSubVarSet` — per-script-scope
-  variable overlays.
-- :class:`MetaCommand` — one entry in the metacommand dispatch table (regex +
-  handler function + flags).
-- :class:`MetaCommandList` — ordered list of :class:`MetaCommand` entries;
-  ``get_match()`` finds the first matching entry for a given line.
-- :class:`SqlStmt` — wraps a single SQL string; ``run()`` executes it via the
-  active database connection.
-- :class:`MetacommandStmt` — wraps a metacommand line; ``run()`` dispatches
-  through :attr:`execsql.state.metacommandlist`.
+"""Script execution engine for execsql.
+
+Classes and functions that load, parse, and drive execution of execsql
+``.sql`` script files.
+
+Classes:
+- :class:`MetaCommand` — one entry in the metacommand dispatch table.
+- :class:`MetaCommandList` — ordered list of :class:`MetaCommand` entries.
+- :class:`SqlStmt` — wraps a single SQL string for execution.
+- :class:`MetacommandStmt` — wraps a metacommand line for dispatch.
 - :class:`ScriptCmd` — pairs a statement with its source-file location.
-- :class:`CommandList` — ordered list of :class:`ScriptCmd` objects plus an
-  execution cursor; ``run_next()`` drives one step of execution.
-- :class:`CommandListWhileLoop` / :class:`CommandListUntilLoop` — loop
-  variants of :class:`CommandList` that re-evaluate a condition each pass.
-- :class:`ScriptFile` — reads and tokenises a ``.sql`` file into
-  :class:`ScriptCmd` objects.
+- :class:`CommandList` — ordered list of :class:`ScriptCmd` objects.
+- :class:`CommandListWhileLoop` — loop variant that repeats while a condition is true.
+- :class:`CommandListUntilLoop` — loop variant that repeats until a condition is true.
+- :class:`ScriptFile` — reads and tokenises a ``.sql`` file.
 - :class:`ScriptExecSpec` — specification for deferred script execution.
 
-Key functions:
-
+Functions:
 - :func:`set_system_vars` — populates built-in ``$VARNAME`` system variables.
 - :func:`substitute_vars` — performs ``!!$VAR!!`` and ``!{$var}!`` expansion.
-- :func:`runscripts` — central execution loop; pops the top
-  :class:`CommandList` from ``_state.commandliststack`` and drives
-  ``run_next()`` until the stack is empty.
-- :func:`current_script_line` — returns the source location of the currently
-  executing command.
-- :func:`read_sqlfile` — parses a SQL script file into a new
-  :class:`CommandList` and pushes it onto ``_state.commandliststack``.
+- :func:`runscripts` — central execution loop.
+- :func:`current_script_line` — returns the source location of the currently executing command.
+- :func:`read_sqlfile` — parses a SQL script file into a new :class:`CommandList`.
+- :func:`read_sqlstring` — parses an inline script string into a new :class:`CommandList`.
 """
 
 import copy
@@ -57,359 +36,24 @@ from typing import Any
 
 import execsql.state as _state
 from execsql.exceptions import ErrInfo
+from execsql.script.variables import LocalSubVarSet, ScriptArgSubVarSet, SubVarSet
 from execsql.utils.errors import exception_desc
 from execsql.utils.fileio import EncodedFile
 
 
-# ---------------------------------------------------------------------------
-# BatchLevels
-# ---------------------------------------------------------------------------
-
-
-class BatchLevels:
-    # A stack to keep a record of the databases used in nested batches.
-    class Batch:
-        def __init__(self) -> None:
-            self.dbs_used: list[Any] = []
-
-    def __init__(self) -> None:
-        self.batchlevels: list[BatchLevels.Batch] = []
-
-    def in_batch(self) -> bool:
-        return len(self.batchlevels) > 0
-
-    def new_batch(self) -> None:
-        self.batchlevels.append(self.Batch())
-
-    def using_db(self, db: Any) -> None:
-        if len(self.batchlevels) > 0 and db not in self.batchlevels[-1].dbs_used:
-            self.batchlevels[-1].dbs_used.append(db)
-
-    def uses_db(self, db: Any) -> bool:
-        if len(self.batchlevels) == 0:
-            return False
-        return any(db in batch.dbs_used for batch in self.batchlevels)
-
-    def rollback_batch(self) -> None:
-        if len(self.batchlevels) > 0:
-            b = self.batchlevels[-1]
-            for db in b.dbs_used:
-                db.rollback()
-
-    def end_batch(self) -> None:
-        b = self.batchlevels.pop()
-        for db in b.dbs_used:
-            db.commit()
-
-
-# ---------------------------------------------------------------------------
-# IfItem / IfLevels
-# ---------------------------------------------------------------------------
-
-
-class IfItem:
-    # An object representing an 'if' level, with context data.
-    def __init__(self, tf_value: bool) -> None:
-        self.tf_value = tf_value
-        self.scriptname, self.scriptline = current_script_line()
-
-    def value(self) -> bool:
-        return self.tf_value
-
-    def invert(self) -> None:
-        self.tf_value = not self.tf_value
-
-    def change_to(self, tf_value: bool) -> None:
-        self.tf_value = tf_value
-
-    def script_line(self) -> tuple:
-        return (self.scriptname, self.scriptline)
-
-
-class IfLevels:
-    # A stack of True/False values corresponding to a nested set of conditionals,
-    # with methods to manipulate and query the set of conditional states.
-    def __init__(self) -> None:
-        self.if_levels: list[IfItem] = []
-
-    def nest(self, tf_value: bool) -> None:
-        self.if_levels.append(IfItem(tf_value))
-
-    def unnest(self) -> None:
-        if len(self.if_levels) == 0:
-            raise ErrInfo(type="error", other_msg="Can't exit an IF block; no IF block is active.")
-        else:
-            self.if_levels.pop()
-
-    def invert(self) -> None:
-        if len(self.if_levels) == 0:
-            raise ErrInfo(type="error", other_msg="Can't change the IF state; no IF block is active.")
-        else:
-            self.if_levels[-1].invert()
-
-    def replace(self, tf_value: bool) -> None:
-        if len(self.if_levels) == 0:
-            raise ErrInfo(type="error", other_msg="Can't change the IF state; no IF block is active.")
-        else:
-            self.if_levels[-1].change_to(tf_value)
-
-    def current(self) -> bool:
-        if len(self.if_levels) == 0:
-            raise ErrInfo(type="error", other_msg="No IF block is active.")
-        else:
-            return self.if_levels[-1].value()
-
-    def all_true(self) -> bool:
-        if self.if_levels == []:
-            return True
-        return all(tf.value() for tf in self.if_levels)
-
-    def only_current_false(self) -> bool:
-        # Returns True if the current if level is false and all higher levels are True.
-        if len(self.if_levels) == 0:
-            return False
-        elif len(self.if_levels) == 1:
-            return not self.if_levels[-1].value()
-        else:
-            return not self.if_levels[-1].value() and all(tf.value() for tf in self.if_levels[:-1])
-
-    def script_lines(self, top_n: int) -> list[tuple]:
-        # Returns a list of tuples containing the script name and line number
-        # for the topmost 'top_n' if levels, in bottom-up order.
-        if len(self.if_levels) < top_n:
-            raise ErrInfo(type="error", other_msg="Invalid IF stack depth reference.")
-        levels = self.if_levels[len(self.if_levels) - top_n :]
-        return [lvl.script_line() for lvl in levels]
-
-
-# ---------------------------------------------------------------------------
-# CounterVars
-# ---------------------------------------------------------------------------
-
-
-class CounterVars:
-    # A dictionary of dynamically created named counter variables.
-    _COUNTER_RX = re.compile(r"!!\$(COUNTER_\d+)!!", re.I)
-
-    def __init__(self) -> None:
-        self.counters: dict[str, int] = {}
-
-    def _ctrid(self, ctr_no: int) -> str:
-        return f"counter_{ctr_no}"
-
-    def set_counter(self, ctr_no: int, ctr_val: int) -> None:
-        self.counters[self._ctrid(ctr_no)] = ctr_val
-
-    def remove_counter(self, ctr_no: int) -> None:
-        ctr_id = self._ctrid(ctr_no)
-        if ctr_id in self.counters:
-            del self.counters[ctr_id]
-
-    def remove_all_counters(self) -> None:
-        self.counters = {}
-
-    def substitute(self, command_str: str) -> tuple:
-        # Substitutes any counter variable references with the counter value and
-        # returns the modified command string and a flag indicating replacements.
-        m = self._COUNTER_RX.search(command_str, re.I)
-        if m:
-            ctr_id = m.group(1).lower()
-            if ctr_id not in self.counters:
-                self.counters[ctr_id] = 0
-            new_count = self.counters[ctr_id] + 1
-            self.counters[ctr_id] = new_count
-            return command_str.replace("!!$" + m.group(1) + "!!", str(new_count)), True
-        return command_str, False
-
-    def substitute_all(self, any_text: str) -> tuple:
-        subbed = True
-        any_subbed = False
-        while subbed:
-            any_text, subbed = self.substitute(any_text)
-            if subbed:
-                any_subbed = True
-        return any_text, any_subbed
-
-
-# ---------------------------------------------------------------------------
-# SubVarSet / LocalSubVarSet / ScriptArgSubVarSet
-# ---------------------------------------------------------------------------
-
-
-class SubVarSet:
-    # A pool of substitution variables.  Each variable consists of a name and
-    # a (string) value.  All variable names are stored as lowercase text.
-    # Internally uses a dict for O(1) lookups; the ``substitutions`` property
-    # exposes the data as a list of ``(name, value)`` tuples for backward
-    # compatibility with external code.
-    def __init__(self) -> None:
-        self._subs_dict: dict[str, Any] = {}
-        self._compiled_patterns: dict[str, tuple] = {}
-        self.prefix_list: list[str] = ["$", "&", "@"]
-        # Don't construct/compile on init because deepcopy() can't handle compiled regexes.
-        self.var_rx = None
-
-    @property
-    def substitutions(self) -> list[tuple]:
-        """Backward-compatible view of substitutions as a list of (name, value) tuples."""
-        return list(self._subs_dict.items())
-
-    @substitutions.setter
-    def substitutions(self, value: Any) -> None:
-        """Accept a list of (name, value) tuples or a dict and populate the internal dict."""
-        if isinstance(value, dict):
-            self._subs_dict = dict(value)
-        else:
-            self._subs_dict = dict(value)
-        self._rebuild_all_patterns()
-
-    def _compile_patterns_for(self, varname: str) -> tuple:
-        """Compile and return the three regex patterns (plain, single-quoted, double-quoted) for *varname*."""
-        match_escaped = "\\" + varname if varname[0] == "$" else varname
-        pat = re.compile(f"!!{match_escaped}!!", re.I)
-        patq = re.compile(f"!'!{match_escaped}!'!", re.I)
-        patdq = re.compile(f'!"!{match_escaped}!"!', re.I)
-        return (pat, patq, patdq)
-
-    def _rebuild_all_patterns(self) -> None:
-        """Rebuild compiled patterns for every variable currently stored."""
-        self._compiled_patterns = {}
-        for varname in self._subs_dict:
-            self._compiled_patterns[varname] = self._compile_patterns_for(varname)
-
-    def compile_var_rx(self) -> None:
-        self.var_rx_str = r"^[" + "".join(self.prefix_list) + r"]?\w+$"
-        self.var_rx = re.compile(self.var_rx_str, re.I)
-
-    def var_name_ok(self, varname: str) -> bool:
-        if self.var_rx is None:
-            self.compile_var_rx()
-        return self.var_rx.match(varname) is not None
-
-    def check_var_name(self, varname: str) -> None:
-        if not self.var_name_ok(varname.lower()):
-            raise ErrInfo("error", other_msg=f"Invalid variable name ({varname}) in this context.")
-
-    def remove_substitution(self, template_str: str) -> None:
-        self.check_var_name(template_str)
-        old_sub = template_str.lower()
-        self._subs_dict.pop(old_sub, None)
-        self._compiled_patterns.pop(old_sub, None)
-
-    def add_substitution(self, varname: str, repl_str: Any) -> None:
-        self.check_var_name(varname)
-        varname = varname.lower()
-        self._subs_dict[varname] = repl_str
-        self._compiled_patterns[varname] = self._compile_patterns_for(varname)
-
-    def append_substitution(self, varname: str, repl_str: str) -> None:
-        self.check_var_name(varname)
-        varname = varname.lower()
-        if varname in self._subs_dict:
-            self.add_substitution(varname, f"{self._subs_dict[varname]}\n{repl_str}")
-        else:
-            self.add_substitution(varname, repl_str)
-
-    def varvalue(self, varname: str) -> str | None:
-        self.check_var_name(varname)
-        return self._subs_dict.get(varname.lower())
-
-    def increment_by(self, varname: str, numeric_increment: Any) -> None:
-        self.check_var_name(varname)
-        varvalue = self.varvalue(varname)
-        if varvalue is None:
-            varvalue = "0"
-            self.add_substitution(varname, varvalue)
-        # Import as_numeric lazily to avoid circular dependency
-        from execsql.utils.numeric import as_numeric
-
-        numvalue = as_numeric(varvalue)
-        numinc = as_numeric(numeric_increment)
-        if numvalue is None or numinc is None:
-            newval = f"{varvalue}+{numeric_increment}"
-        else:
-            newval = str(numvalue + numinc)
-        self.add_substitution(varname, newval)
-
-    def sub_exists(self, template_str: str) -> bool:
-        self.check_var_name(template_str)
-        return template_str.lower() in self._subs_dict
-
-    def merge(self, other_subvars: SubVarSet | None) -> SubVarSet:
-        # Return a new SubVarSet with this object's variables merged with other_subvars.
-        if other_subvars is not None:
-            newsubs = SubVarSet()
-            newsubs._subs_dict = dict(self._subs_dict)
-            newsubs._compiled_patterns = dict(self._compiled_patterns)
-            newsubs.prefix_list = list(set(self.prefix_list + other_subvars.prefix_list))
-            newsubs.compile_var_rx()
-            for varname, value in other_subvars._subs_dict.items():
-                newsubs.add_substitution(varname, value)
-            return newsubs
-        return self
-
-    def substitute(self, command_str: str) -> tuple:
-        # Replace any substitution variables in the command string.
-        if isinstance(command_str, str):
-            for varname, sub in self._subs_dict.items():
-                if sub is None:
-                    sub = ""
-                sub = str(sub)
-                if os.name != "posix":
-                    sub = sub.replace("\\", "\\\\")
-                pat_rx, patq_rx, patdq_rx = self._compiled_patterns[varname]
-                if pat_rx.search(command_str):
-                    return pat_rx.sub(sub, command_str), True
-                if patq_rx.search(command_str):
-                    sub = sub.replace("'", "''")
-                    return patq_rx.sub(sub, command_str), True
-                if patdq_rx.search(command_str):
-                    sub = '"' + sub + '"'
-                    return patdq_rx.sub(sub, command_str), True
-        return command_str, False
-
-    def substitute_all(self, any_text: str) -> tuple:
-        subbed = True
-        any_subbed = False
-        while subbed:
-            any_text, subbed = self.substitute(any_text)
-            if subbed:
-                any_subbed = True
-        return any_text, any_subbed
-
-
-class LocalSubVarSet(SubVarSet):
-    # A pool of local substitution variables.
-    # Only '~' is allowed as a prefix and MUST be present.
-    def __init__(self) -> None:
-        SubVarSet.__init__(self)
-        self.prefix_list = ["~"]
-
-    def compile_var_rx(self) -> None:
-        # Prefix is required, not optional.
-        self.var_rx_str = r"^[" + "".join(self.prefix_list) + r"]\w+$"
-        self.var_rx = re.compile(self.var_rx_str, re.I)
-
-
-class ScriptArgSubVarSet(SubVarSet):
-    # A pool of script argument names.
-    # Only '#' is allowed as a prefix and MUST be present.
-    def __init__(self) -> None:
-        SubVarSet.__init__(self)
-        self.prefix_list = ["#"]
-
-    def compile_var_rx(self) -> None:
-        # Prefix is required, not optional.
-        self.var_rx_str = r"^[" + "".join(self.prefix_list) + r"]\w+$"
-        self.var_rx = re.compile(self.var_rx_str, re.I)
-
-
 # ---------------------------------------------------------------------------
 # MetaCommand / MetaCommandList
 # ---------------------------------------------------------------------------
 
 
 class MetaCommand:
+    """A single entry in the metacommand dispatch table.
+
+    Holds a compiled regex, a handler function, and execution-control flags.
+    Call :meth:`run` with a raw command string to attempt a match and invoke
+    the handler.
+    """
+
     # A compiled metacommand that can be run if it matches a metacommand command string.
     def __init__(
         self,
@@ -419,6 +63,7 @@ class MetaCommand:
         run_in_batch: bool = False,
         run_when_false: bool = False,
         set_error_flag: bool = True,
+        category: str | None = None,
     ) -> None:
         self.rx = rx
         self.exec_fn = exec_func
@@ -426,6 +71,7 @@ class MetaCommand:
         self.run_in_batch = run_in_batch
         self.run_when_false = run_when_false
         self.set_error_flag = set_error_flag
+        self.category = category
         self.hitcount = 0
 
     def __repr__(self) -> str:
@@ -435,6 +81,10 @@ class MetaCommand:
         )
 
     def run(self, cmd_str: str) -> tuple:
+        """Match *cmd_str* against this entry's regex and, if it matches, invoke the handler.
+
+        Returns ``(True, return_value)`` on a match, ``(False, None)`` otherwise.
+        """
         # Runs the metacommand if the command string matches the regex.
         m = self.rx.match(cmd_str.strip())
         if m:
@@ -466,20 +116,49 @@ class MetaCommand:
 
 
 class MetaCommandList:
-    """Ordered list of :class:`MetaCommand` entries.
+    """Ordered list of :class:`MetaCommand` entries with keyword-indexed dispatch.
 
     Commands are stored with the most-recently-added entry first, matching
-    the original linked-list prepend semantics. ``eval()`` and ``get_match()``
-    scan the list linearly; the move-to-front heuristic from the original
-    implementation has been removed for predictable, stable ordering.
+    the original linked-list prepend semantics.  A keyword index
+    (``_by_keyword``) groups entries by their leading keyword so that
+    ``eval()`` and ``get_match()`` test only the small subset of regexes
+    that could possibly match, reducing dispatch from O(N) to O(K) where
+    K is the number of patterns sharing the same leading keyword (typically
+    1–5 vs. 205 total).
     """
 
+    # Regex to extract the leading keyword from a metacommand regex pattern.
+    # Handles ^\s*KEYWORD, ^KEYWORD, and ^\s*(?:PREFIX\s+)?KEYWORD.
+    _KEYWORD_RX = re.compile(
+        r"^\^"
+        r"(?:\\s\*)?(?:\(\?:[^)]+\))?(?:\\s\+)?"
+        r"(?:\\s\*)?"
+        r"([A-Z_]+)",
+    )
+
     def __init__(self) -> None:
         self._commands: list[MetaCommand] = []
+        self._by_keyword: dict[str, list[MetaCommand]] = {}
+        self._unkeyed: list[MetaCommand] = []
 
     def __iter__(self) -> Any:
         return iter(self._commands)
 
+    @staticmethod
+    def _extract_keyword(cmd_str: str) -> str | None:
+        """Extract the leading keyword from a metacommand string."""
+        word = cmd_str.strip().split(None, 1)
+        return word[0].upper() if word else None
+
+    def _index_command(self, mc: MetaCommand, rx_pattern: str) -> None:
+        """Add *mc* to the keyword index based on its regex pattern."""
+        m = self._KEYWORD_RX.match(rx_pattern)
+        if m:
+            kw = m.group(1)
+            self._by_keyword.setdefault(kw, []).insert(0, mc)
+        else:
+            self._unkeyed.insert(0, mc)
+
     def add(
         self,
         matching_regexes: Any,
@@ -488,17 +167,57 @@ class MetaCommandList:
         run_in_batch: bool = False,
         run_when_false: bool = False,
         set_error_flag: bool = True,
+        category: str | None = None,
     ) -> None:
+        """Register one or more regex patterns as a new :class:`MetaCommand` entry.
+
+        *matching_regexes* may be a single pattern string or a list/tuple of
+        patterns; each compiles into a separate :class:`MetaCommand` prepended to
+        the dispatch list so that later registrations take priority.
+        """
         if type(matching_regexes) in (tuple, list):
-            regexes = [re.compile(rx, re.I) for rx in tuple(matching_regexes)]
+            raw_patterns = list(matching_regexes)
+            regexes = [re.compile(rx, re.I) for rx in raw_patterns]
         else:
+            raw_patterns = [matching_regexes]
             regexes = [re.compile(matching_regexes, re.I)]
-        for rx in regexes:
-            # Prepend to preserve "last registered, first checked" ordering.
-            self._commands.insert(
-                0,
-                MetaCommand(rx, exec_func, description, run_in_batch, run_when_false, set_error_flag),
+        for rx, raw in zip(regexes, raw_patterns):
+            mc = MetaCommand(
+                rx,
+                exec_func,
+                description,
+                run_in_batch,
+                run_when_false,
+                set_error_flag,
+                category,
             )
+            # Prepend to preserve "last registered, first checked" ordering.
+            self._commands.insert(0, mc)
+            self._index_command(mc, raw)
+
+    def _candidates(self, cmd_str: str) -> list[MetaCommand]:
+        """Return the subset of commands whose keyword matches *cmd_str*.
+
+        Falls back to the full command list if no keyword match is found.
+        """
+        kw = self._extract_keyword(cmd_str)
+        if kw and kw in self._by_keyword:
+            # Keyword-matched entries plus any unkeyed entries that could match anything.
+            return self._by_keyword[kw] + self._unkeyed
+        return self._commands
+
+    def keywords_by_category(self) -> dict[str, list[str]]:
+        """Return ``{category: [keyword, ...]}`` from entries that have both.
+
+        Used by ``--dump-keywords`` to introspect the dispatch table.
+        """
+        result: dict[str, list[str]] = {}
+        for mc in self._commands:
+            if mc.category and mc.description:
+                kw_list = result.setdefault(mc.category, [])
+                if mc.description not in kw_list:
+                    kw_list.append(mc.description)
+        return result
 
     def eval(self, cmd_str: str) -> tuple:
         """Evaluate *cmd_str* against the registered metacommands.
@@ -506,7 +225,7 @@ class MetaCommandList:
         Returns ``(True, return_value)`` if a matching command was found and
         run, ``(False, None)`` if no command matched.
         """
-        for cmd in self._commands:
+        for cmd in self._candidates(cmd_str):
             if _state.if_stack.all_true() or cmd.run_when_false:
                 success, value = cmd.run(cmd_str)
                 if success:
@@ -517,8 +236,9 @@ class MetaCommandList:
         """Return ``(MetaCommand, re.Match)`` for the first entry matching *cmd*,
         or ``None`` if no entry matches.
         """
-        for node in self._commands:
-            m = node.rx.match(cmd.strip())
+        stripped = cmd.strip()
+        for node in self._candidates(stripped):
+            m = node.rx.match(stripped)
             if m is not None:
                 return (node, m)
         return None
@@ -530,6 +250,8 @@ class MetaCommandList:
 
 
 class SqlStmt:
+    """A single SQL statement ready to be executed against the active database."""
+
     # A SQL statement to be passed to a database to execute.
     def __init__(self, sql_statement: str) -> None:
         self.statement = re.sub(r"\s*;(\s*;\s*)+$", ";", sql_statement)
@@ -538,6 +260,7 @@ class SqlStmt:
         return f"SqlStmt({self.statement})"
 
     def run(self, localvars: SubVarSet | None = None, commit: bool = True) -> None:
+        """Execute the statement on the current database, committing unless in a batch."""
         # Run the SQL statement on the current database.
         if _state.if_stack.all_true():
             e = None
@@ -549,6 +272,10 @@ class SqlStmt:
                 )
             try:
                 db = _state.dbs.current()
+                if _state.conf.log_sql and _state.exec_log:
+                    lno = getattr(_state, "last_command", None)
+                    lno = lno.line_no if lno and hasattr(lno, "line_no") else None
+                    _state.exec_log.log_sql_query(cmd, db.name(), lno)
                 db.execute(cmd)
                 if commit:
                     db.commit()
@@ -569,10 +296,13 @@ class SqlStmt:
             _state.subvars.add_substitution("$LAST_SQL", cmd)
 
     def commandline(self) -> str:
+        """Return the raw SQL statement text."""
         return self.statement
 
 
 class MetacommandStmt:
+    """A single execsql metacommand line ready to be dispatched."""
+
     # A metacommand to be handled by execsql.
     def __init__(self, metacommand_statement: str) -> None:
         self.statement = metacommand_statement
@@ -581,6 +311,7 @@ class MetacommandStmt:
         return f"MetacommandStmt({self.statement})"
 
     def run(self, localvars: SubVarSet | None = None, commit: bool = False) -> Any:
+        """Expand substitution variables then dispatch through the metacommand table."""
         # Tries all metacommands in the dispatch table until one runs.
         errmsg = "Unknown metacommand"
         cmd = substitute_vars(self.statement, localvars)
@@ -609,6 +340,7 @@ class MetacommandStmt:
         return None
 
     def commandline(self) -> str:
+        """Return the metacommand line in its canonical ``-- !x! ...`` form."""
         return "-- !x! " + self.statement
 
 
@@ -618,6 +350,8 @@ class MetacommandStmt:
 
 
 class ScriptCmd:
+    """A parsed script item: either a :class:`SqlStmt` or a :class:`MetacommandStmt`, with source location."""
+
     # A SQL script object that is either a SQL statement or a metacommand.
     def __init__(
         self,
@@ -647,6 +381,12 @@ class ScriptCmd:
 
 
 class CommandList:
+    """Ordered sequence of :class:`ScriptCmd` objects with a forward-only execution cursor.
+
+    Push onto ``_state.commandliststack`` and call :meth:`run_next` in a loop
+    (or let :func:`runscripts` drive it) to execute each command in turn.
+    """
+
     # A list of ScriptCmd objects including execution state.
     def __init__(
         self,
@@ -665,6 +405,7 @@ class CommandList:
         self.init_if_level: int | None = None
 
     def add(self, script_command: ScriptCmd) -> None:
+        """Append *script_command* to the end of this command list."""
         self.cmdlist.append(script_command)
 
     def set_paramvals(self, paramvals: SubVarSet) -> None:
@@ -678,11 +419,13 @@ class CommandList:
                 )
 
     def current_command(self) -> ScriptCmd | None:
+        """Return the :class:`ScriptCmd` at the current cursor position, or ``None`` if exhausted."""
         if self.cmdptr > len(self.cmdlist) - 1:
             return None
         return self.cmdlist[self.cmdptr]
 
     def check_iflevels(self) -> None:
+        """Warn if the IF-stack depth changed during execution of this command list."""
         if_excess = len(_state.if_stack.if_levels) - self.init_if_level
         if if_excess > 0:
             sources = _state.if_stack.script_lines(if_excess)
@@ -731,6 +474,7 @@ class CommandList:
         self.cmdptr += 1
 
     def run_next(self) -> None:
+        """Execute the command at the current cursor and advance; raise StopIteration when done."""
         if self.cmdptr == 0:
             self.init_if_level = len(_state.if_stack.if_levels)
         if self.cmdptr > len(self.cmdlist) - 1:
@@ -750,6 +494,8 @@ class CommandList:
 
 
 class CommandListWhileLoop(CommandList):
+    """A :class:`CommandList` that repeats its commands while a condition evaluates to true."""
+
     # Subclass of CommandList that loops WHILE a condition is met.
     def __init__(
         self,
@@ -776,6 +522,8 @@ class CommandListWhileLoop(CommandList):
 
 
 class CommandListUntilLoop(CommandList):
+    """A :class:`CommandList` that repeats its commands until a condition evaluates to true."""
+
     # Subclass of CommandList that loops UNTIL a condition is met.
     def __init__(
         self,
@@ -807,6 +555,13 @@ class CommandListUntilLoop(CommandList):
 
 
 class ScriptFile(EncodedFile):
+    """An iterable file reader that tracks the current line number.
+
+    Wraps :class:`~execsql.utils.fileio.EncodedFile` and increments
+    :attr:`lno` on each ``next()`` call so that callers always know which
+    source line is being processed.
+    """
+
     # A file reader that returns lines and records the line number.
     def __init__(self, scriptfname: str, file_encoding: str) -> None:
         super().__init__(scriptfname, file_encoding)
@@ -831,6 +586,13 @@ class ScriptFile(EncodedFile):
 
 
 class ScriptExecSpec:
+    """Deferred execution specification for a named SCRIPT block.
+
+    Parses argument expressions and loop-type flags at construction time;
+    call :meth:`execute` to push the resolved :class:`CommandList` onto the
+    execution stack.
+    """
+
     # Stores specifications for executing a SCRIPT, for later use.
     args_rx = re.compile(
         r'(?P<param>#?\w+)\s*=\s*(?P<arg>(?:(?:[^"\'\[][^,\)]*)|(?:"[^"]*")|(?:\'[^\']*\')|(?:\[[^\]]*\])))',
@@ -880,6 +642,7 @@ class ScriptExecSpec:
 
 
 def set_system_vars() -> None:
+    """Refresh all built-in system substitution variables (``$CURRENT_TIME``, ``$DB_NAME``, etc.)."""
     # (Re)define the system substitution variables that are not script-specific.
     _state.subvars.add_substitution("$CANCEL_HALT_STATE", "ON" if _state.status.cancel_halt else "OFF")
     _state.subvars.add_substitution("$ERROR_HALT_STATE", "ON" if _state.status.halt_on_err else "OFF")
@@ -915,7 +678,11 @@ def set_system_vars() -> None:
     _state.subvars.add_substitution("$VERSION3", str(_state.tertiary_vno))
 
 
+_MAX_SUBSTITUTION_DEPTH = 100
+
+
 def substitute_vars(command_str: str, localvars: SubVarSet | None = None) -> str:
+    """Expand all ``!!$VAR!!`` tokens in *command_str*, merging *localvars* when provided."""
     # Substitutes global variables, global counters, and local variables.
     if localvars is not None:
         subs = _state.subvars.merge(localvars)
@@ -923,11 +690,21 @@ def substitute_vars(command_str: str, localvars: SubVarSet | None = None) -> str
         subs = _state.subvars
     cmdstr = copy.copy(command_str)
     subs_made = True
+    iterations = 0
     while subs_made:
         subs_made = False
         cmdstr, subs_made = subs.substitute_all(cmdstr)
         cmdstr, any_subbed = _state.counters.substitute_all(cmdstr)
         subs_made = subs_made or any_subbed
+        iterations += 1
+        if iterations >= _MAX_SUBSTITUTION_DEPTH:
+            raise ErrInfo(
+                type="error",
+                other_msg=(
+                    f"Substitution variable cycle detected: exceeded {_MAX_SUBSTITUTION_DEPTH} "
+                    f"iterations while expanding variables in: {command_str[:200]}"
+                ),
+            )
     m = _state.defer_rx.findall(cmdstr)
     # Substitute any deferred substitution variables with regular substitution var flags.
     if m is not None:
@@ -938,7 +715,7 @@ def substitute_vars(command_str: str, localvars: SubVarSet | None = None) -> str
 
 
 def runscripts() -> None:
-
+    """Drive execution until the command-list stack is empty."""
     # Repeatedly run the next statement from the script at the top of the
     # command list stack until there are no more statements.
     while len(_state.commandliststack) > 0:
@@ -952,12 +729,13 @@ def runscripts() -> None:
             raise
         except ErrInfo:
             raise
-        except Exception:
-            raise ErrInfo(type="exception", exception_msg=exception_desc())
+        except Exception as e:
+            raise ErrInfo(type="exception", exception_msg=exception_desc()) from e
         _state.cmds_run += 1
 
 
 def current_script_line() -> tuple:
+    """Return ``(source_name, line_number)`` for the command currently executing."""
     if len(_state.commandliststack) > 0:
         current_cmds = _state.commandliststack[-1]
         if current_cmds.current_command() is not None:
@@ -968,7 +746,7 @@ def current_script_line() -> tuple:
         return ("", 0)
 
 
-def _parse_script_lines(lines_iter, source_name: str) -> list:
+def _parse_script_lines(lines_iter: Any, source_name: str) -> list:
     # Parse an iterable of lines into a list of ScriptCmd objects.
     from execsql.utils.errors import write_warning
 
@@ -1120,14 +898,18 @@ def _parse_script_lines(lines_iter, source_name: str) -> list:
 
 
 def read_sqlfile(sql_file_name: str) -> None:
+    """Parse a ``.sql`` file and push the resulting :class:`CommandList` onto the execution stack."""
     # Read lines from the given script file, create a list of ScriptCmd objects,
     # and append the list to the top of the stack of script commands.
     from execsql.utils.errors import file_size_date
 
     sz, dt = file_size_date(sql_file_name)
     _state.exec_log.log_status_info(f"Reading script file {sql_file_name} (size: {sz}; date: {dt})")
-    scriptfile_obj = ScriptFile(sql_file_name, _state.conf.script_encoding).open("r")
-    sqllist = _parse_script_lines(scriptfile_obj, sql_file_name)
+    scriptfile_obj = ScriptFile(sql_file_name, _state.conf.script_encoding)
+    try:
+        sqllist = _parse_script_lines(scriptfile_obj, sql_file_name)
+    finally:
+        scriptfile_obj.close()
     if sqllist:
         _state.commandliststack.append(CommandList(sqllist, Path(sql_file_name).name))