execsql2 2.16.18__py3-none-any.whl → 2.17.2__py3-none-any.whl

execsql/metacommands/debug.py

@@ -3,12 +3,24 @@ from __future__ import annotations
 """
 Debug metacommand handlers for execsql.
 
-Provides ``x_debug_write_metacommands``, which implements the
-``WRITE METACOMMANDS`` debug metacommand that prints the full registered
-metacommand list to the log/console for troubleshooting.
-
-Also provides ``x_show_scripts`` for runtime introspection of registered
-SCRIPT blocks.
+Handlers for the ``DEBUG …`` family of introspection metacommands and
+``SHOW SCRIPTS`` script-block listing:
+
+- ``x_debug_log_subvars`` / ``x_debug_write_subvars`` — dump substitution
+  variables to the log file or to terminal/file (``DEBUG LOG SUBVARS`` /
+  ``DEBUG WRITE SUBVARS``). Both accept optional ``LOCAL`` / ``USER``
+  qualifiers.
+- ``x_debug_log_config`` / ``x_debug_write_config`` — dump the merged
+  configuration.
+- ``x_debug_write_odbc_drivers`` — list installed ODBC drivers.
+- ``x_debug_write_metacommands`` — dump the registered metacommand list
+  (``DEBUG WRITE METACOMMANDLIST TO <file>``).
+- ``x_debug_commandliststack`` — dump the current execution stack
+  (``DEBUG WRITE COMMANDLISTSTACK``).
+- ``x_debug_iflevels`` — dump the nested IF condition state
+  (``DEBUG WRITE IFLEVELS``).
+- ``x_show_scripts`` — list registered ``BEGIN SCRIPT`` blocks
+  (``SHOW SCRIPTS [<name>]``), with optional name argument for detail.
 """
 
 from pathlib import Path
@@ -34,23 +46,65 @@ def x_debug_write_metacommands(**kwargs: Any) -> None:
 
 
 def x_debug_commandliststack(**kwargs: Any) -> None:
-    _state.output.write("Command List Stack:\n")
-    pfx = "  "
-    for cl in _state.commandliststack:
-        _state.output.write(pfx + f"Name:             {cl.listname}\n")
-        _state.output.write(pfx + f"Parameters:       {cl.paramnames}\n")
-        _state.output.write(pfx + f"Command pointer:  {cl.cmdptr}\n")
-        pfx = pfx + "  "
+    """Dump the unified AST execution stack.
+
+    Shows every nesting construct the executor is currently inside:
+    ``<main>`` script, ``EXECUTE SCRIPT`` calls, ``INCLUDE``'d files,
+    ``IF``/``ELSEIF``/``ELSE`` branches, ``LOOP`` iterations (with iteration
+    count), and ``BATCH`` blocks. Each frame shows source file and line.
+
+    The legacy ``commandliststack`` is reported separately as a secondary view
+    because it only records SCRIPT call frames; the unified ``ast_exec_stack``
+    is the authoritative debugger view.
+    """
+    stack = getattr(_state, "ast_exec_stack", None) or []
+    _state.output.write(f"Execution Stack (depth: {len(stack)}):\n")
+    if not stack:
+        _state.output.write("  (empty)\n")
+        return None
+
+    for depth, frame in enumerate(stack):
+        kind_label = frame.kind.upper().replace("LOOP_", "LOOP ")
+        # Right-hand description per kind
+        if frame.kind in ("if", "elseif"):
+            desc = f"{kind_label}  {frame.label}"
+        elif frame.kind == "else":
+            desc = kind_label
+        elif frame.kind in ("loop_while", "loop_until"):
+            desc = f"{kind_label}  {frame.label}  iter={frame.iteration}"
+        elif frame.kind == "script":
+            params = ""
+            if frame.params:
+                params = "(" + ", ".join(f"{k}={v!r}" for k, v in frame.params.items()) + ")"
+            iter_suffix = f"  iter={frame.iteration}" if frame.iteration else ""
+            desc = f"SCRIPT {frame.label}{params}{iter_suffix}"
+        elif frame.kind == "include":
+            desc = f"INCLUDE {frame.label}"
+        elif frame.kind == "batch":
+            desc = "BATCH"
+        elif frame.kind == "main":
+            desc = frame.label or "<main>"
+        else:
+            desc = f"{kind_label}  {frame.label}"
+        src = f"  {frame.source}:{frame.line}" if frame.source and frame.line else ""
+        _state.output.write(f"  [{depth}] {desc}{src}\n")
     return None
 
 
 def x_debug_iflevels(**kwargs: Any) -> None:
-    if len(_state.if_stack.if_levels) == 0:
-        _state.output.write("If levels: None\n")
-    else:
-        _state.output.write(
-            "If levels: [{}]\n".format(",".join([str(tf.tf_value) for tf in _state.if_stack.if_levels])),
-        )
+    # Filter the unified AST exec stack to just IF/ELSEIF/ELSE frames.
+    stack = getattr(_state, "ast_exec_stack", None) or []
+    if_frames = [f for f in stack if f.kind in ("if", "elseif", "else")]
+    if not if_frames:
+        _state.output.write("If levels: (no active IF block)\n")
+        return None
+
+    _state.output.write(f"If levels (depth {len(if_frames)}):\n")
+    for depth, frame in enumerate(if_frames):
+        kind_label = frame.kind.upper()
+        label = f"{kind_label} {frame.label}".strip() if frame.label else kind_label
+        src = f"  {frame.source}:{frame.line}" if frame.source and frame.line else ""
+        _state.output.write(f"  [{depth}] {label}{src}\n")
     return None
 
 
@@ -61,13 +115,13 @@ def x_debug_write_odbc_drivers(**kwargs: Any) -> None:
         fatal_error("The pyodbc module is required.")
     output_dest = kwargs["filename"]
     append = kwargs["append"]
+    if output_dest is not None and output_dest != "stdout" and append is None:
+        filewriter_open_as_new(output_dest)
 
     def write(txt: str) -> None:
         if output_dest is None or output_dest == "stdout":
             _state.output.write(txt)
         else:
-            if not append:
-                filewriter_open_as_new(output_dest)
             filewriter_write(output_dest, txt)
 
     for d in pyodbc.drivers():
@@ -77,8 +131,9 @@ def x_debug_write_odbc_drivers(**kwargs: Any) -> None:
 def x_debug_log_subvars(**kwargs: Any) -> None:
     local = kwargs["local"]
     user = kwargs["user"]
-    if _state.commandliststack:
-        for s in _state.commandliststack[-1].localvars.substitutions:
+    localvars = _state.current_localvars()
+    if localvars is not None:
+        for s in localvars.substitutions:
             _state.exec_log.log_status_info(f"Substitution [{s[0]}] = [{s[1]}]")
     if local is None:
         for s in _state.subvars.substitutions:
@@ -86,51 +141,57 @@ def x_debug_log_subvars(**kwargs: Any) -> None:
                 _state.exec_log.log_status_info(f"Substitution [{s[0]}] = [{s[1]}]")
 
 
-def x_debug_log_config(**kwargs: Any) -> None:
+_SENSITIVE_ATTRS = frozenset({"smtp_password", "passwd", "password"})
+
+
+def _config_lines() -> list[str]:
+    """Return the merged config as a list of ``"[<section>] <key> = <value>"`` lines.
+
+    Pulled directly from ``ConfigData._schema``, the registry that ``ConfigData``
+    populates each time it reads an option from the INI files. Adding a new
+    option to ``ConfigData.__init__`` automatically makes it appear here —
+    nothing in this file needs updating.
+
+    Lines are grouped by section in the order each section was first registered
+    so the dump matches the natural reading order of ``execsql.conf``.
+
+    Sensitive values (passwords) are redacted.
+    """
+    from execsql.config import ConfigData
+
     conf = _state.conf
-    _state.exec_log.log_status_info(f"Config; Script encoding = {conf.script_encoding}")
-    _state.exec_log.log_status_info(f"Config; Output encoding = {conf.output_encoding}")
-    _state.exec_log.log_status_info(f"Config; Import encoding = {conf.import_encoding}")
-    _state.exec_log.log_status_info(f"Config; Import common columns only = {conf.import_common_cols_only}")
-    _state.exec_log.log_status_info(f"Config; Use numeric type for Access = {conf.access_use_numeric}")
-    _state.exec_log.log_status_info(f"Config; Max int = {conf.max_int}")
-    _state.exec_log.log_status_info(f"Config; Boolean int = {conf.boolean_int}")
-    _state.exec_log.log_status_info(f"Config; Boolean words = {conf.boolean_words}")
-    _state.exec_log.log_status_info(f"Config; Clean column headers {conf.clean_col_hdrs}")
-    _state.exec_log.log_status_info(f"Config; Create column headers {conf.create_col_hdrs}")
-    _state.exec_log.log_status_info(f"Config; Dedup column headers {conf.dedup_col_hdrs}")
-    _state.exec_log.log_status_info(f"Config; Console wait when done {conf.gui_wait_on_exit}")
-    _state.exec_log.log_status_info(f"Config; Console wait when error {conf.gui_wait_on_error_halt}")
-    _state.exec_log.log_status_info(f"Config; Empty rows = {conf.empty_rows}")
-    _state.exec_log.log_status_info(f"Config; Empty_strings = {conf.empty_strings}")
-    _state.exec_log.log_status_info(f"Config: Trim_strings = {conf.trim_strings}")
-    _state.exec_log.log_status_info(f"Config: Replace_newlines = {conf.replace_newlines}")
-    _state.exec_log.log_status_info(f"Config; Only_strings = {conf.only_strings}")
-    _state.exec_log.log_status_info(f"Config; Scan lines = {conf.scan_lines}")
-    _state.exec_log.log_status_info(f"Config; Import row buffer size = {conf.import_row_buffer}")
-    _state.exec_log.log_status_info(f"Config; Write warnings to console = {conf.write_warnings}")
-    _state.exec_log.log_status_info(f"Config; Write prefix = {conf.write_prefix}")
-    _state.exec_log.log_status_info(f"Config; Write suffix = {conf.write_suffix}")
-    _state.exec_log.log_status_info(f"Config; Log write messages = {conf.tee_write_log}")
-    _state.exec_log.log_status_info(f"Config; Log data variable assignments = {conf.log_datavars}")
-    _state.exec_log.log_status_info(f"Config; GUI level = {conf.gui_level}")
-    _state.exec_log.log_status_info(f"Config; CSS file for HTML export = {conf.css_file}")
-    _state.exec_log.log_status_info(f"Config; CSS styles for HTML export = {conf.css_styles}")
-    _state.exec_log.log_status_info(f"Config; Make export directories = {conf.make_export_dirs}")
-    _state.exec_log.log_status_info(f"Config; Quote all text on export = {conf.quote_all_text}")
-    _state.exec_log.log_status_info(f"Config; Export row buffer size = {conf.export_row_buffer}")
-    _state.exec_log.log_status_info(f"Config; Text length for HDF5 export = {conf.hdf5_text_len}")
-    _state.exec_log.log_status_info(f"Config; Template processor = {conf.template_processor}")
-    _state.exec_log.log_status_info(f"Config; SMTP host = {conf.smtp_host}")
-    _state.exec_log.log_status_info(f"Config; SMTP port = {conf.smtp_port}")
-    _state.exec_log.log_status_info(f"Config; SMTP username = {conf.smtp_username}")
-    _state.exec_log.log_status_info(f"Config; SMTP use SSL = {conf.smtp_ssl}")
-    _state.exec_log.log_status_info(f"Config; SMTP use TLS = {conf.smtp_tls}")
-    _state.exec_log.log_status_info(f"Config; Email format = {conf.email_format}")
-    _state.exec_log.log_status_info(f"Config; Email CSS = {conf.email_css}")
-    _state.exec_log.log_status_info(f"Config; Zip buffer size (Mb) = {conf.zip_buffer_mb}")
-    _state.exec_log.log_status_info(f"Config; DAO flush delay (seconds) = {conf.dao_flush_delay_secs}")
-    _state.exec_log.log_status_info(f"Config; Configuration files read = {', '.join(conf.files_read)}")
+    if conf is None:
+        return ["(no config loaded)"]
+
+    # Group attrs by section, preserving registration order within each section.
+    by_section: dict[str, list[tuple[str, str, str]]] = {}
+    section_order: list[str] = []
+    for attr, (section, ini_key, type_label) in ConfigData._schema.items():
+        if section not in by_section:
+            by_section[section] = []
+            section_order.append(section)
+        by_section[section].append((attr, ini_key, type_label))
+
+    lines: list[str] = []
+    for section in section_order:
+        lines.append(f"[{section}]")
+        for attr, ini_key, _type_label in by_section[section]:
+            value = getattr(conf, attr, "(unset)")
+            if attr in _SENSITIVE_ATTRS and value:
+                value = "***"
+            lines.append(f"  {ini_key} = {value}  (attr: {attr})")
+        lines.append("")
+
+    # Runtime-computed values that aren't read from the INI file.
+    lines.append("[runtime]")
+    files = getattr(conf, "files_read", None) or []
+    lines.append(f"  files_read = {', '.join(files) if files else '(none)'}")
+    return lines
+
+
+def x_debug_log_config(**kwargs: Any) -> None:
+    for line in _config_lines():
+        _state.exec_log.log_status_info(line)
 
 
 def x_debug_write_subvars(**kwargs: Any) -> None:
@@ -147,8 +208,9 @@ def x_debug_write_subvars(**kwargs: Any) -> None:
         else:
             filewriter_write(output_dest, txt)
 
-    if _state.commandliststack:
-        for s in _state.commandliststack[-1].localvars.substitutions:
+    localvars = _state.current_localvars()
+    if localvars is not None:
+        for s in localvars.substitutions:
             write(f"Substitution [{s[0]}] = [{s[1]}]\n")
     if local is None:
         for s in _state.subvars.substitutions:
@@ -159,13 +221,6 @@ def x_debug_write_subvars(**kwargs: Any) -> None:
 def x_debug_write_config(**kwargs: Any) -> None:
     output_dest = kwargs["filename"]
     append = kwargs["append"]
-    conf = _state.conf
-    lines = [
-        f"Config; Script encoding = {conf.script_encoding}",
-        f"Config; Output encoding = {conf.output_encoding}",
-        f"Config; Import encoding = {conf.import_encoding}",
-        f"Config; GUI level = {conf.gui_level}",
-    ]
     if output_dest is not None and output_dest != "stdout" and append is None:
         filewriter_open_as_new(output_dest)
 
@@ -175,7 +230,7 @@ def x_debug_write_config(**kwargs: Any) -> None:
         else:
             filewriter_write(output_dest, txt)
 
-    for line in lines:
+    for line in _config_lines():
         write(f"{line}\n")
 
 

execsql/metacommands/io_export.py

@@ -1,7 +1,19 @@
 """Export metacommand handlers.
 
-Implements ``x_export``, ``x_export_query``, template-based exports,
-ODS multi-sheet export, and export metadata operations.
+- ``x_export`` — EXPORT (table → file in any supported format).
+- ``x_export_query`` — EXPORT QUERY (ad-hoc SELECT → file).
+- ``x_export_with_template`` / ``x_export_query_with_template`` —
+  template-rendered EXPORT (Jinja or string.Template).
+- ``x_export_ods_multiple`` / ``x_export_xlsx_multiple`` — multi-sheet
+  ODS / XLSX workbooks.
+- ``x_export_metadata`` / ``x_export_metadata_table`` — EXPORT_METADATA
+  (dump the in-process export-record log).
+- ``x_export_row_buffer`` — CONFIG EXPORT_ROW_BUFFER (set the per-export
+  row-buffer size).
+
+Per-format dispatch happens inside ``x_export``'s
+``if filefmt in (...): ... elif ...`` chain, which calls the
+appropriate writer in :mod:`execsql.exporters`.
 """
 
 from __future__ import annotations

execsql/metacommands/io_import.py

@@ -1,7 +1,16 @@
 """Import metacommand handlers.
 
-Implements ``x_import``, ``x_import_file``, ODS/XLS/Parquet/Feather
-import handlers, and the import row buffer setting.
+- ``x_import`` — IMPORT (delimited text / CSV; dispatches by extension
+  for .ods/.xls/.xlsx in the same handler).
+- ``x_import_file`` — IMPORT_FILE (insert a binary file blob as a row).
+- ``x_import_ods`` / ``x_import_ods_pattern`` — IMPORT … FROM ODS,
+  single-sheet and SHEETS MATCHING <pattern>.
+- ``x_import_xls`` / ``x_import_xls_pattern`` — same for XLS/XLSX.
+- ``x_import_parquet`` — IMPORT … FROM PARQUET (via polars).
+- ``x_import_feather`` — IMPORT … FROM FEATHER (via polars).
+- ``x_import_json`` — IMPORT … FROM JSON (array of objects or NDJSON).
+- ``x_import_row_buffer`` — CONFIG IMPORT_ROW_BUFFER.
+- ``x_show_progress`` — CONFIG SHOW_PROGRESS (toggle the import progress bar).
 """
 
 from __future__ import annotations

execsql/metacommands/io_write.py

@@ -1,7 +1,14 @@
 """WRITE metacommand handlers.
 
-Implements ``x_write``, ``x_write_create_table`` (CSV, ODS, XLS, alias),
-``x_write_prefix``, ``x_write_suffix``, and ``x_writescript``.
+- ``x_write`` — WRITE "<text>" [TEE TO <file>] (text output).
+- ``x_writescript`` — WRITE SCRIPT <name> [TO <file>] (dump a named
+  script block).
+- ``x_write_create_table`` — WRITE CREATE_TABLE … (emit a CREATE TABLE
+  statement inferred from a data source); plus the format-specific
+  variants ``x_write_create_table_ods``, ``x_write_create_table_xls``,
+  ``x_write_create_table_alias``.
+- ``x_write_prefix`` / ``x_write_suffix`` — CONFIG WRITE_PREFIX /
+  CONFIG WRITE_SUFFIX (set text prepended / appended to every WRITE).
 """
 
 from __future__ import annotations
@@ -222,11 +229,107 @@ def x_write_suffix(**kwargs: Any) -> None:
     return None
 
 
+def _render_script_nodes(nodes: Any, emit: Any) -> None:
+    """Walk a list of AST nodes and emit reconstructed source lines.
+
+    Every node type that can legally appear in a ``ScriptBlock.body`` is
+    handled — nested ``IfBlock`` / ``LoopBlock`` / ``BatchBlock`` /
+    ``SqlBlock`` structures render with their delimiters; ``Comment`` and
+    ``IncludeDirective`` (``INCLUDE`` / ``EXECUTE SCRIPT``) are preserved.
+    Unknown node types raise ``ErrInfo`` rather than silently disappear so
+    future AST additions can't quietly cause data loss.
+    """
+    from execsql.exceptions import ErrInfo
+    from execsql.script.ast import (
+        BatchBlock,
+        Comment,
+        IfBlock,
+        IncludeDirective,
+        LoopBlock,
+        MetaCommandStatement,
+        ScriptBlock,
+        SqlBlock,
+        SqlStatement,
+    )
+
+    for node in nodes:
+        if isinstance(node, SqlStatement):
+            emit(f"{node.text}\n")
+        elif isinstance(node, MetaCommandStatement):
+            emit(f"-- !x! {node.command}\n")
+        elif isinstance(node, Comment):
+            # Preserve comment text verbatim; the parser stores it including
+            # the leading "--" / "/* ... */" delimiters.
+            emit(f"{node.text}\n")
+        elif isinstance(node, IncludeDirective):
+            kw = "EXECUTE SCRIPT" if node.is_execute_script else "INCLUDE"
+            if_exists = " IF EXISTS" if node.if_exists else ""
+            args = f" ({node.arguments})" if node.arguments else ""
+            loop = f" LOOP {node.loop_type} ({node.loop_condition})" if node.loop_type else ""
+            emit(f"-- !x! {kw}{if_exists} {node.target}{args}{loop}\n")
+        elif isinstance(node, IfBlock):
+            emit(f"-- !x! IF ({node.condition})\n")
+            for mod in node.condition_modifiers:
+                kw = "ANDIF" if mod.kind == "AND" else "ORIF"
+                emit(f"-- !x! {kw} ({mod.condition})\n")
+            _render_script_nodes(node.body, emit)
+            for clause in node.elseif_clauses:
+                emit(f"-- !x! ELSEIF ({clause.condition})\n")
+                for mod in clause.condition_modifiers:
+                    kw = "ANDIF" if mod.kind == "AND" else "ORIF"
+                    emit(f"-- !x! {kw} ({mod.condition})\n")
+                _render_script_nodes(clause.body, emit)
+            if node.else_body:
+                emit("-- !x! ELSE\n")
+                _render_script_nodes(node.else_body, emit)
+            emit("-- !x! ENDIF\n")
+        elif isinstance(node, LoopBlock):
+            emit(f"-- !x! LOOP {node.loop_type} ({node.condition})\n")
+            _render_script_nodes(node.body, emit)
+            emit("-- !x! END LOOP\n")
+        elif isinstance(node, BatchBlock):
+            emit("-- !x! BEGIN BATCH\n")
+            _render_script_nodes(node.body, emit)
+            emit("-- !x! END BATCH\n")
+        elif isinstance(node, SqlBlock):
+            emit("-- !x! BEGIN SQL\n")
+            _render_script_nodes(node.body, emit)
+            emit("-- !x! END SQL\n")
+        elif isinstance(node, ScriptBlock):
+            # Nested SCRIPT definition — emit recursively with its own
+            # BEGIN/END SCRIPT framing.
+            param_names = node.param_names
+            if param_names:
+                emit(f"-- !x! BEGIN SCRIPT {node.name} ({', '.join(param_names)})\n")
+            else:
+                emit(f"-- !x! BEGIN SCRIPT {node.name}\n")
+            _render_script_nodes(node.body, emit)
+            emit(f"-- !x! END SCRIPT {node.name}\n")
+        else:
+            raise ErrInfo(
+                type="cmd",
+                other_msg=f"WRITE SCRIPT cannot render AST node of type {type(node).__name__}.",
+            )
+
+
 def x_writescript(**kwargs: Any) -> None:
-    script_id = kwargs["script_id"]
+    """Dump a registered SCRIPT block's source to stdout or a file.
+
+    Reads from the AST script registry (``ctx.ast_scripts``) and walks the
+    full block tree — including nested IF / LOOP / BATCH structures —
+    reconstructing source between ``BEGIN SCRIPT`` and ``END SCRIPT``
+    delimiters so the output is re-includable.
+    """
+    from execsql.exceptions import ErrInfo
+
+    script_id = kwargs["script_id"].lower()
     output_dest = kwargs["filename"]
     append = kwargs["append"]
 
+    block = _state.ast_scripts.get(script_id)
+    if block is None:
+        raise ErrInfo("cmd", other_msg=f"There is no SCRIPT named {script_id}.")
+
     def write(txt: str) -> None:
         if output_dest is None or output_dest == "stdout":
             _state.output.write(txt)
@@ -237,12 +340,11 @@ def x_writescript(**kwargs: Any) -> None:
         check_dir(output_dest)
         if not append:
             filewriter_open_as_new(output_dest)
-    script = _state.savedscripts[script_id]
-    if script.paramnames is not None and len(script.paramnames) > 0:
-        write(f"BEGIN SCRIPT {script_id} ({', '.join(script.paramnames)})\n")
+
+    param_names = block.param_names
+    if param_names:
+        write(f"-- !x! BEGIN SCRIPT {script_id} ({', '.join(param_names)})\n")
     else:
-        write(f"BEGIN SCRIPT {script_id}\n")
-    lines = [c.commandline() for c in script.cmdlist]
-    for line in lines:
-        write(f"{line}\n")
-    write(f"END SCRIPT {script_id}\n")
+        write(f"-- !x! BEGIN SCRIPT {script_id}\n")
+    _render_script_nodes(block.body, write)
+    write(f"-- !x! END SCRIPT {script_id}\n")