execsql2 2.23.1__py3-none-any.whl → 2.25.0__py3-none-any.whl

execsql/debug/repl.py

@@ -1,49 +1,34 @@
 from __future__ import annotations
 
-"""Interactive debug REPL metacommand handler for execsql.
+"""The debug REPL a ``BREAKPOINT`` (or ``run --debug``) opens.
 
-Implements ``x_breakpoint`` — the ``BREAKPOINT`` metacommand — which pauses
-script execution and drops into an interactive read-eval-print loop.
+Input runs as if it were written where the script paused, through
+:mod:`execsql.interactive`: SQL (ending with ``;``, results shown as tables),
+metacommands and blocks, in the paused script's variable scope and
+transaction. A ``SUB`` typed here is still set
+when the script resumes; a ``BREAK`` inside a LOOP leaves the loop; ``HALT``
+ends the run. An error ends only the input that caused it.
 
-The REPL allows the user to:
+Commands to the REPL itself start with ``.``: ``.continue`` and ``.next``
+resume the script, ``.where`` and ``.stack`` show where it paused, ``.quit``
+halts it; ``.vars``, ``.set``, ``.scripts`` and ``.cancel`` inspect and
+change the run.
 
-- Inspect and print substitution variables.
-- Run ad-hoc SQL queries against the current database.
-- Step through the script one statement at a time.
-- Resume or abort execution.
-
-Dispatch is two-way (psql-style): input starting with ``.`` is a REPL
-command (``.continue``, ``.vars [VAR]``, ``.next``, etc.), everything
-else is SQL.  Multi-line SQL is supported: any non-``.`` input opens a
-buffer whose continuation prompt is ``  ...        > ``, accumulating
-until a line ends with ``;``.  ``.cancel`` (or Ctrl-C / EOF) discards
-the partial buffer.
-
-Variable lookup is explicit — ``.vars LOGFILE`` prints one variable;
-``.vars`` lists them all.  There is no bare-identifier lookup, so any
-SQL keyword you type starts a buffer the moment you press Enter.
-
-The trailing ``;`` is the SQL terminator both within one line and across
-multiple lines — the REPL has no read-only mode and DDL on most adapters
-is irreversible, so requiring ``;`` is a small intent gate against
-accidental DML/DDL on mistyped input.  Use SQL ``BEGIN; … ROLLBACK;`` to
-bracket exploratory DML if you need recoverability.
-
-Errors raised by any REPL helper — bad SQL, malformed dot-commands, etc. —
-are caught at the loop level so the session re-prompts instead of escaping
-through ``x_breakpoint`` and being stamped as a "Metacommand error".
+This module also holds the REPL's display helpers (colors, tables, variable
+listings).
 
 In non-interactive environments (CI, piped input, ``sys.stdin.isatty()`` is
 ``False``) the metacommand is silently skipped so automated pipelines are not
 blocked.
 """
 
-import os
 import sys
 from pathlib import Path
 from typing import Any
 
 import execsql.state as _state
+from execsql.interactive import Prompt, common_dot_command, read_eval_loop, unknown_dot_command
+from execsql.utils.color import color_disabled_by_env
 
 __all__ = ["x_breakpoint"]
 
@@ -77,7 +62,7 @@ def _use_color() -> bool:
     global _color_cache  # noqa: PLW0603
     if _color_cache is not None:
         return _color_cache
-    if os.environ.get("NO_COLOR") is not None or os.environ.get("EXECSQL_NO_COLOR") is not None:
+    if color_disabled_by_env():
         _color_cache = False
     else:
         output = _state.output
@@ -123,18 +108,20 @@ _HELP_COMMANDS = [
     (".quit", ".q", "Halt the script (exit 1)"),
     (".vars", ".v", "List all execsql substitution variables"),
     (".vars VAR", ".v VAR", "Print the value of a single variable (e.g. .vars logfile)"),
+    (".vars all", "", "Also list environment (&) variables"),
     (".next", ".n", "Execute the next statement then pause again (step mode)"),
     (".where", ".w", "Show the current script location and upcoming statement"),
     (".stack", "", "Show the command-list stack (script name, line, depth)"),
     (".set VAR VAL", ".s", "Set or update a substitution variable"),
     (".scripts", "", "List all registered SCRIPT definitions"),
     (".scripts NAME", "", "Show detail for a specific SCRIPT"),
-    (".cancel", "", "Discard a partial multi-line SQL buffer"),
+    (".cancel", "", "Discard a statement or block you are typing"),
     (".help", ".h", "Show this help text"),
 ]
 
 _HELP_OTHER = [
-    ("SELECT ...;", "Run SQL — multi-line accepted, terminate with ';' to execute"),
+    ("SELECT ...;", "Run SQL, as the script would here; it may span lines and ends with ';'"),
+    ("-- !x! SUB ...", "Run a metacommand (!x! SUB ... works too); blocks stay open until closed"),
 ]
 
 _HELP_CMD_WIDTH = 13  # width of the command column
@@ -196,8 +183,8 @@ def _write_rule(label: str) -> None:
 def _debug_repl(*, step: bool = False) -> None:
     """Interactive read-eval-print loop for script debugging.
 
-    Reads commands from stdin until the user types ``.continue`` or ``.abort``,
-    or until EOF / KeyboardInterrupt.
+    Reads input until ``.continue``, ``.next`` or ``.quit``, or until Ctrl-D /
+    Ctrl-C on an empty line, which resume the script.
 
     Args:
         step: When ``True``, the entry banner says "Step" instead of
@@ -232,125 +219,37 @@ def _debug_repl(*, step: bool = False) -> None:
     _hint_c = _c(_DIM, "'.c'")
     _write(f"  Type {_hint_help} for commands, {_hint_c} to resume.\n\n")
 
-    sql_buffer: list[str] = []
-
-    while True:
-        prompt = "  ...        > " if sql_buffer else "execsql debug> "
-        try:
-            line = input(prompt).strip()
-        except EOFError:
-            if sql_buffer:
-                sql_buffer.clear()
-                _write("\n  (input discarded)\n")
-                continue
-            _write("\n")
-            return
-        except KeyboardInterrupt:
-            if sql_buffer:
-                sql_buffer.clear()
-                _write("\n  (input discarded)\n")
-                continue
-            _write("\n")
-            return
+    read_eval_loop(_DebugPrompt())
 
-        if not line:
-            continue
-
-        try:
-            if line.startswith("."):
-                cmd = line[1:].strip().lower()
-                if cmd == "cancel":
-                    if sql_buffer:
-                        sql_buffer.clear()
-                        _write("  (input discarded)\n")
-                    continue
-                _handle_dot_command(line)
-                if cmd in ("continue", "c"):
-                    return
-                if cmd in ("abort", "q", "quit"):
-                    return
-                if cmd in ("next", "n"):
-                    return
-                continue
-
-            if sql_buffer:
-                sql_buffer.append(line)
-                joined = " ".join(sql_buffer)
-                if joined.rstrip().endswith(";"):
-                    _run_sql(joined)
-                    sql_buffer.clear()
-                continue
-
-            if line.rstrip().endswith(";"):
-                _run_sql(line)
-                continue
-
-            sql_buffer.append(line)
-        except SystemExit:
-            raise
-        except KeyboardInterrupt:
-            sql_buffer.clear()
-            _write("\n  (interrupted)\n")
-            continue
-        except Exception as exc:
-            sql_buffer.clear()
-            _write(f"  {_c(_RED, 'Error:')} {exc}\n")
-            continue
-
-
-def _handle_dot_command(line: str) -> None:
-    """Dispatch a dot-prefixed REPL command."""
-    # Strip the leading dot and normalize
-    cmd = line[1:].strip().lower()
-
-    if cmd in ("continue", "c"):
-        return  # caller checks and returns from _debug_repl
-    elif cmd in ("abort", "q", "quit"):
+
+class _DebugPrompt(Prompt):
+    name = "execsql debug"
+    source = "<breakpoint>"
+
+    def dot_command(self, command: str, argument: str, line: str) -> bool:
+        return _handle_dot_command(line)
+
+
+def _handle_dot_command(line: str) -> bool:
+    """Dispatch a ``.`` command; return ``False`` when the script should resume."""
+    command, _, argument = line.strip()[1:].partition(" ")
+    command = command.lower()
+    if command in ("continue", "c"):
+        return False
+    if command in ("abort", "q", "quit"):
         raise SystemExit(1)
-    elif cmd in ("help", "h"):
+    if command in ("next", "n"):
+        _enable_step_mode()
+        return False
+    if command in ("help", "h"):
         _write(_format_help())
-    elif cmd in ("vars", "v"):
-        _print_all_vars()
-    elif cmd.startswith("vars ") or cmd.startswith("v "):
-        rest = cmd.split(None, 1)[1].strip() if " " in cmd else ""
-        if rest:
-            _print_var(rest)
-        else:
-            _print_all_vars()
-    elif cmd in ("where", "w"):
+    elif command in ("where", "w"):
         _print_where()
-    elif cmd == "stack":
+    elif command == "stack":
         _print_stack()
-    elif cmd in ("next", "n"):
-        _enable_step_mode()
-    elif cmd.startswith("set ") or cmd == "set":
-        # .set VAR VAL — set or update a substitution variable
-        rest = cmd[4:].strip() if cmd.startswith("set ") else ""
-        if not rest:
-            _write("  Usage: .set VAR VALUE\n")
-        else:
-            parts = rest.split(None, 1)
-            varname = parts[0]
-            value = parts[1] if len(parts) > 1 else ""
-            _set_var(varname, value)
-    elif cmd.startswith("s ") or cmd == "s":
-        # .s VAR VAL — shorthand for .set
-        rest = cmd[2:].strip() if cmd.startswith("s ") else ""
-        if not rest:
-            _write("  Usage: .s VAR VALUE\n")
-        else:
-            parts = rest.split(None, 1)
-            varname = parts[0]
-            value = parts[1] if len(parts) > 1 else ""
-            _set_var(varname, value)
-    elif cmd.startswith("scripts"):
-        rest = cmd[7:].strip()
-        if rest:
-            _print_script_detail(rest)
-        else:
-            _print_scripts()
-    else:
-        _write(f"  {_c(_RED, 'Unknown command:')} {line!r}. Type '.help' for available commands.\n")
+    elif not common_dot_command(command, argument.strip(), line):
+        unknown_dot_command(line)
+    return True
 
 
 # ---------------------------------------------------------------------------
@@ -371,7 +270,7 @@ def _write(text: str) -> None:
 def _print_where() -> None:
     """Print the current script location and the upcoming statement.
 
-    Reads ``_state.last_command`` (a :class:`ScriptCmd`) and displays the
+    Reads ``_state.last_command`` and displays the
     filename, line number, command type, and (truncated) statement text.
     If ``last_command`` is ``None``, reports that the position is unknown.
     """
@@ -531,49 +430,8 @@ def _print_stack() -> None:
     _write("\n")
 
 
-def _run_sql(sql: str) -> None:
-    """Execute ad-hoc SQL and pretty-print results or affected rowcount."""
-    dbs = _state.dbs
-    if dbs is None:
-        _write("  (no database connection is active)\n")
-        return
-    db = dbs.current()
-    if db is None:
-        _write("  (no database connection is active)\n")
-        return
-
-    try:
-        with db._cursor() as curs:
-            try:
-                curs.execute(sql)
-            except Exception as exc:
-                try:
-                    db.rollback()
-                except Exception:
-                    pass
-                _write(f"  {_c(_RED, 'SQL error:')} {exc}\n")
-                return
-
-            try:
-                _state.subvars.add_substitution("$LAST_ROWCOUNT", curs.rowcount)
-            except Exception:
-                pass
-
-            if curs.description is None:
-                rowcount = curs.rowcount if curs.rowcount is not None else -1
-                if rowcount >= 0:
-                    row_word = "row" if rowcount == 1 else "rows"
-                    _write(f"  {_c(_DIM, f'({rowcount} {row_word} affected)')}\n")
-                else:
-                    _write(f"  {_c(_DIM, '(statement executed)')}\n")
-                return
-
-            colnames = [d[0] for d in curs.description]
-            rows = curs.fetchall()
-    except Exception as exc:
-        _write(f"  {_c(_RED, 'SQL error:')} {exc}\n")
-        return
-
+def _print_table(colnames: list[str], rows: list[Any]) -> None:
+    """Print query results as a boxed text table, NULLs dimmed."""
     if not colnames:
         _write("  (query returned no columns)\n")
         return

execsql/exceptions.py

@@ -35,6 +35,7 @@ Note:
 __all__ = [
     "ExecSqlError",
     "ConfigError",
+    "ConfigFileError",
     "ExecSqlTimeoutError",
     "ErrInfo",
     "DataTypeError",
@@ -74,6 +75,14 @@ class ConfigError(ExecSqlError):
     """Raised for invalid or missing execsql configuration values."""
 
 
+class ConfigFileError(ConfigError):
+    """A :class:`ConfigError` found while reading one config file.
+
+    The message starts with the file's path. ``execsql config --validate``
+    reports every problem in the file, with line numbers.
+    """
+
+
 class ExecSqlTimeoutError(ExecSqlError):
     """Timeout during alarm-timer operations.
 

execsql/exporters/delimited.py

@@ -28,7 +28,14 @@ from execsql.utils.errors import exception_desc
 from execsql.utils.fileio import filewriter_close
 from execsql.utils.strings import clean_words, dedup_words, fold_words
 
-__all__ = ["LineDelimiter", "CsvFile", "CsvWriter", "DelimitedWriter", "write_delimited_file"]
+__all__ = [
+    "LineDelimiter",
+    "CsvFile",
+    "CsvWriter",
+    "DelimitedWriter",
+    "blank_lines_as_empty_rows",
+    "write_delimited_file",
+]
 
 _SPACE_DELIM_RX = re.compile(r" +")
 
@@ -618,6 +625,9 @@ class CsvFile(EncodedFile):
                 prev_state = state
                 state = exec_vector[state]()
         end()
+        # An empty result is either a blank line or the end of the file; the
+        # reader needs to know which.
+        self._at_eof = c == ""
         if len(self.parse_errors) > 0:
             raise ErrInfo("error", other_msg=", ".join(self.parse_errors))
         return elements
@@ -653,48 +663,63 @@ class CsvFile(EncodedFile):
                 doublequote=True,
                 strict=False,
             )
-            for elements in csv_reader:
-                if len(elements) == 0:
-                    break
-                # Normalize empty strings to None for parity with the slow reader.
-                normalized_elements = [e if e != "" else None for e in elements]
-                if conf.del_empty_cols and len(self.blank_cols) > 0:
-                    blanks = copy.copy(self.blank_cols)
-                    while len(blanks) > 0:
-                        b = blanks.pop()
-                        del normalized_elements[b]
-                yield normalized_elements
+            # Normalize empty strings to None for parity with the slow reader.
+            raw = ([e if e != "" else None for e in elements] for elements in csv_reader)
+            yield from self._rows(raw, conf)
         finally:
             f.close()
 
     def _slow_reader(self, conf: Any) -> Any:
         """Read using the character-at-a-time state machine (fallback for non-standard formats)."""
         f = self.openclean("rt")
-        line_no = 0
-        try:
+
+        def raw() -> Any:
+            line_no = 0
             while True:
                 line_no += 1
                 try:
                     elements = self.read_and_parse_line(f)
                 except ErrInfo as e:
                     raise ErrInfo("error", other_msg=f"{e.other} on line {line_no}.") from e
-                except:
-                    raise
-                if len(elements) > 0:
-                    if conf.del_empty_cols and len(self.blank_cols) > 0:
-                        blanks = copy.copy(self.blank_cols)
-                        while len(blanks) > 0:
-                            b = blanks.pop()
-                            del elements[b]
-                    yield elements
-                else:
-                    break
+                if not elements and self._at_eof:
+                    return
+                yield elements
+
+        try:
+            yield from self._rows(raw(), conf)
         finally:
             f.close()
 
-    def writer(self, append: bool = False) -> CsvWriter:
-        """Return a :class:`CsvWriter` configured with this file's format settings."""
-        return CsvWriter(self.filename, self.encoding, self.delimiter, self.quotechar, self.escapechar, append)
+    def _rows(self, raw: Any, conf: Any) -> Any:
+        """Yield parsed rows, turning a blank line inside the data into an empty row.
+
+        A blank line (no fields at all) is an all-``None`` row as wide as the
+        header, so ``CONFIG EMPTY_ROWS`` decides whether it is loaded, as it
+        does for a row of empty fields.  Blank lines with no data after them
+        end the file, as they always have, so a trailing newline does not add
+        an empty row.
+        """
+        width = None
+        pending_blank = 0
+        for elements in raw:
+            if not elements:
+                if width is not None:
+                    pending_blank += 1
+                continue
+            if width is None:
+                width = len(elements)
+            for _ in range(pending_blank):
+                yield self._without_blank_cols([None] * width, conf)
+            pending_blank = 0
+            yield self._without_blank_cols(elements, conf)
+
+    def _without_blank_cols(self, elements: list, conf: Any) -> list:
+        if conf.del_empty_cols and len(self.blank_cols) > 0:
+            blanks = copy.copy(self.blank_cols)
+            while len(blanks) > 0:
+                b = blanks.pop()
+                del elements[b]
+        return elements
 
     def _colhdrs(self, inf: Any) -> list[str]:
         conf = _state.conf
@@ -764,6 +789,40 @@ class CsvFile(EncodedFile):
         return self.table_data.create_table(database_type, schemaname, tablename, pretty)
 
 
+def blank_lines_as_empty_rows(f: Any, delimiter: str, quotechar: str | None, columns: int, chunk_size: int) -> Any:
+    """Yield the text of *f* in chunks of about *chunk_size*, with each blank line inside the data made an empty row.
+
+    For a reader of the raw file, such as PostgreSQL's COPY, which rejects a
+    blank line.  The line becomes *columns* empty fields (``,,`` for three),
+    matching what the row-by-row reader loads; blank lines at the end of the
+    file are dropped, as there.  A blank line inside a quoted value is part
+    of that value and is kept: quote characters are counted, and a doubled
+    quote (the only escape a raw CSV reader is given) counts twice.
+    """
+    empty_row = delimiter * (columns - 1) + "\n"
+    in_quotes = False
+    pending_blank = 0
+    buf: list[str] = []
+    size = 0
+    for line in f:
+        if not in_quotes and line in ("\n", "\r\n"):
+            pending_blank += 1
+            continue
+        if pending_blank:
+            buf.append(empty_row * pending_blank)
+            size += len(empty_row) * pending_blank
+            pending_blank = 0
+        if quotechar and line.count(quotechar) % 2:
+            in_quotes = not in_quotes
+        buf.append(line)
+        size += len(line)
+        if size >= chunk_size:
+            yield "".join(buf)
+            buf, size = [], 0
+    if buf:
+        yield "".join(buf)
+
+
 def write_delimited_file(
     outfile: str,
     filefmt: str,

execsql/exporters/xlsx.py

@@ -38,7 +38,9 @@ def _require_openpyxl() -> Any:
 
         return openpyxl
     except ImportError:
-        fatal_error("The openpyxl library is needed to write Excel (.xlsx) spreadsheets (install execsql2[excel]).")
+        fatal_error(
+            "The openpyxl library is needed to write Excel (.xlsx) spreadsheets (install the formats extra, execsql2[formats]).",
+        )
 
 
 def _cell_value(item: Any) -> Any:
@@ -72,6 +74,19 @@ def _cell_value(item: Any) -> Any:
     return str(item)
 
 
+def _write_cell(ws: Any, row: int, column: int, value: Any) -> Any:
+    """Write *value* to a cell as data, never as a formula, and return the cell.
+
+    openpyxl stores any string that starts with ``=`` as a formula, so text
+    from the database such as ``=1+1`` or ``=HYPERLINK(...)`` would be
+    evaluated when the workbook is opened.  Such cells are stored as text.
+    """
+    cell = ws.cell(row=row, column=column, value=value)
+    if cell.data_type == "f":
+        cell.data_type = "s"
+    return cell
+
+
 # ---------------------------------------------------------------------------
 # Public API
 # ---------------------------------------------------------------------------
@@ -145,7 +160,7 @@ def write_query_to_xlsx(
             ("datasheet_name", "created_on", "created_by", "description", "source"),
             start=1,
         ):
-            cell = inv_ws.cell(row=1, column=col_idx, value=hdr)
+            cell = _write_cell(inv_ws, 1, col_idx, hdr)
             cell.font = bold_font
     else:
         inv_ws = wb[inventory_name]
@@ -158,13 +173,13 @@ def write_query_to_xlsx(
 
     # Header row
     for col_idx, hdr in enumerate(hdrs, start=1):
-        cell = ws.cell(row=1, column=col_idx, value=str(hdr))
+        cell = _write_cell(ws, 1, col_idx, str(hdr))
         cell.font = bold_font
 
     # Data rows
     for row_idx, row in enumerate(rows, start=2):
         for col_idx, item in enumerate(row, start=1):
-            ws.cell(row=row_idx, column=col_idx, value=_cell_value(item))
+            _write_cell(ws, row_idx, col_idx, _cell_value(item))
 
     # ------------------------------------------------------------------
     # Update inventory sheet
@@ -185,7 +200,7 @@ def write_query_to_xlsx(
         ),
         start=1,
     ):
-        inv_ws.cell(row=next_row, column=col_idx, value=value)
+        _write_cell(inv_ws, next_row, col_idx, value)
 
     wb.save(outfile)
     wb.close()
@@ -256,7 +271,7 @@ def write_queries_to_xlsx(
             ("datasheet_name", "created_on", "created_by", "description", "source"),
             start=1,
         ):
-            cell = inv_ws.cell(row=1, column=col_idx, value=hdr)
+            cell = _write_cell(inv_ws, 1, col_idx, hdr)
             cell.font = bold_font
     else:
         inv_ws = wb[inventory_name]
@@ -297,12 +312,12 @@ def write_queries_to_xlsx(
         ws = wb.create_sheet(sheet_name)
 
         for col_idx, hdr in enumerate(hdrs, start=1):
-            cell = ws.cell(row=1, column=col_idx, value=str(hdr))
+            cell = _write_cell(ws, 1, col_idx, str(hdr))
             cell.font = bold_font
 
         for row_idx, row in enumerate(rows, start=2):
             for col_idx, item in enumerate(row, start=1):
-                ws.cell(row=row_idx, column=col_idx, value=_cell_value(item))
+                _write_cell(ws, row_idx, col_idx, _cell_value(item))
 
         # Determine per-sheet description.
         if desc is None:
@@ -326,7 +341,7 @@ def write_queries_to_xlsx(
             ),
             start=1,
         ):
-            inv_ws.cell(row=next_row, column=col_idx, value=value)
+            _write_cell(inv_ws, next_row, col_idx, value)
 
         if tee and outfile.lower() != "stdout":
             prettyprint_query(select_stmt, db, "stdout", False, desc=d)

execsql/exporters/zip.py

@@ -54,8 +54,7 @@ class WriteableZipfile:
             date_time=time.localtime(time.time())[:6],
         )
         self.current_zinfo.compress_type = self.zf.compression
-        if sys.version_info.major >= 3 and sys.version_info.minor >= 7:
-            self.current_zinfo._compresslevel = self.zf.compresslevel  # type: ignore[attr-defined]
+        self.current_zinfo._compresslevel = self.zf.compresslevel  # type: ignore[attr-defined]
         # See https://stackoverflow.com/questions/434641/how-do-i-set-permissions-attributes-on-a-file-in-a-zip-file-using-pythons-zip
         self.current_zinfo.external_attr = 0o100755 << 16  # ?rw-rw-rw-
         if sys.platform.startswith("win"):