execsql2 2.23.1__py3-none-any.whl → 2.25.0__py3-none-any.whl

execsql/api.py

@@ -15,7 +15,7 @@ Usage::
     result = run(sql="CREATE TABLE t (id INT); INSERT INTO t VALUES (1);",
                  dsn="sqlite:///my.db")
 
-    # With substitution variables
+    # With substitution variables (referenced as !!$SCHEMA!! and !!$DATE!!)
     result = run(script="etl.sql",
                  dsn="postgresql://user:pass@host/db",
                  variables={"SCHEMA": "public", "DATE": "2026-01-01"})
@@ -29,7 +29,6 @@ Usage::
 
 from __future__ import annotations
 
-import atexit
 import dataclasses
 import datetime
 import io
@@ -57,7 +56,9 @@ class ScriptError:
 
     Attributes:
         message: Human-readable error description.
-        source: Script file path or ``"<inline>"``.
+        source: Script file path or ``"<inline>"`` for an error in the
+            script; ``"<config>"``, ``"<connect>"`` or ``"<output>"`` for a
+            config file, connection or output file that failed.
         line: Source line number, or ``None`` if unknown.
         sql: The SQL statement that caused the error, if applicable.
     """
@@ -77,10 +78,10 @@ class ScriptResult:
         commands_run: Number of SQL statements and metacommands executed.
         elapsed: Wall-clock execution time in seconds.
         errors: List of errors encountered (empty on success).
-        variables: Final state of substitution variables, keyed by name
-            without any sigil. Includes user-defined variables (no prefix)
-            and system variables (the leading ``$`` is stripped, so
-            ``$DATE_TAG`` appears as ``"DATE_TAG"``). Environment (``&``),
+        variables: Final state of substitution variables, keyed by
+            lower-case name without any sigil. Includes user-defined
+            variables (no prefix) and system variables (the leading ``$`` is
+            stripped, so ``$DATE_TAG`` appears as ``"date_tag"``). Environment (``&``),
             column-data (``@``), script-local (``~``), and script-parameter
             (``#``) variables are excluded.
     """
@@ -217,6 +218,8 @@ class _LibraryConfig:
         self.smtp_password: str | None = None
         self.smtp_ssl = False
         self.smtp_tls = False
+        self.smtp_verify_certificate = True
+        self.smtp_ca_file: str | None = None
         self.email_format = "plain"
         self.email_css: str | None = None
 
@@ -352,7 +355,8 @@ def run(
         connection: A pre-existing :class:`~execsql.db.base.Database`
             instance.  ``run()`` will NOT close this connection on exit.
         variables: Substitution variables as ``{"NAME": "value"}``.
-            Keys without a ``$`` prefix get one added automatically.
+            Keys without a ``$`` prefix get one added automatically, so
+            ``{"SCHEMA": "public"}`` is referenced as ``!!$SCHEMA!!``.
         config_file: Optional execsql configuration file to load.
         encoding: Script file encoding (default ``"utf-8"``).
         halt_on_error: If ``True`` (default), stop on the first SQL
@@ -373,7 +377,9 @@ def run(
 
     Returns:
         A :class:`ScriptResult` with execution outcome, timing, errors,
-        and final variable state.
+        and final variable state.  A script that cannot be read, a bad
+        *config_file*, and a *dsn* that cannot be parsed or opened all
+        return ``success=False`` rather than raising.
 
     Raises:
         ValueError: If the argument combination is invalid (e.g. both
@@ -404,14 +410,8 @@ def run(
         else:
             assert sql is not None
             tree = parse_string(sql, source_name="<inline>")
-    except ErrInfo as exc:
-        return ScriptResult(
-            success=False,
-            commands_run=0,
-            elapsed=0.0,
-            errors=[ScriptError(message=exc.errmsg(), source=str(script) if script else "<inline>")],
-            variables={},
-        )
+    except (ErrInfo, OSError) as exc:
+        return _failed(_error_text(exc), str(script) if script else "<inline>")
 
     # ------------------------------------------------------------------
     # Build an isolated RuntimeContext
@@ -425,9 +425,14 @@ def run(
         # Load a real ConfigData with the explicit config file
         from execsql.script.variables import SubVarSet
 
+        if not Path(config_file).is_file():
+            return _failed(f"Config file {str(config_file)!r} does not exist.", "<config>")
         temp_subvars = SubVarSet()
         script_dir = str(Path(script).resolve().parent) if script else os.getcwd()
-        conf = ConfigData(script_dir, temp_subvars, config_file=str(config_file))
+        try:
+            conf = ConfigData(script_dir, temp_subvars, config_file=str(config_file))
+        except Exception as exc:
+            return _failed(_error_text(exc), "<config>")
         # Apply any overrides
         for k, v in conf_overrides.items():
             setattr(conf, k, v)
@@ -492,89 +497,103 @@ def run(
         # Connect to database
         # ------------------------------------------------------------------
         owns_connection = connection is None
-        if dsn is not None:
-            db = _connect_from_dsn(dsn, new_db=new_db)
-        else:
-            db = connection
-
-        assert db is not None
-        assert ctx.dbs is not None
-        ctx.dbs.add("initial", db)
-        ctx.subvars.add_substitution("$CURRENT_DBMS", db.type.dbms_id)
-        ctx.subvars.add_substitution("$CURRENT_DATABASE", db.name())
-        ctx.subvars.add_substitution("$SYSTEM_CMD_EXIT_STATUS", "0")
-
-        # ------------------------------------------------------------------
-        # Execute
-        # ------------------------------------------------------------------
-        from execsql.script.executor import execute
-
-        errors: list[ScriptError] = []
-        t0 = time.perf_counter()
-
-        # WRITE ... TO <file> and TEE are handled by a FileWriter subprocess,
-        # and fileio drops every write when none is running.  Start one here so
-        # the API behaves like the CLI.
-        _ensure_filewriter(conf)
-
+        # Everything from here on runs inside try/finally so an owned
+        # connection is closed and temp files removed however run() exits,
+        # including KeyboardInterrupt in a notebook.
         try:
-            execute(tree, ctx=ctx)
-        except SystemExit:
-            # exit_now() calls sys.exit() — catch and convert to error
-            _capture_errors(ctx, errors)
-        except ErrInfo as exc:
-            errors.append(
-                ScriptError(
-                    message=exc.errmsg(),
-                    source=_last_source(ctx),
-                    line=_last_line(ctx),
-                    sql=getattr(exc, "command_text", None),
-                ),
-            )
-        except Exception as exc:
-            errors.append(ScriptError(message=str(exc), source="<runtime>"))
-
-        # Non-halting errors (halt_on_error=False, METACOMMAND_ERROR_HALT OFF)
-        # are recorded in status.error_history as execution continues; a
-        # halting error never reaches the history, so there is no overlap
-        # with the exception paths above.
-        if ctx.status is not None:
-            for err_source, err_line, err_cmd, err_msg in ctx.status.error_history:
-                errors.append(ScriptError(message=err_msg, source=err_source, line=err_line, sql=err_cmd))
-
-        elapsed = time.perf_counter() - t0
+            if dsn is not None:
+                try:
+                    db = _connect_from_dsn(dsn, new_db=new_db)
+                except Exception as exc:
+                    return _failed(_error_text(exc), "<connect>")
+            else:
+                db = connection
+
+            assert db is not None
+            assert ctx.dbs is not None
+            ctx.dbs.add("initial", db)
+            ctx.subvars.add_substitution("$CURRENT_DBMS", db.type.dbms_id)
+            ctx.subvars.add_substitution("$CURRENT_DATABASE", db.name())
+            ctx.subvars.add_substitution("$SYSTEM_CMD_EXIT_STATUS", "0")
+
+            # ------------------------------------------------------------------
+            # Execute
+            # ------------------------------------------------------------------
+            from execsql.script.executor import execute
+
+            errors: list[ScriptError] = []
+            t0 = time.perf_counter()
+
+            # WRITE ... TO <file> and TEE hand their output to a FileWriter, and
+            # fileio drops every write when none is running.  This run gets its
+            # own, so concurrent runs never share files or a halt.
+            _start_filewriter(ctx, conf)
 
-        # Flush and close every file the script wrote, so the caller can read
-        # them the moment run() returns.  The subprocess itself is left running
-        # for reuse by a later run() and is reaped by the atexit handler; a
-        # writer the caller started is theirs and is never shut down here.
-        from execsql.utils.fileio import filewriter_close_all_after_write
-
-        try:
-            filewriter_close_all_after_write()
-        except Exception:
-            pass  # Best-effort: a failed flush must not mask a script error.
+            try:
+                execute(tree, ctx=ctx)
+            except SystemExit as exc:
+                # exit_now() calls sys.exit() — catch and convert to error
+                _capture_errors(ctx, errors, exc.code)
+            except ErrInfo as exc:
+                errors.append(
+                    ScriptError(
+                        message=exc.errmsg(),
+                        source=_last_source(ctx),
+                        line=_last_line(ctx),
+                        sql=getattr(exc, "command_text", None),
+                    ),
+                )
+            except Exception as exc:
+                errors.append(ScriptError(message=str(exc), source="<runtime>"))
+
+            # Non-halting SQL errors (halt_on_error=False) are recorded in
+            # status.error_history as execution continues; a halting error never
+            # reaches the history, so there is no overlap with the exception
+            # paths above.  They happened first, so they are listed first.
+            if ctx.status is not None:
+                errors[:0] = [
+                    ScriptError(message=err_msg, source=err_source, line=err_line, sql=err_cmd)
+                    for err_source, err_line, err_cmd, err_msg in ctx.status.error_history
+                ]
+
+            elapsed = time.perf_counter() - t0
+
+            # Flush and close every file the script wrote, so the caller can read
+            # them the moment run() returns.  Output to a file that could not be
+            # opened is reported as an error.  The writer is stopped below.
+            from execsql.utils.fileio import filewriter_close_all_after_write
 
-        # ------------------------------------------------------------------
-        # Collect results
-        # ------------------------------------------------------------------
-        final_vars = {}
-        if ctx.subvars is not None:
-            for name, value in ctx.subvars.substitutions:
-                # Include user vars and $-prefixed system vars
-                # Skip environment (&), column (@), local (~), parameter (#) vars
-                if not name or name[0] in ("&", "@", "~", "#"):
-                    continue
-                key = name.lstrip("$")
-                final_vars[key] = str(value) if value is not None else ""
-
-        # Close connection if we own it
-        if owns_connection:
             try:
-                assert ctx.dbs is not None
-                ctx.dbs.closeall()
+                filewriter_close_all_after_write()
+            except ErrInfo as exc:
+                errors.append(ScriptError(message=exc.errmsg(), source="<output>"))
             except Exception:
-                pass
+                pass  # Best-effort: a failed flush must not mask a script error.
+
+            # ------------------------------------------------------------------
+            # Collect results
+            # ------------------------------------------------------------------
+            final_vars = {}
+            if ctx.subvars is not None:
+                for name, value in ctx.subvars.substitutions:
+                    # Include user vars and $-prefixed system vars
+                    # Skip environment (&), column (@), local (~), parameter (#) vars
+                    if not name or name[0] in ("&", "@", "~", "#"):
+                        continue
+                    key = name.lstrip("$")
+                    final_vars[key] = str(value) if value is not None else ""
+        finally:
+            if ctx.filewriter is not None:
+                from execsql.utils.fileio import filewriter_end
+
+                filewriter_end(ctx.filewriter)
+            if owns_connection and ctx.dbs is not None:
+                try:
+                    ctx.dbs.closeall()
+                except Exception:
+                    pass
+            if ctx.tempfiles is not None:
+                ctx.tempfiles.remove_all()
 
     return ScriptResult(
         success=len(errors) == 0,
@@ -588,80 +607,70 @@ def run(
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
-def _ensure_filewriter(conf: Any) -> bool:
-    """Start the FileWriter subprocess if one is not already running.
+def _start_filewriter(ctx: RuntimeContext, conf: Any) -> None:
+    """Give this run its own FileWriter thread, held on *ctx*.
+
+    ``WRITE ... TO <file>`` and ``TEE`` hand their output to the active run's
+    writer, and every entry point in :mod:`execsql.utils.fileio` drops the
+    write when none is running.  A writer per run keeps concurrent runs from
+    sharing open files or reply queues, and a halt in one run (which stops
+    its writer) from discarding another run's output.  :func:`run` stops the
+    writer before it returns.
+    """
+    from execsql.utils.fileio import FileWriter
 
-    ``WRITE ... TO <file>`` and ``TEE`` hand their output to a FileWriter
-    subprocess, and every entry point in :mod:`execsql.utils.fileio` guards on
-    the subprocess being alive — dropping the write when it is not, so that a
-    dead writer cannot deadlock the caller on a full pipe.  The CLI starts one
-    during startup; :func:`run` did not, so every file write under the API was
-    silently discarded and the run still reported success.
+    ctx.filewriter = FileWriter(
+        file_encoding=conf.output_encoding,
+        open_timeout=getattr(conf, "outfile_open_timeout", 10),
+    )
+    ctx.filewriter.start()
 
-    A writer started here outlives the call and is reused by a later
-    :func:`run`, exactly as the CLI keeps one for the life of the process; it
-    is reaped by an ``atexit`` handler.  A writer the caller started themselves
-    is left alone.
 
-    Returns:
-        ``True`` if this call started the writer, ``False`` if one was already
-        running.
-    """
-    import execsql.state as _state
-    import execsql.utils.fileio as _fileio
-    from execsql.utils.fileio import FileWriter, filewriter_end
-
-    if _fileio.filewriter is not None and _fileio.filewriter.is_alive():
-        return False
-
-    # Drain stale messages so a previously-dead subprocess cannot leak
-    # responses or unconsumed commands into the new one.  On macOS (`spawn`)
-    # the OS pipe buffer is small enough that retained entries would deadlock
-    # the next put().
-    for q in (_fileio.fw_input, _fileio.fw_output):
-        try:
-            while True:
-                q.get_nowait()
-        except Exception:
-            pass
+def _capture_errors(ctx: RuntimeContext, errors: list[ScriptError], exit_code: Any) -> None:
+    """Record the error that halted the run, after ``exit_now()`` raised ``SystemExit``.
 
-    try:
-        _fileio.filewriter = _state.filewriter = FileWriter(
-            _fileio.fw_input,
-            _fileio.fw_output,
-            file_encoding=conf.output_encoding,
-            open_timeout=getattr(conf, "outfile_open_timeout", 10),
-        )
-        _state.filewriter.start()
-    except Exception:
-        # Under the `spawn` start method (macOS, Windows) the subprocess
-        # re-imports the parent's __main__, which fails from a REPL, a
-        # notebook, or `python -c`.  Leave the writer unset: fileio's guards
-        # then warn on the first dropped write rather than deadlocking.
-        _fileio.filewriter = _state.filewriter = None
-        return False
-    atexit.register(filewriter_end)
-    return True
-
-
-def _capture_errors(ctx: RuntimeContext, errors: list[ScriptError]) -> None:
-    """Extract error info from the current context after a SystemExit."""
-    last_error = None
-    error_msg = None
-    if ctx.subvars is not None:
-        subs = dict(ctx.subvars.substitutions)
-        last_error = subs.get("$LAST_ERROR")
-        error_msg = subs.get("$ERROR_MESSAGE")
-
-    msg = error_msg or last_error or "Script execution failed"
+    ``exit_now`` leaves the halting :class:`ErrInfo` on the context, and any
+    file output lost at the halt as a second error.  A failed
+    SQL statement's text is in ``$LAST_ERROR``, which the executor sets just
+    before halting; it is reported only when the halt came from SQL, so a
+    metacommand error is never paired with an earlier statement.  A ``HALT``
+    carries no error, only its exit status.
+    """
+    err = ctx.halt_error
+    if err is None:
+        msg = f"Script halted by HALT with exit status {exit_code}."
+    else:
+        msg = err.errmsg()
+    sql = None
+    lc = ctx.last_command
+    if err is not None and lc is not None and lc.command_type == "sql" and ctx.subvars is not None:
+        sql = ctx.subvars.varvalue("$LAST_ERROR") or None
     errors.append(
         ScriptError(
-            message=str(msg),
+            message=msg,
             source=_last_source(ctx),
             line=_last_line(ctx),
-            sql=str(last_error) if last_error else None,
+            sql=sql,
         ),
     )
+    if ctx.halt_output_error is not None:
+        errors.append(ScriptError(message=ctx.halt_output_error.errmsg(), source="<output>"))
+
+
+def _failed(message: str, source: str) -> ScriptResult:
+    """A result for a run that failed before any of the script ran."""
+    return ScriptResult(
+        success=False,
+        commands_run=0,
+        elapsed=0.0,
+        errors=[ScriptError(message=message, source=source)],
+        variables={},
+    )
+
+
+def _error_text(exc: BaseException) -> str:
+    """The message of *exc*: the full text for an :class:`ErrInfo`, else ``str()``."""
+    return exc.errmsg() if isinstance(exc, ErrInfo) else str(exc)
 
 
 def _last_source(ctx: RuntimeContext) -> str: