execsql2 2.12.2__tar.gz → 2.12.5__tar.gz

{execsql2-2.12.2 → execsql2-2.12.5}/CHANGELOG.md

@@ -13,6 +13,29 @@ ______________________________________________________________________
 
 ______________________________________________________________________
 
+## [2.12.5] - 2026-04-03
+
+______________________________________________________________________
+
+## [2.12.4] - 2026-04-03
+
+### Added
+
+- New `PG_UPSERT` metacommand for QA-checked, FK-dependency-ordered upserts from a staging schema to a base schema on PostgreSQL. Integrates [pg-upsert](https://pg-upsert.readthedocs.io/) as an optional dependency (`pip install execsql2[upsert]`). Three modes: full pipeline (`PG_UPSERT FROM ... TO ... TABLES ...`), QA-only (`PG_UPSERT QA ...`), and schema check (`PG_UPSERT CHECK ...`). Supports `METHOD`, `COMMIT`, `INTERACTIVE`, `COMPACT`, `EXCLUDE`, `EXCLUDE_NULL`, and `LOGFILE` keywords. Sets 12 `$PG_UPSERT_*` substitution variables after execution.
+
+______________________________________________________________________
+
+## [2.12.3] - 2026-04-02
+
+### Changed
+
+- Performance: split `set_system_vars()` into static (once per script + on CONNECT/CHDIR) and dynamic (per statement) — eliminates ~14 redundant `add_substitution` calls and 2 `Path.resolve()` filesystem syscalls per statement.
+- Performance: `$RANDOM` and `$UUID` are now lazy — computed only when actually referenced in a statement, not generated unconditionally for every statement.
+- Performance: `LineDelimiter.delimited()` caches `quote_all_text` at construction time instead of reading `_state.conf` via module proxy on every row during export.
+- Performance: CSV/TSV import uses Python's `csv` module as a fast path for standard delimited formats (comma, tab, semicolon, pipe) with doubled-quote escaping. Falls back to the character-at-a-time parser for non-standard formats (space-delimiter collapsing, escape characters).
+
+______________________________________________________________________
+
 ## [2.12.2] - 2026-04-02
 
 ### Added

{execsql2-2.12.2 → execsql2-2.12.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: execsql2
-Version: 2.12.2
+Version: 2.12.5
 Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
 Project-URL: Homepage, https://execsql2.readthedocs.io
 Project-URL: Repository, https://github.com/geocoug/execsql
@@ -51,6 +51,7 @@ Requires-Dist: keyring; extra == 'all'
 Requires-Dist: odfpy; extra == 'all'
 Requires-Dist: openpyxl; extra == 'all'
 Requires-Dist: oracledb; extra == 'all'
+Requires-Dist: pg-upsert>=1.17.0; extra == 'all'
 Requires-Dist: polars; extra == 'all'
 Requires-Dist: psycopg2-binary; extra == 'all'
 Requires-Dist: pymysql; extra == 'all'
@@ -107,6 +108,8 @@ Provides-Extra: oracle
 Requires-Dist: oracledb; extra == 'oracle'
 Provides-Extra: postgres
 Requires-Dist: psycopg2-binary; extra == 'postgres'
+Provides-Extra: upsert
+Requires-Dist: pg-upsert>=1.17.0; extra == 'upsert'
 Description-Content-Type: text/markdown
 
 > [!NOTE]

{execsql2-2.12.2 → execsql2-2.12.5}/docs/about/divergence.md

@@ -41,12 +41,13 @@ ______________________________________________________________________
 
 ### Metacommands
 
-| Metacommand            | Description                                                                                                                                                           |
-| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `ASSERT`               | Evaluate a condition and raise an error (halting the script) if it is false. Supports all IF conditions. Optional quoted failure message. Skipped in false IF blocks. |
-| `BREAKPOINT`           | Pause script execution and drop into an interactive debug REPL. See [Debugging](#debugging) below for full details.                                                   |
-| `CONFIG SHOW_PROGRESS` | Enable the Rich progress bar for IMPORT operations at runtime.                                                                                                        |
-| `CONFIG LOG_SQL`       | Enable SQL query audit logging — writes executed SQL to the log file.                                                                                                 |
+| Metacommand            | Description                                                                                                                                                                                                                      |
+| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `ASSERT`               | Evaluate a condition and raise an error (halting the script) if it is false. Supports all IF conditions. Optional quoted failure message. Skipped in false IF blocks.                                                            |
+| `BREAKPOINT`           | Pause script execution and drop into an interactive debug REPL. See [Debugging](#debugging) below for full details.                                                                                                              |
+| `CONFIG SHOW_PROGRESS` | Enable the Rich progress bar for IMPORT operations at runtime.                                                                                                                                                                   |
+| `CONFIG LOG_SQL`       | Enable SQL query audit logging — writes executed SQL to the log file.                                                                                                                                                            |
+| `PG_UPSERT`            | QA-checked, FK-dependency-ordered upserts from staging to base schema on PostgreSQL. Integrates [pg-upsert](https://pg-upsert.readthedocs.io/) as an optional dependency. Three modes: full pipeline, QA-only, and schema check. |
 
 ### Conditional Tests
 
@@ -163,6 +164,12 @@ All 33 mutable runtime globals in `state.py` have been consolidated into a `Runt
 
 - **Cycle detection** — `substitute_vars()` raises an error after 100 iterations to prevent infinite loops when variables reference each other cyclically. Upstream had no protection.
 - **O(1) substitution** — Variable substitution uses a single combined regex and dict lookup instead of O(V) per-variable regex passes. Behavior is identical; performance is improved.
+- **Lazy `$RANDOM`/`$UUID`** — These system variables are now computed on first access rather than generated unconditionally for every statement. Behavior is identical when referenced; scripts that never reference them skip the computation entirely.
+- **Static/dynamic system var split** — System substitution variables are split into static (set once per script and refreshed on CONNECT/CHDIR) and dynamic (refreshed per statement). Eliminates redundant `Path.resolve()` syscalls and database pool lookups per statement.
+
+### CSV Import
+
+- **Fast-path CSV reader** — Standard delimited imports (comma, tab, semicolon, pipe with doubled-quote escaping) now use Python's `csv` module. Non-standard formats (space-delimiter collapsing, escape characters) fall back to the original character-at-a-time parser.
 
 ### Database Adapters
 

{execsql2-2.12.2 → execsql2-2.12.5}/docs/getting-started/installation.md

@@ -25,6 +25,7 @@ pip install "execsql2[duckdb]"      # DuckDB
 pip install "execsql2[mssql]"       # MS SQL Server / ODBC
 pip install "execsql2[formats]"     # ODS, Excel, Jinja2, Feather, Parquet, HDF5
 pip install "execsql2[auth]"        # OS keyring integration
+pip install "execsql2[upsert]"      # PG_UPSERT metacommand (pg-upsert)
 pip install "execsql2[all-db]"      # All database drivers
 pip install "execsql2[all]"         # Everything
 ```

{execsql2-2.12.2 → execsql2-2.12.5}/docs/reference/metacommands.md

@@ -2066,6 +2066,122 @@ If the "HALT" action is taken, either as a result of user input or as a result o
 Double quotes (as shown above), apostrophes, or square brackets can be used to delimit the text.
 
 
+## PG_UPSERT { #pg_upsert }
+
+```
+PG_UPSERT FROM <staging_schema> TO <base_schema> TABLES <table1>, <table2> [options]
+PG_UPSERT QA FROM <staging_schema> TO <base_schema> TABLES <table1>, <table2> [options]
+PG_UPSERT CHECK FROM <staging_schema> TO <base_schema> TABLES <table1>, <table2>
+```
+
+Performs QA-checked, FK-dependency-ordered upserts from a staging schema to a base schema on PostgreSQL. Integrates [pg-upsert](https://pg-upsert.readthedocs.io/) as an optional dependency.
+
+**Requires:** `pip install execsql2[upsert]`
+
+**Requires:** A PostgreSQL connection. Raises an error if the current DBMS is not PostgreSQL.
+
+### Modes
+
+- **Full pipeline** (`PG_UPSERT FROM ... TO ... TABLES ...`): Runs all QA checks, then upserts data (update + insert by default), with optional commit.
+- **QA only** (`PG_UPSERT QA FROM ... TO ... TABLES ...`): Runs all QA checks without upserting. Never commits.
+- **Schema check** (`PG_UPSERT CHECK FROM ... TO ... TABLES ...`): Checks column existence and type compatibility only. Never commits.
+
+### Optional keywords
+
+Keywords can appear in any order after the table list.
+
+| Keyword | Description |
+|---------|-------------|
+| `METHOD upsert\|update\|insert` | Upsert strategy. `upsert` (default) updates existing + inserts new rows. `update` only updates. `insert` only inserts. Full mode only. |
+| `COMMIT` | Commit changes if QA passes. Without it, changes are rolled back (dry-run). Full mode only. |
+| `EXCLUDE col1, col2` | Columns to skip during upsert. |
+| `EXCLUDE_NULL col1, col2` | Columns to skip in null QA checks. |
+| `INTERACTIVE` | Enable pg-upsert's interactive UI dialogs (tkinter or textual) for reviewing QA failures. Without it, runs non-interactively. |
+| `COMPACT` | Use compact grid format for QA summary instead of detailed per-table panels. |
+| `LOGFILE <path>` | Append pg-upsert's plain-text log output to the given file. Supports quoted paths for spaces: `LOGFILE "path/to/log.txt"`. |
+
+### Substitution variables
+
+Set after every `PG_UPSERT` execution:
+
+| Variable | Type | Description |
+|----------|------|-------------|
+| `$PG_UPSERT_QA_PASSED` | TRUE/FALSE | Did all QA checks pass? |
+| `$PG_UPSERT_ROWS_UPDATED` | integer | Total rows updated across all tables |
+| `$PG_UPSERT_ROWS_INSERTED` | integer | Total rows inserted across all tables |
+| `$PG_UPSERT_COMMITTED` | TRUE/FALSE | Were changes committed? |
+| `$PG_UPSERT_STAGING_SCHEMA` | string | Staging schema name used |
+| `$PG_UPSERT_BASE_SCHEMA` | string | Base schema name used |
+| `$PG_UPSERT_TABLES` | string | Comma-separated list of table names processed |
+| `$PG_UPSERT_METHOD` | string | Upsert method used |
+| `$PG_UPSERT_DURATION` | float | Elapsed time in seconds |
+| `$PG_UPSERT_STARTED_AT` | string | ISO 8601 start timestamp |
+| `$PG_UPSERT_FINISHED_AT` | string | ISO 8601 end timestamp |
+| `$PG_UPSERT_RESULT_JSON` | JSON string | Full result for detailed inspection |
+
+!!! note "Using `$PG_UPSERT_RESULT_JSON` with WRITE"
+    The JSON value is stored as compact single-line JSON. Because it contains double quotes (`"`), square brackets (`[]`), and apostrophes may appear in data, use tilde or backtick delimiters with WRITE:
+
+    ```sql
+    -- !x! WRITE ~!!$PG_UPSERT_RESULT_JSON!!~
+    -- !x! WRITE `!!$PG_UPSERT_RESULT_JSON!!`
+    ```
+
+### Temporary objects
+
+pg-upsert creates temporary tables and views (all prefixed `ups_`) that persist after the metacommand completes. Users can query these for debugging and inspection — for example, `SELECT * FROM ups_control` shows per-table QA results and row counts.
+
+For the full list of temporary objects and their schemas, see the [pg-upsert Temporary Objects Reference](https://pg-upsert.readthedocs.io/en/latest/architecture/#temporary-objects-reference).
+
+### Examples
+
+```sql
+-- Full pipeline: QA + upsert + commit
+-- !x! PG_UPSERT FROM staging TO public TABLES books, authors COMMIT
+
+-- Dry run (no commit) with update-only method
+-- !x! PG_UPSERT FROM staging TO public TABLES books METHOD update
+
+-- QA only with interactive review
+-- !x! PG_UPSERT QA FROM staging TO public TABLES books, authors INTERACTIVE
+
+-- Schema check only
+-- !x! PG_UPSERT CHECK FROM staging TO public TABLES books, authors
+
+-- Log pg-upsert output to a file
+-- !x! PG_UPSERT FROM staging TO public TABLES books, authors LOGFILE "upsert.log" COMMIT
+
+-- Use substitution variables after upsert
+-- !x! PG_UPSERT FROM staging TO public TABLES books COMMIT
+-- !x! WRITE "Updated !!$PG_UPSERT_ROWS_UPDATED!! rows, inserted !!$PG_UPSERT_ROWS_INSERTED!! in !!$PG_UPSERT_DURATION!! seconds"
+
+-- Inspect pg-upsert's control table after a run
+-- !x! PG_UPSERT FROM staging TO public TABLES books, authors, genres COMMIT
+-- !x! EXPORT ups_control TO stdout AS txt
+
+-- Export QA results to a CSV file for review
+-- !x! PG_UPSERT QA FROM staging TO public TABLES books, authors
+-- !x! EXPORT ups_control TO "qa_results.csv" AS csv
+
+-- Check for type mismatches between schemas and display them
+-- !x! PG_UPSERT CHECK FROM staging TO public TABLES books, authors
+-- !x! EXPORT ups_type_mismatches TO stdout AS txt
+
+-- Conditional logic based on QA results
+-- !x! PG_UPSERT FROM staging TO public TABLES books, authors
+-- !x! IF !!$PG_UPSERT_QA_PASSED!! = TRUE
+-- !x!   WRITE "All QA checks passed — proceeding with commit"
+-- !x! ELSE
+-- !x!   WRITE "QA failed — inspect ups_control for details"
+-- !x!   EXPORT ups_control TO stdout AS txt
+-- !x! ENDIF
+
+-- Write the full JSON result using tilde delimiters (JSON contains " and [])
+-- !x! PG_UPSERT FROM staging TO public TABLES books COMMIT
+-- !x! WRITE ~!!$PG_UPSERT_RESULT_JSON!!~
+```
+
+
 ## PG_VACUUM
 
 ```

{execsql2-2.12.2 → execsql2-2.12.5}/extras/vscode-execsql/syntaxes/execsql.tmLanguage.json

@@ -93,7 +93,7 @@
     },
     "action-keywords": {
       "comment": "sub, write, execute script, export, etc.",
-      "match": "(?i)\\b(reset\\s+dialog_canceled|write\\s+create_table|export_metadata|sub_querystring|execute\\s+script|append\\s+script|extend\\s+script|reset\\s+counter|export\\s+query|sub_tempfile|write\\s+script|import_file|set\\s+counter|sub_decrypt|sub_encrypt|autocommit|breakpoint|copy\\s+query|disconnect|select_sub|sub_append|system_cmd|pg_vacuum|sub_empty|sub_local|connect|include|max_int|rm_file|subdata|sub_add|sub_ini|assert|export|import|rm_sub|debug|email|serve|write|copy|log|run|sub|use|zip|cd)\\b",
+      "match": "(?i)\\b(reset\\s+dialog_canceled|write\\s+create_table|export_metadata|pg_upsert\\s+check|sub_querystring|execute\\s+script|append\\s+script|extend\\s+script|reset\\s+counter|export\\s+query|pg_upsert\\s+qa|sub_tempfile|write\\s+script|import_file|set\\s+counter|sub_decrypt|sub_encrypt|autocommit|breakpoint|copy\\s+query|disconnect|select_sub|sub_append|system_cmd|pg_upsert|pg_vacuum|sub_empty|sub_local|connect|include|max_int|rm_file|subdata|sub_add|sub_ini|assert|export|import|rm_sub|debug|email|serve|write|copy|log|run|sub|use|zip|cd)\\b",
       "name": "keyword.other.execsql"
     },
     "config-event-keywords": {

{execsql2-2.12.2 → execsql2-2.12.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "execsql2"
-version = "2.12.2"
+version = "2.12.5"
 description = "Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables."
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE.txt" }
@@ -58,6 +58,7 @@ odbc = ["pyodbc"]
 # Feature bundles
 formats = ["odfpy", "xlrd", "openpyxl", "Jinja2", "polars", "tables", "PyYAML"]
 auth = ["keyring"]
+upsert = ["pg-upsert>=1.17.0"]
 # Convenience groups
 all-db = [
     "psycopg2-binary",
@@ -71,6 +72,7 @@ all = [
     "execsql2[all-db]",
     "execsql2[formats]",
     "execsql2[auth]",
+    "execsql2[upsert]",
 ]
 dev = [
     "build>=1.2.2.post1",
@@ -159,7 +161,7 @@ skip-magic-trailing-comma = false
 line-ending = "auto"
 
 [tool.bumpversion]
-current_version = "2.12.2"
+current_version = "2.12.5"
 commit = true
 commit_args = "--no-verify"
 tag = true

{execsql2-2.12.2 → execsql2-2.12.5}/src/execsql/db/base.py

@@ -686,6 +686,13 @@ class DatabasePool:
             )
             self.pool[db_alias].close()
         self.pool[db_alias] = db_obj
+        # Refresh static system vars so $DB_NAME, $DB_USER, etc. reflect the new connection.
+        try:
+            from execsql.script.engine import set_static_system_vars
+
+            set_static_system_vars()
+        except Exception:
+            pass  # Engine not yet initialized (early startup).
 
     def aliases(self) -> list[str]:
         """Return a list of all currently registered database aliases."""

{execsql2-2.12.2 → execsql2-2.12.5}/src/execsql/exporters/delimited.py

@@ -40,6 +40,7 @@ class LineDelimiter:
         self.delimiter = delim
         self.joinchar = delim if delim else ""
         self.quotechar = quote
+        self.quote_all_text = _state.conf.quote_all_text if _state.conf else False
         if quote:
             if escchar:
                 self.quotedquote = escchar + quote
@@ -50,13 +51,12 @@ class LineDelimiter:
 
     def delimited(self, datarow: Any, add_newline: bool = True) -> str:
         """Format a sequence of values as a single delimited text line."""
-        conf = _state.conf
         if self.quotechar:
             d_row = []
             for e in datarow:
                 if isinstance(e, str):
                     if (
-                        conf.quote_all_text
+                        self.quote_all_text
                         or (self.quotechar in e)
                         or (self.delimiter is not None and self.delimiter in e)
                         or ("\n" in e)
@@ -609,10 +609,52 @@ class CsvFile(EncodedFile):
             raise ErrInfo("error", other_msg=", ".join(self.parse_errors))
         return elements
 
+    def _can_use_fast_csv_reader(self) -> bool:
+        """Return True if the detected format is compatible with Python's csv module."""
+        # The csv module handles comma/tab delimiters with doubled-quote escaping.
+        # It cannot handle: space-delimiter collapsing, escape chars, or no delimiter.
+        if self.delimiter is None or self.delimiter == " ":
+            return False
+        return self.escapechar is None
+
     def reader(self) -> Any:
         """Yield parsed rows from the file as lists of field values."""
         conf = _state.conf
         self.evaluate_line_format()
+        if self._can_use_fast_csv_reader():
+            yield from self._fast_reader(conf)
+        else:
+            yield from self._slow_reader(conf)
+
+    def _fast_reader(self, conf: Any) -> Any:
+        """Read using Python's csv module (fast path for standard delimited formats)."""
+        import csv
+
+        f = self.openclean("rt")
+        try:
+            csv_reader = csv.reader(
+                f,
+                delimiter=self.delimiter,
+                quotechar=self.quotechar,
+                doublequote=True,
+                strict=False,
+            )
+            for elements in csv_reader:
+                if len(elements) == 0:
+                    break
+                # Normalize empty strings to None for parity with the slow reader.
+                elements = [e if e != "" else None for e in elements]
+                if conf.del_empty_cols and len(self.blank_cols) > 0:
+                    blanks = copy.copy(self.blank_cols)
+                    while len(blanks) > 0:
+                        b = blanks.pop()
+                        del elements[b]
+                yield elements
+        finally:
+            f.close()
+
+    def _slow_reader(self, conf: Any) -> Any:
+        """Read using the character-at-a-time state machine (fallback for non-standard formats)."""
         f = self.openclean("rt")
         line_no = 0
         try:

{execsql2-2.12.2 → execsql2-2.12.5}/src/execsql/metacommands/__init__.py

@@ -168,6 +168,11 @@ from execsql.metacommands.script_ext import (
     x_extendscript_sql,
     x_executescript,
 )
+from execsql.metacommands.upsert import (
+    x_pg_upsert,
+    x_pg_upsert_check,
+    x_pg_upsert_qa,
+)
 from execsql.metacommands.system import (
     x_system_cmd,
     x_email,
@@ -401,6 +406,10 @@ __all__ = [
     "x_write_warnings",
     "x_gui_level",
     "x_execute",
+    # upsert handlers
+    "x_pg_upsert",
+    "x_pg_upsert_check",
+    "x_pg_upsert_qa",
     # regex helpers
     "ins_rxs",
     "ins_quoted_rx",

{execsql2-2.12.2 → execsql2-2.12.5}/src/execsql/metacommands/dispatch.py

@@ -168,6 +168,11 @@ from execsql.metacommands.script_ext import (
     x_extendscript_metacommand,
     x_extendscript_sql,
 )
+from execsql.metacommands.upsert import (
+    x_pg_upsert,
+    x_pg_upsert_check,
+    x_pg_upsert_qa,
+)
 from execsql.metacommands.system import (
     x_cancel_halt,
     x_cancel_halt_email,
@@ -2142,6 +2147,31 @@ def build_dispatch_table() -> MetaCommandList:
         x_write,
     )
 
+    # ------------------------------------------------------------------
+    # PG_UPSERT — pg-upsert integration (optional dependency)
+    # ------------------------------------------------------------------
+    # Order matters: CHECK and QA patterns must precede the general pattern
+    # so that "PG_UPSERT CHECK ..." and "PG_UPSERT QA ..." are matched
+    # before "PG_UPSERT FROM ...".
+    mcl.add(
+        r"^\s*PG_UPSERT\s+CHECK\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
+        x_pg_upsert_check,
+        description="PG_UPSERT CHECK",
+        category="action",
+    )
+    mcl.add(
+        r"^\s*PG_UPSERT\s+QA\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
+        x_pg_upsert_qa,
+        description="PG_UPSERT QA",
+        category="action",
+    )
+    mcl.add(
+        r"^\s*PG_UPSERT\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
+        x_pg_upsert,
+        description="PG_UPSERT",
+        category="action",
+    )
+
     # ------------------------------------------------------------------
     # SUB (top-level variable assignment — kept near end so more specific
     # SUB_* patterns above take precedence)

{execsql2-2.12.2 → execsql2-2.12.5}/src/execsql/metacommands/io_fileops.py

@@ -241,6 +241,10 @@ def x_cd(**kwargs: Any) -> None:
     os.chdir(new_dir)
     script, lno = current_script_line()
     _state.exec_log.log_status_info(f"Current directory changed to {new_dir} at line {lno} of {script}")
+    if _state.subvars is not None:
+        from execsql.script.engine import set_static_system_vars
+
+        set_static_system_vars()
     return None