execsql2 2.12.3__tar.gz → 2.12.6__tar.gz

{execsql2-2.12.3 → execsql2-2.12.6}/CHANGELOG.md

@@ -13,6 +13,31 @@ ______________________________________________________________________
 
 ______________________________________________________________________
 
+## [2.12.6] - 2026-04-03
+
+### Added
+
+- `PG_UPSERT` now supports per-table progress via pg-upsert's callback API. New substitution variables `$PG_UPSERT_CURRENT_TABLE`, `$PG_UPSERT_TABLE_QA_PASSED`, `$PG_UPSERT_TABLE_ROWS_UPDATED`, and `$PG_UPSERT_TABLE_ROWS_INSERTED` are updated as each table is processed.
+- New `CLEANUP` keyword for `PG_UPSERT` — drops all `ups_*` temporary tables and views after execution. Without it, temp objects persist for inspection (default).
+
+______________________________________________________________________
+
+## [2.12.5] - 2026-04-03
+
+### Fixed
+
+- Fixed CLI test `test_nonexistent_file_error_message_is_clear` failing with `ValueError: stderr not separately captured` when CliRunner mixes stderr into stdout.
+
+______________________________________________________________________
+
+## [2.12.4] - 2026-04-03
+
+### Added
+
+- New `PG_UPSERT` metacommand for QA-checked, FK-dependency-ordered upserts from a staging schema to a base schema on PostgreSQL. Integrates [pg-upsert](https://pg-upsert.readthedocs.io/) as an optional dependency (`pip install execsql2[upsert]`). Three modes: full pipeline (`PG_UPSERT FROM ... TO ... TABLES ...`), QA-only (`PG_UPSERT QA ...`), and schema check (`PG_UPSERT CHECK ...`). Supports `METHOD`, `COMMIT`, `INTERACTIVE`, `COMPACT`, `EXCLUDE`, `EXCLUDE_NULL`, and `LOGFILE` keywords. Sets 12 `$PG_UPSERT_*` substitution variables after execution.
+
+______________________________________________________________________
+
 ## [2.12.3] - 2026-04-02
 
 ### Changed

{execsql2-2.12.3 → execsql2-2.12.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: execsql2
-Version: 2.12.3
+Version: 2.12.6
 Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
 Project-URL: Homepage, https://execsql2.readthedocs.io
 Project-URL: Repository, https://github.com/geocoug/execsql
@@ -51,6 +51,7 @@ Requires-Dist: keyring; extra == 'all'
 Requires-Dist: odfpy; extra == 'all'
 Requires-Dist: openpyxl; extra == 'all'
 Requires-Dist: oracledb; extra == 'all'
+Requires-Dist: pg-upsert>=1.18.0; extra == 'all'
 Requires-Dist: polars; extra == 'all'
 Requires-Dist: psycopg2-binary; extra == 'all'
 Requires-Dist: pymysql; extra == 'all'
@@ -107,6 +108,8 @@ Provides-Extra: oracle
 Requires-Dist: oracledb; extra == 'oracle'
 Provides-Extra: postgres
 Requires-Dist: psycopg2-binary; extra == 'postgres'
+Provides-Extra: upsert
+Requires-Dist: pg-upsert>=1.18.0; extra == 'upsert'
 Description-Content-Type: text/markdown
 
 > [!NOTE]

{execsql2-2.12.3 → execsql2-2.12.6}/docs/about/divergence.md

@@ -41,12 +41,13 @@ ______________________________________________________________________
 
 ### Metacommands
 
-| Metacommand            | Description                                                                                                                                                           |
-| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `ASSERT`               | Evaluate a condition and raise an error (halting the script) if it is false. Supports all IF conditions. Optional quoted failure message. Skipped in false IF blocks. |
-| `BREAKPOINT`           | Pause script execution and drop into an interactive debug REPL. See [Debugging](#debugging) below for full details.                                                   |
-| `CONFIG SHOW_PROGRESS` | Enable the Rich progress bar for IMPORT operations at runtime.                                                                                                        |
-| `CONFIG LOG_SQL`       | Enable SQL query audit logging — writes executed SQL to the log file.                                                                                                 |
+| Metacommand            | Description                                                                                                                                                                                                                      |
+| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `ASSERT`               | Evaluate a condition and raise an error (halting the script) if it is false. Supports all IF conditions. Optional quoted failure message. Skipped in false IF blocks.                                                            |
+| `BREAKPOINT`           | Pause script execution and drop into an interactive debug REPL. See [Debugging](#debugging) below for full details.                                                                                                              |
+| `CONFIG SHOW_PROGRESS` | Enable the Rich progress bar for IMPORT operations at runtime.                                                                                                                                                                   |
+| `CONFIG LOG_SQL`       | Enable SQL query audit logging — writes executed SQL to the log file.                                                                                                                                                            |
+| `PG_UPSERT`            | QA-checked, FK-dependency-ordered upserts from staging to base schema on PostgreSQL. Integrates [pg-upsert](https://pg-upsert.readthedocs.io/) as an optional dependency. Three modes: full pipeline, QA-only, and schema check. |
 
 ### Conditional Tests
 

{execsql2-2.12.3 → execsql2-2.12.6}/docs/getting-started/installation.md

@@ -25,6 +25,7 @@ pip install "execsql2[duckdb]"      # DuckDB
 pip install "execsql2[mssql]"       # MS SQL Server / ODBC
 pip install "execsql2[formats]"     # ODS, Excel, Jinja2, Feather, Parquet, HDF5
 pip install "execsql2[auth]"        # OS keyring integration
+pip install "execsql2[upsert]"      # PG_UPSERT metacommand (pg-upsert)
 pip install "execsql2[all-db]"      # All database drivers
 pip install "execsql2[all]"         # Everything
 ```

{execsql2-2.12.3 → execsql2-2.12.6}/docs/reference/metacommands.md

@@ -2066,6 +2066,127 @@ If the "HALT" action is taken, either as a result of user input or as a result o
 Double quotes (as shown above), apostrophes, or square brackets can be used to delimit the text.
 
 
+## PG_UPSERT { #pg_upsert }
+
+```
+PG_UPSERT FROM <staging_schema> TO <base_schema> TABLES <table1>, <table2> [options]
+PG_UPSERT QA FROM <staging_schema> TO <base_schema> TABLES <table1>, <table2> [options]
+PG_UPSERT CHECK FROM <staging_schema> TO <base_schema> TABLES <table1>, <table2>
+```
+
+Performs QA-checked, FK-dependency-ordered upserts from a staging schema to a base schema on PostgreSQL. Integrates [pg-upsert](https://pg-upsert.readthedocs.io/) as an optional dependency.
+
+**Requires:** `pip install execsql2[upsert]`
+
+**Requires:** A PostgreSQL connection. Raises an error if the current DBMS is not PostgreSQL.
+
+### Modes
+
+- **Full pipeline** (`PG_UPSERT FROM ... TO ... TABLES ...`): Runs all QA checks, then upserts data (update + insert by default), with optional commit.
+- **QA only** (`PG_UPSERT QA FROM ... TO ... TABLES ...`): Runs all QA checks without upserting. Never commits.
+- **Schema check** (`PG_UPSERT CHECK FROM ... TO ... TABLES ...`): Checks column existence and type compatibility only. Never commits.
+
+### Optional keywords
+
+Keywords can appear in any order after the table list.
+
+| Keyword | Description |
+|---------|-------------|
+| `METHOD upsert\|update\|insert` | Upsert strategy. `upsert` (default) updates existing + inserts new rows. `update` only updates. `insert` only inserts. Full mode only. |
+| `COMMIT` | Commit changes if QA passes. Without it, changes are rolled back (dry-run). Full mode only. |
+| `EXCLUDE col1, col2` | Columns to skip during upsert. |
+| `EXCLUDE_NULL col1, col2` | Columns to skip in null QA checks. |
+| `INTERACTIVE` | Enable pg-upsert's interactive UI dialogs (tkinter or textual) for reviewing QA failures. Without it, runs non-interactively. |
+| `COMPACT` | Use compact grid format for QA summary instead of detailed per-table panels. |
+| `LOGFILE <path>` | Append pg-upsert's plain-text log output to the given file. Supports quoted paths for spaces: `LOGFILE "path/to/log.txt"`. |
+| `CLEANUP` | Drop all `ups_*` temporary tables and views after execution. Without it, temp objects persist for inspection (default). |
+
+### Substitution variables
+
+Set after every `PG_UPSERT` execution:
+
+| Variable | Type | Description |
+|----------|------|-------------|
+| `$PG_UPSERT_QA_PASSED` | TRUE/FALSE | Did all QA checks pass? |
+| `$PG_UPSERT_ROWS_UPDATED` | integer | Total rows updated across all tables |
+| `$PG_UPSERT_ROWS_INSERTED` | integer | Total rows inserted across all tables |
+| `$PG_UPSERT_COMMITTED` | TRUE/FALSE | Were changes committed? |
+| `$PG_UPSERT_STAGING_SCHEMA` | string | Staging schema name used |
+| `$PG_UPSERT_BASE_SCHEMA` | string | Base schema name used |
+| `$PG_UPSERT_TABLES` | string | Comma-separated list of table names processed |
+| `$PG_UPSERT_METHOD` | string | Upsert method used |
+| `$PG_UPSERT_DURATION` | float | Elapsed time in seconds |
+| `$PG_UPSERT_STARTED_AT` | string | ISO 8601 start timestamp |
+| `$PG_UPSERT_FINISHED_AT` | string | ISO 8601 end timestamp |
+| `$PG_UPSERT_RESULT_JSON` | JSON string | Full result for detailed inspection |
+| `$PG_UPSERT_CURRENT_TABLE` | string | Last table processed (updated per table during execution) |
+| `$PG_UPSERT_TABLE_QA_PASSED` | TRUE/FALSE | QA result for the current table (updated per table) |
+| `$PG_UPSERT_TABLE_ROWS_UPDATED` | integer | Rows updated for the current table (updated per table) |
+| `$PG_UPSERT_TABLE_ROWS_INSERTED` | integer | Rows inserted for the current table (updated per table) |
+
+!!! note "Using `$PG_UPSERT_RESULT_JSON` with WRITE"
+    The JSON value is stored as compact single-line JSON. Because it contains double quotes (`"`), square brackets (`[]`), and apostrophes may appear in data, use tilde or backtick delimiters with WRITE:
+
+    ```sql
+    -- !x! WRITE ~!!$PG_UPSERT_RESULT_JSON!!~
+    -- !x! WRITE `!!$PG_UPSERT_RESULT_JSON!!`
+    ```
+
+### Temporary objects
+
+pg-upsert creates temporary tables and views (all prefixed `ups_`) that persist after the metacommand completes. Users can query these for debugging and inspection — for example, `SELECT * FROM ups_control` shows per-table QA results and row counts.
+
+For the full list of temporary objects and their schemas, see the [pg-upsert Temporary Objects Reference](https://pg-upsert.readthedocs.io/en/latest/architecture/#temporary-objects-reference).
+
+### Examples
+
+```sql
+-- Full pipeline: QA + upsert + commit
+-- !x! PG_UPSERT FROM staging TO public TABLES books, authors COMMIT
+
+-- Dry run (no commit) with update-only method
+-- !x! PG_UPSERT FROM staging TO public TABLES books METHOD update
+
+-- QA only with interactive review
+-- !x! PG_UPSERT QA FROM staging TO public TABLES books, authors INTERACTIVE
+
+-- Schema check only
+-- !x! PG_UPSERT CHECK FROM staging TO public TABLES books, authors
+
+-- Log pg-upsert output to a file
+-- !x! PG_UPSERT FROM staging TO public TABLES books, authors LOGFILE "upsert.log" COMMIT
+
+-- Use substitution variables after upsert
+-- !x! PG_UPSERT FROM staging TO public TABLES books COMMIT
+-- !x! WRITE "Updated !!$PG_UPSERT_ROWS_UPDATED!! rows, inserted !!$PG_UPSERT_ROWS_INSERTED!! in !!$PG_UPSERT_DURATION!! seconds"
+
+-- Inspect pg-upsert's control table after a run
+-- !x! PG_UPSERT FROM staging TO public TABLES books, authors, genres COMMIT
+-- !x! EXPORT ups_control TO stdout AS txt
+
+-- Export QA results to a CSV file for review
+-- !x! PG_UPSERT QA FROM staging TO public TABLES books, authors
+-- !x! EXPORT ups_control TO "qa_results.csv" AS csv
+
+-- Check for type mismatches between schemas and display them
+-- !x! PG_UPSERT CHECK FROM staging TO public TABLES books, authors
+-- !x! EXPORT ups_type_mismatches TO stdout AS txt
+
+-- Conditional logic based on QA results
+-- !x! PG_UPSERT FROM staging TO public TABLES books, authors
+-- !x! IF !!$PG_UPSERT_QA_PASSED!! = TRUE
+-- !x!   WRITE "All QA checks passed — proceeding with commit"
+-- !x! ELSE
+-- !x!   WRITE "QA failed — inspect ups_control for details"
+-- !x!   EXPORT ups_control TO stdout AS txt
+-- !x! ENDIF
+
+-- Write the full JSON result using tilde delimiters (JSON contains " and [])
+-- !x! PG_UPSERT FROM staging TO public TABLES books COMMIT
+-- !x! WRITE ~!!$PG_UPSERT_RESULT_JSON!!~
+```
+
+
 ## PG_VACUUM
 
 ```

{execsql2-2.12.3 → execsql2-2.12.6}/extras/vscode-execsql/syntaxes/execsql.tmLanguage.json

@@ -93,7 +93,7 @@
     },
     "action-keywords": {
       "comment": "sub, write, execute script, export, etc.",
-      "match": "(?i)\\b(reset\\s+dialog_canceled|write\\s+create_table|export_metadata|sub_querystring|execute\\s+script|append\\s+script|extend\\s+script|reset\\s+counter|export\\s+query|sub_tempfile|write\\s+script|import_file|set\\s+counter|sub_decrypt|sub_encrypt|autocommit|breakpoint|copy\\s+query|disconnect|select_sub|sub_append|system_cmd|pg_vacuum|sub_empty|sub_local|connect|include|max_int|rm_file|subdata|sub_add|sub_ini|assert|export|import|rm_sub|debug|email|serve|write|copy|log|run|sub|use|zip|cd)\\b",
+      "match": "(?i)\\b(reset\\s+dialog_canceled|write\\s+create_table|export_metadata|pg_upsert\\s+check|sub_querystring|execute\\s+script|append\\s+script|extend\\s+script|reset\\s+counter|export\\s+query|pg_upsert\\s+qa|sub_tempfile|write\\s+script|import_file|set\\s+counter|sub_decrypt|sub_encrypt|autocommit|breakpoint|copy\\s+query|disconnect|select_sub|sub_append|system_cmd|pg_upsert|pg_vacuum|sub_empty|sub_local|connect|include|max_int|rm_file|subdata|sub_add|sub_ini|assert|export|import|rm_sub|debug|email|serve|write|copy|log|run|sub|use|zip|cd)\\b",
       "name": "keyword.other.execsql"
     },
     "config-event-keywords": {

{execsql2-2.12.3 → execsql2-2.12.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "execsql2"
-version = "2.12.3"
+version = "2.12.6"
 description = "Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables."
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE.txt" }
@@ -58,6 +58,7 @@ odbc = ["pyodbc"]
 # Feature bundles
 formats = ["odfpy", "xlrd", "openpyxl", "Jinja2", "polars", "tables", "PyYAML"]
 auth = ["keyring"]
+upsert = ["pg-upsert>=1.18.0"]
 # Convenience groups
 all-db = [
     "psycopg2-binary",
@@ -71,6 +72,7 @@ all = [
     "execsql2[all-db]",
     "execsql2[formats]",
     "execsql2[auth]",
+    "execsql2[upsert]",
 ]
 dev = [
     "build>=1.2.2.post1",
@@ -159,7 +161,7 @@ skip-magic-trailing-comma = false
 line-ending = "auto"
 
 [tool.bumpversion]
-current_version = "2.12.3"
+current_version = "2.12.6"
 commit = true
 commit_args = "--no-verify"
 tag = true

{execsql2-2.12.3 → execsql2-2.12.6}/src/execsql/metacommands/__init__.py

@@ -168,6 +168,11 @@ from execsql.metacommands.script_ext import (
     x_extendscript_sql,
     x_executescript,
 )
+from execsql.metacommands.upsert import (
+    x_pg_upsert,
+    x_pg_upsert_check,
+    x_pg_upsert_qa,
+)
 from execsql.metacommands.system import (
     x_system_cmd,
     x_email,
@@ -401,6 +406,10 @@ __all__ = [
     "x_write_warnings",
     "x_gui_level",
     "x_execute",
+    # upsert handlers
+    "x_pg_upsert",
+    "x_pg_upsert_check",
+    "x_pg_upsert_qa",
     # regex helpers
     "ins_rxs",
     "ins_quoted_rx",

{execsql2-2.12.3 → execsql2-2.12.6}/src/execsql/metacommands/dispatch.py

@@ -168,6 +168,11 @@ from execsql.metacommands.script_ext import (
     x_extendscript_metacommand,
     x_extendscript_sql,
 )
+from execsql.metacommands.upsert import (
+    x_pg_upsert,
+    x_pg_upsert_check,
+    x_pg_upsert_qa,
+)
 from execsql.metacommands.system import (
     x_cancel_halt,
     x_cancel_halt_email,
@@ -2142,6 +2147,31 @@ def build_dispatch_table() -> MetaCommandList:
         x_write,
     )
 
+    # ------------------------------------------------------------------
+    # PG_UPSERT — pg-upsert integration (optional dependency)
+    # ------------------------------------------------------------------
+    # Order matters: CHECK and QA patterns must precede the general pattern
+    # so that "PG_UPSERT CHECK ..." and "PG_UPSERT QA ..." are matched
+    # before "PG_UPSERT FROM ...".
+    mcl.add(
+        r"^\s*PG_UPSERT\s+CHECK\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
+        x_pg_upsert_check,
+        description="PG_UPSERT CHECK",
+        category="action",
+    )
+    mcl.add(
+        r"^\s*PG_UPSERT\s+QA\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
+        x_pg_upsert_qa,
+        description="PG_UPSERT QA",
+        category="action",
+    )
+    mcl.add(
+        r"^\s*PG_UPSERT\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
+        x_pg_upsert,
+        description="PG_UPSERT",
+        category="action",
+    )
+
     # ------------------------------------------------------------------
     # SUB (top-level variable assignment — kept near end so more specific
     # SUB_* patterns above take precedence)