execsql2 2.14.1__py3-none-any.whl → 2.15.0__py3-none-any.whl

execsql/metacommands/upsert.py

@@ -26,11 +26,11 @@ from execsql.utils.errors import exception_desc
 
 _KW_METHOD = re.compile(r"\bMETHOD\s+(upsert|update|insert)\b", re.IGNORECASE)
 _KW_EXCLUDE = re.compile(
-    r"\bEXCLUDE\s+([\w\s,]+?)(?=\s+(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE_NULL|LOGFILE|CLEANUP)\b|\s*$)",
+    r"\bEXCLUDE\s+([\w\s,]+?)(?=\s+(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE_NULL|LOGFILE|CLEANUP|EXPORT_FAILURES|EXPORT_FORMAT|EXPORT_MAX_ROWS)\b|\s*$)",
     re.IGNORECASE,
 )
 _KW_EXCLUDE_NULL = re.compile(
-    r"\bEXCLUDE_NULL\s+([\w\s,]+?)(?=\s+(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE|LOGFILE|CLEANUP)\b|\s*$)",
+    r"\bEXCLUDE_NULL\s+([\w\s,]+?)(?=\s+(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE|LOGFILE|CLEANUP|EXPORT_FAILURES|EXPORT_FORMAT|EXPORT_MAX_ROWS)\b|\s*$)",
     re.IGNORECASE,
 )
 _KW_COMMIT = re.compile(r"\bCOMMIT\b", re.IGNORECASE)
@@ -38,13 +38,21 @@ _KW_INTERACTIVE = re.compile(r"\bINTERACTIVE\b", re.IGNORECASE)
 _KW_COMPACT = re.compile(r"\bCOMPACT\b", re.IGNORECASE)
 _KW_CLEANUP = re.compile(r"\bCLEANUP\b", re.IGNORECASE)
 _KW_LOGFILE = re.compile(r"""\bLOGFILE\s+(?:"([^"]+)"|'([^']+)'|(\S+))""", re.IGNORECASE)
+_KW_EXPORT_FAILURES = re.compile(
+    r"""\bEXPORT_FAILURES\s+(?:"([^"]+)"|'([^']+)'|(\S+))""",
+    re.IGNORECASE,
+)
+_KW_EXPORT_FORMAT = re.compile(r"\bEXPORT_FORMAT\s+(\S+)", re.IGNORECASE)
+_KW_EXPORT_MAX_ROWS = re.compile(r"\bEXPORT_MAX_ROWS\s+(\S+)", re.IGNORECASE)
 
 # All recognized keywords — used to split table names from options.
 _ALL_KEYWORDS = re.compile(
-    r"\b(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE_NULL|EXCLUDE|LOGFILE|CLEANUP)\b",
+    r"\b(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE_NULL|EXCLUDE|LOGFILE|CLEANUP|EXPORT_FAILURES|EXPORT_FORMAT|EXPORT_MAX_ROWS)\b",
     re.IGNORECASE,
 )
 
+_VALID_EXPORT_FORMATS = ("csv", "json", "xlsx")
+
 
 def _parse_tables_and_options(tail: str) -> dict[str, Any]:
     """Parse the trailing text after ``TABLES`` into table names and options.
@@ -90,6 +98,40 @@ def _parse_tables_and_options(tail: str) -> dict[str, Any]:
     if m:
         logfile = m.group(1) or m.group(2) or m.group(3)
 
+    export_failures: str | None = None
+    m = _KW_EXPORT_FAILURES.search(opts_part)
+    if m:
+        export_failures = m.group(1) or m.group(2) or m.group(3)
+
+    export_format = "csv"
+    m = _KW_EXPORT_FORMAT.search(opts_part)
+    if m:
+        fmt = m.group(1).lower()
+        if fmt not in _VALID_EXPORT_FORMATS:
+            raise ErrInfo(
+                "cmd",
+                other_msg=(
+                    f"PG_UPSERT: unsupported EXPORT_FORMAT {m.group(1)!r}. "
+                    f"Supported: {', '.join(_VALID_EXPORT_FORMATS)}"
+                ),
+            )
+        export_format = fmt
+
+    export_max_rows = 1000
+    m = _KW_EXPORT_MAX_ROWS.search(opts_part)
+    if m:
+        raw = m.group(1)
+        try:
+            val = int(raw)
+            if val <= 0:
+                raise ValueError
+        except ValueError as exc:
+            raise ErrInfo(
+                "cmd",
+                other_msg=(f"PG_UPSERT: EXPORT_MAX_ROWS must be a positive integer, got {raw!r}"),
+            ) from exc
+        export_max_rows = val
+
     return {
         "tables": tables,
         "method": method,
@@ -100,6 +142,9 @@ def _parse_tables_and_options(tail: str) -> dict[str, Any]:
         "exclude_null_check_cols": exclude_null,
         "logfile": logfile,
         "cleanup": bool(_KW_CLEANUP.search(opts_part)),
+        "export_failures": export_failures,
+        "export_format": export_format,
+        "export_max_rows": export_max_rows,
     }
 
 
@@ -157,6 +202,9 @@ def _set_subvars(result: Any) -> None:
     sv("$PG_UPSERT_STARTED_AT", result.started_at)
     sv("$PG_UPSERT_FINISHED_AT", result.finished_at)
     sv("$PG_UPSERT_RESULT_JSON", json.dumps(result.to_dict(), separators=(",", ":")))
+    # Default export path subvar to empty; _export_failures_if_requested
+    # will overwrite it with the actual path if an export was produced.
+    sv("$PG_UPSERT_EXPORT_PATH", "")
 
 
 def _qa_failure_msg(result: Any) -> str:
@@ -241,20 +289,26 @@ def _create_pgupsert(
     if _state.conf:
         ui_mode = _state.conf.gui_framework
 
-    ups = PgUpsert(
-        conn=db.conn,
-        staging_schema=staging_schema,
-        base_schema=base_schema,
-        tables=opts["tables"],
-        do_commit=opts["commit"],
-        interactive=opts["interactive"],
-        compact=opts["compact"],
-        upsert_method=opts["method"],
-        exclude_cols=opts["exclude_cols"],
-        exclude_null_check_cols=opts["exclude_null_check_cols"],
-        ui_mode=ui_mode,
-        callback=_make_callback(),
-    )
+    kwargs: dict[str, Any] = {
+        "conn": db.conn,
+        "staging_schema": staging_schema,
+        "base_schema": base_schema,
+        "tables": opts["tables"],
+        "do_commit": opts["commit"],
+        "interactive": opts["interactive"],
+        "compact": opts["compact"],
+        "upsert_method": opts["method"],
+        "exclude_cols": opts["exclude_cols"],
+        "exclude_null_check_cols": opts["exclude_null_check_cols"],
+        "ui_mode": ui_mode,
+        "callback": _make_callback(),
+    }
+    # Only pass fix-sheet capture args when an export was requested, so the
+    # metacommand stays compatible with any pg-upsert build that lacks them.
+    if opts.get("export_failures"):
+        kwargs["capture_detail_rows"] = True
+        kwargs["max_export_rows"] = opts.get("export_max_rows", 1000)
+    ups = PgUpsert(**kwargs)
     return ups
 
 
@@ -328,6 +382,57 @@ def _run_with_autocommit_guard(db: Any, fn: Any) -> Any:
             db.autocommit_on()
 
 
+def _export_failures_if_requested(
+    result: Any,
+    opts: dict[str, Any],
+    metacommandline: str | None,
+) -> None:
+    """Export a QA fix sheet if EXPORT_FAILURES was given in the metacommand.
+
+    Always called after ``_set_subvars(result)`` so ``$PG_UPSERT_EXPORT_PATH``
+    is initialized to empty first, then overwritten here on a successful
+    export.  Called even when QA failed — that's the whole point of the
+    fix sheet.
+    """
+    path = opts.get("export_failures")
+    if not path:
+        return
+    fmt = opts["export_format"]
+    try:
+        exported = result.export_failures(path, fmt=fmt)
+    except Exception as exc:
+        raise ErrInfo(
+            "exception",
+            exception_msg=exception_desc(),
+            other_msg=(f"PG_UPSERT failed to export failure sheet to {path}"),
+        ) from exc
+    _state.subvars.add_substitution(
+        "$PG_UPSERT_EXPORT_PATH",
+        str(exported) if exported else "",
+    )
+    if exported:
+        msg = f"PG_UPSERT: exported QA failures to {exported} ({fmt})"
+    else:
+        msg = f"PG_UPSERT: no QA failures to export (EXPORT_FAILURES {path} skipped)"
+    _state.exec_log.log_user_msg(msg)
+    try:
+        _state.output.write(msg + "\n")
+    except Exception:
+        # Output sink may be unavailable in some contexts (e.g. tests);
+        # the log message above is sufficient in that case.
+        pass
+    # Tee to the LOGFILE keyword target if one was given, matching how the
+    # pg-upsert display output is routed there via _FileWriterHandler.
+    logfile = opts.get("logfile")
+    if logfile:
+        try:
+            from execsql.utils.fileio import filewriter_write
+
+            filewriter_write(logfile, msg + "\n")
+        except Exception:
+            pass
+
+
 def _handle_pg_upsert_errors(fn: Any, metacommandline: str | None) -> Any:
     """Run *fn*, translating pg-upsert exceptions to ErrInfo."""
     from pg_upsert import UserCancelledError
@@ -381,6 +486,7 @@ def x_pg_upsert(**kwargs: Any) -> None:
         _detach_log_handlers(loggers, handlers, prev_levels)
 
     _set_subvars(result)
+    _export_failures_if_requested(result, opts, metacommandline)
     if opts.get("cleanup"):
         ups.cleanup()
 
@@ -420,6 +526,7 @@ def x_pg_upsert_qa(**kwargs: Any) -> None:
 
     result = _build_result_from_qa_errors(ups)
     _set_subvars(result)
+    _export_failures_if_requested(result, opts, metacommandline)
     if opts.get("cleanup"):
         ups.cleanup()
 
@@ -462,6 +569,7 @@ def x_pg_upsert_check(**kwargs: Any) -> None:
 
     result = _build_result_from_qa_errors(ups)
     _set_subvars(result)
+    _export_failures_if_requested(result, opts, metacommandline)
     if opts.get("cleanup"):
         ups.cleanup()
 

execsql/utils/gui.py

@@ -498,10 +498,10 @@ def gui_credentials(
     cmd:
         The originating metacommand line (for logging only).
     """
+    import getpass as _getpass
     import queue as _queue
 
     import execsql.state as _state
-    from execsql.utils.auth import get_password
 
     gui_level = _state.conf.gui_level if _state.conf else 0
     if gui_level > 0 and _state.gui_manager_thread is not None and _state.gui_manager_thread.is_alive():
@@ -515,7 +515,7 @@ def gui_credentials(
         if message:
             print(message, file=sys.stderr)
         uname = input("Username: ")
-        passwd = get_password(f"Password for {uname}: ")
+        passwd = _getpass.getpass(f"Password for {uname}: ")
 
     if username:
         _state.subvars.add_substitution(username, uname)

{execsql2-2.14.1.dist-info → execsql2-2.15.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: execsql2
-Version: 2.14.1
+Version: 2.15.0
 Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
 Project-URL: Homepage, https://execsql2.readthedocs.io
 Project-URL: Repository, https://github.com/geocoug/execsql
@@ -51,7 +51,7 @@ Requires-Dist: keyring; extra == 'all'
 Requires-Dist: odfpy; extra == 'all'
 Requires-Dist: openpyxl; extra == 'all'
 Requires-Dist: oracledb; extra == 'all'
-Requires-Dist: pg-upsert>=1.20.0; extra == 'all'
+Requires-Dist: pg-upsert>=1.21.0; extra == 'all'
 Requires-Dist: polars; extra == 'all'
 Requires-Dist: psycopg2-binary; extra == 'all'
 Requires-Dist: pymysql; extra == 'all'
@@ -109,7 +109,7 @@ Requires-Dist: oracledb; extra == 'oracle'
 Provides-Extra: postgres
 Requires-Dist: psycopg2-binary; extra == 'postgres'
 Provides-Extra: upsert
-Requires-Dist: pg-upsert>=1.20.0; extra == 'upsert'
+Requires-Dist: pg-upsert>=1.21.0; extra == 'upsert'
 Description-Content-Type: text/markdown
 
 > [!NOTE]

{execsql2-2.14.1.dist-info → execsql2-2.15.0.dist-info}/RECORD

@@ -77,7 +77,7 @@ execsql/metacommands/io_write.py,sha256=NpL2aYGfBpbqmPpYsqniYltYfd_SCA1EQz3_4qSd
 execsql/metacommands/prompt.py,sha256=E2e7q4pxbl_wEBrhco0B2gm5hO_HG3rNIF75PLdTgGg,36767
 execsql/metacommands/script_ext.py,sha256=TUgAldB2LSJAwZrCvDDi804hQ1d9BDQD2GDqHNPVOcM,2280
 execsql/metacommands/system.py,sha256=AeyxbMLZVOi2nThbHiZ2F_UAIrZ1r4kjI3G80TtnSD4,7385
-execsql/metacommands/upsert.py,sha256=P2935aQHDZPiVwnXi0fGQ7Guxrm-Sy_YunyuSqVSegI,15880
+execsql/metacommands/upsert.py,sha256=W_5EEcp4QbDpq1LdS7Dz9eTa6do1D1a9aro2OCYZaMY,20032
 execsql/script/__init__.py,sha256=HbVQmQEVn4gBtzwy5_nlbDGuRnbWd4dI4nG-q1KyBxs,3498
 execsql/script/control.py,sha256=s-1eZdGARM6H1FwZ6VDdO_f50j7bvvRtTHesfUm9tbc,6144
 execsql/script/engine.py,sha256=6LYabzy1LI-_ISjYzTJos0BrLO62QF6FEKdqcN0YzK4,42995
@@ -88,30 +88,30 @@ execsql/utils/crypto.py,sha256=2OnBWwn9bCBGc1ZkyRv16TvhottoCNYtXqgbE3mG3Sg,2960
 execsql/utils/datetime.py,sha256=V_itd5vVvUPjT86P_z_hh4mlerMDGhDzI5MwPMDBaI4,7715
 execsql/utils/errors.py,sha256=YKhYD27-3timuZavc2vIrRIfHa71vzih-KVPsAKgvkU,8163
 execsql/utils/fileio.py,sha256=RGaMUe-e0xIw32OeprNJCezh0Jr9gQimQNqOXBEaJ2M,24338
-execsql/utils/gui.py,sha256=UFtwrXPNqNvZCJFCbumZ1aG2d9B-vyaJXIG83fDHteo,18409
+execsql/utils/gui.py,sha256=eZeFJm8EaWnzeHIw_O-tn9hO8sxGjZRX_aUFDtGQp4w,18396
 execsql/utils/mail.py,sha256=Sd7vWj-dz3w0XDSFU9PM8gmy41pojk-Vsgbfven2DMk,5786
 execsql/utils/numeric.py,sha256=xh02ANSRk3nUpQ-rtm66ILoMqoi7HtzCoRMIOT9U8QI,1570
 execsql/utils/regex.py,sha256=diEzTZqU_HHwVMadPAvN1Vgzhl7I03eVaEFGCXyGGL8,3770
 execsql/utils/strings.py,sha256=5Dvzrk-9SIw2lpxXZQkiJbNyo1sy7iXXAtSULlZ0KG8,8488
 execsql/utils/timer.py,sha256=eDYf5VzCNFk7oo90InJucUm3XcBdhYMogjZMqeg9xzc,1899
-execsql2-2.14.1.data/data/execsql2_extras/README.md,sha256=sxwVyU0ZahCfANv56LahkyuM505kFjrMhe-1SvWE69E,4845
-execsql2-2.14.1.data/data/execsql2_extras/config_settings.sqlite,sha256=aY5cxR7Q7J6zJ4bC9lu5mHUrhy211Cq3MNKPQVCt02E,20480
-execsql2-2.14.1.data/data/execsql2_extras/example_config_prompt.sql,sha256=SY3Jxn1qcVm4kPW9xmmTfknHfvURXmeEYTbRjYrjGSw,7487
-execsql2-2.14.1.data/data/execsql2_extras/execsql.conf,sha256=_45iJ-KWZnB8uMW_gEg067MM5pmGJ-dVl7VbAZMunAE,9530
-execsql2-2.14.1.data/data/execsql2_extras/make_config_db.sql,sha256=WwyC6dK-Eh5CAVppiBCDHqiI1_wEI9U95Ytpr4lsZkg,8726
-execsql2-2.14.1.data/data/execsql2_extras/md_compare.sql,sha256=B8Wd7LZ0vnMY2qvA139JIEBkPObgRH2i5xj6PejTQt8,24092
-execsql2-2.14.1.data/data/execsql2_extras/md_glossary.sql,sha256=DJRHcU5NbFpxTTX-IwH3yRlsboj1q6BBGrUAHKn4Cuo,10796
-execsql2-2.14.1.data/data/execsql2_extras/md_upsert.sql,sha256=v_7GbWh_N1mBTmw3gvTrkagOVp2q0KmXvM8hE-DlFxY,112524
-execsql2-2.14.1.data/data/execsql2_extras/pg_compare.sql,sha256=9dWa8hnfy5dVJI-z2iGpd9JzQmI4j2ziMlEdpnr66ro,24352
-execsql2-2.14.1.data/data/execsql2_extras/pg_glossary.sql,sha256=pKjIIDsROAgJq2H-1qNEcRMAWManivcZ_AEVHzUUlic,9908
-execsql2-2.14.1.data/data/execsql2_extras/pg_upsert.sql,sha256=k7AFiGTLBy3nf-qO5QIaZrEYTAKvdxxU3JDLx9jqkzs,108315
-execsql2-2.14.1.data/data/execsql2_extras/script_template.sql,sha256=1Estacb_vm1FgK41k_G9nuduP1yiA-fQ1Kn4Z4mv5Ao,11153
-execsql2-2.14.1.data/data/execsql2_extras/ss_compare.sql,sha256=TsVxWm3cEpR5-EiMYXNhtaY0arSNeKZhsJdHdLA7xeI,24833
-execsql2-2.14.1.data/data/execsql2_extras/ss_glossary.sql,sha256=cLM7nN8JOIu9ZVP9oY9qdSK3hrnWJiDcX6nZmQQbQWI,13065
-execsql2-2.14.1.data/data/execsql2_extras/ss_upsert.sql,sha256=BCqmBykXBF-BpCgOFeG1qhf2XfScKsxPD17wd1hYfHw,120647
-execsql2-2.14.1.dist-info/METADATA,sha256=4H299Jz_SqnoHxbxOxobQ42CCgTOv0ynOarTVEQxMqo,17566
-execsql2-2.14.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-execsql2-2.14.1.dist-info/entry_points.txt,sha256=sUOxkM-dN1eBGGpSpDLsAaE0yNXYQKWZAfxPOlMkQyk,90
-execsql2-2.14.1.dist-info/licenses/LICENSE.txt,sha256=LBdhuxejF8_bLCHZ2kWfmDXpDGUu914Gbd6_3JjCRe0,676
-execsql2-2.14.1.dist-info/licenses/NOTICE,sha256=sqVrM73Ys9zfvWC_P797lHfTnoPW_ETeBSrUTFaob0A,339
-execsql2-2.14.1.dist-info/RECORD,,
+execsql2-2.15.0.data/data/execsql2_extras/README.md,sha256=sxwVyU0ZahCfANv56LahkyuM505kFjrMhe-1SvWE69E,4845
+execsql2-2.15.0.data/data/execsql2_extras/config_settings.sqlite,sha256=aY5cxR7Q7J6zJ4bC9lu5mHUrhy211Cq3MNKPQVCt02E,20480
+execsql2-2.15.0.data/data/execsql2_extras/example_config_prompt.sql,sha256=SY3Jxn1qcVm4kPW9xmmTfknHfvURXmeEYTbRjYrjGSw,7487
+execsql2-2.15.0.data/data/execsql2_extras/execsql.conf,sha256=_45iJ-KWZnB8uMW_gEg067MM5pmGJ-dVl7VbAZMunAE,9530
+execsql2-2.15.0.data/data/execsql2_extras/make_config_db.sql,sha256=WwyC6dK-Eh5CAVppiBCDHqiI1_wEI9U95Ytpr4lsZkg,8726
+execsql2-2.15.0.data/data/execsql2_extras/md_compare.sql,sha256=B8Wd7LZ0vnMY2qvA139JIEBkPObgRH2i5xj6PejTQt8,24092
+execsql2-2.15.0.data/data/execsql2_extras/md_glossary.sql,sha256=DJRHcU5NbFpxTTX-IwH3yRlsboj1q6BBGrUAHKn4Cuo,10796
+execsql2-2.15.0.data/data/execsql2_extras/md_upsert.sql,sha256=v_7GbWh_N1mBTmw3gvTrkagOVp2q0KmXvM8hE-DlFxY,112524
+execsql2-2.15.0.data/data/execsql2_extras/pg_compare.sql,sha256=9dWa8hnfy5dVJI-z2iGpd9JzQmI4j2ziMlEdpnr66ro,24352
+execsql2-2.15.0.data/data/execsql2_extras/pg_glossary.sql,sha256=pKjIIDsROAgJq2H-1qNEcRMAWManivcZ_AEVHzUUlic,9908
+execsql2-2.15.0.data/data/execsql2_extras/pg_upsert.sql,sha256=k7AFiGTLBy3nf-qO5QIaZrEYTAKvdxxU3JDLx9jqkzs,108315
+execsql2-2.15.0.data/data/execsql2_extras/script_template.sql,sha256=1Estacb_vm1FgK41k_G9nuduP1yiA-fQ1Kn4Z4mv5Ao,11153
+execsql2-2.15.0.data/data/execsql2_extras/ss_compare.sql,sha256=TsVxWm3cEpR5-EiMYXNhtaY0arSNeKZhsJdHdLA7xeI,24833
+execsql2-2.15.0.data/data/execsql2_extras/ss_glossary.sql,sha256=cLM7nN8JOIu9ZVP9oY9qdSK3hrnWJiDcX6nZmQQbQWI,13065
+execsql2-2.15.0.data/data/execsql2_extras/ss_upsert.sql,sha256=BCqmBykXBF-BpCgOFeG1qhf2XfScKsxPD17wd1hYfHw,120647
+execsql2-2.15.0.dist-info/METADATA,sha256=4EszZ5kfmBXU3czXlw0mfPpf7vNyPYLd69_Y20nlzMw,17566
+execsql2-2.15.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+execsql2-2.15.0.dist-info/entry_points.txt,sha256=sUOxkM-dN1eBGGpSpDLsAaE0yNXYQKWZAfxPOlMkQyk,90
+execsql2-2.15.0.dist-info/licenses/LICENSE.txt,sha256=LBdhuxejF8_bLCHZ2kWfmDXpDGUu914Gbd6_3JjCRe0,676
+execsql2-2.15.0.dist-info/licenses/NOTICE,sha256=sqVrM73Ys9zfvWC_P797lHfTnoPW_ETeBSrUTFaob0A,339
+execsql2-2.15.0.dist-info/RECORD,,