execsql2 2.12.3__py3-none-any.whl → 2.12.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/metacommands/__init__.py +9 -0
- execsql/metacommands/dispatch.py +30 -0
- execsql/metacommands/upsert.py +448 -0
- {execsql2-2.12.3.dist-info → execsql2-2.12.5.dist-info}/METADATA +4 -1
- {execsql2-2.12.3.dist-info → execsql2-2.12.5.dist-info}/RECORD +24 -23
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/README.md +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/execsql.conf +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/pg_upsert.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.12.3.dist-info → execsql2-2.12.5.dist-info}/WHEEL +0 -0
- {execsql2-2.12.3.dist-info → execsql2-2.12.5.dist-info}/entry_points.txt +0 -0
- {execsql2-2.12.3.dist-info → execsql2-2.12.5.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.12.3.dist-info → execsql2-2.12.5.dist-info}/licenses/NOTICE +0 -0
execsql/metacommands/__init__.py
CHANGED
|
@@ -168,6 +168,11 @@ from execsql.metacommands.script_ext import (
|
|
|
168
168
|
x_extendscript_sql,
|
|
169
169
|
x_executescript,
|
|
170
170
|
)
|
|
171
|
+
from execsql.metacommands.upsert import (
|
|
172
|
+
x_pg_upsert,
|
|
173
|
+
x_pg_upsert_check,
|
|
174
|
+
x_pg_upsert_qa,
|
|
175
|
+
)
|
|
171
176
|
from execsql.metacommands.system import (
|
|
172
177
|
x_system_cmd,
|
|
173
178
|
x_email,
|
|
@@ -401,6 +406,10 @@ __all__ = [
|
|
|
401
406
|
"x_write_warnings",
|
|
402
407
|
"x_gui_level",
|
|
403
408
|
"x_execute",
|
|
409
|
+
# upsert handlers
|
|
410
|
+
"x_pg_upsert",
|
|
411
|
+
"x_pg_upsert_check",
|
|
412
|
+
"x_pg_upsert_qa",
|
|
404
413
|
# regex helpers
|
|
405
414
|
"ins_rxs",
|
|
406
415
|
"ins_quoted_rx",
|
execsql/metacommands/dispatch.py
CHANGED
|
@@ -168,6 +168,11 @@ from execsql.metacommands.script_ext import (
|
|
|
168
168
|
x_extendscript_metacommand,
|
|
169
169
|
x_extendscript_sql,
|
|
170
170
|
)
|
|
171
|
+
from execsql.metacommands.upsert import (
|
|
172
|
+
x_pg_upsert,
|
|
173
|
+
x_pg_upsert_check,
|
|
174
|
+
x_pg_upsert_qa,
|
|
175
|
+
)
|
|
171
176
|
from execsql.metacommands.system import (
|
|
172
177
|
x_cancel_halt,
|
|
173
178
|
x_cancel_halt_email,
|
|
@@ -2142,6 +2147,31 @@ def build_dispatch_table() -> MetaCommandList:
|
|
|
2142
2147
|
x_write,
|
|
2143
2148
|
)
|
|
2144
2149
|
|
|
2150
|
+
# ------------------------------------------------------------------
|
|
2151
|
+
# PG_UPSERT — pg-upsert integration (optional dependency)
|
|
2152
|
+
# ------------------------------------------------------------------
|
|
2153
|
+
# Order matters: CHECK and QA patterns must precede the general pattern
|
|
2154
|
+
# so that "PG_UPSERT CHECK ..." and "PG_UPSERT QA ..." are matched
|
|
2155
|
+
# before "PG_UPSERT FROM ...".
|
|
2156
|
+
mcl.add(
|
|
2157
|
+
r"^\s*PG_UPSERT\s+CHECK\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
|
|
2158
|
+
x_pg_upsert_check,
|
|
2159
|
+
description="PG_UPSERT CHECK",
|
|
2160
|
+
category="action",
|
|
2161
|
+
)
|
|
2162
|
+
mcl.add(
|
|
2163
|
+
r"^\s*PG_UPSERT\s+QA\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
|
|
2164
|
+
x_pg_upsert_qa,
|
|
2165
|
+
description="PG_UPSERT QA",
|
|
2166
|
+
category="action",
|
|
2167
|
+
)
|
|
2168
|
+
mcl.add(
|
|
2169
|
+
r"^\s*PG_UPSERT\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
|
|
2170
|
+
x_pg_upsert,
|
|
2171
|
+
description="PG_UPSERT",
|
|
2172
|
+
category="action",
|
|
2173
|
+
)
|
|
2174
|
+
|
|
2145
2175
|
# ------------------------------------------------------------------
|
|
2146
2176
|
# SUB (top-level variable assignment — kept near end so more specific
|
|
2147
2177
|
# SUB_* patterns above take precedence)
|
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
"""PG_UPSERT metacommand handler.
|
|
2
|
+
|
|
3
|
+
Integrates pg-upsert (https://pg-upsert.readthedocs.io/) as an optional
|
|
4
|
+
dependency, providing QA-checked, FK-dependency-ordered upserts from a
|
|
5
|
+
staging schema to a base schema on PostgreSQL.
|
|
6
|
+
|
|
7
|
+
Requires: ``pip install execsql2[upsert]``
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import logging
|
|
14
|
+
import re
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import execsql.state as _state
|
|
18
|
+
from execsql.exceptions import ErrInfo
|
|
19
|
+
from execsql.types import dbt_postgres
|
|
20
|
+
from execsql.utils.errors import exception_desc
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---------------------------------------------------------------------------
|
|
24
|
+
# Keyword parser for the trailing portion after TABLES
|
|
25
|
+
# ---------------------------------------------------------------------------
|
|
26
|
+
|
|
27
|
+
_KW_METHOD = re.compile(r"\bMETHOD\s+(upsert|update|insert)\b", re.IGNORECASE)
|
|
28
|
+
_KW_EXCLUDE = re.compile(
|
|
29
|
+
r"\bEXCLUDE\s+([\w\s,]+?)(?=\s+(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE_NULL|LOGFILE)\b|\s*$)",
|
|
30
|
+
re.IGNORECASE,
|
|
31
|
+
)
|
|
32
|
+
_KW_EXCLUDE_NULL = re.compile(
|
|
33
|
+
r"\bEXCLUDE_NULL\s+([\w\s,]+?)(?=\s+(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE|LOGFILE)\b|\s*$)",
|
|
34
|
+
re.IGNORECASE,
|
|
35
|
+
)
|
|
36
|
+
_KW_COMMIT = re.compile(r"\bCOMMIT\b", re.IGNORECASE)
|
|
37
|
+
_KW_INTERACTIVE = re.compile(r"\bINTERACTIVE\b", re.IGNORECASE)
|
|
38
|
+
_KW_COMPACT = re.compile(r"\bCOMPACT\b", re.IGNORECASE)
|
|
39
|
+
_KW_LOGFILE = re.compile(r"""\bLOGFILE\s+(?:"([^"]+)"|'([^']+)'|(\S+))""", re.IGNORECASE)
|
|
40
|
+
|
|
41
|
+
# All recognized keywords — used to split table names from options.
|
|
42
|
+
_ALL_KEYWORDS = re.compile(
|
|
43
|
+
r"\b(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE_NULL|EXCLUDE|LOGFILE)\b",
|
|
44
|
+
re.IGNORECASE,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _parse_tables_and_options(tail: str) -> dict[str, Any]:
|
|
49
|
+
"""Parse the trailing text after ``TABLES`` into table names and options.
|
|
50
|
+
|
|
51
|
+
Parameters
|
|
52
|
+
----------
|
|
53
|
+
tail:
|
|
54
|
+
Everything captured after the ``TABLES`` keyword in the regex.
|
|
55
|
+
|
|
56
|
+
Returns
|
|
57
|
+
-------
|
|
58
|
+
dict with keys: tables, method, commit, interactive, compact,
|
|
59
|
+
exclude_cols, exclude_null_check_cols.
|
|
60
|
+
"""
|
|
61
|
+
# Split at the first keyword to isolate the table list.
|
|
62
|
+
kw_match = _ALL_KEYWORDS.search(tail)
|
|
63
|
+
if kw_match:
|
|
64
|
+
table_part = tail[: kw_match.start()]
|
|
65
|
+
opts_part = tail[kw_match.start() :]
|
|
66
|
+
else:
|
|
67
|
+
table_part = tail
|
|
68
|
+
opts_part = ""
|
|
69
|
+
|
|
70
|
+
tables = [t.strip() for t in table_part.split(",") if t.strip()]
|
|
71
|
+
|
|
72
|
+
method = "upsert"
|
|
73
|
+
m = _KW_METHOD.search(opts_part)
|
|
74
|
+
if m:
|
|
75
|
+
method = m.group(1).lower()
|
|
76
|
+
|
|
77
|
+
exclude_cols: list[str] = []
|
|
78
|
+
m = _KW_EXCLUDE.search(opts_part)
|
|
79
|
+
if m:
|
|
80
|
+
exclude_cols = [c.strip() for c in m.group(1).split(",") if c.strip()]
|
|
81
|
+
|
|
82
|
+
exclude_null: list[str] = []
|
|
83
|
+
m = _KW_EXCLUDE_NULL.search(opts_part)
|
|
84
|
+
if m:
|
|
85
|
+
exclude_null = [c.strip() for c in m.group(1).split(",") if c.strip()]
|
|
86
|
+
|
|
87
|
+
logfile: str | None = None
|
|
88
|
+
m = _KW_LOGFILE.search(opts_part)
|
|
89
|
+
if m:
|
|
90
|
+
logfile = m.group(1) or m.group(2) or m.group(3)
|
|
91
|
+
|
|
92
|
+
return {
|
|
93
|
+
"tables": tables,
|
|
94
|
+
"method": method,
|
|
95
|
+
"commit": bool(_KW_COMMIT.search(opts_part)),
|
|
96
|
+
"interactive": bool(_KW_INTERACTIVE.search(opts_part)),
|
|
97
|
+
"compact": bool(_KW_COMPACT.search(opts_part)),
|
|
98
|
+
"exclude_cols": exclude_cols,
|
|
99
|
+
"exclude_null_check_cols": exclude_null,
|
|
100
|
+
"logfile": logfile,
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# ---------------------------------------------------------------------------
|
|
105
|
+
# Logging bridge: pg_upsert.display → execsql exec_log
|
|
106
|
+
# ---------------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class _ExecLogHandler(logging.Handler):
|
|
110
|
+
"""Route pg_upsert's plain-text file logger to execsql's exec_log."""
|
|
111
|
+
|
|
112
|
+
def __init__(self, exec_log: Any) -> None:
|
|
113
|
+
super().__init__()
|
|
114
|
+
self._exec_log = exec_log
|
|
115
|
+
|
|
116
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
117
|
+
self._exec_log.log_user_msg(self.format(record))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class _FileWriterHandler(logging.Handler):
|
|
121
|
+
"""Route pg_upsert log messages through execsql's async FileWriter.
|
|
122
|
+
|
|
123
|
+
This ensures that pg-upsert log output and execsql WRITE TEE output
|
|
124
|
+
arrive in the same order they were issued, since both go through the
|
|
125
|
+
same FileWriter queue.
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
def __init__(self, filename: str) -> None:
|
|
129
|
+
super().__init__()
|
|
130
|
+
self._filename = filename
|
|
131
|
+
|
|
132
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
133
|
+
from execsql.utils.fileio import filewriter_write
|
|
134
|
+
|
|
135
|
+
filewriter_write(self._filename, self.format(record) + "\n")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# ---------------------------------------------------------------------------
|
|
139
|
+
# Result → substitution variables
|
|
140
|
+
# ---------------------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _set_subvars(result: Any) -> None:
|
|
144
|
+
"""Populate ``$PG_UPSERT_*`` substitution variables from an UpsertResult."""
|
|
145
|
+
sv = _state.subvars.add_substitution
|
|
146
|
+
sv("$PG_UPSERT_QA_PASSED", str(result.qa_passed).upper())
|
|
147
|
+
sv("$PG_UPSERT_ROWS_UPDATED", str(result.total_updated))
|
|
148
|
+
sv("$PG_UPSERT_ROWS_INSERTED", str(result.total_inserted))
|
|
149
|
+
sv("$PG_UPSERT_COMMITTED", str(result.committed).upper())
|
|
150
|
+
sv("$PG_UPSERT_STAGING_SCHEMA", result.staging_schema)
|
|
151
|
+
sv("$PG_UPSERT_BASE_SCHEMA", result.base_schema)
|
|
152
|
+
sv("$PG_UPSERT_TABLES", ", ".join(t.table_name for t in result.tables))
|
|
153
|
+
sv("$PG_UPSERT_METHOD", result.upsert_method)
|
|
154
|
+
sv("$PG_UPSERT_DURATION", str(result.duration_seconds))
|
|
155
|
+
sv("$PG_UPSERT_STARTED_AT", result.started_at)
|
|
156
|
+
sv("$PG_UPSERT_FINISHED_AT", result.finished_at)
|
|
157
|
+
sv("$PG_UPSERT_RESULT_JSON", json.dumps(result.to_dict(), separators=(",", ":")))
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _qa_failure_msg(result: Any) -> str:
|
|
161
|
+
"""Build a concise QA failure message listing which tables failed."""
|
|
162
|
+
failed = [t.table_name for t in result.tables if not t.qa_passed]
|
|
163
|
+
if failed:
|
|
164
|
+
return f"PG_UPSERT QA failed for: {', '.join(failed)}"
|
|
165
|
+
return "PG_UPSERT QA checks failed."
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# ---------------------------------------------------------------------------
|
|
169
|
+
# Import guard + helpers
|
|
170
|
+
# ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _require_pg_upsert() -> None:
|
|
174
|
+
"""Raise ErrInfo if pg_upsert is not installed."""
|
|
175
|
+
try:
|
|
176
|
+
import pg_upsert # noqa: F401
|
|
177
|
+
except ImportError as exc:
|
|
178
|
+
raise ErrInfo(
|
|
179
|
+
"exception",
|
|
180
|
+
other_msg=("PG_UPSERT requires the pg-upsert package. Install it with: pip install execsql2[upsert]"),
|
|
181
|
+
) from exc
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _require_postgres(db: Any, metacommandline: str | None) -> None:
|
|
185
|
+
"""Raise ErrInfo if the current connection is not PostgreSQL."""
|
|
186
|
+
if db.type != dbt_postgres:
|
|
187
|
+
raise ErrInfo(
|
|
188
|
+
"cmd",
|
|
189
|
+
command_text=metacommandline,
|
|
190
|
+
other_msg=(f"PG_UPSERT requires a PostgreSQL connection. Current DBMS: {db.type.dbms_id}"),
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _build_result_from_qa_errors(ups: Any) -> Any:
|
|
195
|
+
"""Build an UpsertResult from ``ups.qa_errors`` after a QA/CHECK run."""
|
|
196
|
+
from pg_upsert.models import TableResult, UpsertResult
|
|
197
|
+
|
|
198
|
+
table_results: dict[str, Any] = {}
|
|
199
|
+
for table_name in ups.tables:
|
|
200
|
+
table_results[table_name] = TableResult(table_name=table_name)
|
|
201
|
+
for err in ups.qa_errors:
|
|
202
|
+
if err.table in table_results:
|
|
203
|
+
table_results[err.table].qa_errors.append(err)
|
|
204
|
+
return UpsertResult(
|
|
205
|
+
tables=list(table_results.values()),
|
|
206
|
+
committed=False,
|
|
207
|
+
staging_schema=ups.staging_schema,
|
|
208
|
+
base_schema=ups.base_schema,
|
|
209
|
+
upsert_method=ups.upsert_method,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _create_pgupsert(
|
|
214
|
+
db: Any,
|
|
215
|
+
staging_schema: str,
|
|
216
|
+
base_schema: str,
|
|
217
|
+
opts: dict[str, Any],
|
|
218
|
+
) -> Any:
|
|
219
|
+
"""Create and return a PgUpsert instance with execsql's connection."""
|
|
220
|
+
from pg_upsert import PgUpsert
|
|
221
|
+
|
|
222
|
+
ui_mode = "tkinter"
|
|
223
|
+
if _state.conf:
|
|
224
|
+
ui_mode = _state.conf.gui_framework
|
|
225
|
+
|
|
226
|
+
ups = PgUpsert(
|
|
227
|
+
conn=db.conn,
|
|
228
|
+
staging_schema=staging_schema,
|
|
229
|
+
base_schema=base_schema,
|
|
230
|
+
tables=opts["tables"],
|
|
231
|
+
do_commit=opts["commit"],
|
|
232
|
+
interactive=opts["interactive"],
|
|
233
|
+
compact=opts["compact"],
|
|
234
|
+
upsert_method=opts["method"],
|
|
235
|
+
exclude_cols=opts["exclude_cols"],
|
|
236
|
+
exclude_null_check_cols=opts["exclude_null_check_cols"],
|
|
237
|
+
ui_mode=ui_mode,
|
|
238
|
+
)
|
|
239
|
+
return ups
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _attach_log_handlers(
|
|
243
|
+
logfile: str | None = None,
|
|
244
|
+
) -> tuple[list[logging.Logger], list[logging.Handler], dict[str, int]]:
|
|
245
|
+
"""Attach logging handlers to pg_upsert loggers.
|
|
246
|
+
|
|
247
|
+
Always attaches the exec_log bridge to ``pg_upsert.display``.
|
|
248
|
+
If *logfile* is given, also attaches a FileHandler (append mode) to both
|
|
249
|
+
``pg_upsert`` and ``pg_upsert.display`` — matching pg-upsert CLI behavior.
|
|
250
|
+
|
|
251
|
+
Returns (loggers, handlers) so the caller can detach in a finally block.
|
|
252
|
+
"""
|
|
253
|
+
display_logger = logging.getLogger("pg_upsert.display")
|
|
254
|
+
# pg-upsert's display logger has propagate=False and level=NOTSET, which
|
|
255
|
+
# gives it an effective level of WARNING (inherited from root). Its messages
|
|
256
|
+
# are logged at INFO, so we must explicitly lower the level.
|
|
257
|
+
prev_display_level = display_logger.level
|
|
258
|
+
if display_logger.getEffectiveLevel() > logging.INFO:
|
|
259
|
+
display_logger.setLevel(logging.INFO)
|
|
260
|
+
|
|
261
|
+
exec_handler = _ExecLogHandler(_state.exec_log)
|
|
262
|
+
display_logger.addHandler(exec_handler)
|
|
263
|
+
|
|
264
|
+
loggers: list[logging.Logger] = [display_logger]
|
|
265
|
+
handlers: list[logging.Handler] = [exec_handler]
|
|
266
|
+
prev_levels: dict[str, int] = {"pg_upsert.display": prev_display_level}
|
|
267
|
+
|
|
268
|
+
if logfile:
|
|
269
|
+
file_handler = _FileWriterHandler(logfile)
|
|
270
|
+
file_handler.setFormatter(logging.Formatter("%(message)s"))
|
|
271
|
+
# Attach to both loggers, same as pg-upsert CLI does
|
|
272
|
+
main_logger = logging.getLogger("pg_upsert")
|
|
273
|
+
prev_levels["pg_upsert"] = main_logger.level
|
|
274
|
+
if main_logger.getEffectiveLevel() > logging.INFO:
|
|
275
|
+
main_logger.setLevel(logging.INFO)
|
|
276
|
+
main_logger.addHandler(file_handler)
|
|
277
|
+
display_logger.addHandler(file_handler)
|
|
278
|
+
loggers.append(main_logger)
|
|
279
|
+
handlers.append(file_handler)
|
|
280
|
+
|
|
281
|
+
return loggers, handlers, prev_levels
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _detach_log_handlers(
|
|
285
|
+
loggers: list[logging.Logger],
|
|
286
|
+
handlers: list[logging.Handler],
|
|
287
|
+
prev_levels: dict[str, int],
|
|
288
|
+
) -> None:
|
|
289
|
+
"""Remove all handlers added by ``_attach_log_handlers``."""
|
|
290
|
+
for handler in handlers:
|
|
291
|
+
for lgr in loggers:
|
|
292
|
+
lgr.removeHandler(handler)
|
|
293
|
+
if hasattr(handler, "close"):
|
|
294
|
+
handler.close()
|
|
295
|
+
# Restore original logger levels
|
|
296
|
+
for name, level in prev_levels.items():
|
|
297
|
+
logging.getLogger(name).setLevel(level)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _run_with_autocommit_guard(db: Any, fn: Any) -> Any:
|
|
301
|
+
"""Temporarily disable autocommit, run *fn*, then restore."""
|
|
302
|
+
was_autocommit = db.autocommit
|
|
303
|
+
if was_autocommit:
|
|
304
|
+
db.autocommit_off()
|
|
305
|
+
try:
|
|
306
|
+
return fn()
|
|
307
|
+
finally:
|
|
308
|
+
if was_autocommit:
|
|
309
|
+
db.autocommit_on()
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _handle_pg_upsert_errors(fn: Any, metacommandline: str | None) -> Any:
|
|
313
|
+
"""Run *fn*, translating pg-upsert exceptions to ErrInfo."""
|
|
314
|
+
from pg_upsert import UserCancelledError
|
|
315
|
+
|
|
316
|
+
try:
|
|
317
|
+
return fn()
|
|
318
|
+
except UserCancelledError as exc:
|
|
319
|
+
raise ErrInfo(
|
|
320
|
+
"cmd",
|
|
321
|
+
command_text=metacommandline,
|
|
322
|
+
other_msg="PG_UPSERT cancelled by user.",
|
|
323
|
+
) from exc
|
|
324
|
+
except ErrInfo:
|
|
325
|
+
raise
|
|
326
|
+
except Exception as exc:
|
|
327
|
+
raise ErrInfo(
|
|
328
|
+
"exception",
|
|
329
|
+
exception_msg=exception_desc(),
|
|
330
|
+
other_msg="PG_UPSERT failed unexpectedly.",
|
|
331
|
+
) from exc
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
# ---------------------------------------------------------------------------
|
|
335
|
+
# Metacommand handlers
|
|
336
|
+
# ---------------------------------------------------------------------------
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def x_pg_upsert(**kwargs: Any) -> None:
|
|
340
|
+
"""PG_UPSERT FROM <staging> TO <base> TABLES <t1>, <t2> [options]
|
|
341
|
+
|
|
342
|
+
Full pipeline: QA checks → upsert → optional commit.
|
|
343
|
+
"""
|
|
344
|
+
_require_pg_upsert()
|
|
345
|
+
db = _state.dbs.current()
|
|
346
|
+
metacommandline = kwargs.get("metacommandline")
|
|
347
|
+
_require_postgres(db, metacommandline)
|
|
348
|
+
|
|
349
|
+
staging = kwargs["staging_schema"]
|
|
350
|
+
base = kwargs["base_schema"]
|
|
351
|
+
opts = _parse_tables_and_options(kwargs["tail"])
|
|
352
|
+
|
|
353
|
+
ups = _create_pgupsert(db, staging, base, opts)
|
|
354
|
+
loggers, handlers, prev_levels = _attach_log_handlers(opts.get("logfile"))
|
|
355
|
+
|
|
356
|
+
try:
|
|
357
|
+
result = _run_with_autocommit_guard(
|
|
358
|
+
db,
|
|
359
|
+
lambda: _handle_pg_upsert_errors(ups.run, metacommandline),
|
|
360
|
+
)
|
|
361
|
+
finally:
|
|
362
|
+
_detach_log_handlers(loggers, handlers, prev_levels)
|
|
363
|
+
|
|
364
|
+
_set_subvars(result)
|
|
365
|
+
|
|
366
|
+
if not result.qa_passed:
|
|
367
|
+
raise ErrInfo(
|
|
368
|
+
"cmd",
|
|
369
|
+
command_text=metacommandline,
|
|
370
|
+
other_msg=_qa_failure_msg(result),
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def x_pg_upsert_qa(**kwargs: Any) -> None:
|
|
375
|
+
"""PG_UPSERT QA FROM <staging> TO <base> TABLES <t1>, <t2> [options]
|
|
376
|
+
|
|
377
|
+
QA-only mode: run all QA checks without upserting.
|
|
378
|
+
"""
|
|
379
|
+
_require_pg_upsert()
|
|
380
|
+
db = _state.dbs.current()
|
|
381
|
+
metacommandline = kwargs.get("metacommandline")
|
|
382
|
+
_require_postgres(db, metacommandline)
|
|
383
|
+
|
|
384
|
+
staging = kwargs["staging_schema"]
|
|
385
|
+
base = kwargs["base_schema"]
|
|
386
|
+
opts = _parse_tables_and_options(kwargs["tail"])
|
|
387
|
+
opts["commit"] = False # QA-only never commits
|
|
388
|
+
|
|
389
|
+
ups = _create_pgupsert(db, staging, base, opts)
|
|
390
|
+
loggers, handlers, prev_levels = _attach_log_handlers(opts.get("logfile"))
|
|
391
|
+
|
|
392
|
+
try:
|
|
393
|
+
_run_with_autocommit_guard(
|
|
394
|
+
db,
|
|
395
|
+
lambda: _handle_pg_upsert_errors(ups.qa_all, metacommandline),
|
|
396
|
+
)
|
|
397
|
+
finally:
|
|
398
|
+
_detach_log_handlers(loggers, handlers, prev_levels)
|
|
399
|
+
|
|
400
|
+
result = _build_result_from_qa_errors(ups)
|
|
401
|
+
_set_subvars(result)
|
|
402
|
+
|
|
403
|
+
if not result.qa_passed:
|
|
404
|
+
raise ErrInfo(
|
|
405
|
+
"cmd",
|
|
406
|
+
command_text=metacommandline,
|
|
407
|
+
other_msg=_qa_failure_msg(result),
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def x_pg_upsert_check(**kwargs: Any) -> None:
|
|
412
|
+
"""PG_UPSERT CHECK FROM <staging> TO <base> TABLES <t1>, <t2>
|
|
413
|
+
|
|
414
|
+
Schema check only: column existence + type mismatch.
|
|
415
|
+
"""
|
|
416
|
+
_require_pg_upsert()
|
|
417
|
+
db = _state.dbs.current()
|
|
418
|
+
metacommandline = kwargs.get("metacommandline")
|
|
419
|
+
_require_postgres(db, metacommandline)
|
|
420
|
+
|
|
421
|
+
staging = kwargs["staging_schema"]
|
|
422
|
+
base = kwargs["base_schema"]
|
|
423
|
+
opts = _parse_tables_and_options(kwargs["tail"])
|
|
424
|
+
opts["commit"] = False
|
|
425
|
+
|
|
426
|
+
ups = _create_pgupsert(db, staging, base, opts)
|
|
427
|
+
loggers, handlers, prev_levels = _attach_log_handlers(opts.get("logfile"))
|
|
428
|
+
|
|
429
|
+
try:
|
|
430
|
+
_run_with_autocommit_guard(
|
|
431
|
+
db,
|
|
432
|
+
lambda: _handle_pg_upsert_errors(
|
|
433
|
+
lambda: ups.qa_column_existence().qa_type_mismatch(),
|
|
434
|
+
metacommandline,
|
|
435
|
+
),
|
|
436
|
+
)
|
|
437
|
+
finally:
|
|
438
|
+
_detach_log_handlers(loggers, handlers, prev_levels)
|
|
439
|
+
|
|
440
|
+
result = _build_result_from_qa_errors(ups)
|
|
441
|
+
_set_subvars(result)
|
|
442
|
+
|
|
443
|
+
if not result.qa_passed:
|
|
444
|
+
raise ErrInfo(
|
|
445
|
+
"cmd",
|
|
446
|
+
command_text=metacommandline,
|
|
447
|
+
other_msg=_qa_failure_msg(result),
|
|
448
|
+
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: execsql2
|
|
3
|
-
Version: 2.12.
|
|
3
|
+
Version: 2.12.5
|
|
4
4
|
Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
|
|
5
5
|
Project-URL: Homepage, https://execsql2.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/geocoug/execsql
|
|
@@ -51,6 +51,7 @@ Requires-Dist: keyring; extra == 'all'
|
|
|
51
51
|
Requires-Dist: odfpy; extra == 'all'
|
|
52
52
|
Requires-Dist: openpyxl; extra == 'all'
|
|
53
53
|
Requires-Dist: oracledb; extra == 'all'
|
|
54
|
+
Requires-Dist: pg-upsert>=1.17.0; extra == 'all'
|
|
54
55
|
Requires-Dist: polars; extra == 'all'
|
|
55
56
|
Requires-Dist: psycopg2-binary; extra == 'all'
|
|
56
57
|
Requires-Dist: pymysql; extra == 'all'
|
|
@@ -107,6 +108,8 @@ Provides-Extra: oracle
|
|
|
107
108
|
Requires-Dist: oracledb; extra == 'oracle'
|
|
108
109
|
Provides-Extra: postgres
|
|
109
110
|
Requires-Dist: psycopg2-binary; extra == 'postgres'
|
|
111
|
+
Provides-Extra: upsert
|
|
112
|
+
Requires-Dist: pg-upsert>=1.17.0; extra == 'upsert'
|
|
110
113
|
Description-Content-Type: text/markdown
|
|
111
114
|
|
|
112
115
|
> [!NOTE]
|
|
@@ -61,13 +61,13 @@ execsql/importers/csv.py,sha256=Mu848WNzuhVO1ade-WurPyxqGOuVNRO8UwRF3-bav_I,4845
|
|
|
61
61
|
execsql/importers/feather.py,sha256=g2B69d2uv9vmnXcmjFyTVsMP40LYEzFYkhk3gD26mGw,1900
|
|
62
62
|
execsql/importers/ods.py,sha256=MJsdsjropzCvxAA3DDZfAL_AnmZ4yij7DnrjGyDJqHQ,2843
|
|
63
63
|
execsql/importers/xls.py,sha256=e0Zfe47ZiCpA1Ae3XDJ1ko3sCiH3-8U6XLKi6NvD0jQ,3683
|
|
64
|
-
execsql/metacommands/__init__.py,sha256=
|
|
64
|
+
execsql/metacommands/__init__.py,sha256=jrp5PvAZyQYvZ0a_qYNxjv1ZqxDlGYE_P3dyqRTgVSM,11394
|
|
65
65
|
execsql/metacommands/conditions.py,sha256=Fzrk83-pWbFOoKahYdQW7CZjQeh3zByDUbfgpTM_bjQ,29259
|
|
66
66
|
execsql/metacommands/connect.py,sha256=Nsm0D91i3RX-R2rzQQ-Br-gULaI6Uvdn9fqb7DOAVfE,14804
|
|
67
67
|
execsql/metacommands/control.py,sha256=PAZFK1ck5SDSm5QdFV1ctif3KpEiyYWIXdDceRWgQ6k,8513
|
|
68
68
|
execsql/metacommands/data.py,sha256=tRQBGTAuW-eJ2tBNWaoZI9OjTyNNyHJISo7gOdL-sm8,11370
|
|
69
69
|
execsql/metacommands/debug.py,sha256=pnT24dfvfOx8xFu86mO5czfVCGKbcvgBLyXnqaMWO4w,8184
|
|
70
|
-
execsql/metacommands/dispatch.py,sha256=
|
|
70
|
+
execsql/metacommands/dispatch.py,sha256=2Ec7dJmOFstbY3tEuFoTODV8yaHpR-Sfa-ZydOmvIxo,85205
|
|
71
71
|
execsql/metacommands/io.py,sha256=Duh60caM4go9JczbGYNMKKYpcMimwPzF6EQ_tshKxdE,2971
|
|
72
72
|
execsql/metacommands/io_export.py,sha256=7lkCSnPhXy9FVau9_hT1u68NOVdG2DsWmvUh9hM1QWI,18359
|
|
73
73
|
execsql/metacommands/io_fileops.py,sha256=RrcJTh_cgj7bJ-bezjo0yNl-fN3CoWV-aZ71z1KHYZs,9803
|
|
@@ -76,6 +76,7 @@ execsql/metacommands/io_write.py,sha256=NpL2aYGfBpbqmPpYsqniYltYfd_SCA1EQz3_4qSd
|
|
|
76
76
|
execsql/metacommands/prompt.py,sha256=xd3mAkdbn4AE4hUPuSfN5DgZGUZmk-Db23iL-JJPwXs,36918
|
|
77
77
|
execsql/metacommands/script_ext.py,sha256=TUgAldB2LSJAwZrCvDDi804hQ1d9BDQD2GDqHNPVOcM,2280
|
|
78
78
|
execsql/metacommands/system.py,sha256=sUR5kLL7idTVg8WXIMdd-Kv7nkERIiaeL0beWsz8NyY,7293
|
|
79
|
+
execsql/metacommands/upsert.py,sha256=Ze_FhbCRM-8tjvqPKfL3E9abZVRGFfwPI0_NAihiEMU,14906
|
|
79
80
|
execsql/script/__init__.py,sha256=HbVQmQEVn4gBtzwy5_nlbDGuRnbWd4dI4nG-q1KyBxs,3498
|
|
80
81
|
execsql/script/control.py,sha256=s-1eZdGARM6H1FwZ6VDdO_f50j7bvvRtTHesfUm9tbc,6144
|
|
81
82
|
execsql/script/engine.py,sha256=6LYabzy1LI-_ISjYzTJos0BrLO62QF6FEKdqcN0YzK4,42995
|
|
@@ -92,24 +93,24 @@ execsql/utils/numeric.py,sha256=xh02ANSRk3nUpQ-rtm66ILoMqoi7HtzCoRMIOT9U8QI,1570
|
|
|
92
93
|
execsql/utils/regex.py,sha256=diEzTZqU_HHwVMadPAvN1Vgzhl7I03eVaEFGCXyGGL8,3770
|
|
93
94
|
execsql/utils/strings.py,sha256=5Dvzrk-9SIw2lpxXZQkiJbNyo1sy7iXXAtSULlZ0KG8,8488
|
|
94
95
|
execsql/utils/timer.py,sha256=eDYf5VzCNFk7oo90InJucUm3XcBdhYMogjZMqeg9xzc,1899
|
|
95
|
-
execsql2-2.12.
|
|
96
|
-
execsql2-2.12.
|
|
97
|
-
execsql2-2.12.
|
|
98
|
-
execsql2-2.12.
|
|
99
|
-
execsql2-2.12.
|
|
100
|
-
execsql2-2.12.
|
|
101
|
-
execsql2-2.12.
|
|
102
|
-
execsql2-2.12.
|
|
103
|
-
execsql2-2.12.
|
|
104
|
-
execsql2-2.12.
|
|
105
|
-
execsql2-2.12.
|
|
106
|
-
execsql2-2.12.
|
|
107
|
-
execsql2-2.12.
|
|
108
|
-
execsql2-2.12.
|
|
109
|
-
execsql2-2.12.
|
|
110
|
-
execsql2-2.12.
|
|
111
|
-
execsql2-2.12.
|
|
112
|
-
execsql2-2.12.
|
|
113
|
-
execsql2-2.12.
|
|
114
|
-
execsql2-2.12.
|
|
115
|
-
execsql2-2.12.
|
|
96
|
+
execsql2-2.12.5.data/data/execsql2_extras/README.md,sha256=sxwVyU0ZahCfANv56LahkyuM505kFjrMhe-1SvWE69E,4845
|
|
97
|
+
execsql2-2.12.5.data/data/execsql2_extras/config_settings.sqlite,sha256=aY5cxR7Q7J6zJ4bC9lu5mHUrhy211Cq3MNKPQVCt02E,20480
|
|
98
|
+
execsql2-2.12.5.data/data/execsql2_extras/example_config_prompt.sql,sha256=SY3Jxn1qcVm4kPW9xmmTfknHfvURXmeEYTbRjYrjGSw,7487
|
|
99
|
+
execsql2-2.12.5.data/data/execsql2_extras/execsql.conf,sha256=_45iJ-KWZnB8uMW_gEg067MM5pmGJ-dVl7VbAZMunAE,9530
|
|
100
|
+
execsql2-2.12.5.data/data/execsql2_extras/make_config_db.sql,sha256=WwyC6dK-Eh5CAVppiBCDHqiI1_wEI9U95Ytpr4lsZkg,8726
|
|
101
|
+
execsql2-2.12.5.data/data/execsql2_extras/md_compare.sql,sha256=B8Wd7LZ0vnMY2qvA139JIEBkPObgRH2i5xj6PejTQt8,24092
|
|
102
|
+
execsql2-2.12.5.data/data/execsql2_extras/md_glossary.sql,sha256=DJRHcU5NbFpxTTX-IwH3yRlsboj1q6BBGrUAHKn4Cuo,10796
|
|
103
|
+
execsql2-2.12.5.data/data/execsql2_extras/md_upsert.sql,sha256=v_7GbWh_N1mBTmw3gvTrkagOVp2q0KmXvM8hE-DlFxY,112524
|
|
104
|
+
execsql2-2.12.5.data/data/execsql2_extras/pg_compare.sql,sha256=9dWa8hnfy5dVJI-z2iGpd9JzQmI4j2ziMlEdpnr66ro,24352
|
|
105
|
+
execsql2-2.12.5.data/data/execsql2_extras/pg_glossary.sql,sha256=pKjIIDsROAgJq2H-1qNEcRMAWManivcZ_AEVHzUUlic,9908
|
|
106
|
+
execsql2-2.12.5.data/data/execsql2_extras/pg_upsert.sql,sha256=k7AFiGTLBy3nf-qO5QIaZrEYTAKvdxxU3JDLx9jqkzs,108315
|
|
107
|
+
execsql2-2.12.5.data/data/execsql2_extras/script_template.sql,sha256=1Estacb_vm1FgK41k_G9nuduP1yiA-fQ1Kn4Z4mv5Ao,11153
|
|
108
|
+
execsql2-2.12.5.data/data/execsql2_extras/ss_compare.sql,sha256=TsVxWm3cEpR5-EiMYXNhtaY0arSNeKZhsJdHdLA7xeI,24833
|
|
109
|
+
execsql2-2.12.5.data/data/execsql2_extras/ss_glossary.sql,sha256=cLM7nN8JOIu9ZVP9oY9qdSK3hrnWJiDcX6nZmQQbQWI,13065
|
|
110
|
+
execsql2-2.12.5.data/data/execsql2_extras/ss_upsert.sql,sha256=BCqmBykXBF-BpCgOFeG1qhf2XfScKsxPD17wd1hYfHw,120647
|
|
111
|
+
execsql2-2.12.5.dist-info/METADATA,sha256=Z0cWFXEpF8ujF1fqnSsBO33IIylqc9t583xJ9H33Xss,17560
|
|
112
|
+
execsql2-2.12.5.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
113
|
+
execsql2-2.12.5.dist-info/entry_points.txt,sha256=sUOxkM-dN1eBGGpSpDLsAaE0yNXYQKWZAfxPOlMkQyk,90
|
|
114
|
+
execsql2-2.12.5.dist-info/licenses/LICENSE.txt,sha256=LBdhuxejF8_bLCHZ2kWfmDXpDGUu914Gbd6_3JjCRe0,676
|
|
115
|
+
execsql2-2.12.5.dist-info/licenses/NOTICE,sha256=sqVrM73Ys9zfvWC_P797lHfTnoPW_ETeBSrUTFaob0A,339
|
|
116
|
+
execsql2-2.12.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{execsql2-2.12.3.data → execsql2-2.12.5.data}/data/execsql2_extras/example_config_prompt.sql
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|