execsql2 2.12.2__py3-none-any.whl → 2.12.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/db/base.py +7 -0
- execsql/exporters/delimited.py +44 -2
- execsql/metacommands/__init__.py +9 -0
- execsql/metacommands/dispatch.py +30 -0
- execsql/metacommands/io_fileops.py +4 -0
- execsql/metacommands/upsert.py +448 -0
- execsql/script/__init__.py +4 -0
- execsql/script/engine.py +55 -23
- execsql/script/variables.py +35 -2
- {execsql2-2.12.2.dist-info → execsql2-2.12.5.dist-info}/METADATA +4 -1
- {execsql2-2.12.2.dist-info → execsql2-2.12.5.dist-info}/RECORD +30 -29
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/README.md +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/execsql.conf +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/pg_upsert.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.12.2.dist-info → execsql2-2.12.5.dist-info}/WHEEL +0 -0
- {execsql2-2.12.2.dist-info → execsql2-2.12.5.dist-info}/entry_points.txt +0 -0
- {execsql2-2.12.2.dist-info → execsql2-2.12.5.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.12.2.dist-info → execsql2-2.12.5.dist-info}/licenses/NOTICE +0 -0
execsql/db/base.py
CHANGED
|
@@ -686,6 +686,13 @@ class DatabasePool:
|
|
|
686
686
|
)
|
|
687
687
|
self.pool[db_alias].close()
|
|
688
688
|
self.pool[db_alias] = db_obj
|
|
689
|
+
# Refresh static system vars so $DB_NAME, $DB_USER, etc. reflect the new connection.
|
|
690
|
+
try:
|
|
691
|
+
from execsql.script.engine import set_static_system_vars
|
|
692
|
+
|
|
693
|
+
set_static_system_vars()
|
|
694
|
+
except Exception:
|
|
695
|
+
pass # Engine not yet initialized (early startup).
|
|
689
696
|
|
|
690
697
|
def aliases(self) -> list[str]:
|
|
691
698
|
"""Return a list of all currently registered database aliases."""
|
execsql/exporters/delimited.py
CHANGED
|
@@ -40,6 +40,7 @@ class LineDelimiter:
|
|
|
40
40
|
self.delimiter = delim
|
|
41
41
|
self.joinchar = delim if delim else ""
|
|
42
42
|
self.quotechar = quote
|
|
43
|
+
self.quote_all_text = _state.conf.quote_all_text if _state.conf else False
|
|
43
44
|
if quote:
|
|
44
45
|
if escchar:
|
|
45
46
|
self.quotedquote = escchar + quote
|
|
@@ -50,13 +51,12 @@ class LineDelimiter:
|
|
|
50
51
|
|
|
51
52
|
def delimited(self, datarow: Any, add_newline: bool = True) -> str:
|
|
52
53
|
"""Format a sequence of values as a single delimited text line."""
|
|
53
|
-
conf = _state.conf
|
|
54
54
|
if self.quotechar:
|
|
55
55
|
d_row = []
|
|
56
56
|
for e in datarow:
|
|
57
57
|
if isinstance(e, str):
|
|
58
58
|
if (
|
|
59
|
-
|
|
59
|
+
self.quote_all_text
|
|
60
60
|
or (self.quotechar in e)
|
|
61
61
|
or (self.delimiter is not None and self.delimiter in e)
|
|
62
62
|
or ("\n" in e)
|
|
@@ -609,10 +609,52 @@ class CsvFile(EncodedFile):
|
|
|
609
609
|
raise ErrInfo("error", other_msg=", ".join(self.parse_errors))
|
|
610
610
|
return elements
|
|
611
611
|
|
|
612
|
+
def _can_use_fast_csv_reader(self) -> bool:
|
|
613
|
+
"""Return True if the detected format is compatible with Python's csv module."""
|
|
614
|
+
# The csv module handles comma/tab delimiters with doubled-quote escaping.
|
|
615
|
+
# It cannot handle: space-delimiter collapsing, escape chars, or no delimiter.
|
|
616
|
+
if self.delimiter is None or self.delimiter == " ":
|
|
617
|
+
return False
|
|
618
|
+
return self.escapechar is None
|
|
619
|
+
|
|
612
620
|
def reader(self) -> Any:
|
|
613
621
|
"""Yield parsed rows from the file as lists of field values."""
|
|
614
622
|
conf = _state.conf
|
|
615
623
|
self.evaluate_line_format()
|
|
624
|
+
if self._can_use_fast_csv_reader():
|
|
625
|
+
yield from self._fast_reader(conf)
|
|
626
|
+
else:
|
|
627
|
+
yield from self._slow_reader(conf)
|
|
628
|
+
|
|
629
|
+
def _fast_reader(self, conf: Any) -> Any:
|
|
630
|
+
"""Read using Python's csv module (fast path for standard delimited formats)."""
|
|
631
|
+
import csv
|
|
632
|
+
|
|
633
|
+
f = self.openclean("rt")
|
|
634
|
+
try:
|
|
635
|
+
csv_reader = csv.reader(
|
|
636
|
+
f,
|
|
637
|
+
delimiter=self.delimiter,
|
|
638
|
+
quotechar=self.quotechar,
|
|
639
|
+
doublequote=True,
|
|
640
|
+
strict=False,
|
|
641
|
+
)
|
|
642
|
+
for elements in csv_reader:
|
|
643
|
+
if len(elements) == 0:
|
|
644
|
+
break
|
|
645
|
+
# Normalize empty strings to None for parity with the slow reader.
|
|
646
|
+
elements = [e if e != "" else None for e in elements]
|
|
647
|
+
if conf.del_empty_cols and len(self.blank_cols) > 0:
|
|
648
|
+
blanks = copy.copy(self.blank_cols)
|
|
649
|
+
while len(blanks) > 0:
|
|
650
|
+
b = blanks.pop()
|
|
651
|
+
del elements[b]
|
|
652
|
+
yield elements
|
|
653
|
+
finally:
|
|
654
|
+
f.close()
|
|
655
|
+
|
|
656
|
+
def _slow_reader(self, conf: Any) -> Any:
|
|
657
|
+
"""Read using the character-at-a-time state machine (fallback for non-standard formats)."""
|
|
616
658
|
f = self.openclean("rt")
|
|
617
659
|
line_no = 0
|
|
618
660
|
try:
|
execsql/metacommands/__init__.py
CHANGED
|
@@ -168,6 +168,11 @@ from execsql.metacommands.script_ext import (
|
|
|
168
168
|
x_extendscript_sql,
|
|
169
169
|
x_executescript,
|
|
170
170
|
)
|
|
171
|
+
from execsql.metacommands.upsert import (
|
|
172
|
+
x_pg_upsert,
|
|
173
|
+
x_pg_upsert_check,
|
|
174
|
+
x_pg_upsert_qa,
|
|
175
|
+
)
|
|
171
176
|
from execsql.metacommands.system import (
|
|
172
177
|
x_system_cmd,
|
|
173
178
|
x_email,
|
|
@@ -401,6 +406,10 @@ __all__ = [
|
|
|
401
406
|
"x_write_warnings",
|
|
402
407
|
"x_gui_level",
|
|
403
408
|
"x_execute",
|
|
409
|
+
# upsert handlers
|
|
410
|
+
"x_pg_upsert",
|
|
411
|
+
"x_pg_upsert_check",
|
|
412
|
+
"x_pg_upsert_qa",
|
|
404
413
|
# regex helpers
|
|
405
414
|
"ins_rxs",
|
|
406
415
|
"ins_quoted_rx",
|
execsql/metacommands/dispatch.py
CHANGED
|
@@ -168,6 +168,11 @@ from execsql.metacommands.script_ext import (
|
|
|
168
168
|
x_extendscript_metacommand,
|
|
169
169
|
x_extendscript_sql,
|
|
170
170
|
)
|
|
171
|
+
from execsql.metacommands.upsert import (
|
|
172
|
+
x_pg_upsert,
|
|
173
|
+
x_pg_upsert_check,
|
|
174
|
+
x_pg_upsert_qa,
|
|
175
|
+
)
|
|
171
176
|
from execsql.metacommands.system import (
|
|
172
177
|
x_cancel_halt,
|
|
173
178
|
x_cancel_halt_email,
|
|
@@ -2142,6 +2147,31 @@ def build_dispatch_table() -> MetaCommandList:
|
|
|
2142
2147
|
x_write,
|
|
2143
2148
|
)
|
|
2144
2149
|
|
|
2150
|
+
# ------------------------------------------------------------------
|
|
2151
|
+
# PG_UPSERT — pg-upsert integration (optional dependency)
|
|
2152
|
+
# ------------------------------------------------------------------
|
|
2153
|
+
# Order matters: CHECK and QA patterns must precede the general pattern
|
|
2154
|
+
# so that "PG_UPSERT CHECK ..." and "PG_UPSERT QA ..." are matched
|
|
2155
|
+
# before "PG_UPSERT FROM ...".
|
|
2156
|
+
mcl.add(
|
|
2157
|
+
r"^\s*PG_UPSERT\s+CHECK\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
|
|
2158
|
+
x_pg_upsert_check,
|
|
2159
|
+
description="PG_UPSERT CHECK",
|
|
2160
|
+
category="action",
|
|
2161
|
+
)
|
|
2162
|
+
mcl.add(
|
|
2163
|
+
r"^\s*PG_UPSERT\s+QA\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
|
|
2164
|
+
x_pg_upsert_qa,
|
|
2165
|
+
description="PG_UPSERT QA",
|
|
2166
|
+
category="action",
|
|
2167
|
+
)
|
|
2168
|
+
mcl.add(
|
|
2169
|
+
r"^\s*PG_UPSERT\s+FROM\s+(?P<staging_schema>\S+)\s+TO\s+(?P<base_schema>\S+)\s+TABLES\s+(?P<tail>.+)$",
|
|
2170
|
+
x_pg_upsert,
|
|
2171
|
+
description="PG_UPSERT",
|
|
2172
|
+
category="action",
|
|
2173
|
+
)
|
|
2174
|
+
|
|
2145
2175
|
# ------------------------------------------------------------------
|
|
2146
2176
|
# SUB (top-level variable assignment — kept near end so more specific
|
|
2147
2177
|
# SUB_* patterns above take precedence)
|
|
@@ -241,6 +241,10 @@ def x_cd(**kwargs: Any) -> None:
|
|
|
241
241
|
os.chdir(new_dir)
|
|
242
242
|
script, lno = current_script_line()
|
|
243
243
|
_state.exec_log.log_status_info(f"Current directory changed to {new_dir} at line {lno} of {script}")
|
|
244
|
+
if _state.subvars is not None:
|
|
245
|
+
from execsql.script.engine import set_static_system_vars
|
|
246
|
+
|
|
247
|
+
set_static_system_vars()
|
|
244
248
|
return None
|
|
245
249
|
|
|
246
250
|
|
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
"""PG_UPSERT metacommand handler.
|
|
2
|
+
|
|
3
|
+
Integrates pg-upsert (https://pg-upsert.readthedocs.io/) as an optional
|
|
4
|
+
dependency, providing QA-checked, FK-dependency-ordered upserts from a
|
|
5
|
+
staging schema to a base schema on PostgreSQL.
|
|
6
|
+
|
|
7
|
+
Requires: ``pip install execsql2[upsert]``
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import logging
|
|
14
|
+
import re
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import execsql.state as _state
|
|
18
|
+
from execsql.exceptions import ErrInfo
|
|
19
|
+
from execsql.types import dbt_postgres
|
|
20
|
+
from execsql.utils.errors import exception_desc
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---------------------------------------------------------------------------
|
|
24
|
+
# Keyword parser for the trailing portion after TABLES
|
|
25
|
+
# ---------------------------------------------------------------------------
|
|
26
|
+
|
|
27
|
+
_KW_METHOD = re.compile(r"\bMETHOD\s+(upsert|update|insert)\b", re.IGNORECASE)
|
|
28
|
+
_KW_EXCLUDE = re.compile(
|
|
29
|
+
r"\bEXCLUDE\s+([\w\s,]+?)(?=\s+(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE_NULL|LOGFILE)\b|\s*$)",
|
|
30
|
+
re.IGNORECASE,
|
|
31
|
+
)
|
|
32
|
+
_KW_EXCLUDE_NULL = re.compile(
|
|
33
|
+
r"\bEXCLUDE_NULL\s+([\w\s,]+?)(?=\s+(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE|LOGFILE)\b|\s*$)",
|
|
34
|
+
re.IGNORECASE,
|
|
35
|
+
)
|
|
36
|
+
_KW_COMMIT = re.compile(r"\bCOMMIT\b", re.IGNORECASE)
|
|
37
|
+
_KW_INTERACTIVE = re.compile(r"\bINTERACTIVE\b", re.IGNORECASE)
|
|
38
|
+
_KW_COMPACT = re.compile(r"\bCOMPACT\b", re.IGNORECASE)
|
|
39
|
+
_KW_LOGFILE = re.compile(r"""\bLOGFILE\s+(?:"([^"]+)"|'([^']+)'|(\S+))""", re.IGNORECASE)
|
|
40
|
+
|
|
41
|
+
# All recognized keywords — used to split table names from options.
|
|
42
|
+
_ALL_KEYWORDS = re.compile(
|
|
43
|
+
r"\b(?:METHOD|COMMIT|INTERACTIVE|COMPACT|EXCLUDE_NULL|EXCLUDE|LOGFILE)\b",
|
|
44
|
+
re.IGNORECASE,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _parse_tables_and_options(tail: str) -> dict[str, Any]:
|
|
49
|
+
"""Parse the trailing text after ``TABLES`` into table names and options.
|
|
50
|
+
|
|
51
|
+
Parameters
|
|
52
|
+
----------
|
|
53
|
+
tail:
|
|
54
|
+
Everything captured after the ``TABLES`` keyword in the regex.
|
|
55
|
+
|
|
56
|
+
Returns
|
|
57
|
+
-------
|
|
58
|
+
dict with keys: tables, method, commit, interactive, compact,
|
|
59
|
+
exclude_cols, exclude_null_check_cols.
|
|
60
|
+
"""
|
|
61
|
+
# Split at the first keyword to isolate the table list.
|
|
62
|
+
kw_match = _ALL_KEYWORDS.search(tail)
|
|
63
|
+
if kw_match:
|
|
64
|
+
table_part = tail[: kw_match.start()]
|
|
65
|
+
opts_part = tail[kw_match.start() :]
|
|
66
|
+
else:
|
|
67
|
+
table_part = tail
|
|
68
|
+
opts_part = ""
|
|
69
|
+
|
|
70
|
+
tables = [t.strip() for t in table_part.split(",") if t.strip()]
|
|
71
|
+
|
|
72
|
+
method = "upsert"
|
|
73
|
+
m = _KW_METHOD.search(opts_part)
|
|
74
|
+
if m:
|
|
75
|
+
method = m.group(1).lower()
|
|
76
|
+
|
|
77
|
+
exclude_cols: list[str] = []
|
|
78
|
+
m = _KW_EXCLUDE.search(opts_part)
|
|
79
|
+
if m:
|
|
80
|
+
exclude_cols = [c.strip() for c in m.group(1).split(",") if c.strip()]
|
|
81
|
+
|
|
82
|
+
exclude_null: list[str] = []
|
|
83
|
+
m = _KW_EXCLUDE_NULL.search(opts_part)
|
|
84
|
+
if m:
|
|
85
|
+
exclude_null = [c.strip() for c in m.group(1).split(",") if c.strip()]
|
|
86
|
+
|
|
87
|
+
logfile: str | None = None
|
|
88
|
+
m = _KW_LOGFILE.search(opts_part)
|
|
89
|
+
if m:
|
|
90
|
+
logfile = m.group(1) or m.group(2) or m.group(3)
|
|
91
|
+
|
|
92
|
+
return {
|
|
93
|
+
"tables": tables,
|
|
94
|
+
"method": method,
|
|
95
|
+
"commit": bool(_KW_COMMIT.search(opts_part)),
|
|
96
|
+
"interactive": bool(_KW_INTERACTIVE.search(opts_part)),
|
|
97
|
+
"compact": bool(_KW_COMPACT.search(opts_part)),
|
|
98
|
+
"exclude_cols": exclude_cols,
|
|
99
|
+
"exclude_null_check_cols": exclude_null,
|
|
100
|
+
"logfile": logfile,
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# ---------------------------------------------------------------------------
|
|
105
|
+
# Logging bridge: pg_upsert.display → execsql exec_log
|
|
106
|
+
# ---------------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class _ExecLogHandler(logging.Handler):
|
|
110
|
+
"""Route pg_upsert's plain-text file logger to execsql's exec_log."""
|
|
111
|
+
|
|
112
|
+
def __init__(self, exec_log: Any) -> None:
|
|
113
|
+
super().__init__()
|
|
114
|
+
self._exec_log = exec_log
|
|
115
|
+
|
|
116
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
117
|
+
self._exec_log.log_user_msg(self.format(record))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class _FileWriterHandler(logging.Handler):
|
|
121
|
+
"""Route pg_upsert log messages through execsql's async FileWriter.
|
|
122
|
+
|
|
123
|
+
This ensures that pg-upsert log output and execsql WRITE TEE output
|
|
124
|
+
arrive in the same order they were issued, since both go through the
|
|
125
|
+
same FileWriter queue.
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
def __init__(self, filename: str) -> None:
|
|
129
|
+
super().__init__()
|
|
130
|
+
self._filename = filename
|
|
131
|
+
|
|
132
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
133
|
+
from execsql.utils.fileio import filewriter_write
|
|
134
|
+
|
|
135
|
+
filewriter_write(self._filename, self.format(record) + "\n")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# ---------------------------------------------------------------------------
|
|
139
|
+
# Result → substitution variables
|
|
140
|
+
# ---------------------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _set_subvars(result: Any) -> None:
|
|
144
|
+
"""Populate ``$PG_UPSERT_*`` substitution variables from an UpsertResult."""
|
|
145
|
+
sv = _state.subvars.add_substitution
|
|
146
|
+
sv("$PG_UPSERT_QA_PASSED", str(result.qa_passed).upper())
|
|
147
|
+
sv("$PG_UPSERT_ROWS_UPDATED", str(result.total_updated))
|
|
148
|
+
sv("$PG_UPSERT_ROWS_INSERTED", str(result.total_inserted))
|
|
149
|
+
sv("$PG_UPSERT_COMMITTED", str(result.committed).upper())
|
|
150
|
+
sv("$PG_UPSERT_STAGING_SCHEMA", result.staging_schema)
|
|
151
|
+
sv("$PG_UPSERT_BASE_SCHEMA", result.base_schema)
|
|
152
|
+
sv("$PG_UPSERT_TABLES", ", ".join(t.table_name for t in result.tables))
|
|
153
|
+
sv("$PG_UPSERT_METHOD", result.upsert_method)
|
|
154
|
+
sv("$PG_UPSERT_DURATION", str(result.duration_seconds))
|
|
155
|
+
sv("$PG_UPSERT_STARTED_AT", result.started_at)
|
|
156
|
+
sv("$PG_UPSERT_FINISHED_AT", result.finished_at)
|
|
157
|
+
sv("$PG_UPSERT_RESULT_JSON", json.dumps(result.to_dict(), separators=(",", ":")))
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _qa_failure_msg(result: Any) -> str:
|
|
161
|
+
"""Build a concise QA failure message listing which tables failed."""
|
|
162
|
+
failed = [t.table_name for t in result.tables if not t.qa_passed]
|
|
163
|
+
if failed:
|
|
164
|
+
return f"PG_UPSERT QA failed for: {', '.join(failed)}"
|
|
165
|
+
return "PG_UPSERT QA checks failed."
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# ---------------------------------------------------------------------------
|
|
169
|
+
# Import guard + helpers
|
|
170
|
+
# ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _require_pg_upsert() -> None:
|
|
174
|
+
"""Raise ErrInfo if pg_upsert is not installed."""
|
|
175
|
+
try:
|
|
176
|
+
import pg_upsert # noqa: F401
|
|
177
|
+
except ImportError as exc:
|
|
178
|
+
raise ErrInfo(
|
|
179
|
+
"exception",
|
|
180
|
+
other_msg=("PG_UPSERT requires the pg-upsert package. Install it with: pip install execsql2[upsert]"),
|
|
181
|
+
) from exc
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _require_postgres(db: Any, metacommandline: str | None) -> None:
|
|
185
|
+
"""Raise ErrInfo if the current connection is not PostgreSQL."""
|
|
186
|
+
if db.type != dbt_postgres:
|
|
187
|
+
raise ErrInfo(
|
|
188
|
+
"cmd",
|
|
189
|
+
command_text=metacommandline,
|
|
190
|
+
other_msg=(f"PG_UPSERT requires a PostgreSQL connection. Current DBMS: {db.type.dbms_id}"),
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _build_result_from_qa_errors(ups: Any) -> Any:
|
|
195
|
+
"""Build an UpsertResult from ``ups.qa_errors`` after a QA/CHECK run."""
|
|
196
|
+
from pg_upsert.models import TableResult, UpsertResult
|
|
197
|
+
|
|
198
|
+
table_results: dict[str, Any] = {}
|
|
199
|
+
for table_name in ups.tables:
|
|
200
|
+
table_results[table_name] = TableResult(table_name=table_name)
|
|
201
|
+
for err in ups.qa_errors:
|
|
202
|
+
if err.table in table_results:
|
|
203
|
+
table_results[err.table].qa_errors.append(err)
|
|
204
|
+
return UpsertResult(
|
|
205
|
+
tables=list(table_results.values()),
|
|
206
|
+
committed=False,
|
|
207
|
+
staging_schema=ups.staging_schema,
|
|
208
|
+
base_schema=ups.base_schema,
|
|
209
|
+
upsert_method=ups.upsert_method,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _create_pgupsert(
|
|
214
|
+
db: Any,
|
|
215
|
+
staging_schema: str,
|
|
216
|
+
base_schema: str,
|
|
217
|
+
opts: dict[str, Any],
|
|
218
|
+
) -> Any:
|
|
219
|
+
"""Create and return a PgUpsert instance with execsql's connection."""
|
|
220
|
+
from pg_upsert import PgUpsert
|
|
221
|
+
|
|
222
|
+
ui_mode = "tkinter"
|
|
223
|
+
if _state.conf:
|
|
224
|
+
ui_mode = _state.conf.gui_framework
|
|
225
|
+
|
|
226
|
+
ups = PgUpsert(
|
|
227
|
+
conn=db.conn,
|
|
228
|
+
staging_schema=staging_schema,
|
|
229
|
+
base_schema=base_schema,
|
|
230
|
+
tables=opts["tables"],
|
|
231
|
+
do_commit=opts["commit"],
|
|
232
|
+
interactive=opts["interactive"],
|
|
233
|
+
compact=opts["compact"],
|
|
234
|
+
upsert_method=opts["method"],
|
|
235
|
+
exclude_cols=opts["exclude_cols"],
|
|
236
|
+
exclude_null_check_cols=opts["exclude_null_check_cols"],
|
|
237
|
+
ui_mode=ui_mode,
|
|
238
|
+
)
|
|
239
|
+
return ups
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _attach_log_handlers(
|
|
243
|
+
logfile: str | None = None,
|
|
244
|
+
) -> tuple[list[logging.Logger], list[logging.Handler], dict[str, int]]:
|
|
245
|
+
"""Attach logging handlers to pg_upsert loggers.
|
|
246
|
+
|
|
247
|
+
Always attaches the exec_log bridge to ``pg_upsert.display``.
|
|
248
|
+
If *logfile* is given, also attaches a FileHandler (append mode) to both
|
|
249
|
+
``pg_upsert`` and ``pg_upsert.display`` — matching pg-upsert CLI behavior.
|
|
250
|
+
|
|
251
|
+
Returns (loggers, handlers) so the caller can detach in a finally block.
|
|
252
|
+
"""
|
|
253
|
+
display_logger = logging.getLogger("pg_upsert.display")
|
|
254
|
+
# pg-upsert's display logger has propagate=False and level=NOTSET, which
|
|
255
|
+
# gives it an effective level of WARNING (inherited from root). Its messages
|
|
256
|
+
# are logged at INFO, so we must explicitly lower the level.
|
|
257
|
+
prev_display_level = display_logger.level
|
|
258
|
+
if display_logger.getEffectiveLevel() > logging.INFO:
|
|
259
|
+
display_logger.setLevel(logging.INFO)
|
|
260
|
+
|
|
261
|
+
exec_handler = _ExecLogHandler(_state.exec_log)
|
|
262
|
+
display_logger.addHandler(exec_handler)
|
|
263
|
+
|
|
264
|
+
loggers: list[logging.Logger] = [display_logger]
|
|
265
|
+
handlers: list[logging.Handler] = [exec_handler]
|
|
266
|
+
prev_levels: dict[str, int] = {"pg_upsert.display": prev_display_level}
|
|
267
|
+
|
|
268
|
+
if logfile:
|
|
269
|
+
file_handler = _FileWriterHandler(logfile)
|
|
270
|
+
file_handler.setFormatter(logging.Formatter("%(message)s"))
|
|
271
|
+
# Attach to both loggers, same as pg-upsert CLI does
|
|
272
|
+
main_logger = logging.getLogger("pg_upsert")
|
|
273
|
+
prev_levels["pg_upsert"] = main_logger.level
|
|
274
|
+
if main_logger.getEffectiveLevel() > logging.INFO:
|
|
275
|
+
main_logger.setLevel(logging.INFO)
|
|
276
|
+
main_logger.addHandler(file_handler)
|
|
277
|
+
display_logger.addHandler(file_handler)
|
|
278
|
+
loggers.append(main_logger)
|
|
279
|
+
handlers.append(file_handler)
|
|
280
|
+
|
|
281
|
+
return loggers, handlers, prev_levels
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _detach_log_handlers(
|
|
285
|
+
loggers: list[logging.Logger],
|
|
286
|
+
handlers: list[logging.Handler],
|
|
287
|
+
prev_levels: dict[str, int],
|
|
288
|
+
) -> None:
|
|
289
|
+
"""Remove all handlers added by ``_attach_log_handlers``."""
|
|
290
|
+
for handler in handlers:
|
|
291
|
+
for lgr in loggers:
|
|
292
|
+
lgr.removeHandler(handler)
|
|
293
|
+
if hasattr(handler, "close"):
|
|
294
|
+
handler.close()
|
|
295
|
+
# Restore original logger levels
|
|
296
|
+
for name, level in prev_levels.items():
|
|
297
|
+
logging.getLogger(name).setLevel(level)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _run_with_autocommit_guard(db: Any, fn: Any) -> Any:
|
|
301
|
+
"""Temporarily disable autocommit, run *fn*, then restore."""
|
|
302
|
+
was_autocommit = db.autocommit
|
|
303
|
+
if was_autocommit:
|
|
304
|
+
db.autocommit_off()
|
|
305
|
+
try:
|
|
306
|
+
return fn()
|
|
307
|
+
finally:
|
|
308
|
+
if was_autocommit:
|
|
309
|
+
db.autocommit_on()
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _handle_pg_upsert_errors(fn: Any, metacommandline: str | None) -> Any:
|
|
313
|
+
"""Run *fn*, translating pg-upsert exceptions to ErrInfo."""
|
|
314
|
+
from pg_upsert import UserCancelledError
|
|
315
|
+
|
|
316
|
+
try:
|
|
317
|
+
return fn()
|
|
318
|
+
except UserCancelledError as exc:
|
|
319
|
+
raise ErrInfo(
|
|
320
|
+
"cmd",
|
|
321
|
+
command_text=metacommandline,
|
|
322
|
+
other_msg="PG_UPSERT cancelled by user.",
|
|
323
|
+
) from exc
|
|
324
|
+
except ErrInfo:
|
|
325
|
+
raise
|
|
326
|
+
except Exception as exc:
|
|
327
|
+
raise ErrInfo(
|
|
328
|
+
"exception",
|
|
329
|
+
exception_msg=exception_desc(),
|
|
330
|
+
other_msg="PG_UPSERT failed unexpectedly.",
|
|
331
|
+
) from exc
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
# ---------------------------------------------------------------------------
|
|
335
|
+
# Metacommand handlers
|
|
336
|
+
# ---------------------------------------------------------------------------
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def x_pg_upsert(**kwargs: Any) -> None:
|
|
340
|
+
"""PG_UPSERT FROM <staging> TO <base> TABLES <t1>, <t2> [options]
|
|
341
|
+
|
|
342
|
+
Full pipeline: QA checks → upsert → optional commit.
|
|
343
|
+
"""
|
|
344
|
+
_require_pg_upsert()
|
|
345
|
+
db = _state.dbs.current()
|
|
346
|
+
metacommandline = kwargs.get("metacommandline")
|
|
347
|
+
_require_postgres(db, metacommandline)
|
|
348
|
+
|
|
349
|
+
staging = kwargs["staging_schema"]
|
|
350
|
+
base = kwargs["base_schema"]
|
|
351
|
+
opts = _parse_tables_and_options(kwargs["tail"])
|
|
352
|
+
|
|
353
|
+
ups = _create_pgupsert(db, staging, base, opts)
|
|
354
|
+
loggers, handlers, prev_levels = _attach_log_handlers(opts.get("logfile"))
|
|
355
|
+
|
|
356
|
+
try:
|
|
357
|
+
result = _run_with_autocommit_guard(
|
|
358
|
+
db,
|
|
359
|
+
lambda: _handle_pg_upsert_errors(ups.run, metacommandline),
|
|
360
|
+
)
|
|
361
|
+
finally:
|
|
362
|
+
_detach_log_handlers(loggers, handlers, prev_levels)
|
|
363
|
+
|
|
364
|
+
_set_subvars(result)
|
|
365
|
+
|
|
366
|
+
if not result.qa_passed:
|
|
367
|
+
raise ErrInfo(
|
|
368
|
+
"cmd",
|
|
369
|
+
command_text=metacommandline,
|
|
370
|
+
other_msg=_qa_failure_msg(result),
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def x_pg_upsert_qa(**kwargs: Any) -> None:
|
|
375
|
+
"""PG_UPSERT QA FROM <staging> TO <base> TABLES <t1>, <t2> [options]
|
|
376
|
+
|
|
377
|
+
QA-only mode: run all QA checks without upserting.
|
|
378
|
+
"""
|
|
379
|
+
_require_pg_upsert()
|
|
380
|
+
db = _state.dbs.current()
|
|
381
|
+
metacommandline = kwargs.get("metacommandline")
|
|
382
|
+
_require_postgres(db, metacommandline)
|
|
383
|
+
|
|
384
|
+
staging = kwargs["staging_schema"]
|
|
385
|
+
base = kwargs["base_schema"]
|
|
386
|
+
opts = _parse_tables_and_options(kwargs["tail"])
|
|
387
|
+
opts["commit"] = False # QA-only never commits
|
|
388
|
+
|
|
389
|
+
ups = _create_pgupsert(db, staging, base, opts)
|
|
390
|
+
loggers, handlers, prev_levels = _attach_log_handlers(opts.get("logfile"))
|
|
391
|
+
|
|
392
|
+
try:
|
|
393
|
+
_run_with_autocommit_guard(
|
|
394
|
+
db,
|
|
395
|
+
lambda: _handle_pg_upsert_errors(ups.qa_all, metacommandline),
|
|
396
|
+
)
|
|
397
|
+
finally:
|
|
398
|
+
_detach_log_handlers(loggers, handlers, prev_levels)
|
|
399
|
+
|
|
400
|
+
result = _build_result_from_qa_errors(ups)
|
|
401
|
+
_set_subvars(result)
|
|
402
|
+
|
|
403
|
+
if not result.qa_passed:
|
|
404
|
+
raise ErrInfo(
|
|
405
|
+
"cmd",
|
|
406
|
+
command_text=metacommandline,
|
|
407
|
+
other_msg=_qa_failure_msg(result),
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def x_pg_upsert_check(**kwargs: Any) -> None:
|
|
412
|
+
"""PG_UPSERT CHECK FROM <staging> TO <base> TABLES <t1>, <t2>
|
|
413
|
+
|
|
414
|
+
Schema check only: column existence + type mismatch.
|
|
415
|
+
"""
|
|
416
|
+
_require_pg_upsert()
|
|
417
|
+
db = _state.dbs.current()
|
|
418
|
+
metacommandline = kwargs.get("metacommandline")
|
|
419
|
+
_require_postgres(db, metacommandline)
|
|
420
|
+
|
|
421
|
+
staging = kwargs["staging_schema"]
|
|
422
|
+
base = kwargs["base_schema"]
|
|
423
|
+
opts = _parse_tables_and_options(kwargs["tail"])
|
|
424
|
+
opts["commit"] = False
|
|
425
|
+
|
|
426
|
+
ups = _create_pgupsert(db, staging, base, opts)
|
|
427
|
+
loggers, handlers, prev_levels = _attach_log_handlers(opts.get("logfile"))
|
|
428
|
+
|
|
429
|
+
try:
|
|
430
|
+
_run_with_autocommit_guard(
|
|
431
|
+
db,
|
|
432
|
+
lambda: _handle_pg_upsert_errors(
|
|
433
|
+
lambda: ups.qa_column_existence().qa_type_mismatch(),
|
|
434
|
+
metacommandline,
|
|
435
|
+
),
|
|
436
|
+
)
|
|
437
|
+
finally:
|
|
438
|
+
_detach_log_handlers(loggers, handlers, prev_levels)
|
|
439
|
+
|
|
440
|
+
result = _build_result_from_qa_errors(ups)
|
|
441
|
+
_set_subvars(result)
|
|
442
|
+
|
|
443
|
+
if not result.qa_passed:
|
|
444
|
+
raise ErrInfo(
|
|
445
|
+
"cmd",
|
|
446
|
+
command_text=metacommandline,
|
|
447
|
+
other_msg=_qa_failure_msg(result),
|
|
448
|
+
)
|
execsql/script/__init__.py
CHANGED
|
@@ -63,6 +63,8 @@ from execsql.script.engine import (
|
|
|
63
63
|
read_sqlfile,
|
|
64
64
|
read_sqlstring,
|
|
65
65
|
runscripts,
|
|
66
|
+
set_dynamic_system_vars,
|
|
67
|
+
set_static_system_vars,
|
|
66
68
|
set_system_vars,
|
|
67
69
|
substitute_vars,
|
|
68
70
|
)
|
|
@@ -86,6 +88,8 @@ __all__ = [
|
|
|
86
88
|
"CommandListUntilLoop",
|
|
87
89
|
"ScriptFile",
|
|
88
90
|
"ScriptExecSpec",
|
|
91
|
+
"set_dynamic_system_vars",
|
|
92
|
+
"set_static_system_vars",
|
|
89
93
|
"set_system_vars",
|
|
90
94
|
"substitute_vars",
|
|
91
95
|
"runscripts",
|
execsql/script/engine.py
CHANGED
|
@@ -704,9 +704,45 @@ class ScriptExecSpec:
|
|
|
704
704
|
# ---------------------------------------------------------------------------
|
|
705
705
|
|
|
706
706
|
|
|
707
|
-
def
|
|
708
|
-
"""
|
|
709
|
-
|
|
707
|
+
def set_static_system_vars() -> None:
|
|
708
|
+
"""Set system substitution variables that only change on CONNECT or CHDIR.
|
|
709
|
+
|
|
710
|
+
Called once before the execution loop. These values are expensive to compute
|
|
711
|
+
(filesystem syscalls, database pool lookups) but rarely change — only on
|
|
712
|
+
``CONNECT``, ``USE``, or ``CHDIR`` metacommands. The ``runscripts()`` loop
|
|
713
|
+
calls this once up front; metacommand handlers that change the connection or
|
|
714
|
+
working directory should call it again afterward.
|
|
715
|
+
"""
|
|
716
|
+
import random
|
|
717
|
+
|
|
718
|
+
cwd = str(Path(".").resolve())
|
|
719
|
+
_state.subvars.add_substitution("$CURRENT_DIR", cwd)
|
|
720
|
+
_state.subvars.add_substitution("$CURRENT_PATH", cwd + os.sep)
|
|
721
|
+
_state.subvars.add_substitution("$CURRENT_ALIAS", _state.dbs.current_alias())
|
|
722
|
+
db = _state.dbs.current()
|
|
723
|
+
_state.subvars.add_substitution("$DB_USER", db.user if db.user else "")
|
|
724
|
+
_state.subvars.add_substitution(
|
|
725
|
+
"$DB_SERVER",
|
|
726
|
+
db.server_name if db.server_name else "",
|
|
727
|
+
)
|
|
728
|
+
_state.subvars.add_substitution("$DB_NAME", db.db_name)
|
|
729
|
+
_state.subvars.add_substitution("$DB_NEED_PWD", "TRUE" if db.need_passwd else "FALSE")
|
|
730
|
+
_state.subvars.add_substitution("$VERSION1", str(_state.primary_vno))
|
|
731
|
+
_state.subvars.add_substitution("$VERSION2", str(_state.secondary_vno))
|
|
732
|
+
_state.subvars.add_substitution("$VERSION3", str(_state.tertiary_vno))
|
|
733
|
+
# Register lazy providers for $RANDOM and $UUID — computed only when referenced.
|
|
734
|
+
_state.subvars.register_lazy("$random", lambda: str(random.random()))
|
|
735
|
+
_state.subvars.register_lazy("$uuid", lambda: str(uuid.uuid4()))
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def set_dynamic_system_vars() -> None:
|
|
739
|
+
"""Refresh system substitution variables that change every statement.
|
|
740
|
+
|
|
741
|
+
Called once per statement in the execution loop. Includes cheap boolean-to-string
|
|
742
|
+
conversions for halt states and autocommit (which can change on any CONFIG or
|
|
743
|
+
AUTOCOMMIT metacommand) plus ``$TIMER`` and lazy-variable cache reset.
|
|
744
|
+
"""
|
|
745
|
+
# Halt/config state vars — cheap to set, can change on any CONFIG metacommand.
|
|
710
746
|
_state.subvars.add_substitution("$CANCEL_HALT_STATE", "ON" if _state.status.cancel_halt else "OFF")
|
|
711
747
|
_state.subvars.add_substitution("$ERROR_HALT_STATE", "ON" if _state.status.halt_on_err else "OFF")
|
|
712
748
|
_state.subvars.add_substitution(
|
|
@@ -718,27 +754,22 @@ def set_system_vars() -> None:
|
|
|
718
754
|
"ON" if _state.conf.gui_wait_on_error_halt else "OFF",
|
|
719
755
|
)
|
|
720
756
|
_state.subvars.add_substitution("$CONSOLE_WAIT_WHEN_DONE_STATE", "ON" if _state.conf.gui_wait_on_exit else "OFF")
|
|
721
|
-
# $CURRENT_TIME is set per-statement in run_and_increment() for accuracy.
|
|
722
|
-
_state.subvars.add_substitution("$CURRENT_DIR", str(Path(".").resolve()))
|
|
723
|
-
_state.subvars.add_substitution("$CURRENT_PATH", str(Path(".").resolve()) + os.sep)
|
|
724
|
-
_state.subvars.add_substitution("$CURRENT_ALIAS", _state.dbs.current_alias())
|
|
725
757
|
db = _state.dbs.current()
|
|
726
758
|
_state.subvars.add_substitution("$AUTOCOMMIT_STATE", "ON" if db.autocommit else "OFF")
|
|
759
|
+
# $CURRENT_TIME is set per-statement in run_and_increment() for accuracy.
|
|
727
760
|
_state.subvars.add_substitution("$TIMER", str(datetime.timedelta(seconds=_state.timer.elapsed())))
|
|
728
|
-
_state.subvars.
|
|
729
|
-
_state.subvars.add_substitution(
|
|
730
|
-
"$DB_SERVER",
|
|
731
|
-
db.server_name if db.server_name else "",
|
|
732
|
-
)
|
|
733
|
-
_state.subvars.add_substitution("$DB_NAME", db.db_name)
|
|
734
|
-
_state.subvars.add_substitution("$DB_NEED_PWD", "TRUE" if db.need_passwd else "FALSE")
|
|
735
|
-
import random
|
|
761
|
+
_state.subvars.clear_lazy_cache()
|
|
736
762
|
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
763
|
+
|
|
764
|
+
def set_system_vars() -> None:
|
|
765
|
+
"""Refresh all built-in system substitution variables.
|
|
766
|
+
|
|
767
|
+
Convenience wrapper that calls both :func:`set_static_system_vars` and
|
|
768
|
+
:func:`set_dynamic_system_vars`. Retained for backward compatibility with
|
|
769
|
+
tests and any external callers.
|
|
770
|
+
"""
|
|
771
|
+
set_static_system_vars()
|
|
772
|
+
set_dynamic_system_vars()
|
|
742
773
|
|
|
743
774
|
|
|
744
775
|
_MAX_SUBSTITUTION_DEPTH = 100
|
|
@@ -779,11 +810,12 @@ def substitute_vars(command_str: str, localvars: SubVarSet | None = None) -> str
|
|
|
779
810
|
|
|
780
811
|
def runscripts() -> None:
|
|
781
812
|
"""Drive execution until the command-list stack is empty."""
|
|
782
|
-
#
|
|
783
|
-
#
|
|
813
|
+
# Set static vars once before the loop; they are refreshed by metacommand
|
|
814
|
+
# handlers (CONNECT, CONFIG, AUTOCOMMIT, CHDIR) when state changes.
|
|
815
|
+
set_static_system_vars()
|
|
784
816
|
while len(_state.commandliststack) > 0:
|
|
785
817
|
current_cmds = _state.commandliststack[-1]
|
|
786
|
-
|
|
818
|
+
set_dynamic_system_vars()
|
|
787
819
|
try:
|
|
788
820
|
current_cmds.run_next()
|
|
789
821
|
except StopIteration:
|
execsql/script/variables.py
CHANGED
|
@@ -89,6 +89,7 @@ class SubVarSet:
|
|
|
89
89
|
# compatibility with external code.
|
|
90
90
|
def __init__(self) -> None:
|
|
91
91
|
self._subs_dict: dict[str, Any] = {}
|
|
92
|
+
self._lazy_providers: dict[str, Any] = {}
|
|
92
93
|
self.prefix_list: list[str] = ["$", "&", "@"]
|
|
93
94
|
# Don't construct/compile on init because deepcopy() can't handle compiled regexes.
|
|
94
95
|
self.var_rx = None
|
|
@@ -120,6 +121,30 @@ class SubVarSet:
|
|
|
120
121
|
if not self.var_name_ok(varname.lower()):
|
|
121
122
|
raise ErrInfo("error", other_msg=f"Invalid variable name ({varname}) in this context.")
|
|
122
123
|
|
|
124
|
+
def register_lazy(self, varname: str, provider: Any) -> None:
|
|
125
|
+
"""Register a lazy variable whose value is computed on first access per cycle.
|
|
126
|
+
|
|
127
|
+
The *provider* callable is invoked only when the variable is actually
|
|
128
|
+
referenced (via :meth:`substitute`, :meth:`varvalue`, etc.). The result
|
|
129
|
+
is cached in ``_subs_dict`` until :meth:`clear_lazy_cache` is called.
|
|
130
|
+
"""
|
|
131
|
+
self.check_var_name(varname)
|
|
132
|
+
self._lazy_providers[varname.lower()] = provider
|
|
133
|
+
|
|
134
|
+
def clear_lazy_cache(self) -> None:
|
|
135
|
+
"""Remove materialized lazy values so they regenerate on next access."""
|
|
136
|
+
for key in self._lazy_providers:
|
|
137
|
+
self._subs_dict.pop(key, None)
|
|
138
|
+
|
|
139
|
+
def _materialize_lazy(self, varname: str) -> str | None:
|
|
140
|
+
"""If *varname* has a lazy provider, invoke it, cache the result, and return it."""
|
|
141
|
+
provider = self._lazy_providers.get(varname)
|
|
142
|
+
if provider is not None:
|
|
143
|
+
value = str(provider())
|
|
144
|
+
self._subs_dict[varname] = value
|
|
145
|
+
return value
|
|
146
|
+
return None
|
|
147
|
+
|
|
123
148
|
def remove_substitution(self, template_str: str) -> None:
|
|
124
149
|
"""Remove the variable named *template_str* from the substitution pool."""
|
|
125
150
|
self.check_var_name(template_str)
|
|
@@ -143,7 +168,11 @@ class SubVarSet:
|
|
|
143
168
|
def varvalue(self, varname: str) -> str | None:
|
|
144
169
|
"""Return the value of *varname*, or ``None`` if it is not defined."""
|
|
145
170
|
self.check_var_name(varname)
|
|
146
|
-
|
|
171
|
+
key = varname.lower()
|
|
172
|
+
val = self._subs_dict.get(key)
|
|
173
|
+
if val is None and key in self._lazy_providers:
|
|
174
|
+
return self._materialize_lazy(key)
|
|
175
|
+
return val
|
|
147
176
|
|
|
148
177
|
def increment_by(self, varname: str, numeric_increment: Any) -> None:
|
|
149
178
|
self.check_var_name(varname)
|
|
@@ -165,13 +194,15 @@ class SubVarSet:
|
|
|
165
194
|
def sub_exists(self, template_str: str) -> bool:
|
|
166
195
|
"""Return True if the variable named *template_str* is defined."""
|
|
167
196
|
self.check_var_name(template_str)
|
|
168
|
-
|
|
197
|
+
key = template_str.lower()
|
|
198
|
+
return key in self._subs_dict or key in self._lazy_providers
|
|
169
199
|
|
|
170
200
|
def merge(self, other_subvars: SubVarSet | None) -> SubVarSet:
|
|
171
201
|
"""Return a new SubVarSet with this object's variables merged with other_subvars."""
|
|
172
202
|
if other_subvars is not None:
|
|
173
203
|
newsubs = SubVarSet()
|
|
174
204
|
newsubs._subs_dict = {**self._subs_dict, **other_subvars._subs_dict}
|
|
205
|
+
newsubs._lazy_providers = {**self._lazy_providers, **other_subvars._lazy_providers}
|
|
175
206
|
newsubs.prefix_list = list(set(self.prefix_list + other_subvars.prefix_list))
|
|
176
207
|
newsubs.compile_var_rx()
|
|
177
208
|
return newsubs
|
|
@@ -201,6 +232,8 @@ class SubVarSet:
|
|
|
201
232
|
m = self._TOKEN_RX.search(command_str)
|
|
202
233
|
while m:
|
|
203
234
|
varname = m.group("varname").lower()
|
|
235
|
+
if varname not in self._subs_dict and varname in self._lazy_providers:
|
|
236
|
+
self._materialize_lazy(varname)
|
|
204
237
|
if varname in self._subs_dict:
|
|
205
238
|
sub = self._subs_dict[varname]
|
|
206
239
|
if sub is None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: execsql2
|
|
3
|
-
Version: 2.12.
|
|
3
|
+
Version: 2.12.5
|
|
4
4
|
Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
|
|
5
5
|
Project-URL: Homepage, https://execsql2.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/geocoug/execsql
|
|
@@ -51,6 +51,7 @@ Requires-Dist: keyring; extra == 'all'
|
|
|
51
51
|
Requires-Dist: odfpy; extra == 'all'
|
|
52
52
|
Requires-Dist: openpyxl; extra == 'all'
|
|
53
53
|
Requires-Dist: oracledb; extra == 'all'
|
|
54
|
+
Requires-Dist: pg-upsert>=1.17.0; extra == 'all'
|
|
54
55
|
Requires-Dist: polars; extra == 'all'
|
|
55
56
|
Requires-Dist: psycopg2-binary; extra == 'all'
|
|
56
57
|
Requires-Dist: pymysql; extra == 'all'
|
|
@@ -107,6 +108,8 @@ Provides-Extra: oracle
|
|
|
107
108
|
Requires-Dist: oracledb; extra == 'oracle'
|
|
108
109
|
Provides-Extra: postgres
|
|
109
110
|
Requires-Dist: psycopg2-binary; extra == 'postgres'
|
|
111
|
+
Provides-Extra: upsert
|
|
112
|
+
Requires-Dist: pg-upsert>=1.17.0; extra == 'upsert'
|
|
110
113
|
Description-Content-Type: text/markdown
|
|
111
114
|
|
|
112
115
|
> [!NOTE]
|
|
@@ -16,7 +16,7 @@ execsql/cli/lint.py,sha256=XWuVcEsheZ8ql48VFWqICWEkAUezB2nIePX6SUiKSg8,16109
|
|
|
16
16
|
execsql/cli/run.py,sha256=JGfndnBnJMkEqbz26pflhEdXDScZNIdGu6b6jTRLYl8,30681
|
|
17
17
|
execsql/db/__init__.py,sha256=jTbuafuKOqYtXFR1wvCOoKK5Lr3l1uErfaIbIr6UywI,1063
|
|
18
18
|
execsql/db/access.py,sha256=L79gUnAnnM9EJ_f4k42jr7DI0qGcKtLOnJTlBC7uPm0,17879
|
|
19
|
-
execsql/db/base.py,sha256=
|
|
19
|
+
execsql/db/base.py,sha256=Gywel7cnF9vWdDGENYZ_psNOg339z3-Mro5pwGyly4I,31256
|
|
20
20
|
execsql/db/dsn.py,sha256=TgQUedVCxnEYA3vae7JETyhb8kK23qkNbPxsMQrNUN8,5368
|
|
21
21
|
execsql/db/duckdb.py,sha256=cKeMwiSlYPyPDn1VLaJgbUD6_IEEaNqtUToLcmq7QaE,3189
|
|
22
22
|
execsql/db/factory.py,sha256=YR1m_c2Hhj_GXVGGkWoSEPZBpgNu_c4FxRnbp-xV3rs,5230
|
|
@@ -30,7 +30,7 @@ execsql/debug/__init__.py,sha256=j6EGUR0dHzUhWN1mHHtf1-Lhjq3Sb1V-vmnq2Ztgj1M,178
|
|
|
30
30
|
execsql/debug/repl.py,sha256=HeQ9emFKUjo7UTouxuTcmpGCTJIR1nOLxKkRJ5mvd2c,16669
|
|
31
31
|
execsql/exporters/__init__.py,sha256=-Cnji-OgodJV8ftcDcOyTof0kQMy9J5kKVC8GVFpc3o,670
|
|
32
32
|
execsql/exporters/base.py,sha256=W9USFyk_2eztjJ51X6CJh7-chE1i3cSx-STOtbHXCNI,6373
|
|
33
|
-
execsql/exporters/delimited.py,sha256=
|
|
33
|
+
execsql/exporters/delimited.py,sha256=URvEQo1IRF_tfdVHL3uBwEonihC-XfDm0f1argQPf4M,32088
|
|
34
34
|
execsql/exporters/duckdb.py,sha256=Wc9I5uiV4MzmVQzCX-vgVHQUL7U3ZWdOkFVFWBv5SXM,2911
|
|
35
35
|
execsql/exporters/feather.py,sha256=w2qZAnewzeiRMnmPXECvkgD-6KtyxaiQwjokRT7Awrc,4167
|
|
36
36
|
execsql/exporters/html.py,sha256=ISQBOr7AJ5koKlebXSvWqzEvl1nXriCRGeKmk-bzkrc,9335
|
|
@@ -61,25 +61,26 @@ execsql/importers/csv.py,sha256=Mu848WNzuhVO1ade-WurPyxqGOuVNRO8UwRF3-bav_I,4845
|
|
|
61
61
|
execsql/importers/feather.py,sha256=g2B69d2uv9vmnXcmjFyTVsMP40LYEzFYkhk3gD26mGw,1900
|
|
62
62
|
execsql/importers/ods.py,sha256=MJsdsjropzCvxAA3DDZfAL_AnmZ4yij7DnrjGyDJqHQ,2843
|
|
63
63
|
execsql/importers/xls.py,sha256=e0Zfe47ZiCpA1Ae3XDJ1ko3sCiH3-8U6XLKi6NvD0jQ,3683
|
|
64
|
-
execsql/metacommands/__init__.py,sha256=
|
|
64
|
+
execsql/metacommands/__init__.py,sha256=jrp5PvAZyQYvZ0a_qYNxjv1ZqxDlGYE_P3dyqRTgVSM,11394
|
|
65
65
|
execsql/metacommands/conditions.py,sha256=Fzrk83-pWbFOoKahYdQW7CZjQeh3zByDUbfgpTM_bjQ,29259
|
|
66
66
|
execsql/metacommands/connect.py,sha256=Nsm0D91i3RX-R2rzQQ-Br-gULaI6Uvdn9fqb7DOAVfE,14804
|
|
67
67
|
execsql/metacommands/control.py,sha256=PAZFK1ck5SDSm5QdFV1ctif3KpEiyYWIXdDceRWgQ6k,8513
|
|
68
68
|
execsql/metacommands/data.py,sha256=tRQBGTAuW-eJ2tBNWaoZI9OjTyNNyHJISo7gOdL-sm8,11370
|
|
69
69
|
execsql/metacommands/debug.py,sha256=pnT24dfvfOx8xFu86mO5czfVCGKbcvgBLyXnqaMWO4w,8184
|
|
70
|
-
execsql/metacommands/dispatch.py,sha256=
|
|
70
|
+
execsql/metacommands/dispatch.py,sha256=2Ec7dJmOFstbY3tEuFoTODV8yaHpR-Sfa-ZydOmvIxo,85205
|
|
71
71
|
execsql/metacommands/io.py,sha256=Duh60caM4go9JczbGYNMKKYpcMimwPzF6EQ_tshKxdE,2971
|
|
72
72
|
execsql/metacommands/io_export.py,sha256=7lkCSnPhXy9FVau9_hT1u68NOVdG2DsWmvUh9hM1QWI,18359
|
|
73
|
-
execsql/metacommands/io_fileops.py,sha256=
|
|
73
|
+
execsql/metacommands/io_fileops.py,sha256=RrcJTh_cgj7bJ-bezjo0yNl-fN3CoWV-aZ71z1KHYZs,9803
|
|
74
74
|
execsql/metacommands/io_import.py,sha256=wyxJJdlW07P5ZIhweejhXyyGANAvEhY5uMjKZ200Jyc,12983
|
|
75
75
|
execsql/metacommands/io_write.py,sha256=NpL2aYGfBpbqmPpYsqniYltYfd_SCA1EQz3_4qSdNbo,8279
|
|
76
76
|
execsql/metacommands/prompt.py,sha256=xd3mAkdbn4AE4hUPuSfN5DgZGUZmk-Db23iL-JJPwXs,36918
|
|
77
77
|
execsql/metacommands/script_ext.py,sha256=TUgAldB2LSJAwZrCvDDi804hQ1d9BDQD2GDqHNPVOcM,2280
|
|
78
78
|
execsql/metacommands/system.py,sha256=sUR5kLL7idTVg8WXIMdd-Kv7nkERIiaeL0beWsz8NyY,7293
|
|
79
|
-
execsql/
|
|
79
|
+
execsql/metacommands/upsert.py,sha256=Ze_FhbCRM-8tjvqPKfL3E9abZVRGFfwPI0_NAihiEMU,14906
|
|
80
|
+
execsql/script/__init__.py,sha256=HbVQmQEVn4gBtzwy5_nlbDGuRnbWd4dI4nG-q1KyBxs,3498
|
|
80
81
|
execsql/script/control.py,sha256=s-1eZdGARM6H1FwZ6VDdO_f50j7bvvRtTHesfUm9tbc,6144
|
|
81
|
-
execsql/script/engine.py,sha256=
|
|
82
|
-
execsql/script/variables.py,sha256=
|
|
82
|
+
execsql/script/engine.py,sha256=6LYabzy1LI-_ISjYzTJos0BrLO62QF6FEKdqcN0YzK4,42995
|
|
83
|
+
execsql/script/variables.py,sha256=gTCCWY64LFmQUna-63CM1GAbupcaOTSS4cn6HDaHk9Q,11923
|
|
83
84
|
execsql/utils/__init__.py,sha256=0uR6JwVJQRX3vceByNBduCAf5dd5assKjeqJUWvpZoA,278
|
|
84
85
|
execsql/utils/auth.py,sha256=onXzNkNZQZxGC5w7eey06sjvAIAX_Lf9g7nUJtcsel0,7009
|
|
85
86
|
execsql/utils/crypto.py,sha256=2OnBWwn9bCBGc1ZkyRv16TvhottoCNYtXqgbE3mG3Sg,2960
|
|
@@ -92,24 +93,24 @@ execsql/utils/numeric.py,sha256=xh02ANSRk3nUpQ-rtm66ILoMqoi7HtzCoRMIOT9U8QI,1570
|
|
|
92
93
|
execsql/utils/regex.py,sha256=diEzTZqU_HHwVMadPAvN1Vgzhl7I03eVaEFGCXyGGL8,3770
|
|
93
94
|
execsql/utils/strings.py,sha256=5Dvzrk-9SIw2lpxXZQkiJbNyo1sy7iXXAtSULlZ0KG8,8488
|
|
94
95
|
execsql/utils/timer.py,sha256=eDYf5VzCNFk7oo90InJucUm3XcBdhYMogjZMqeg9xzc,1899
|
|
95
|
-
execsql2-2.12.
|
|
96
|
-
execsql2-2.12.
|
|
97
|
-
execsql2-2.12.
|
|
98
|
-
execsql2-2.12.
|
|
99
|
-
execsql2-2.12.
|
|
100
|
-
execsql2-2.12.
|
|
101
|
-
execsql2-2.12.
|
|
102
|
-
execsql2-2.12.
|
|
103
|
-
execsql2-2.12.
|
|
104
|
-
execsql2-2.12.
|
|
105
|
-
execsql2-2.12.
|
|
106
|
-
execsql2-2.12.
|
|
107
|
-
execsql2-2.12.
|
|
108
|
-
execsql2-2.12.
|
|
109
|
-
execsql2-2.12.
|
|
110
|
-
execsql2-2.12.
|
|
111
|
-
execsql2-2.12.
|
|
112
|
-
execsql2-2.12.
|
|
113
|
-
execsql2-2.12.
|
|
114
|
-
execsql2-2.12.
|
|
115
|
-
execsql2-2.12.
|
|
96
|
+
execsql2-2.12.5.data/data/execsql2_extras/README.md,sha256=sxwVyU0ZahCfANv56LahkyuM505kFjrMhe-1SvWE69E,4845
|
|
97
|
+
execsql2-2.12.5.data/data/execsql2_extras/config_settings.sqlite,sha256=aY5cxR7Q7J6zJ4bC9lu5mHUrhy211Cq3MNKPQVCt02E,20480
|
|
98
|
+
execsql2-2.12.5.data/data/execsql2_extras/example_config_prompt.sql,sha256=SY3Jxn1qcVm4kPW9xmmTfknHfvURXmeEYTbRjYrjGSw,7487
|
|
99
|
+
execsql2-2.12.5.data/data/execsql2_extras/execsql.conf,sha256=_45iJ-KWZnB8uMW_gEg067MM5pmGJ-dVl7VbAZMunAE,9530
|
|
100
|
+
execsql2-2.12.5.data/data/execsql2_extras/make_config_db.sql,sha256=WwyC6dK-Eh5CAVppiBCDHqiI1_wEI9U95Ytpr4lsZkg,8726
|
|
101
|
+
execsql2-2.12.5.data/data/execsql2_extras/md_compare.sql,sha256=B8Wd7LZ0vnMY2qvA139JIEBkPObgRH2i5xj6PejTQt8,24092
|
|
102
|
+
execsql2-2.12.5.data/data/execsql2_extras/md_glossary.sql,sha256=DJRHcU5NbFpxTTX-IwH3yRlsboj1q6BBGrUAHKn4Cuo,10796
|
|
103
|
+
execsql2-2.12.5.data/data/execsql2_extras/md_upsert.sql,sha256=v_7GbWh_N1mBTmw3gvTrkagOVp2q0KmXvM8hE-DlFxY,112524
|
|
104
|
+
execsql2-2.12.5.data/data/execsql2_extras/pg_compare.sql,sha256=9dWa8hnfy5dVJI-z2iGpd9JzQmI4j2ziMlEdpnr66ro,24352
|
|
105
|
+
execsql2-2.12.5.data/data/execsql2_extras/pg_glossary.sql,sha256=pKjIIDsROAgJq2H-1qNEcRMAWManivcZ_AEVHzUUlic,9908
|
|
106
|
+
execsql2-2.12.5.data/data/execsql2_extras/pg_upsert.sql,sha256=k7AFiGTLBy3nf-qO5QIaZrEYTAKvdxxU3JDLx9jqkzs,108315
|
|
107
|
+
execsql2-2.12.5.data/data/execsql2_extras/script_template.sql,sha256=1Estacb_vm1FgK41k_G9nuduP1yiA-fQ1Kn4Z4mv5Ao,11153
|
|
108
|
+
execsql2-2.12.5.data/data/execsql2_extras/ss_compare.sql,sha256=TsVxWm3cEpR5-EiMYXNhtaY0arSNeKZhsJdHdLA7xeI,24833
|
|
109
|
+
execsql2-2.12.5.data/data/execsql2_extras/ss_glossary.sql,sha256=cLM7nN8JOIu9ZVP9oY9qdSK3hrnWJiDcX6nZmQQbQWI,13065
|
|
110
|
+
execsql2-2.12.5.data/data/execsql2_extras/ss_upsert.sql,sha256=BCqmBykXBF-BpCgOFeG1qhf2XfScKsxPD17wd1hYfHw,120647
|
|
111
|
+
execsql2-2.12.5.dist-info/METADATA,sha256=Z0cWFXEpF8ujF1fqnSsBO33IIylqc9t583xJ9H33Xss,17560
|
|
112
|
+
execsql2-2.12.5.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
113
|
+
execsql2-2.12.5.dist-info/entry_points.txt,sha256=sUOxkM-dN1eBGGpSpDLsAaE0yNXYQKWZAfxPOlMkQyk,90
|
|
114
|
+
execsql2-2.12.5.dist-info/licenses/LICENSE.txt,sha256=LBdhuxejF8_bLCHZ2kWfmDXpDGUu914Gbd6_3JjCRe0,676
|
|
115
|
+
execsql2-2.12.5.dist-info/licenses/NOTICE,sha256=sqVrM73Ys9zfvWC_P797lHfTnoPW_ETeBSrUTFaob0A,339
|
|
116
|
+
execsql2-2.12.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{execsql2-2.12.2.data → execsql2-2.12.5.data}/data/execsql2_extras/example_config_prompt.sql
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|