execsql2 2.28.0__py3-none-any.whl → 2.30.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/__main__.py +4 -5
- execsql/api.py +5 -0
- execsql/cli/__init__.py +3 -14
- execsql/cli/application.py +5 -3
- execsql/cli/commands/lint.py +4 -1
- execsql/cli/commands/lsp.py +42 -0
- execsql/cli/dsn.py +3 -1
- execsql/cli/lint.py +14 -6
- execsql/cli/run.py +17 -9
- execsql/config.py +32 -6
- execsql/db/base.py +6 -9
- execsql/db/duckdb.py +96 -14
- execsql/db/mysql.py +19 -3
- execsql/db/postgres.py +1 -1
- execsql/exporters/xls.py +3 -2
- execsql/gui/__init__.py +3 -31
- execsql/lsp/_util.py +21 -0
- execsql/lsp/cache.py +73 -0
- execsql/lsp/code_actions.py +10 -14
- execsql/lsp/completion.py +8 -15
- execsql/lsp/diagnostics.py +14 -2
- execsql/lsp/document.py +139 -41
- execsql/lsp/hover.py +5 -13
- execsql/lsp/navigation.py +7 -10
- execsql/lsp/server.py +111 -29
- execsql/manifest.py +29 -9
- execsql/metacommands/__init__.py +0 -2
- execsql/metacommands/io_fileops.py +5 -0
- execsql/script/executor.py +2 -21
- execsql/script/variables.py +42 -10
- execsql/state.py +6 -3
- execsql/state.pyi +2 -0
- execsql/types.py +8 -0
- execsql/utils/errors.py +19 -36
- execsql/utils/fileio.py +44 -48
- execsql/utils/regex.py +0 -5
- {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/METADATA +2 -4
- {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/RECORD +56 -54
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/README.md +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/pg_upsert.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/WHEEL +0 -0
- {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/licenses/NOTICE +0 -0
execsql/manifest.py
CHANGED
|
@@ -10,7 +10,11 @@ Files are recorded as the executor runs each metacommand, after variable
|
|
|
10
10
|
substitution, so paths are the ones actually used. What a metacommand touches
|
|
11
11
|
is decided by :mod:`execsql.metacommands.effects`. Variable values and passwords are never written:
|
|
12
12
|
the run log hides ``-a`` and ``--var`` values because they can be secrets, and
|
|
13
|
-
the manifest lists only their names.
|
|
13
|
+
the manifest lists only their names. A password or decrypted value that reaches
|
|
14
|
+
a path, a connection target or an error message is written as ``***``, as are
|
|
15
|
+
common secret shapes (a URL's password, ``password=...``, API tokens). ``-a`` and
|
|
16
|
+
``--var`` values, which the log also masks, are kept: they are usually part of
|
|
17
|
+
the paths the manifest exists to report.
|
|
14
18
|
|
|
15
19
|
One recorder is active per run; :func:`current` returns it, or ``None`` when
|
|
16
20
|
no manifest was asked for, so the executor's hooks cost one check. Writing the
|
|
@@ -81,6 +85,22 @@ def _now() -> datetime.datetime:
|
|
|
81
85
|
return datetime.datetime.now(tz=datetime.timezone.utc)
|
|
82
86
|
|
|
83
87
|
|
|
88
|
+
def _masked(value: Any) -> Any:
|
|
89
|
+
"""*value* with the run's passwords and common secret shapes replaced by ``***`` in every string it holds.
|
|
90
|
+
|
|
91
|
+
``-a`` / ``--var`` values are left in place; times, counts and line numbers are not strings.
|
|
92
|
+
"""
|
|
93
|
+
from execsql.utils.fileio import redact
|
|
94
|
+
|
|
95
|
+
if isinstance(value, str):
|
|
96
|
+
return redact(value, arguments=False)
|
|
97
|
+
if isinstance(value, list):
|
|
98
|
+
return [_masked(item) for item in value]
|
|
99
|
+
if isinstance(value, dict):
|
|
100
|
+
return {key: _masked(item) for key, item in value.items()}
|
|
101
|
+
return value
|
|
102
|
+
|
|
103
|
+
|
|
84
104
|
class RunManifest:
|
|
85
105
|
"""Collects what one run does and writes it as JSON when the run ends."""
|
|
86
106
|
|
|
@@ -166,20 +186,20 @@ class RunManifest:
|
|
|
166
186
|
finished = _now()
|
|
167
187
|
data = {
|
|
168
188
|
"execsql_version": __version__,
|
|
169
|
-
"script": self.script,
|
|
189
|
+
"script": _masked(self.script),
|
|
170
190
|
"started": self.started.isoformat(timespec="seconds"),
|
|
171
191
|
"finished": finished.isoformat(timespec="seconds"),
|
|
172
192
|
"duration_s": round((finished - self.started).total_seconds(), 3),
|
|
173
193
|
"exit_status": exit_status,
|
|
174
|
-
"config_files": self.config_files,
|
|
175
|
-
"database": self.database,
|
|
194
|
+
"config_files": _masked(self.config_files),
|
|
195
|
+
"database": _masked(self.database),
|
|
176
196
|
"variables": self.variables,
|
|
177
197
|
"statements": {"sql": self.sql_statements, "metacommands": self.metacommands},
|
|
178
|
-
"files_read": self.reads,
|
|
179
|
-
"files_written": self.writes,
|
|
180
|
-
"files_deleted": self.deletes,
|
|
181
|
-
"connections": self.connections,
|
|
182
|
-
"errors": self.errors,
|
|
198
|
+
"files_read": _masked(self.reads),
|
|
199
|
+
"files_written": _masked(self.writes),
|
|
200
|
+
"files_deleted": _masked(self.deletes),
|
|
201
|
+
"connections": _masked(self.connections),
|
|
202
|
+
"errors": _masked(self.errors),
|
|
183
203
|
}
|
|
184
204
|
self.written = True
|
|
185
205
|
import execsql.state as _state
|
execsql/metacommands/__init__.py
CHANGED
|
@@ -242,7 +242,6 @@ from execsql.metacommands.system import (
|
|
|
242
242
|
# Regex helper functions (from utils/regex.py)
|
|
243
243
|
from execsql.utils.regex import (
|
|
244
244
|
ins_rxs,
|
|
245
|
-
ins_quoted_rx,
|
|
246
245
|
ins_schema_rxs,
|
|
247
246
|
ins_table_rxs,
|
|
248
247
|
ins_table_list_rxs,
|
|
@@ -443,7 +442,6 @@ __all__ = [
|
|
|
443
442
|
"x_pg_upsert_qa",
|
|
444
443
|
# regex helpers
|
|
445
444
|
"ins_rxs",
|
|
446
|
-
"ins_quoted_rx",
|
|
447
445
|
"ins_schema_rxs",
|
|
448
446
|
"ins_table_rxs",
|
|
449
447
|
"ins_table_list_rxs",
|
|
@@ -307,6 +307,11 @@ def x_cd(**kwargs: Any) -> None:
|
|
|
307
307
|
command_text=kwargs["metacommandline"],
|
|
308
308
|
other_msg="Directory does not exist",
|
|
309
309
|
)
|
|
310
|
+
if _state.cd_from is None:
|
|
311
|
+
try:
|
|
312
|
+
_state.cd_from = os.getcwd()
|
|
313
|
+
except OSError: # the directory is gone: nowhere to return to
|
|
314
|
+
pass
|
|
310
315
|
os.chdir(new_dir)
|
|
311
316
|
script, lno = current_script_line()
|
|
312
317
|
_state.exec_log.log_status_info(f"Current directory changed to {new_dir} at line {lno} of {script}")
|
execsql/script/executor.py
CHANGED
|
@@ -957,16 +957,15 @@ class ExecutingStatement:
|
|
|
957
957
|
rather than copying fields out of it, so there is nothing to keep in sync
|
|
958
958
|
and no second source of truth about what is executing. Error reporting
|
|
959
959
|
(:mod:`execsql.utils.errors`), the debug REPL, and ``api.run()`` read
|
|
960
|
-
|
|
960
|
+
its attributes; everything is derived from ``node`` on access, so
|
|
961
961
|
constructing one costs a single small object per statement rather than
|
|
962
962
|
the two classes the previous shim built each time.
|
|
963
963
|
"""
|
|
964
964
|
|
|
965
|
-
__slots__ = ("node",
|
|
965
|
+
__slots__ = ("node",)
|
|
966
966
|
|
|
967
967
|
def __init__(self, node: Node) -> None:
|
|
968
968
|
self.node = node
|
|
969
|
-
self._source_dir: str | None = None
|
|
970
969
|
|
|
971
970
|
@property
|
|
972
971
|
def source(self) -> str:
|
|
@@ -976,18 +975,6 @@ class ExecutingStatement:
|
|
|
976
975
|
def line_no(self) -> int:
|
|
977
976
|
return self.node.span.start_line
|
|
978
977
|
|
|
979
|
-
@property
|
|
980
|
-
def source_name(self) -> str:
|
|
981
|
-
return Path(self.node.span.file).name
|
|
982
|
-
|
|
983
|
-
@property
|
|
984
|
-
def source_dir(self) -> str:
|
|
985
|
-
# resolve() touches the filesystem, so it is done once per statement
|
|
986
|
-
# object rather than on every read.
|
|
987
|
-
if self._source_dir is None:
|
|
988
|
-
self._source_dir = str(Path(self.node.span.file).resolve().parent) + os.sep
|
|
989
|
-
return self._source_dir
|
|
990
|
-
|
|
991
978
|
@property
|
|
992
979
|
def command_type(self) -> str:
|
|
993
980
|
return "sql" if isinstance(self.node, SqlStatement) else "cmd"
|
|
@@ -1004,12 +991,6 @@ class ExecutingStatement:
|
|
|
1004
991
|
def __repr__(self) -> str:
|
|
1005
992
|
return f"ExecutingStatement({self.source}:{self.line_no}, {self.command_type})"
|
|
1006
993
|
|
|
1007
|
-
def current_script_line(self) -> tuple[str, int]:
|
|
1008
|
-
return (self.source, self.line_no)
|
|
1009
|
-
|
|
1010
|
-
def commandline(self) -> str:
|
|
1011
|
-
return cast(str, self.command.commandline())
|
|
1012
|
-
|
|
1013
994
|
|
|
1014
995
|
# ---------------------------------------------------------------------------
|
|
1015
996
|
# Node type/text helpers for profiling
|
execsql/script/variables.py
CHANGED
|
@@ -17,6 +17,42 @@ from execsql.exceptions import ErrInfo
|
|
|
17
17
|
__all__ = ["CounterVars", "SubVarSet", "LocalSubVarSet", "ScriptArgSubVarSet"]
|
|
18
18
|
|
|
19
19
|
|
|
20
|
+
def _after_escape_string_prefix(command: str, start: int) -> bool:
|
|
21
|
+
"""Whether the ``!'!`` token at *start* follows an ``E`` that makes it a PostgreSQL escape string (``E!'!v!'!``).
|
|
22
|
+
|
|
23
|
+
The ``E`` must stand alone: in ``TYPE!'!v!'!`` it ends a word.
|
|
24
|
+
"""
|
|
25
|
+
if start < 1 or command[start - 1] not in "Ee":
|
|
26
|
+
return False
|
|
27
|
+
return start < 2 or not (command[start - 2].isalnum() or command[start - 2] == "_")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _sql_string_literal(value: str, *, escape_string: bool = False) -> str:
|
|
31
|
+
"""*value* as the string literal ``!'!var!'!`` substitutes, in the current connection's form.
|
|
32
|
+
|
|
33
|
+
An apostrophe is doubled everywhere; a backslash only where the DBMS
|
|
34
|
+
reads it as an escape (MySQL/MariaDB), or inside an escape string the
|
|
35
|
+
script wrote as ``E!'!var!'!`` (*escape_string*), so ``C:\\data`` is
|
|
36
|
+
stored as written in every case. Without a connection, the standard SQL
|
|
37
|
+
form. (Upstream doubled backslashes only on Windows, to protect
|
|
38
|
+
``re.sub``'s replacement template, not the SQL.)
|
|
39
|
+
"""
|
|
40
|
+
if escape_string:
|
|
41
|
+
return "'" + value.replace("\\", "\\\\").replace("'", "''") + "'"
|
|
42
|
+
db = None
|
|
43
|
+
try:
|
|
44
|
+
import execsql.state as _state
|
|
45
|
+
|
|
46
|
+
dbs = _state.dbs
|
|
47
|
+
if dbs is not None and dbs.current_db is not None:
|
|
48
|
+
db = dbs.current()
|
|
49
|
+
except Exception:
|
|
50
|
+
db = None
|
|
51
|
+
if db is not None:
|
|
52
|
+
return db.quote_literal(value)
|
|
53
|
+
return "'" + value.replace("'", "''") + "'"
|
|
54
|
+
|
|
55
|
+
|
|
20
56
|
# ---------------------------------------------------------------------------
|
|
21
57
|
# CounterVars
|
|
22
58
|
# ---------------------------------------------------------------------------
|
|
@@ -253,13 +289,10 @@ class SubVarSet:
|
|
|
253
289
|
)
|
|
254
290
|
quote = m.group("q")
|
|
255
291
|
if quote == "'":
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
# branch (``os.name != 'posix'``) applied the escape on
|
|
261
|
-
# the wrong axis — host OS vs target DBMS.
|
|
262
|
-
sub = "'" + sub.replace("\\", "\\\\").replace("'", "''") + "'"
|
|
292
|
+
sub = _sql_string_literal(
|
|
293
|
+
sub,
|
|
294
|
+
escape_string=_after_escape_string_prefix(command_str, m.start()),
|
|
295
|
+
)
|
|
263
296
|
elif quote == '"':
|
|
264
297
|
# B07a/F001: wrap in double quotes, doubling embedded
|
|
265
298
|
# ``"`` so a value containing ``"; DROP TABLE x; --``
|
|
@@ -306,13 +339,12 @@ class SubVarSet:
|
|
|
306
339
|
if idx != -1:
|
|
307
340
|
_check_nul(varname, sub)
|
|
308
341
|
return command_str[:idx] + sub + command_str[idx + len(token) :], True
|
|
309
|
-
# Single-quote-wrapped token: !'!varname!'! —
|
|
310
|
-
# double embedded ``'`` (see substitute_one for rationale).
|
|
342
|
+
# Single-quote-wrapped token: !'!varname!'! — a string literal.
|
|
311
343
|
tokenq = f"!'!{varname}!'!"
|
|
312
344
|
idxq = cmd_lower.find(tokenq)
|
|
313
345
|
if idxq != -1:
|
|
314
346
|
_check_nul(varname, sub)
|
|
315
|
-
wrapped =
|
|
347
|
+
wrapped = _sql_string_literal(sub, escape_string=_after_escape_string_prefix(command_str, idxq))
|
|
316
348
|
return (
|
|
317
349
|
command_str[:idxq] + wrapped + command_str[idxq + len(tokenq) :],
|
|
318
350
|
True,
|
execsql/state.py
CHANGED
|
@@ -89,8 +89,6 @@ class ExecFrame:
|
|
|
89
89
|
|
|
90
90
|
|
|
91
91
|
if TYPE_CHECKING:
|
|
92
|
-
import multiprocessing as _mp
|
|
93
|
-
|
|
94
92
|
from execsql.config import ConfigData, StatObj, WriteHooks
|
|
95
93
|
from execsql.db.base import DatabasePool
|
|
96
94
|
from execsql.exporters.base import ExportMetadata, WriteSpec
|
|
@@ -230,6 +228,7 @@ _CONTEXT_ATTRS: frozenset[str] = frozenset(
|
|
|
230
228
|
"manifest",
|
|
231
229
|
"halt_error",
|
|
232
230
|
"halt_output_error",
|
|
231
|
+
"cd_from",
|
|
233
232
|
# Lazy singletons
|
|
234
233
|
"counters",
|
|
235
234
|
"timer",
|
|
@@ -288,6 +287,7 @@ class RuntimeContext:
|
|
|
288
287
|
"manifest",
|
|
289
288
|
"halt_error",
|
|
290
289
|
"halt_output_error",
|
|
290
|
+
"cd_from",
|
|
291
291
|
# Lazy singletons
|
|
292
292
|
"counters",
|
|
293
293
|
"timer",
|
|
@@ -344,6 +344,9 @@ class RuntimeContext:
|
|
|
344
344
|
self.halt_error: ErrInfo | None = None
|
|
345
345
|
# File output lost when the run halted (set by ``exit_now``), or None.
|
|
346
346
|
self.halt_output_error: ErrInfo | None = None
|
|
347
|
+
# The working directory before this run's first CD, or None: the
|
|
348
|
+
# library's run() returns its caller there.
|
|
349
|
+
self.cd_from: str | None = None
|
|
347
350
|
|
|
348
351
|
# Lazy singletons
|
|
349
352
|
self.counters: CounterVars | None = None
|
|
@@ -356,7 +359,7 @@ class RuntimeContext:
|
|
|
356
359
|
|
|
357
360
|
# GUI
|
|
358
361
|
self.gui_console: Any = None
|
|
359
|
-
self.gui_manager_queue:
|
|
362
|
+
self.gui_manager_queue: Any = None # a queue.Queue or a _TkinterSyncQueue; only put() is used
|
|
360
363
|
self.gui_manager_thread: threading.Thread | None = None
|
|
361
364
|
|
|
362
365
|
# Profiling — None means profiling is disabled; a list means it is enabled.
|
execsql/state.pyi
CHANGED
|
@@ -59,6 +59,7 @@ filewriter: FileWriter | None
|
|
|
59
59
|
manifest: RunManifest | None
|
|
60
60
|
halt_error: ErrInfo | None
|
|
61
61
|
halt_output_error: ErrInfo | None
|
|
62
|
+
cd_from: str | None
|
|
62
63
|
counters: CounterVars
|
|
63
64
|
timer: Timer
|
|
64
65
|
dbs: DatabasePool
|
|
@@ -97,6 +98,7 @@ class RuntimeContext:
|
|
|
97
98
|
manifest: RunManifest | None
|
|
98
99
|
halt_error: ErrInfo | None
|
|
99
100
|
halt_output_error: ErrInfo | None
|
|
101
|
+
cd_from: str | None
|
|
100
102
|
counters: CounterVars | None
|
|
101
103
|
timer: Timer | None
|
|
102
104
|
dbs: DatabasePool | None
|
execsql/types.py
CHANGED
|
@@ -884,6 +884,14 @@ dbt_oracle.name_datatype(DT_TimestampTZ, "TIMESTAMP WITH TIME ZONE")
|
|
|
884
884
|
dbt_oracle.name_datatype(DT_Timestamp, "TIMESTAMP", casting_name="TIMESTAMP")
|
|
885
885
|
dbt_oracle.name_datatype(DT_Date, "DATE", casting_name="DATE")
|
|
886
886
|
dbt_oracle.name_datatype(DT_Time_Oracle, "VARCHAR2", True, casting_name="VARCHAR(20)")
|
|
887
|
+
# Oracle has no time-of-day type, and oracledb cannot bind a datetime.time: a
|
|
888
|
+
# time column is stored as its ISO text (at most 15 characters, HH:MM:SS.ffffff).
|
|
889
|
+
dbt_oracle.name_datatype(
|
|
890
|
+
DT_Time,
|
|
891
|
+
"VARCHAR2(15)",
|
|
892
|
+
casting_name="VARCHAR(20)",
|
|
893
|
+
conv_mod_fn=lambda x: x.isoformat() if x is not None else None,
|
|
894
|
+
)
|
|
887
895
|
dbt_oracle.name_datatype(DT_Integer, "NUMBER")
|
|
888
896
|
dbt_oracle.name_datatype(DT_Long, "NUMBER")
|
|
889
897
|
dbt_oracle.name_datatype(DT_Float, "FLOAT")
|
execsql/utils/errors.py
CHANGED
|
@@ -6,8 +6,6 @@ Error handling and reporting utilities for execsql.
|
|
|
6
6
|
Provides the functions that form the error-reporting pipeline used
|
|
7
7
|
throughout the codebase:
|
|
8
8
|
|
|
9
|
-
- :func:`exception_info` — extracts type, value, and source location
|
|
10
|
-
from the current exception.
|
|
11
9
|
- :func:`exception_desc` — ``Type: message`` for the exception being
|
|
12
10
|
handled, as shown on screen.
|
|
13
11
|
- :func:`error_origin` / :func:`logged_error` — where in execsql an error
|
|
@@ -28,7 +26,6 @@ import execsql.state as _state
|
|
|
28
26
|
from execsql.exceptions import ErrInfo
|
|
29
27
|
|
|
30
28
|
__all__ = [
|
|
31
|
-
"exception_info",
|
|
32
29
|
"error_origin",
|
|
33
30
|
"exception_desc",
|
|
34
31
|
"logged_error",
|
|
@@ -56,43 +53,29 @@ def install_hint(extra: str) -> str:
|
|
|
56
53
|
)
|
|
57
54
|
|
|
58
55
|
|
|
59
|
-
def exception_info() -> tuple:
|
|
60
|
-
# Returns the exception type, value, source file name, source line number, and source line text.
|
|
61
|
-
strace = traceback.extract_tb(sys.exc_info()[2])[-1:]
|
|
62
|
-
traces = traceback.extract_tb(sys.exc_info()[2])
|
|
63
|
-
xline = 0
|
|
64
|
-
for trace in traces:
|
|
65
|
-
if "execsql" in trace[0]:
|
|
66
|
-
xline = trace[1]
|
|
67
|
-
exc_message = ""
|
|
68
|
-
exc_param = sys.exc_info()[1]
|
|
69
|
-
if isinstance(exc_param, str):
|
|
70
|
-
exc_message = exc_param
|
|
71
|
-
elif exc_param is not None:
|
|
72
|
-
message = getattr(exc_param, "message", None)
|
|
73
|
-
value = getattr(exc_param, "value", None)
|
|
74
|
-
if isinstance(message, str) and len(message) > 0:
|
|
75
|
-
exc_message = message
|
|
76
|
-
elif isinstance(value, str) and len(value) > 0:
|
|
77
|
-
exc_message = value
|
|
78
|
-
else:
|
|
79
|
-
exc_message = str(exc_param)
|
|
80
|
-
try:
|
|
81
|
-
exc_message = str(exc_message)
|
|
82
|
-
except Exception:
|
|
83
|
-
exc_message = repr(exc_message)
|
|
84
|
-
xinfo = sys.exc_info()[0]
|
|
85
|
-
xname = getattr(xinfo, "__name__", "")
|
|
86
|
-
return xname, exc_message, strace[0][0], xline, strace[0][3]
|
|
87
|
-
|
|
88
|
-
|
|
89
56
|
def exception_desc() -> str:
|
|
90
57
|
"""``Type: message`` for the exception being handled, as shown to the person running the script.
|
|
91
58
|
|
|
92
|
-
|
|
59
|
+
The message is the exception's ``message`` or ``value`` attribute when it
|
|
60
|
+
has a non-empty string one (as some drivers' exceptions do), else its
|
|
61
|
+
``str()``. Where in execsql it was raised is for the log: see
|
|
62
|
+
:func:`error_origin`.
|
|
93
63
|
"""
|
|
94
|
-
exc_type,
|
|
95
|
-
|
|
64
|
+
exc_type, exc = sys.exc_info()[:2]
|
|
65
|
+
if isinstance(exc, str):
|
|
66
|
+
message = exc
|
|
67
|
+
elif exc is None:
|
|
68
|
+
message = ""
|
|
69
|
+
else:
|
|
70
|
+
attr = next(
|
|
71
|
+
(v for v in (getattr(exc, "message", None), getattr(exc, "value", None)) if isinstance(v, str) and v),
|
|
72
|
+
None,
|
|
73
|
+
)
|
|
74
|
+
try:
|
|
75
|
+
message = attr if attr is not None else str(exc)
|
|
76
|
+
except Exception:
|
|
77
|
+
message = repr(exc)
|
|
78
|
+
return f"{getattr(exc_type, '__name__', '')}: {message}"
|
|
96
79
|
|
|
97
80
|
|
|
98
81
|
def error_origin(exc: BaseException) -> str | None:
|
execsql/utils/fileio.py
CHANGED
|
@@ -34,7 +34,6 @@ import threading
|
|
|
34
34
|
import time
|
|
35
35
|
import weakref
|
|
36
36
|
from collections.abc import Callable
|
|
37
|
-
from encodings.aliases import aliases as codec_dict
|
|
38
37
|
from typing import Any, NamedTuple, cast
|
|
39
38
|
|
|
40
39
|
from execsql.exceptions import ErrInfo
|
|
@@ -47,14 +46,11 @@ __all__ = [
|
|
|
47
46
|
"EncodedFile",
|
|
48
47
|
"Logger",
|
|
49
48
|
"TempFileMgr",
|
|
50
|
-
"list_encodings",
|
|
51
49
|
"filewriter_filestatus",
|
|
52
50
|
"filewriter_write",
|
|
53
51
|
"filewriter_open_as_new",
|
|
54
52
|
"filewriter_close",
|
|
55
53
|
"filewriter_close_all_after_write",
|
|
56
|
-
"filewriter_closeall",
|
|
57
|
-
"filewriter_shutdown",
|
|
58
54
|
"filewriter_end",
|
|
59
55
|
"redact",
|
|
60
56
|
"register_secret",
|
|
@@ -216,9 +212,8 @@ class FileWriter(threading.Thread):
|
|
|
216
212
|
CMD_OPEN_AS_NEW,
|
|
217
213
|
CMD_CLOSE_ALL_AFTER_WRITE,
|
|
218
214
|
CMD_CLOSED_STATUS,
|
|
219
|
-
CMD_PING,
|
|
220
215
|
CMD_OPEN_FAILURES,
|
|
221
|
-
) = range(
|
|
216
|
+
) = range(9)
|
|
222
217
|
|
|
223
218
|
class FileControl:
|
|
224
219
|
STATUS_OPEN, STATUS_WAITING, STATUS_UNOPENED, STATUS_CLOSED, STATUS_OPENFAILURE, STATUS_WRITEFAILURE = range(6)
|
|
@@ -375,7 +370,6 @@ class FileWriter(threading.Thread):
|
|
|
375
370
|
self.open_as_new,
|
|
376
371
|
self.close_all_after_write,
|
|
377
372
|
self.closed_status,
|
|
378
|
-
self.ping,
|
|
379
373
|
self.open_failures,
|
|
380
374
|
)
|
|
381
375
|
|
|
@@ -417,10 +411,6 @@ class FileWriter(threading.Thread):
|
|
|
417
411
|
if fc.status != self.FileControl.STATUS_CLOSED and fc.status not in fc.FAILED:
|
|
418
412
|
self._on_file(fc, fc.clean_close)
|
|
419
413
|
|
|
420
|
-
def ping(self, token: object) -> None:
|
|
421
|
-
"""Echo *token* back on the return queue — used to synchronize callers."""
|
|
422
|
-
self.return_msg_queue.put(token)
|
|
423
|
-
|
|
424
414
|
def retry_waiting(self) -> None:
|
|
425
415
|
"""Re-attempt opening every file still waiting on a lock.
|
|
426
416
|
|
|
@@ -526,7 +516,7 @@ class FileWriter(threading.Thread):
|
|
|
526
516
|
the rest, so one bad command cannot stop the thread (and with it
|
|
527
517
|
every later write, to any file).
|
|
528
518
|
"""
|
|
529
|
-
filename = argtuple[0] if
|
|
519
|
+
filename = argtuple[0] if argtuple and isinstance(argtuple[0], str) else ""
|
|
530
520
|
lost = LostOutput(filename, 0, str(exc), "writer")
|
|
531
521
|
if command == self.CMD_OPEN_FAILURES:
|
|
532
522
|
self.return_msg_queue.put([lost])
|
|
@@ -535,7 +525,6 @@ class FileWriter(threading.Thread):
|
|
|
535
525
|
replies: dict[int, Any] = {
|
|
536
526
|
self.CMD_GET_STATUS: self.FileControl.STATUS_CLOSED,
|
|
537
527
|
self.CMD_CLOSED_STATUS: self.FileControl.STATUS_CLOSED,
|
|
538
|
-
self.CMD_PING: argtuple[0] if argtuple else None,
|
|
539
528
|
}
|
|
540
529
|
if command in replies:
|
|
541
530
|
self.return_msg_queue.put(replies[command])
|
|
@@ -745,20 +734,6 @@ def _lost_output_message(lost: LostOutput, open_timeout: int) -> str:
|
|
|
745
734
|
return f"{cause}; {lost.dropped} line(s) of output to it were discarded" if lost.dropped else cause
|
|
746
735
|
|
|
747
736
|
|
|
748
|
-
def filewriter_closeall() -> None:
|
|
749
|
-
fw = _active_writer()
|
|
750
|
-
if fw is None:
|
|
751
|
-
return
|
|
752
|
-
fw.input_queue.put((FileWriter.CMD_CLOSE_ALL, ()))
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
def filewriter_shutdown() -> None:
|
|
756
|
-
fw = _active_writer()
|
|
757
|
-
if fw is None:
|
|
758
|
-
return
|
|
759
|
-
fw.input_queue.put((FileWriter.CMD_SHUTDOWN, ()))
|
|
760
|
-
|
|
761
|
-
|
|
762
737
|
def filewriter_end(writer: FileWriter | None = None) -> None:
|
|
763
738
|
"""Close every file and stop *writer*, or the active run's writer if omitted.
|
|
764
739
|
|
|
@@ -894,6 +869,8 @@ class Logger:
|
|
|
894
869
|
self.server_name = server_name
|
|
895
870
|
self.cmdline_options = cmdline_options
|
|
896
871
|
self._redaction_values: list[str] = []
|
|
872
|
+
# Of those, the -a / --var values: masked in the log, not in the manifest.
|
|
873
|
+
self._argument_values: set[str] = set()
|
|
897
874
|
if log_file_name:
|
|
898
875
|
self.log_file_name = log_file_name
|
|
899
876
|
else:
|
|
@@ -956,9 +933,9 @@ class Logger:
|
|
|
956
933
|
)
|
|
957
934
|
self.writelog(self._redact(msg) or "")
|
|
958
935
|
if server_name:
|
|
959
|
-
msg = f"run_db_server\t{self.run_id}\t{server_name}\t{db_name}\n"
|
|
936
|
+
msg = f"run_db_server\t{self.run_id}\t{self._redact(server_name)}\t{self._redact(db_name)}\n"
|
|
960
937
|
else:
|
|
961
|
-
msg = f"run_db_file\t{self.run_id}\t{db_name}\n"
|
|
938
|
+
msg = f"run_db_file\t{self.run_id}\t{self._redact(db_name)}\n"
|
|
962
939
|
self.writelog(msg)
|
|
963
940
|
self.seq_no = 0
|
|
964
941
|
atexit.register(self.close)
|
|
@@ -974,20 +951,41 @@ class Logger:
|
|
|
974
951
|
|
|
975
952
|
return _datetime.datetime.now().isoformat(timespec="seconds")
|
|
976
953
|
|
|
977
|
-
def add_redaction_value(self, value: str | None) -> None:
|
|
954
|
+
def add_redaction_value(self, value: str | None, *, argument: bool = False) -> None:
|
|
955
|
+
"""Mask *value* in every later log record.
|
|
956
|
+
|
|
957
|
+
*argument* marks an ``-a`` / ``--var`` value: it may be a secret, so
|
|
958
|
+
the log masks it, but it is just as often part of a path, so the
|
|
959
|
+
``--manifest`` JSON (``redact(..., arguments=False)``) does not. A
|
|
960
|
+
value registered both ways is a secret.
|
|
961
|
+
"""
|
|
978
962
|
if value is None:
|
|
979
963
|
return
|
|
980
964
|
value = str(value)
|
|
981
|
-
if not value
|
|
965
|
+
if not value:
|
|
966
|
+
return
|
|
967
|
+
if not argument:
|
|
968
|
+
self._argument_values.discard(value)
|
|
969
|
+
if value in self._redaction_values:
|
|
982
970
|
return
|
|
983
971
|
self._redaction_values.append(value)
|
|
972
|
+
if argument:
|
|
973
|
+
self._argument_values.add(value)
|
|
974
|
+
|
|
975
|
+
def _redact(self, msg: str | None, *, arguments: bool = True) -> str | None:
|
|
976
|
+
"""*msg* with registered secrets and common secret shapes replaced by ``***``.
|
|
984
977
|
|
|
985
|
-
|
|
978
|
+
Every record applies it to its free-text fields (messages, names,
|
|
979
|
+
paths), never to a whole line: a registered value can be as short as
|
|
980
|
+
a ``-a 2024`` argument, which would otherwise mask timestamps and
|
|
981
|
+
sequence numbers.
|
|
982
|
+
"""
|
|
986
983
|
if msg is None:
|
|
987
984
|
return None
|
|
988
985
|
redacted = str(msg)
|
|
989
986
|
for value in sorted(self._redaction_values, key=len, reverse=True):
|
|
990
|
-
|
|
987
|
+
if arguments or value not in self._argument_values:
|
|
988
|
+
redacted = redacted.replace(value, "***")
|
|
991
989
|
for pattern in self._COMMON_SECRET_PATTERNS:
|
|
992
990
|
if pattern.pattern.startswith("(?i)(://"):
|
|
993
991
|
redacted = pattern.sub(r"\1***\3", redacted)
|
|
@@ -1026,12 +1024,13 @@ class Logger:
|
|
|
1026
1024
|
|
|
1027
1025
|
def log_db_connect(self, db: Any) -> None:
|
|
1028
1026
|
self.seq_no += 1
|
|
1029
|
-
msg = f"connect\t{self.run_id}\t{self.seq_no}\t{self._ts()}\t{db.name()}\n"
|
|
1027
|
+
msg = f"connect\t{self.run_id}\t{self.seq_no}\t{self._ts()}\t{self._redact(db.name())}\n"
|
|
1030
1028
|
self.writelog(msg)
|
|
1031
1029
|
|
|
1032
1030
|
def log_action_export(self, line_no: int, query_name: str, export_file_name: str) -> None:
|
|
1033
1031
|
self.seq_no += 1
|
|
1034
|
-
|
|
1032
|
+
description = self._redact(f"Query {query_name} exported to {export_file_name}")
|
|
1033
|
+
msg = f"action\t{self.run_id}\t{self.seq_no}\t{self._ts()}\texport\t{line_no}\t{description}\n"
|
|
1035
1034
|
self.writelog(msg)
|
|
1036
1035
|
|
|
1037
1036
|
def log_status_exception(self, msg: str | None) -> None:
|
|
@@ -1087,7 +1086,7 @@ class Logger:
|
|
|
1087
1086
|
self.exit_type = "halt"
|
|
1088
1087
|
self.exit_scriptfile = script_file_name
|
|
1089
1088
|
self.exit_lno = line_no
|
|
1090
|
-
self.exit_description = msg
|
|
1089
|
+
self.exit_description = None if not msg else msg.replace("\n", "")
|
|
1091
1090
|
|
|
1092
1091
|
def log_exit_interrupted(self) -> None:
|
|
1093
1092
|
# Save values to be used by exit() function triggered on program exit
|
|
@@ -1121,9 +1120,9 @@ class Logger:
|
|
|
1121
1120
|
wmsg = "exit\t{}\t{}\t{}({})\t{}\t{:.1f}s\n".format(
|
|
1122
1121
|
self.run_id,
|
|
1123
1122
|
self.exit_type,
|
|
1124
|
-
self.exit_scriptfile or "",
|
|
1123
|
+
self._redact(self.exit_scriptfile) or "",
|
|
1125
1124
|
str(self.exit_lno or ""),
|
|
1126
|
-
self.exit_description or "",
|
|
1125
|
+
self._redact(self.exit_description) or "", # a HALT MESSAGE can hold any variable's value
|
|
1127
1126
|
elapsed,
|
|
1128
1127
|
)
|
|
1129
1128
|
self.writelog(wmsg)
|
|
@@ -1144,12 +1143,16 @@ def register_secret(value: str | None) -> None:
|
|
|
1144
1143
|
add(value)
|
|
1145
1144
|
|
|
1146
1145
|
|
|
1147
|
-
def redact(text: str) -> str:
|
|
1148
|
-
"""*text* with the active run's registered secrets and common secret shapes replaced by ``***``.
|
|
1146
|
+
def redact(text: str, *, arguments: bool = True) -> str:
|
|
1147
|
+
"""*text* with the active run's registered secrets and common secret shapes replaced by ``***``.
|
|
1148
|
+
|
|
1149
|
+
With *arguments* false, ``-a`` / ``--var`` values are left as they are
|
|
1150
|
+
(see :meth:`Logger.add_redaction_value`).
|
|
1151
|
+
"""
|
|
1149
1152
|
import execsql.state as _state
|
|
1150
1153
|
|
|
1151
1154
|
redactor = getattr(_state.exec_log, "_redact", None)
|
|
1152
|
-
return redactor(text) if redactor is not None else text
|
|
1155
|
+
return (redactor(text, arguments=arguments) or "") if redactor is not None else text
|
|
1153
1156
|
|
|
1154
1157
|
|
|
1155
1158
|
class TempFileMgr:
|
|
@@ -1188,10 +1191,3 @@ _live_tempfile_mgrs: weakref.WeakSet[TempFileMgr] = weakref.WeakSet()
|
|
|
1188
1191
|
def _remove_all_temp_files() -> None:
|
|
1189
1192
|
for mgr in list(_live_tempfile_mgrs):
|
|
1190
1193
|
mgr.remove_all()
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
def list_encodings() -> None:
|
|
1194
|
-
enc = list(codec_dict.keys())
|
|
1195
|
-
enc.sort()
|
|
1196
|
-
msg = f"Encodings: {', '.join(enc)}\n"
|
|
1197
|
-
print(msg)
|
execsql/utils/regex.py
CHANGED
|
@@ -16,7 +16,6 @@ import os
|
|
|
16
16
|
|
|
17
17
|
__all__ = [
|
|
18
18
|
"ins_rxs",
|
|
19
|
-
"ins_quoted_rx",
|
|
20
19
|
"ins_schema_rxs",
|
|
21
20
|
"ins_table_rxs",
|
|
22
21
|
"ins_table_list_rxs",
|
|
@@ -42,10 +41,6 @@ def ins_rxs(rx_list: tuple[str, ...], fragment1: object, fragment2: object) -> t
|
|
|
42
41
|
return tuple(rv)
|
|
43
42
|
|
|
44
43
|
|
|
45
|
-
def ins_quoted_rx(fragment1: object, fragment2: object, rx: str) -> tuple[str, ...]:
|
|
46
|
-
return ins_rxs((rx, rf'"{rx}"'), fragment1, fragment2)
|
|
47
|
-
|
|
48
|
-
|
|
49
44
|
def ins_schema_rxs(fragment1: object, fragment2: object, suffix: str | None = None) -> tuple[str, ...]:
|
|
50
45
|
schema_exprs: tuple[str, ...] = (
|
|
51
46
|
r'"(?P<schema>[A-Za-z0-9_\- ]+)"',
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: execsql2
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.30.0
|
|
4
4
|
Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
|
|
5
5
|
Project-URL: Homepage, https://execsql2.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/geocoug/execsql
|
|
@@ -83,7 +83,6 @@ Provides-Extra: auth-plaintext
|
|
|
83
83
|
Requires-Dist: keyring>=25.0; extra == 'auth-plaintext'
|
|
84
84
|
Requires-Dist: keyrings-alt>=5.0; extra == 'auth-plaintext'
|
|
85
85
|
Provides-Extra: dev
|
|
86
|
-
Requires-Dist: build>=1.2.2.post1; extra == 'dev'
|
|
87
86
|
Requires-Dist: bump-my-version>=1.2.7; extra == 'dev'
|
|
88
87
|
Requires-Dist: defusedxml>=0.7; extra == 'dev'
|
|
89
88
|
Requires-Dist: jinja2>=3.1; extra == 'dev'
|
|
@@ -104,7 +103,6 @@ Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
|
104
103
|
Requires-Dist: sqlglot<30.22,>=30.21.0; extra == 'dev'
|
|
105
104
|
Requires-Dist: tables>=3.10; extra == 'dev'
|
|
106
105
|
Requires-Dist: tox-uv>=1.13.1; extra == 'dev'
|
|
107
|
-
Requires-Dist: twine>=6.1.0; extra == 'dev'
|
|
108
106
|
Requires-Dist: xlrd>=2.0; extra == 'dev'
|
|
109
107
|
Requires-Dist: zensical==0.0.67; extra == 'dev'
|
|
110
108
|
Provides-Extra: duckdb
|
|
@@ -466,7 +464,7 @@ execsql format --no-sql --in-place scripts/
|
|
|
466
464
|
```yaml
|
|
467
465
|
repos:
|
|
468
466
|
- repo: https://github.com/geocoug/execsql
|
|
469
|
-
rev: v2.
|
|
467
|
+
rev: v2.30.0
|
|
470
468
|
hooks:
|
|
471
469
|
- id: execsql-format
|
|
472
470
|
- id: execsql-lint
|