execsql2 2.28.0__py3-none-any.whl → 2.30.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. execsql/__main__.py +4 -5
  2. execsql/api.py +5 -0
  3. execsql/cli/__init__.py +3 -14
  4. execsql/cli/application.py +5 -3
  5. execsql/cli/commands/lint.py +4 -1
  6. execsql/cli/commands/lsp.py +42 -0
  7. execsql/cli/dsn.py +3 -1
  8. execsql/cli/lint.py +14 -6
  9. execsql/cli/run.py +17 -9
  10. execsql/config.py +32 -6
  11. execsql/db/base.py +6 -9
  12. execsql/db/duckdb.py +96 -14
  13. execsql/db/mysql.py +19 -3
  14. execsql/db/postgres.py +1 -1
  15. execsql/exporters/xls.py +3 -2
  16. execsql/gui/__init__.py +3 -31
  17. execsql/lsp/_util.py +21 -0
  18. execsql/lsp/cache.py +73 -0
  19. execsql/lsp/code_actions.py +10 -14
  20. execsql/lsp/completion.py +8 -15
  21. execsql/lsp/diagnostics.py +14 -2
  22. execsql/lsp/document.py +139 -41
  23. execsql/lsp/hover.py +5 -13
  24. execsql/lsp/navigation.py +7 -10
  25. execsql/lsp/server.py +111 -29
  26. execsql/manifest.py +29 -9
  27. execsql/metacommands/__init__.py +0 -2
  28. execsql/metacommands/io_fileops.py +5 -0
  29. execsql/script/executor.py +2 -21
  30. execsql/script/variables.py +42 -10
  31. execsql/state.py +6 -3
  32. execsql/state.pyi +2 -0
  33. execsql/types.py +8 -0
  34. execsql/utils/errors.py +19 -36
  35. execsql/utils/fileio.py +44 -48
  36. execsql/utils/regex.py +0 -5
  37. {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/METADATA +2 -4
  38. {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/RECORD +56 -54
  39. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/README.md +0 -0
  40. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
  41. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
  42. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
  43. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/md_compare.sql +0 -0
  44. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
  45. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
  46. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
  47. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
  48. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/pg_upsert.sql +0 -0
  49. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/script_template.sql +0 -0
  50. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
  51. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
  52. {execsql2-2.28.0.data → execsql2-2.30.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
  53. {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/WHEEL +0 -0
  54. {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/entry_points.txt +0 -0
  55. {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/licenses/LICENSE.txt +0 -0
  56. {execsql2-2.28.0.dist-info → execsql2-2.30.0.dist-info}/licenses/NOTICE +0 -0
execsql/manifest.py CHANGED
@@ -10,7 +10,11 @@ Files are recorded as the executor runs each metacommand, after variable
10
10
  substitution, so paths are the ones actually used. What a metacommand touches
11
11
  is decided by :mod:`execsql.metacommands.effects`. Variable values and passwords are never written:
12
12
  the run log hides ``-a`` and ``--var`` values because they can be secrets, and
13
- the manifest lists only their names.
13
+ the manifest lists only their names. A password or decrypted value that reaches
14
+ a path, a connection target or an error message is written as ``***``, as are
15
+ common secret shapes (a URL's password, ``password=...``, API tokens). ``-a`` and
16
+ ``--var`` values, which the log also masks, are kept: they are usually part of
17
+ the paths the manifest exists to report.
14
18
 
15
19
  One recorder is active per run; :func:`current` returns it, or ``None`` when
16
20
  no manifest was asked for, so the executor's hooks cost one check. Writing the
@@ -81,6 +85,22 @@ def _now() -> datetime.datetime:
81
85
  return datetime.datetime.now(tz=datetime.timezone.utc)
82
86
 
83
87
 
88
+ def _masked(value: Any) -> Any:
89
+ """*value* with the run's passwords and common secret shapes replaced by ``***`` in every string it holds.
90
+
91
+ ``-a`` / ``--var`` values are left in place; times, counts and line numbers are not strings.
92
+ """
93
+ from execsql.utils.fileio import redact
94
+
95
+ if isinstance(value, str):
96
+ return redact(value, arguments=False)
97
+ if isinstance(value, list):
98
+ return [_masked(item) for item in value]
99
+ if isinstance(value, dict):
100
+ return {key: _masked(item) for key, item in value.items()}
101
+ return value
102
+
103
+
84
104
  class RunManifest:
85
105
  """Collects what one run does and writes it as JSON when the run ends."""
86
106
 
@@ -166,20 +186,20 @@ class RunManifest:
166
186
  finished = _now()
167
187
  data = {
168
188
  "execsql_version": __version__,
169
- "script": self.script,
189
+ "script": _masked(self.script),
170
190
  "started": self.started.isoformat(timespec="seconds"),
171
191
  "finished": finished.isoformat(timespec="seconds"),
172
192
  "duration_s": round((finished - self.started).total_seconds(), 3),
173
193
  "exit_status": exit_status,
174
- "config_files": self.config_files,
175
- "database": self.database,
194
+ "config_files": _masked(self.config_files),
195
+ "database": _masked(self.database),
176
196
  "variables": self.variables,
177
197
  "statements": {"sql": self.sql_statements, "metacommands": self.metacommands},
178
- "files_read": self.reads,
179
- "files_written": self.writes,
180
- "files_deleted": self.deletes,
181
- "connections": self.connections,
182
- "errors": self.errors,
198
+ "files_read": _masked(self.reads),
199
+ "files_written": _masked(self.writes),
200
+ "files_deleted": _masked(self.deletes),
201
+ "connections": _masked(self.connections),
202
+ "errors": _masked(self.errors),
183
203
  }
184
204
  self.written = True
185
205
  import execsql.state as _state
@@ -242,7 +242,6 @@ from execsql.metacommands.system import (
242
242
  # Regex helper functions (from utils/regex.py)
243
243
  from execsql.utils.regex import (
244
244
  ins_rxs,
245
- ins_quoted_rx,
246
245
  ins_schema_rxs,
247
246
  ins_table_rxs,
248
247
  ins_table_list_rxs,
@@ -443,7 +442,6 @@ __all__ = [
443
442
  "x_pg_upsert_qa",
444
443
  # regex helpers
445
444
  "ins_rxs",
446
- "ins_quoted_rx",
447
445
  "ins_schema_rxs",
448
446
  "ins_table_rxs",
449
447
  "ins_table_list_rxs",
@@ -307,6 +307,11 @@ def x_cd(**kwargs: Any) -> None:
307
307
  command_text=kwargs["metacommandline"],
308
308
  other_msg="Directory does not exist",
309
309
  )
310
+ if _state.cd_from is None:
311
+ try:
312
+ _state.cd_from = os.getcwd()
313
+ except OSError: # the directory is gone: nowhere to return to
314
+ pass
310
315
  os.chdir(new_dir)
311
316
  script, lno = current_script_line()
312
317
  _state.exec_log.log_status_info(f"Current directory changed to {new_dir} at line {lno} of {script}")
@@ -957,16 +957,15 @@ class ExecutingStatement:
957
957
  rather than copying fields out of it, so there is nothing to keep in sync
958
958
  and no second source of truth about what is executing. Error reporting
959
959
  (:mod:`execsql.utils.errors`), the debug REPL, and ``api.run()`` read
960
- these five attributes; everything is derived from ``node`` on access, so
960
+ its attributes; everything is derived from ``node`` on access, so
961
961
  constructing one costs a single small object per statement rather than
962
962
  the two classes the previous shim built each time.
963
963
  """
964
964
 
965
- __slots__ = ("node", "_source_dir")
965
+ __slots__ = ("node",)
966
966
 
967
967
  def __init__(self, node: Node) -> None:
968
968
  self.node = node
969
- self._source_dir: str | None = None
970
969
 
971
970
  @property
972
971
  def source(self) -> str:
@@ -976,18 +975,6 @@ class ExecutingStatement:
976
975
  def line_no(self) -> int:
977
976
  return self.node.span.start_line
978
977
 
979
- @property
980
- def source_name(self) -> str:
981
- return Path(self.node.span.file).name
982
-
983
- @property
984
- def source_dir(self) -> str:
985
- # resolve() touches the filesystem, so it is done once per statement
986
- # object rather than on every read.
987
- if self._source_dir is None:
988
- self._source_dir = str(Path(self.node.span.file).resolve().parent) + os.sep
989
- return self._source_dir
990
-
991
978
  @property
992
979
  def command_type(self) -> str:
993
980
  return "sql" if isinstance(self.node, SqlStatement) else "cmd"
@@ -1004,12 +991,6 @@ class ExecutingStatement:
1004
991
  def __repr__(self) -> str:
1005
992
  return f"ExecutingStatement({self.source}:{self.line_no}, {self.command_type})"
1006
993
 
1007
- def current_script_line(self) -> tuple[str, int]:
1008
- return (self.source, self.line_no)
1009
-
1010
- def commandline(self) -> str:
1011
- return cast(str, self.command.commandline())
1012
-
1013
994
 
1014
995
  # ---------------------------------------------------------------------------
1015
996
  # Node type/text helpers for profiling
@@ -17,6 +17,42 @@ from execsql.exceptions import ErrInfo
17
17
  __all__ = ["CounterVars", "SubVarSet", "LocalSubVarSet", "ScriptArgSubVarSet"]
18
18
 
19
19
 
20
+ def _after_escape_string_prefix(command: str, start: int) -> bool:
21
+ """Whether the ``!'!`` token at *start* follows an ``E`` that makes it a PostgreSQL escape string (``E!'!v!'!``).
22
+
23
+ The ``E`` must stand alone: in ``TYPE!'!v!'!`` it ends a word.
24
+ """
25
+ if start < 1 or command[start - 1] not in "Ee":
26
+ return False
27
+ return start < 2 or not (command[start - 2].isalnum() or command[start - 2] == "_")
28
+
29
+
30
+ def _sql_string_literal(value: str, *, escape_string: bool = False) -> str:
31
+ """*value* as the string literal ``!'!var!'!`` substitutes, in the current connection's form.
32
+
33
+ An apostrophe is doubled everywhere; a backslash only where the DBMS
34
+ reads it as an escape (MySQL/MariaDB), or inside an escape string the
35
+ script wrote as ``E!'!var!'!`` (*escape_string*), so ``C:\\data`` is
36
+ stored as written in every case. Without a connection, the standard SQL
37
+ form. (Upstream doubled backslashes only on Windows, to protect
38
+ ``re.sub``'s replacement template, not the SQL.)
39
+ """
40
+ if escape_string:
41
+ return "'" + value.replace("\\", "\\\\").replace("'", "''") + "'"
42
+ db = None
43
+ try:
44
+ import execsql.state as _state
45
+
46
+ dbs = _state.dbs
47
+ if dbs is not None and dbs.current_db is not None:
48
+ db = dbs.current()
49
+ except Exception:
50
+ db = None
51
+ if db is not None:
52
+ return db.quote_literal(value)
53
+ return "'" + value.replace("'", "''") + "'"
54
+
55
+
20
56
  # ---------------------------------------------------------------------------
21
57
  # CounterVars
22
58
  # ---------------------------------------------------------------------------
@@ -253,13 +289,10 @@ class SubVarSet:
253
289
  )
254
290
  quote = m.group("q")
255
291
  if quote == "'":
256
- # B07a/F002: wrap in single quotes, doubling embedded
257
- # apostrophes AND escaping backslashes so MySQL default
258
- # mode and PostgreSQL E-string literals can't end the
259
- # quoted region via ``\'``. The previous Windows-only
260
- # branch (``os.name != 'posix'``) applied the escape on
261
- # the wrong axis — host OS vs target DBMS.
262
- sub = "'" + sub.replace("\\", "\\\\").replace("'", "''") + "'"
292
+ sub = _sql_string_literal(
293
+ sub,
294
+ escape_string=_after_escape_string_prefix(command_str, m.start()),
295
+ )
263
296
  elif quote == '"':
264
297
  # B07a/F001: wrap in double quotes, doubling embedded
265
298
  # ``"`` so a value containing ``"; DROP TABLE x; --``
@@ -306,13 +339,12 @@ class SubVarSet:
306
339
  if idx != -1:
307
340
  _check_nul(varname, sub)
308
341
  return command_str[:idx] + sub + command_str[idx + len(token) :], True
309
- # Single-quote-wrapped token: !'!varname!'! — escape ``\`` and
310
- # double embedded ``'`` (see substitute_one for rationale).
342
+ # Single-quote-wrapped token: !'!varname!'! — a string literal.
311
343
  tokenq = f"!'!{varname}!'!"
312
344
  idxq = cmd_lower.find(tokenq)
313
345
  if idxq != -1:
314
346
  _check_nul(varname, sub)
315
- wrapped = "'" + sub.replace("\\", "\\\\").replace("'", "''") + "'"
347
+ wrapped = _sql_string_literal(sub, escape_string=_after_escape_string_prefix(command_str, idxq))
316
348
  return (
317
349
  command_str[:idxq] + wrapped + command_str[idxq + len(tokenq) :],
318
350
  True,
execsql/state.py CHANGED
@@ -89,8 +89,6 @@ class ExecFrame:
89
89
 
90
90
 
91
91
  if TYPE_CHECKING:
92
- import multiprocessing as _mp
93
-
94
92
  from execsql.config import ConfigData, StatObj, WriteHooks
95
93
  from execsql.db.base import DatabasePool
96
94
  from execsql.exporters.base import ExportMetadata, WriteSpec
@@ -230,6 +228,7 @@ _CONTEXT_ATTRS: frozenset[str] = frozenset(
230
228
  "manifest",
231
229
  "halt_error",
232
230
  "halt_output_error",
231
+ "cd_from",
233
232
  # Lazy singletons
234
233
  "counters",
235
234
  "timer",
@@ -288,6 +287,7 @@ class RuntimeContext:
288
287
  "manifest",
289
288
  "halt_error",
290
289
  "halt_output_error",
290
+ "cd_from",
291
291
  # Lazy singletons
292
292
  "counters",
293
293
  "timer",
@@ -344,6 +344,9 @@ class RuntimeContext:
344
344
  self.halt_error: ErrInfo | None = None
345
345
  # File output lost when the run halted (set by ``exit_now``), or None.
346
346
  self.halt_output_error: ErrInfo | None = None
347
+ # The working directory before this run's first CD, or None: the
348
+ # library's run() returns its caller there.
349
+ self.cd_from: str | None = None
347
350
 
348
351
  # Lazy singletons
349
352
  self.counters: CounterVars | None = None
@@ -356,7 +359,7 @@ class RuntimeContext:
356
359
 
357
360
  # GUI
358
361
  self.gui_console: Any = None
359
- self.gui_manager_queue: _mp.Queue | None = None
362
+ self.gui_manager_queue: Any = None # a queue.Queue or a _TkinterSyncQueue; only put() is used
360
363
  self.gui_manager_thread: threading.Thread | None = None
361
364
 
362
365
  # Profiling — None means profiling is disabled; a list means it is enabled.
execsql/state.pyi CHANGED
@@ -59,6 +59,7 @@ filewriter: FileWriter | None
59
59
  manifest: RunManifest | None
60
60
  halt_error: ErrInfo | None
61
61
  halt_output_error: ErrInfo | None
62
+ cd_from: str | None
62
63
  counters: CounterVars
63
64
  timer: Timer
64
65
  dbs: DatabasePool
@@ -97,6 +98,7 @@ class RuntimeContext:
97
98
  manifest: RunManifest | None
98
99
  halt_error: ErrInfo | None
99
100
  halt_output_error: ErrInfo | None
101
+ cd_from: str | None
100
102
  counters: CounterVars | None
101
103
  timer: Timer | None
102
104
  dbs: DatabasePool | None
execsql/types.py CHANGED
@@ -884,6 +884,14 @@ dbt_oracle.name_datatype(DT_TimestampTZ, "TIMESTAMP WITH TIME ZONE")
884
884
  dbt_oracle.name_datatype(DT_Timestamp, "TIMESTAMP", casting_name="TIMESTAMP")
885
885
  dbt_oracle.name_datatype(DT_Date, "DATE", casting_name="DATE")
886
886
  dbt_oracle.name_datatype(DT_Time_Oracle, "VARCHAR2", True, casting_name="VARCHAR(20)")
887
+ # Oracle has no time-of-day type, and oracledb cannot bind a datetime.time: a
888
+ # time column is stored as its ISO text (at most 15 characters, HH:MM:SS.ffffff).
889
+ dbt_oracle.name_datatype(
890
+ DT_Time,
891
+ "VARCHAR2(15)",
892
+ casting_name="VARCHAR(20)",
893
+ conv_mod_fn=lambda x: x.isoformat() if x is not None else None,
894
+ )
887
895
  dbt_oracle.name_datatype(DT_Integer, "NUMBER")
888
896
  dbt_oracle.name_datatype(DT_Long, "NUMBER")
889
897
  dbt_oracle.name_datatype(DT_Float, "FLOAT")
execsql/utils/errors.py CHANGED
@@ -6,8 +6,6 @@ Error handling and reporting utilities for execsql.
6
6
  Provides the functions that form the error-reporting pipeline used
7
7
  throughout the codebase:
8
8
 
9
- - :func:`exception_info` — extracts type, value, and source location
10
- from the current exception.
11
9
  - :func:`exception_desc` — ``Type: message`` for the exception being
12
10
  handled, as shown on screen.
13
11
  - :func:`error_origin` / :func:`logged_error` — where in execsql an error
@@ -28,7 +26,6 @@ import execsql.state as _state
28
26
  from execsql.exceptions import ErrInfo
29
27
 
30
28
  __all__ = [
31
- "exception_info",
32
29
  "error_origin",
33
30
  "exception_desc",
34
31
  "logged_error",
@@ -56,43 +53,29 @@ def install_hint(extra: str) -> str:
56
53
  )
57
54
 
58
55
 
59
- def exception_info() -> tuple:
60
- # Returns the exception type, value, source file name, source line number, and source line text.
61
- strace = traceback.extract_tb(sys.exc_info()[2])[-1:]
62
- traces = traceback.extract_tb(sys.exc_info()[2])
63
- xline = 0
64
- for trace in traces:
65
- if "execsql" in trace[0]:
66
- xline = trace[1]
67
- exc_message = ""
68
- exc_param = sys.exc_info()[1]
69
- if isinstance(exc_param, str):
70
- exc_message = exc_param
71
- elif exc_param is not None:
72
- message = getattr(exc_param, "message", None)
73
- value = getattr(exc_param, "value", None)
74
- if isinstance(message, str) and len(message) > 0:
75
- exc_message = message
76
- elif isinstance(value, str) and len(value) > 0:
77
- exc_message = value
78
- else:
79
- exc_message = str(exc_param)
80
- try:
81
- exc_message = str(exc_message)
82
- except Exception:
83
- exc_message = repr(exc_message)
84
- xinfo = sys.exc_info()[0]
85
- xname = getattr(xinfo, "__name__", "")
86
- return xname, exc_message, strace[0][0], xline, strace[0][3]
87
-
88
-
89
56
  def exception_desc() -> str:
90
57
  """``Type: message`` for the exception being handled, as shown to the person running the script.
91
58
 
92
- Where in execsql it was raised is for the log: see :func:`error_origin`.
59
+ The message is the exception's ``message`` or ``value`` attribute when it
60
+ has a non-empty string one (as some drivers' exceptions do), else its
61
+ ``str()``. Where in execsql it was raised is for the log: see
62
+ :func:`error_origin`.
93
63
  """
94
- exc_type, exc_strval, _filename, _lineno, _linetext = exception_info()
95
- return f"{exc_type}: {exc_strval}"
64
+ exc_type, exc = sys.exc_info()[:2]
65
+ if isinstance(exc, str):
66
+ message = exc
67
+ elif exc is None:
68
+ message = ""
69
+ else:
70
+ attr = next(
71
+ (v for v in (getattr(exc, "message", None), getattr(exc, "value", None)) if isinstance(v, str) and v),
72
+ None,
73
+ )
74
+ try:
75
+ message = attr if attr is not None else str(exc)
76
+ except Exception:
77
+ message = repr(exc)
78
+ return f"{getattr(exc_type, '__name__', '')}: {message}"
96
79
 
97
80
 
98
81
  def error_origin(exc: BaseException) -> str | None:
execsql/utils/fileio.py CHANGED
@@ -34,7 +34,6 @@ import threading
34
34
  import time
35
35
  import weakref
36
36
  from collections.abc import Callable
37
- from encodings.aliases import aliases as codec_dict
38
37
  from typing import Any, NamedTuple, cast
39
38
 
40
39
  from execsql.exceptions import ErrInfo
@@ -47,14 +46,11 @@ __all__ = [
47
46
  "EncodedFile",
48
47
  "Logger",
49
48
  "TempFileMgr",
50
- "list_encodings",
51
49
  "filewriter_filestatus",
52
50
  "filewriter_write",
53
51
  "filewriter_open_as_new",
54
52
  "filewriter_close",
55
53
  "filewriter_close_all_after_write",
56
- "filewriter_closeall",
57
- "filewriter_shutdown",
58
54
  "filewriter_end",
59
55
  "redact",
60
56
  "register_secret",
@@ -216,9 +212,8 @@ class FileWriter(threading.Thread):
216
212
  CMD_OPEN_AS_NEW,
217
213
  CMD_CLOSE_ALL_AFTER_WRITE,
218
214
  CMD_CLOSED_STATUS,
219
- CMD_PING,
220
215
  CMD_OPEN_FAILURES,
221
- ) = range(10)
216
+ ) = range(9)
222
217
 
223
218
  class FileControl:
224
219
  STATUS_OPEN, STATUS_WAITING, STATUS_UNOPENED, STATUS_CLOSED, STATUS_OPENFAILURE, STATUS_WRITEFAILURE = range(6)
@@ -375,7 +370,6 @@ class FileWriter(threading.Thread):
375
370
  self.open_as_new,
376
371
  self.close_all_after_write,
377
372
  self.closed_status,
378
- self.ping,
379
373
  self.open_failures,
380
374
  )
381
375
 
@@ -417,10 +411,6 @@ class FileWriter(threading.Thread):
417
411
  if fc.status != self.FileControl.STATUS_CLOSED and fc.status not in fc.FAILED:
418
412
  self._on_file(fc, fc.clean_close)
419
413
 
420
- def ping(self, token: object) -> None:
421
- """Echo *token* back on the return queue — used to synchronize callers."""
422
- self.return_msg_queue.put(token)
423
-
424
414
  def retry_waiting(self) -> None:
425
415
  """Re-attempt opening every file still waiting on a lock.
426
416
 
@@ -526,7 +516,7 @@ class FileWriter(threading.Thread):
526
516
  the rest, so one bad command cannot stop the thread (and with it
527
517
  every later write, to any file).
528
518
  """
529
- filename = argtuple[0] if command != self.CMD_PING and argtuple and isinstance(argtuple[0], str) else ""
519
+ filename = argtuple[0] if argtuple and isinstance(argtuple[0], str) else ""
530
520
  lost = LostOutput(filename, 0, str(exc), "writer")
531
521
  if command == self.CMD_OPEN_FAILURES:
532
522
  self.return_msg_queue.put([lost])
@@ -535,7 +525,6 @@ class FileWriter(threading.Thread):
535
525
  replies: dict[int, Any] = {
536
526
  self.CMD_GET_STATUS: self.FileControl.STATUS_CLOSED,
537
527
  self.CMD_CLOSED_STATUS: self.FileControl.STATUS_CLOSED,
538
- self.CMD_PING: argtuple[0] if argtuple else None,
539
528
  }
540
529
  if command in replies:
541
530
  self.return_msg_queue.put(replies[command])
@@ -745,20 +734,6 @@ def _lost_output_message(lost: LostOutput, open_timeout: int) -> str:
745
734
  return f"{cause}; {lost.dropped} line(s) of output to it were discarded" if lost.dropped else cause
746
735
 
747
736
 
748
- def filewriter_closeall() -> None:
749
- fw = _active_writer()
750
- if fw is None:
751
- return
752
- fw.input_queue.put((FileWriter.CMD_CLOSE_ALL, ()))
753
-
754
-
755
- def filewriter_shutdown() -> None:
756
- fw = _active_writer()
757
- if fw is None:
758
- return
759
- fw.input_queue.put((FileWriter.CMD_SHUTDOWN, ()))
760
-
761
-
762
737
  def filewriter_end(writer: FileWriter | None = None) -> None:
763
738
  """Close every file and stop *writer*, or the active run's writer if omitted.
764
739
 
@@ -894,6 +869,8 @@ class Logger:
894
869
  self.server_name = server_name
895
870
  self.cmdline_options = cmdline_options
896
871
  self._redaction_values: list[str] = []
872
+ # Of those, the -a / --var values: masked in the log, not in the manifest.
873
+ self._argument_values: set[str] = set()
897
874
  if log_file_name:
898
875
  self.log_file_name = log_file_name
899
876
  else:
@@ -956,9 +933,9 @@ class Logger:
956
933
  )
957
934
  self.writelog(self._redact(msg) or "")
958
935
  if server_name:
959
- msg = f"run_db_server\t{self.run_id}\t{server_name}\t{db_name}\n"
936
+ msg = f"run_db_server\t{self.run_id}\t{self._redact(server_name)}\t{self._redact(db_name)}\n"
960
937
  else:
961
- msg = f"run_db_file\t{self.run_id}\t{db_name}\n"
938
+ msg = f"run_db_file\t{self.run_id}\t{self._redact(db_name)}\n"
962
939
  self.writelog(msg)
963
940
  self.seq_no = 0
964
941
  atexit.register(self.close)
@@ -974,20 +951,41 @@ class Logger:
974
951
 
975
952
  return _datetime.datetime.now().isoformat(timespec="seconds")
976
953
 
977
- def add_redaction_value(self, value: str | None) -> None:
954
+ def add_redaction_value(self, value: str | None, *, argument: bool = False) -> None:
955
+ """Mask *value* in every later log record.
956
+
957
+ *argument* marks an ``-a`` / ``--var`` value: it may be a secret, so
958
+ the log masks it, but it is just as often part of a path, so the
959
+ ``--manifest`` JSON (``redact(..., arguments=False)``) does not. A
960
+ value registered both ways is a secret.
961
+ """
978
962
  if value is None:
979
963
  return
980
964
  value = str(value)
981
- if not value or value in self._redaction_values:
965
+ if not value:
966
+ return
967
+ if not argument:
968
+ self._argument_values.discard(value)
969
+ if value in self._redaction_values:
982
970
  return
983
971
  self._redaction_values.append(value)
972
+ if argument:
973
+ self._argument_values.add(value)
974
+
975
+ def _redact(self, msg: str | None, *, arguments: bool = True) -> str | None:
976
+ """*msg* with registered secrets and common secret shapes replaced by ``***``.
984
977
 
985
- def _redact(self, msg: str | None) -> str | None:
978
+ Every record applies it to its free-text fields (messages, names,
979
+ paths), never to a whole line: a registered value can be as short as
980
+ a ``-a 2024`` argument, which would otherwise mask timestamps and
981
+ sequence numbers.
982
+ """
986
983
  if msg is None:
987
984
  return None
988
985
  redacted = str(msg)
989
986
  for value in sorted(self._redaction_values, key=len, reverse=True):
990
- redacted = redacted.replace(value, "***")
987
+ if arguments or value not in self._argument_values:
988
+ redacted = redacted.replace(value, "***")
991
989
  for pattern in self._COMMON_SECRET_PATTERNS:
992
990
  if pattern.pattern.startswith("(?i)(://"):
993
991
  redacted = pattern.sub(r"\1***\3", redacted)
@@ -1026,12 +1024,13 @@ class Logger:
1026
1024
 
1027
1025
  def log_db_connect(self, db: Any) -> None:
1028
1026
  self.seq_no += 1
1029
- msg = f"connect\t{self.run_id}\t{self.seq_no}\t{self._ts()}\t{db.name()}\n"
1027
+ msg = f"connect\t{self.run_id}\t{self.seq_no}\t{self._ts()}\t{self._redact(db.name())}\n"
1030
1028
  self.writelog(msg)
1031
1029
 
1032
1030
  def log_action_export(self, line_no: int, query_name: str, export_file_name: str) -> None:
1033
1031
  self.seq_no += 1
1034
- msg = f"action\t{self.run_id}\t{self.seq_no}\t{self._ts()}\texport\t{line_no}\tQuery {query_name} exported to {export_file_name}\n"
1032
+ description = self._redact(f"Query {query_name} exported to {export_file_name}")
1033
+ msg = f"action\t{self.run_id}\t{self.seq_no}\t{self._ts()}\texport\t{line_no}\t{description}\n"
1035
1034
  self.writelog(msg)
1036
1035
 
1037
1036
  def log_status_exception(self, msg: str | None) -> None:
@@ -1087,7 +1086,7 @@ class Logger:
1087
1086
  self.exit_type = "halt"
1088
1087
  self.exit_scriptfile = script_file_name
1089
1088
  self.exit_lno = line_no
1090
- self.exit_description = msg
1089
+ self.exit_description = None if not msg else msg.replace("\n", "")
1091
1090
 
1092
1091
  def log_exit_interrupted(self) -> None:
1093
1092
  # Save values to be used by exit() function triggered on program exit
@@ -1121,9 +1120,9 @@ class Logger:
1121
1120
  wmsg = "exit\t{}\t{}\t{}({})\t{}\t{:.1f}s\n".format(
1122
1121
  self.run_id,
1123
1122
  self.exit_type,
1124
- self.exit_scriptfile or "",
1123
+ self._redact(self.exit_scriptfile) or "",
1125
1124
  str(self.exit_lno or ""),
1126
- self.exit_description or "",
1125
+ self._redact(self.exit_description) or "", # a HALT MESSAGE can hold any variable's value
1127
1126
  elapsed,
1128
1127
  )
1129
1128
  self.writelog(wmsg)
@@ -1144,12 +1143,16 @@ def register_secret(value: str | None) -> None:
1144
1143
  add(value)
1145
1144
 
1146
1145
 
1147
- def redact(text: str) -> str:
1148
- """*text* with the active run's registered secrets and common secret shapes replaced by ``***``."""
1146
+ def redact(text: str, *, arguments: bool = True) -> str:
1147
+ """*text* with the active run's registered secrets and common secret shapes replaced by ``***``.
1148
+
1149
+ With *arguments* false, ``-a`` / ``--var`` values are left as they are
1150
+ (see :meth:`Logger.add_redaction_value`).
1151
+ """
1149
1152
  import execsql.state as _state
1150
1153
 
1151
1154
  redactor = getattr(_state.exec_log, "_redact", None)
1152
- return redactor(text) if redactor is not None else text
1155
+ return (redactor(text, arguments=arguments) or "") if redactor is not None else text
1153
1156
 
1154
1157
 
1155
1158
  class TempFileMgr:
@@ -1188,10 +1191,3 @@ _live_tempfile_mgrs: weakref.WeakSet[TempFileMgr] = weakref.WeakSet()
1188
1191
  def _remove_all_temp_files() -> None:
1189
1192
  for mgr in list(_live_tempfile_mgrs):
1190
1193
  mgr.remove_all()
1191
-
1192
-
1193
- def list_encodings() -> None:
1194
- enc = list(codec_dict.keys())
1195
- enc.sort()
1196
- msg = f"Encodings: {', '.join(enc)}\n"
1197
- print(msg)
execsql/utils/regex.py CHANGED
@@ -16,7 +16,6 @@ import os
16
16
 
17
17
  __all__ = [
18
18
  "ins_rxs",
19
- "ins_quoted_rx",
20
19
  "ins_schema_rxs",
21
20
  "ins_table_rxs",
22
21
  "ins_table_list_rxs",
@@ -42,10 +41,6 @@ def ins_rxs(rx_list: tuple[str, ...], fragment1: object, fragment2: object) -> t
42
41
  return tuple(rv)
43
42
 
44
43
 
45
- def ins_quoted_rx(fragment1: object, fragment2: object, rx: str) -> tuple[str, ...]:
46
- return ins_rxs((rx, rf'"{rx}"'), fragment1, fragment2)
47
-
48
-
49
44
  def ins_schema_rxs(fragment1: object, fragment2: object, suffix: str | None = None) -> tuple[str, ...]:
50
45
  schema_exprs: tuple[str, ...] = (
51
46
  r'"(?P<schema>[A-Za-z0-9_\- ]+)"',
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: execsql2
3
- Version: 2.28.0
3
+ Version: 2.30.0
4
4
  Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
5
5
  Project-URL: Homepage, https://execsql2.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/geocoug/execsql
@@ -83,7 +83,6 @@ Provides-Extra: auth-plaintext
83
83
  Requires-Dist: keyring>=25.0; extra == 'auth-plaintext'
84
84
  Requires-Dist: keyrings-alt>=5.0; extra == 'auth-plaintext'
85
85
  Provides-Extra: dev
86
- Requires-Dist: build>=1.2.2.post1; extra == 'dev'
87
86
  Requires-Dist: bump-my-version>=1.2.7; extra == 'dev'
88
87
  Requires-Dist: defusedxml>=0.7; extra == 'dev'
89
88
  Requires-Dist: jinja2>=3.1; extra == 'dev'
@@ -104,7 +103,6 @@ Requires-Dist: ruff>=0.4; extra == 'dev'
104
103
  Requires-Dist: sqlglot<30.22,>=30.21.0; extra == 'dev'
105
104
  Requires-Dist: tables>=3.10; extra == 'dev'
106
105
  Requires-Dist: tox-uv>=1.13.1; extra == 'dev'
107
- Requires-Dist: twine>=6.1.0; extra == 'dev'
108
106
  Requires-Dist: xlrd>=2.0; extra == 'dev'
109
107
  Requires-Dist: zensical==0.0.67; extra == 'dev'
110
108
  Provides-Extra: duckdb
@@ -466,7 +464,7 @@ execsql format --no-sql --in-place scripts/
466
464
  ```yaml
467
465
  repos:
468
466
  - repo: https://github.com/geocoug/execsql
469
- rev: v2.28.0
467
+ rev: v2.30.0
470
468
  hooks:
471
469
  - id: execsql-format
472
470
  - id: execsql-lint