execsql2 2.30.0__py3-none-any.whl → 2.31.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +1 -0
- execsql/cli/application.py +1 -1
- execsql/cli/commands/lint.py +1 -1
- execsql/cli/dsn.py +35 -1
- execsql/cli/lint.py +10 -1
- execsql/cli/run.py +8 -3
- execsql/config.py +26 -4
- execsql/data/execsql.conf.template +8 -2
- execsql/db/access.py +7 -5
- execsql/db/base.py +18 -0
- execsql/db/factory.py +11 -1
- execsql/db/mysql.py +56 -3
- execsql/db/odbc.py +45 -0
- execsql/db/postgres.py +1 -1
- execsql/db/sqlserver.py +37 -21
- execsql/exporters/ods.py +146 -41
- execsql/exporters/values.py +166 -9
- execsql/importers/base.py +1 -1
- execsql/importers/csv.py +1 -1
- execsql/importers/ods.py +2 -0
- execsql/lsp/diagnostics.py +1 -1
- execsql/lsp/document.py +3 -0
- execsql/metacommands/io_fileops.py +11 -5
- execsql/metacommands/reference.py +1 -1
- execsql/metacommands/upsert.py +10 -1
- execsql/utils/auth.py +21 -14
- execsql/utils/errors.py +1 -1
- execsql/utils/paths.py +31 -0
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/README.md +9 -8
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/md_compare.sql +53 -29
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/md_glossary.sql +20 -20
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/md_upsert.sql +59 -66
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/pg_compare.sql +19 -22
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/pg_glossary.sql +13 -17
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/script_template.sql +302 -299
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/ss_compare.sql +39 -29
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/ss_glossary.sql +13 -17
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/ss_upsert.sql +56 -66
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/METADATA +17 -41
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/RECORD +47 -46
- execsql2-2.30.0.data/data/execsql2_extras/pg_upsert.sql +0 -2791
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/WHEEL +0 -0
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/entry_points.txt +0 -0
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/licenses/NOTICE +0 -0
execsql/api.py
CHANGED
execsql/cli/application.py
CHANGED
|
@@ -282,7 +282,7 @@ def _online_help_callback(value: bool) -> None:
|
|
|
282
282
|
if value:
|
|
283
283
|
import webbrowser
|
|
284
284
|
|
|
285
|
-
webbrowser.open("https://execsql2.readthedocs.io/en/
|
|
285
|
+
webbrowser.open("https://execsql2.readthedocs.io/en/stable/", new=2, autoraise=True)
|
|
286
286
|
raise typer.Exit()
|
|
287
287
|
|
|
288
288
|
|
execsql/cli/commands/lint.py
CHANGED
|
@@ -143,7 +143,7 @@ class LintFormat(str, Enum):
|
|
|
143
143
|
"Every issue names its rule code, which --select and --ignore accept, as a full "
|
|
144
144
|
"code (V001) or a prefix (V). Parse errors (P001) are always reported. Exits 1 "
|
|
145
145
|
"when any error is found; warnings alone exit 0 unless --strict is given.\n\n"
|
|
146
|
-
"Rules: https://execsql2.readthedocs.io/en/
|
|
146
|
+
"Rules: https://execsql2.readthedocs.io/en/stable/reference/lint/"
|
|
147
147
|
),
|
|
148
148
|
)
|
|
149
149
|
def lint_cmd(
|
execsql/cli/dsn.py
CHANGED
|
@@ -40,7 +40,9 @@ def _parse_connection_string(dsn: str) -> dict:
|
|
|
40
40
|
written ``%40``, ``%3A``, ``%2F``, ``%3F``, ``%23``, ``%25``.
|
|
41
41
|
|
|
42
42
|
Returns a dict with keys: ``db_type``, ``server``, ``db``, ``db_file``,
|
|
43
|
-
``user``, ``password``, ``port
|
|
43
|
+
``user``, ``password``, ``port`` and ``trust_server_certificate`` (from
|
|
44
|
+
``?trust_server_certificate=yes`` on an ``mssql://`` URL). Absent
|
|
45
|
+
components are ``None``.
|
|
44
46
|
|
|
45
47
|
Raises :class:`~execsql.exceptions.ConfigError` for an unrecognised
|
|
46
48
|
URL scheme or a completely un-parseable string.
|
|
@@ -86,6 +88,8 @@ def _parse_connection_string(dsn: str) -> dict:
|
|
|
86
88
|
# Remove leading "/"
|
|
87
89
|
db = raw_path.lstrip("/") or None
|
|
88
90
|
|
|
91
|
+
trust_server_certificate = _query_options(parsed.query, db_type)
|
|
92
|
+
|
|
89
93
|
return {
|
|
90
94
|
"db_type": db_type,
|
|
91
95
|
"server": server,
|
|
@@ -94,4 +98,34 @@ def _parse_connection_string(dsn: str) -> dict:
|
|
|
94
98
|
"user": user,
|
|
95
99
|
"password": password,
|
|
96
100
|
"port": port,
|
|
101
|
+
"trust_server_certificate": trust_server_certificate,
|
|
97
102
|
}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
_TRUE = ("yes", "true", "on", "1")
|
|
106
|
+
_FALSE = ("no", "false", "off", "0")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _query_options(query: str, db_type: str) -> bool | None:
|
|
110
|
+
"""Return ``trust_server_certificate`` from a DSN's query string, or None if absent.
|
|
111
|
+
|
|
112
|
+
It is the only query parameter execsql reads, and only for SQL Server.
|
|
113
|
+
Anything else is an error rather than ignored, so a misspelt option
|
|
114
|
+
cannot silently leave a connection without the setting it asked for.
|
|
115
|
+
"""
|
|
116
|
+
from urllib.parse import parse_qsl
|
|
117
|
+
|
|
118
|
+
trust: bool | None = None
|
|
119
|
+
for key, value in parse_qsl(query, keep_blank_values=True):
|
|
120
|
+
if key.lower() != "trust_server_certificate" or db_type != "s":
|
|
121
|
+
raise ConfigError(
|
|
122
|
+
f"Unrecognised connection-string parameter {key!r}. "
|
|
123
|
+
"The only one execsql reads is trust_server_certificate, for mssql:// URLs.",
|
|
124
|
+
)
|
|
125
|
+
if value.lower() in _TRUE:
|
|
126
|
+
trust = True
|
|
127
|
+
elif value.lower() in _FALSE:
|
|
128
|
+
trust = False
|
|
129
|
+
else:
|
|
130
|
+
raise ConfigError(f"trust_server_certificate must be yes or no, not {value!r}.")
|
|
131
|
+
return trust
|
execsql/cli/lint.py
CHANGED
|
@@ -56,6 +56,7 @@ from collections.abc import Iterable
|
|
|
56
56
|
from pathlib import Path
|
|
57
57
|
from typing import TYPE_CHECKING, NamedTuple
|
|
58
58
|
|
|
59
|
+
from execsql.utils.paths import is_network_path
|
|
59
60
|
from execsql.exceptions import ErrInfo
|
|
60
61
|
from execsql.script.ast import (
|
|
61
62
|
BatchBlock,
|
|
@@ -461,6 +462,8 @@ def _read_ini_vars(
|
|
|
461
462
|
language server; a file that is unreadable or not valid INI defines
|
|
462
463
|
nothing.
|
|
463
464
|
"""
|
|
465
|
+
if is_network_path(ini_file): # see _check_include_path
|
|
466
|
+
return
|
|
464
467
|
p = Path(ini_file)
|
|
465
468
|
if not p.is_absolute() and script_dir is not None:
|
|
466
469
|
p = script_dir / p
|
|
@@ -582,7 +585,13 @@ def _check_include_path(
|
|
|
582
585
|
line_no: int,
|
|
583
586
|
issues: list[_Issue],
|
|
584
587
|
) -> None:
|
|
585
|
-
"""Warn if the INCLUDE target does not exist on disk.
|
|
588
|
+
"""Warn if the INCLUDE target does not exist on disk.
|
|
589
|
+
|
|
590
|
+
A target written as a network path is not checked: on Windows, even
|
|
591
|
+
asking whether it exists contacts the host it names.
|
|
592
|
+
"""
|
|
593
|
+
if is_network_path(raw_path):
|
|
594
|
+
return
|
|
586
595
|
p = Path(raw_path)
|
|
587
596
|
if not p.is_absolute() and script_dir is not None:
|
|
588
597
|
p = script_dir / p
|
execsql/cli/run.py
CHANGED
|
@@ -281,6 +281,7 @@ def _load_config(
|
|
|
281
281
|
subvars: SubVarSet,
|
|
282
282
|
config_file: str | None,
|
|
283
283
|
start_dir: str | None = None,
|
|
284
|
+
local_only: bool = False,
|
|
284
285
|
) -> ConfigData:
|
|
285
286
|
"""Load and merge configuration files for the given script.
|
|
286
287
|
|
|
@@ -291,7 +292,7 @@ def _load_config(
|
|
|
291
292
|
script_path = str(Path(script_name).resolve().parent)
|
|
292
293
|
else: # resolved, as os.getcwd() is, so the start directory's execsql.conf is not read twice
|
|
293
294
|
script_path = str(Path(start_dir).resolve()) if start_dir else os.getcwd()
|
|
294
|
-
return ConfigData(script_path, subvars, config_file=config_file, start_dir=start_dir)
|
|
295
|
+
return ConfigData(script_path, subvars, config_file=config_file, start_dir=start_dir, local_only=local_only)
|
|
295
296
|
|
|
296
297
|
|
|
297
298
|
def _tool_config(config_file: str | None, start_dir: str | None = None) -> ConfigData:
|
|
@@ -301,9 +302,11 @@ def _tool_config(config_file: str | None, start_dir: str | None = None) -> Confi
|
|
|
301
302
|
locations plus *config_file* — not per script, so a script directory's
|
|
302
303
|
own ``execsql.conf`` applies only when passed with ``--config``. The
|
|
303
304
|
language server passes a workspace folder as *start_dir*: the settings
|
|
304
|
-
those commands would read when run from it.
|
|
305
|
+
those commands would read when run from it. Config chains written as
|
|
306
|
+
network paths are skipped untouched (``local_only``): these tools read
|
|
307
|
+
config from repositories other people write.
|
|
305
308
|
"""
|
|
306
|
-
return _load_config(None, _seed_early_subvars(start_dir), config_file, start_dir)
|
|
309
|
+
return _load_config(None, _seed_early_subvars(start_dir), config_file, start_dir, local_only=True)
|
|
307
310
|
|
|
308
311
|
|
|
309
312
|
def _configured_script_encoding(conf: ConfigData) -> str | None:
|
|
@@ -366,6 +369,8 @@ def _apply_dsn(dsn: str, conf: ConfigData, db_type: str | None) -> tuple[str | N
|
|
|
366
369
|
conf.db = parsed["db"]
|
|
367
370
|
if parsed["db_file"]:
|
|
368
371
|
conf.db_file = parsed["db_file"]
|
|
372
|
+
if parsed["trust_server_certificate"] is not None:
|
|
373
|
+
conf.trust_server_certificate = parsed["trust_server_certificate"]
|
|
369
374
|
user = parsed["user"] # may be None
|
|
370
375
|
if parsed["password"]:
|
|
371
376
|
conf.db_password = parsed["password"]
|
execsql/config.py
CHANGED
|
@@ -23,6 +23,7 @@ from configparser import ConfigParser
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
from typing import Protocol
|
|
25
25
|
|
|
26
|
+
from execsql.utils.paths import is_network_path
|
|
26
27
|
from execsql.exceptions import ConfigError, ConfigFileError
|
|
27
28
|
from execsql.utils.crypto import Encrypt
|
|
28
29
|
|
|
@@ -281,6 +282,7 @@ class ConfigData:
|
|
|
281
282
|
self._get_bool(cp, self._CONNECT_SECTION, "password_prompt", "passwd_prompt")
|
|
282
283
|
self._get_bool(cp, self._CONNECT_SECTION, "use_keyring", "use_keyring")
|
|
283
284
|
self._get_bool(cp, self._CONNECT_SECTION, "new_db", "new_db")
|
|
285
|
+
self._get_bool(cp, self._CONNECT_SECTION, "trust_server_certificate", "trust_server_certificate")
|
|
284
286
|
# --- [encoding] ---
|
|
285
287
|
self._get_str(cp, self._ENCODING_SECTION, "database", "db_encoding")
|
|
286
288
|
self._get_str(cp, self._ENCODING_SECTION, "script", "script_encoding", required=True)
|
|
@@ -402,6 +404,7 @@ class ConfigData:
|
|
|
402
404
|
*,
|
|
403
405
|
config_file: str | None = None,
|
|
404
406
|
start_dir: str | None = None,
|
|
407
|
+
local_only: bool = False,
|
|
405
408
|
) -> None:
|
|
406
409
|
"""Load and merge all discoverable execsql.conf files for the given script path.
|
|
407
410
|
|
|
@@ -420,8 +423,13 @@ class ConfigData:
|
|
|
420
423
|
``[config]`` chains) resolve against it. Defaults to the
|
|
421
424
|
working directory. The language server passes each
|
|
422
425
|
workspace folder, so it never changes the process's directory.
|
|
426
|
+
local_only: Skip a ``config_file`` chain or ``[include_*]`` path
|
|
427
|
+
written as a network path (:func:`~execsql.utils.paths.is_network_path`)
|
|
428
|
+
without touching it. The static tools pass it: they read
|
|
429
|
+
config files a cloned repository may supply.
|
|
423
430
|
"""
|
|
424
431
|
self._start_dir = start_dir
|
|
432
|
+
self._local_only = local_only
|
|
425
433
|
self._set_defaults()
|
|
426
434
|
current_script = str(Path(sys.argv[0]).resolve())
|
|
427
435
|
from collections import deque
|
|
@@ -464,6 +472,8 @@ class ConfigData:
|
|
|
464
472
|
|
|
465
473
|
#: Where relative paths resolve; ``None`` is the working directory.
|
|
466
474
|
_start_dir: str | None = None
|
|
475
|
+
#: Whether paths written as network paths are skipped (see ``__init__``).
|
|
476
|
+
_local_only: bool = False
|
|
467
477
|
|
|
468
478
|
#: Guard against circular config_file references.
|
|
469
479
|
_MAX_CONFIG_CHAIN = 20
|
|
@@ -481,6 +491,7 @@ class ConfigData:
|
|
|
481
491
|
self.use_keyring = True
|
|
482
492
|
self.db_file: str | None = None
|
|
483
493
|
self.new_db = False
|
|
494
|
+
self.trust_server_certificate = False
|
|
484
495
|
self.user_logfile = False
|
|
485
496
|
self.db_encoding: str | None = None
|
|
486
497
|
self.script_encoding = "utf8"
|
|
@@ -602,12 +613,15 @@ class ConfigData:
|
|
|
602
613
|
*,
|
|
603
614
|
expand_home: bool,
|
|
604
615
|
start_dir: str | None = None,
|
|
616
|
+
local_only: bool = False,
|
|
605
617
|
) -> str:
|
|
606
618
|
"""Resolve a ``config_file``-style value to the file it names.
|
|
607
619
|
|
|
608
620
|
A leading ``~`` is expanded when *expand_home* is true, substitution
|
|
609
621
|
variables are replaced, and a directory means the ``execsql.conf``
|
|
610
622
|
inside it. The result may not exist; callers decide what that means.
|
|
623
|
+
With *local_only*, a value that is a network path once substituted
|
|
624
|
+
gives ``""`` without being touched.
|
|
611
625
|
"""
|
|
612
626
|
conffile = value
|
|
613
627
|
if expand_home and conffile and conffile[0] == "~":
|
|
@@ -616,12 +630,18 @@ class ConfigData:
|
|
|
616
630
|
elif len(conffile) > 1 and conffile[1] == os.sep:
|
|
617
631
|
conffile = str(Path("~").expanduser() / conffile[2:])
|
|
618
632
|
conffile = variable_pool.substitute(conffile)[0]
|
|
633
|
+
if local_only and is_network_path(conffile):
|
|
634
|
+
return ""
|
|
619
635
|
if start_dir and not Path(conffile).is_absolute():
|
|
620
636
|
conffile = str(Path(start_dir) / conffile)
|
|
621
637
|
if not Path(conffile).is_file():
|
|
622
638
|
conffile = str(Path(conffile) / cls.config_file_name)
|
|
623
639
|
return conffile
|
|
624
640
|
|
|
641
|
+
def _skips(self, path: str) -> bool:
|
|
642
|
+
"""Whether *path* is left untouched: a network path when ``local_only``."""
|
|
643
|
+
return self._local_only and is_network_path(path)
|
|
644
|
+
|
|
625
645
|
def _apply_file(self, cp: ConfigParser, variable_pool: _VariablePool, current_script: str) -> list[str]:
|
|
626
646
|
"""Apply one parsed config file to ``self``; return the files it chains to.
|
|
627
647
|
|
|
@@ -672,8 +692,9 @@ class ConfigData:
|
|
|
672
692
|
variable_pool,
|
|
673
693
|
expand_home=os.name == "posix",
|
|
674
694
|
start_dir=self._start_dir,
|
|
695
|
+
local_only=self._local_only,
|
|
675
696
|
)
|
|
676
|
-
if Path(conffile).is_file():
|
|
697
|
+
if conffile and Path(conffile).is_file():
|
|
677
698
|
# Silently ignore a non-existent file, for cross-OS compatibility.
|
|
678
699
|
chained.append(conffile)
|
|
679
700
|
# OS-specific additional config files.
|
|
@@ -690,8 +711,9 @@ class ConfigData:
|
|
|
690
711
|
variable_pool,
|
|
691
712
|
expand_home=True,
|
|
692
713
|
start_dir=self._start_dir,
|
|
714
|
+
local_only=self._local_only,
|
|
693
715
|
)
|
|
694
|
-
if Path(conffile).is_file():
|
|
716
|
+
if conffile and Path(conffile).is_file():
|
|
695
717
|
chained.append(conffile)
|
|
696
718
|
# dao_flush_delay_secs has a specific error message — keep inline
|
|
697
719
|
if cp.has_option(self._CONFIG_SECTION, "dao_flush_delay_secs"):
|
|
@@ -727,7 +749,7 @@ class ConfigData:
|
|
|
727
749
|
if cp.has_section(self._INCLUDE_REQ_SECTION):
|
|
728
750
|
imp_items = cp.items(self._INCLUDE_REQ_SECTION)
|
|
729
751
|
ord_items = sorted([(int(i[0]), i[1]) for i in imp_items], key=lambda x: x[0])
|
|
730
|
-
newfiles = [str(Path(f[1]).resolve()) for f in ord_items]
|
|
752
|
+
newfiles = [str(Path(f[1]).resolve()) for f in ord_items if not self._skips(f[1])]
|
|
731
753
|
u_files = []
|
|
732
754
|
for f in newfiles:
|
|
733
755
|
if not (f in u_files or f in self.include_req or f in self.include_opt) and f != current_script:
|
|
@@ -738,7 +760,7 @@ class ConfigData:
|
|
|
738
760
|
if cp.has_section(self._INCLUDE_OPT_SECTION):
|
|
739
761
|
imp_items = cp.items(self._INCLUDE_OPT_SECTION)
|
|
740
762
|
ord_items = sorted([(int(i[0]), i[1]) for i in imp_items], key=lambda x: x[0])
|
|
741
|
-
newfiles = [str(Path(f[1]).resolve()) for f in ord_items]
|
|
763
|
+
newfiles = [str(Path(f[1]).resolve()) for f in ord_items if not self._skips(f[1])]
|
|
742
764
|
u_files = []
|
|
743
765
|
for f in newfiles:
|
|
744
766
|
if (
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# execsql.conf — Configuration file for execsql
|
|
2
2
|
#
|
|
3
|
-
# Documentation: https://execsql2.readthedocs.io/en/
|
|
3
|
+
# Documentation: https://execsql2.readthedocs.io/en/stable/reference/configuration/
|
|
4
4
|
#
|
|
5
5
|
# This file uses INI format. Section names are case-sensitive (lowercase).
|
|
6
6
|
# Property names are not case-sensitive. Lines starting with # are comments.
|
|
@@ -54,6 +54,12 @@
|
|
|
54
54
|
# Values: Yes or No. Default: No.
|
|
55
55
|
#new_db=No
|
|
56
56
|
|
|
57
|
+
# SQL Server only: accept the server's TLS certificate without verifying it,
|
|
58
|
+
# for a server with a self-signed certificate. ODBC Driver 18 otherwise
|
|
59
|
+
# refuses one it cannot verify. Applies to CONNECT as well.
|
|
60
|
+
# Values: Yes or No. Default: No.
|
|
61
|
+
#trust_server_certificate=No
|
|
62
|
+
|
|
57
63
|
|
|
58
64
|
[encoding]
|
|
59
65
|
# Character encoding for text input and output.
|
|
@@ -279,7 +285,7 @@
|
|
|
279
285
|
#message_css=
|
|
280
286
|
|
|
281
287
|
# Obfuscated password (XOR, not cryptographically secure — use keyring instead).
|
|
282
|
-
# See: https://execsql2.readthedocs.io/en/
|
|
288
|
+
# See: https://execsql2.readthedocs.io/en/stable/reference/security/#credentials
|
|
283
289
|
#enc_password=
|
|
284
290
|
|
|
285
291
|
|
execsql/db/access.py
CHANGED
|
@@ -15,6 +15,7 @@ from pathlib import Path
|
|
|
15
15
|
from typing import Any, cast
|
|
16
16
|
|
|
17
17
|
from execsql.db.base import Database, _fetched
|
|
18
|
+
from execsql.db.odbc import odbc_value
|
|
18
19
|
from execsql.db.tiers import SupportTier
|
|
19
20
|
from execsql.exceptions import ErrInfo
|
|
20
21
|
from execsql.utils.errors import exception_desc, fatal_error, install_hint
|
|
@@ -119,10 +120,11 @@ class AccessDatabase(Database):
|
|
|
119
120
|
def _try_odbc_drivers():
|
|
120
121
|
db_name = str(Path(self.db_name).resolve())
|
|
121
122
|
for cs, jet4flag in self.connection_strings:
|
|
123
|
+
connstr = cs % odbc_value(db_name)
|
|
124
|
+
logstr = connstr
|
|
122
125
|
if self.need_passwd:
|
|
123
|
-
connstr
|
|
124
|
-
|
|
125
|
-
connstr = cs % db_name
|
|
126
|
+
connstr += f" Uid={odbc_value(self.user)}; Pwd={odbc_value(self.password)};"
|
|
127
|
+
logstr += f" Uid={odbc_value(self.user)}; Pwd=***;"
|
|
126
128
|
try:
|
|
127
129
|
# Access is file-based but a UNC path on an
|
|
128
130
|
# unreachable share can still block forever; cap
|
|
@@ -133,12 +135,12 @@ class AccessDatabase(Database):
|
|
|
133
135
|
errors.append(e)
|
|
134
136
|
if _state.exec_log is not None:
|
|
135
137
|
_state.exec_log.log_status_info(
|
|
136
|
-
f"Could not connect via ODBC using: {
|
|
138
|
+
f"Could not connect via ODBC using: {logstr}",
|
|
137
139
|
)
|
|
138
140
|
else:
|
|
139
141
|
if _state.exec_log is not None:
|
|
140
142
|
_state.exec_log.log_status_info(
|
|
141
|
-
f"Connected via ODBC using: {
|
|
143
|
+
f"Connected via ODBC using: {logstr}",
|
|
142
144
|
)
|
|
143
145
|
self.jet4 = jet4flag
|
|
144
146
|
return True
|
execsql/db/base.py
CHANGED
|
@@ -641,6 +641,24 @@ class Database(ABC):
|
|
|
641
641
|
|
|
642
642
|
raise DatabaseNotImplementedError(self.name(), "role_exists")
|
|
643
643
|
|
|
644
|
+
def drop_table_for_replacement(self, schema_name: str | None, table_name: str) -> None:
|
|
645
|
+
"""Drop the table a ``TO REPLACEMENT`` import or copy is about to recreate, if it is there.
|
|
646
|
+
|
|
647
|
+
Not every DBMS has ``DROP TABLE IF EXISTS``: SQL Server, Oracle,
|
|
648
|
+
Firebird and Access refuse to drop a table that does not exist. The
|
|
649
|
+
drop is tried first, and its error is ignored only when the table is
|
|
650
|
+
confirmed absent, so a drop that would have succeeded always runs.
|
|
651
|
+
"""
|
|
652
|
+
try:
|
|
653
|
+
self.drop_table(self.schema_qualified_table_name(schema_name, table_name))
|
|
654
|
+
except Exception:
|
|
655
|
+
try:
|
|
656
|
+
there = self.table_exists_where_created(table_name, schema_name)
|
|
657
|
+
except Exception:
|
|
658
|
+
there = True
|
|
659
|
+
if there:
|
|
660
|
+
raise
|
|
661
|
+
|
|
644
662
|
def drop_table(self, tablename: str) -> None:
|
|
645
663
|
"""Drop *tablename* if it exists; *tablename* must already be schema-qualified and quoted."""
|
|
646
664
|
# The 'tablename' argument should be schema-qualified and quoted as necessary.
|
execsql/db/factory.py
CHANGED
|
@@ -106,9 +106,19 @@ def db_SqlServer(
|
|
|
106
106
|
port: int | None = None,
|
|
107
107
|
encoding: str | None = None,
|
|
108
108
|
password: str | None = None,
|
|
109
|
+
trust_server_certificate: bool | None = None,
|
|
109
110
|
) -> SqlServerDatabase:
|
|
110
111
|
"""Open a Microsoft SQL Server connection via pyodbc."""
|
|
111
|
-
return SqlServerDatabase(
|
|
112
|
+
return SqlServerDatabase(
|
|
113
|
+
server_name,
|
|
114
|
+
database_name,
|
|
115
|
+
user,
|
|
116
|
+
pw_needed,
|
|
117
|
+
port,
|
|
118
|
+
encoding,
|
|
119
|
+
password=password,
|
|
120
|
+
trust_server_certificate=trust_server_certificate,
|
|
121
|
+
)
|
|
112
122
|
|
|
113
123
|
|
|
114
124
|
def db_MySQL(
|
execsql/db/mysql.py
CHANGED
|
@@ -7,7 +7,9 @@ Implements :class:`MySQLDatabase`, which connects to MySQL and MariaDB
|
|
|
7
7
|
servers via ``pymysql``. Corresponds to ``-t m`` on the CLI.
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
+
import os
|
|
10
11
|
import re
|
|
12
|
+
import tempfile
|
|
11
13
|
from typing import Any, cast
|
|
12
14
|
|
|
13
15
|
from execsql.db.base import Database
|
|
@@ -53,6 +55,49 @@ def _crlf_terminated(filename: str) -> bool:
|
|
|
53
55
|
return i > 0 and chunk[i - 1 : i] == b"\r"
|
|
54
56
|
|
|
55
57
|
|
|
58
|
+
def _data_length(filename: str) -> int | None:
|
|
59
|
+
"""Return the file's length without its trailing blank lines, or None if it has none.
|
|
60
|
+
|
|
61
|
+
LOAD DATA reads each blank line at the end of a file as a row of NULLs;
|
|
62
|
+
every other import path drops them. The file is read backwards from the
|
|
63
|
+
end, as bytes, so no decoding can hide a line ending.
|
|
64
|
+
"""
|
|
65
|
+
try:
|
|
66
|
+
with open(filename, "rb") as f:
|
|
67
|
+
size = f.seek(0, 2)
|
|
68
|
+
pos = size
|
|
69
|
+
content_end = 0
|
|
70
|
+
while pos > 0:
|
|
71
|
+
start = max(0, pos - 65536)
|
|
72
|
+
f.seek(start)
|
|
73
|
+
block = f.read(pos - start)
|
|
74
|
+
stripped = len(block.rstrip(b"\r\n"))
|
|
75
|
+
if stripped:
|
|
76
|
+
content_end = start + stripped
|
|
77
|
+
break
|
|
78
|
+
pos = start
|
|
79
|
+
f.seek(content_end)
|
|
80
|
+
ending = f.read(2)
|
|
81
|
+
except OSError:
|
|
82
|
+
return None
|
|
83
|
+
# Keep the line ending of the last line of data.
|
|
84
|
+
keep = content_end + (2 if ending == b"\r\n" else 1 if ending[:1] in (b"\r", b"\n") else 0)
|
|
85
|
+
return keep if keep < size else None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _copy_head(filename: str, length: int) -> str:
|
|
89
|
+
"""Copy the first *length* bytes of *filename* to a temporary file; return its path."""
|
|
90
|
+
with open(filename, "rb") as src, tempfile.NamedTemporaryFile("wb", suffix=".csv", delete=False) as dst:
|
|
91
|
+
remaining = length
|
|
92
|
+
while remaining > 0:
|
|
93
|
+
chunk = src.read(min(remaining, 1 << 20))
|
|
94
|
+
if not chunk:
|
|
95
|
+
break
|
|
96
|
+
dst.write(chunk)
|
|
97
|
+
remaining -= len(chunk)
|
|
98
|
+
return dst.name
|
|
99
|
+
|
|
100
|
+
|
|
56
101
|
class MySQLDatabase(Database):
|
|
57
102
|
"""MySQL and MariaDB adapter using the pymysql package."""
|
|
58
103
|
|
|
@@ -353,7 +398,11 @@ class MySQLDatabase(Database):
|
|
|
353
398
|
# String literals in this session treat a backslash as an escape,
|
|
354
399
|
# so the file name, delimiter and quote character are quoted with
|
|
355
400
|
# quote_literal, which doubles it.
|
|
356
|
-
|
|
401
|
+
load_file = csv_file_obj.csvfname
|
|
402
|
+
data_length = _data_length(load_file)
|
|
403
|
+
if data_length is not None:
|
|
404
|
+
load_file = _copy_head(load_file, data_length)
|
|
405
|
+
import_sql = f"load data local infile {self.quote_literal(load_file)} into table {sq_name}"
|
|
357
406
|
if csv_file_obj.encoding:
|
|
358
407
|
charset = _PYTHON_TO_MYSQL_CHARSET.get(csv_file_obj.encoding.lower(), csv_file_obj.encoding)
|
|
359
408
|
import_sql = f"{import_sql} character set {charset}"
|
|
@@ -385,7 +434,11 @@ class MySQLDatabase(Database):
|
|
|
385
434
|
_state.exec_log.log_status_info(
|
|
386
435
|
f"IMPORTing {csv_file_obj.csvfname} using the DBMS' fast file reading routine",
|
|
387
436
|
)
|
|
388
|
-
|
|
437
|
+
try:
|
|
438
|
+
self.execute(import_sql)
|
|
439
|
+
finally:
|
|
440
|
+
if load_file != csv_file_obj.csvfname:
|
|
441
|
+
os.unlink(load_file)
|
|
389
442
|
else:
|
|
390
443
|
data_indexes = [csv_file_cols.index(col) for col in import_cols]
|
|
391
444
|
paramspec = ",".join(["%s"] * len(import_cols))
|
|
@@ -450,7 +503,7 @@ class MySQLDatabase(Database):
|
|
|
450
503
|
if not _state.conf.empty_strings and line[i].strip() == "":
|
|
451
504
|
line[i] = None
|
|
452
505
|
# Pad short line with nulls
|
|
453
|
-
line.extend([None] * (len(
|
|
506
|
+
line.extend([None] * (len(csv_file_cols) - len(line)))
|
|
454
507
|
linedata = [line[ix] for ix in data_indexes]
|
|
455
508
|
add_line = True
|
|
456
509
|
if not _state.conf.empty_rows:
|
execsql/db/odbc.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ODBC connection strings for the SQL Server and Access adapters.
|
|
3
|
+
|
|
4
|
+
An ODBC connection string is a ``;``-separated list of ``KEY=value``
|
|
5
|
+
attributes, so a value containing ``;`` (a password, a file path, a name
|
|
6
|
+
taken from a substitution variable) would end its attribute early and
|
|
7
|
+
start another. :func:`odbc_value` brace-quotes such values;
|
|
8
|
+
:func:`connection_string` joins the attributes, and :func:`loggable`
|
|
9
|
+
gives the same string with the password masked for the log.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
__all__ = ["connection_string", "loggable", "odbc_value"]
|
|
15
|
+
|
|
16
|
+
_SECRET_KEYS = frozenset({"pwd", "password"})
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def odbc_value(value: object) -> str:
|
|
20
|
+
"""Return *value* as an ODBC attribute value, brace-quoted when it needs it.
|
|
21
|
+
|
|
22
|
+
A ``;`` ends an attribute and a leading ``{`` opens a quoted value, so a
|
|
23
|
+
value with either is wrapped in ``{…}`` with each ``}`` doubled. Other
|
|
24
|
+
values are written as they are, because older drivers do not all accept
|
|
25
|
+
braces around every attribute.
|
|
26
|
+
"""
|
|
27
|
+
text = str(value)
|
|
28
|
+
if ";" in text or text.startswith("{"):
|
|
29
|
+
return "{" + text.replace("}", "}}") + "}"
|
|
30
|
+
return text
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def connection_string(driver: str, attributes: list[tuple[str, object]]) -> str:
|
|
34
|
+
"""Return ``DRIVER={driver};KEY=value;…`` with every value quoted as needed."""
|
|
35
|
+
parts = [f"DRIVER={{{driver}}}"]
|
|
36
|
+
parts.extend(f"{key}={odbc_value(value)}" for key, value in attributes)
|
|
37
|
+
return ";".join(parts) + ";"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def loggable(driver: str, attributes: list[tuple[str, object]]) -> str:
|
|
41
|
+
"""Return the connection string with the password replaced by ``***``."""
|
|
42
|
+
return connection_string(
|
|
43
|
+
driver,
|
|
44
|
+
[(key, "***" if key.lower() in _SECRET_KEYS else value) for key, value in attributes],
|
|
45
|
+
)
|
execsql/db/postgres.py
CHANGED
|
@@ -462,7 +462,7 @@ class PostgresDatabase(Database):
|
|
|
462
462
|
if not _state.conf.empty_strings and line[i].strip() == "":
|
|
463
463
|
line[i] = None
|
|
464
464
|
# Pad short line with nulls
|
|
465
|
-
line.extend([None] * (len(
|
|
465
|
+
line.extend([None] * (len(csv_file_cols) - len(line)))
|
|
466
466
|
linedata = [line[ix] for ix in data_indexes]
|
|
467
467
|
add_line = True
|
|
468
468
|
if not _state.conf.empty_rows:
|
execsql/db/sqlserver.py
CHANGED
|
@@ -7,9 +7,8 @@ Implements :class:`SqlServerDatabase`, which connects to Microsoft SQL
|
|
|
7
7
|
Server via ``pyodbc``. Corresponds to ``-t s`` on the CLI.
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
-
import re
|
|
11
|
-
|
|
12
10
|
from execsql.db.base import Database
|
|
11
|
+
from execsql.db.odbc import connection_string, loggable
|
|
13
12
|
from execsql.exceptions import ErrInfo
|
|
14
13
|
from execsql.utils.errors import fatal_error, install_hint
|
|
15
14
|
from execsql.utils.auth import (
|
|
@@ -34,9 +33,10 @@ class SqlServerDatabase(Database):
|
|
|
34
33
|
db_name: str,
|
|
35
34
|
user_name: str | None,
|
|
36
35
|
need_passwd: bool = False,
|
|
37
|
-
port: int | None =
|
|
36
|
+
port: int | None = None,
|
|
38
37
|
encoding: str | None = "latin1",
|
|
39
38
|
password: str | None = None,
|
|
39
|
+
trust_server_certificate: bool | None = None,
|
|
40
40
|
) -> None:
|
|
41
41
|
try:
|
|
42
42
|
import pyodbc # noqa: F401
|
|
@@ -51,6 +51,15 @@ class SqlServerDatabase(Database):
|
|
|
51
51
|
need_passwd=need_passwd,
|
|
52
52
|
port=port if port else 1433,
|
|
53
53
|
)
|
|
54
|
+
# Only a port that was asked for goes into the connection string: a
|
|
55
|
+
# named instance (``host\instance``) is found through SQL Server
|
|
56
|
+
# Browser, which an explicit port bypasses.
|
|
57
|
+
self.explicit_port = port or None
|
|
58
|
+
# None: the [connect] trust_server_certificate setting, which also
|
|
59
|
+
# covers connections a script opens with CONNECT.
|
|
60
|
+
if trust_server_certificate is None:
|
|
61
|
+
trust_server_certificate = bool(getattr(_state.conf, "trust_server_certificate", False))
|
|
62
|
+
self.trust_server_certificate = trust_server_certificate
|
|
54
63
|
self.type = dbt_sqlserver
|
|
55
64
|
self.password = password
|
|
56
65
|
self.encoding = encoding or "latin1" # Default on installation of SQL Server
|
|
@@ -79,6 +88,12 @@ class SqlServerDatabase(Database):
|
|
|
79
88
|
"""
|
|
80
89
|
return "[" + identifier.replace("]", "]]") + "]"
|
|
81
90
|
|
|
91
|
+
def _server_attribute(self) -> str:
|
|
92
|
+
"""Return the ``SERVER`` value: ``host,port`` when a port was given."""
|
|
93
|
+
if self.explicit_port and "," not in self.server_name:
|
|
94
|
+
return f"{self.server_name},{self.explicit_port}"
|
|
95
|
+
return str(self.server_name)
|
|
96
|
+
|
|
82
97
|
def open_db(self) -> None:
|
|
83
98
|
"""Open a connection to the SQL Server database via pyodbc."""
|
|
84
99
|
import pyodbc
|
|
@@ -95,6 +110,7 @@ class SqlServerDatabase(Database):
|
|
|
95
110
|
)
|
|
96
111
|
# Use pyodbc to connect. Try different driver versions from newest to oldest.
|
|
97
112
|
ssdrivers = (
|
|
113
|
+
"ODBC Driver 18 for SQL Server",
|
|
98
114
|
"ODBC Driver 17 for SQL Server",
|
|
99
115
|
"ODBC Driver 13.1 for SQL Server",
|
|
100
116
|
"ODBC Driver 13 for SQL Server",
|
|
@@ -108,23 +124,23 @@ class SqlServerDatabase(Database):
|
|
|
108
124
|
errors: list[BaseException] = []
|
|
109
125
|
|
|
110
126
|
def _try_drivers():
|
|
127
|
+
attributes: list[tuple[str, object]] = [
|
|
128
|
+
("SERVER", self._server_attribute()),
|
|
129
|
+
("MARS_Connection", "Yes"),
|
|
130
|
+
("DATABASE", self.db_name),
|
|
131
|
+
]
|
|
132
|
+
if self.user:
|
|
133
|
+
attributes.append(("Uid", self.user))
|
|
134
|
+
if self.password:
|
|
135
|
+
attributes.append(("Pwd", self.password))
|
|
136
|
+
else:
|
|
137
|
+
attributes.append(("Trusted_Connection", "yes"))
|
|
138
|
+
if self.trust_server_certificate:
|
|
139
|
+
# Driver 18 encrypts and verifies the server's certificate
|
|
140
|
+
# by default; this accepts a self-signed one.
|
|
141
|
+
attributes.append(("TrustServerCertificate", "yes"))
|
|
111
142
|
for drv in ssdrivers:
|
|
112
|
-
|
|
113
|
-
if self.password:
|
|
114
|
-
connstr = (
|
|
115
|
-
f"DRIVER={{{drv}}};SERVER={self.server_name};MARS_Connection=Yes; "
|
|
116
|
-
f"DATABASE={self.db_name};Uid={self.user};Pwd={self.password}"
|
|
117
|
-
)
|
|
118
|
-
else:
|
|
119
|
-
connstr = (
|
|
120
|
-
f"DRIVER={{{drv}}};SERVER={self.server_name};MARS_Connection=Yes; "
|
|
121
|
-
f"DATABASE={self.db_name};Uid={self.user}"
|
|
122
|
-
)
|
|
123
|
-
else:
|
|
124
|
-
connstr = (
|
|
125
|
-
f"DRIVER={{{drv}}};SERVER={self.server_name};MARS_Connection=Yes; "
|
|
126
|
-
f"DATABASE={self.db_name};Trusted_Connection=yes"
|
|
127
|
-
)
|
|
143
|
+
connstr = connection_string(drv, attributes)
|
|
128
144
|
try:
|
|
129
145
|
# 30 s connect timeout matches the Postgres adapter
|
|
130
146
|
# default; pyodbc treats `timeout` as the login-and-
|
|
@@ -134,12 +150,12 @@ class SqlServerDatabase(Database):
|
|
|
134
150
|
errors.append(e)
|
|
135
151
|
if _state.exec_log is not None:
|
|
136
152
|
_state.exec_log.log_status_info(
|
|
137
|
-
f"Could not connect using: {
|
|
153
|
+
f"Could not connect using: {loggable(drv, attributes)}",
|
|
138
154
|
)
|
|
139
155
|
else:
|
|
140
156
|
if _state.exec_log is not None:
|
|
141
157
|
_state.exec_log.log_status_info(
|
|
142
|
-
f"Connected using: {
|
|
158
|
+
f"Connected using: {loggable(drv, attributes)}",
|
|
143
159
|
)
|
|
144
160
|
return True
|
|
145
161
|
return False
|