execsql2 2.30.0__py3-none-any.whl → 2.31.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. execsql/api.py +1 -0
  2. execsql/cli/application.py +1 -1
  3. execsql/cli/commands/lint.py +1 -1
  4. execsql/cli/dsn.py +35 -1
  5. execsql/cli/lint.py +10 -1
  6. execsql/cli/run.py +8 -3
  7. execsql/config.py +26 -4
  8. execsql/data/execsql.conf.template +8 -2
  9. execsql/db/access.py +7 -5
  10. execsql/db/base.py +18 -0
  11. execsql/db/factory.py +11 -1
  12. execsql/db/mysql.py +56 -3
  13. execsql/db/odbc.py +45 -0
  14. execsql/db/postgres.py +1 -1
  15. execsql/db/sqlserver.py +37 -21
  16. execsql/exporters/ods.py +146 -41
  17. execsql/exporters/values.py +166 -9
  18. execsql/importers/base.py +1 -1
  19. execsql/importers/csv.py +1 -1
  20. execsql/importers/ods.py +2 -0
  21. execsql/lsp/diagnostics.py +1 -1
  22. execsql/lsp/document.py +3 -0
  23. execsql/metacommands/io_fileops.py +11 -5
  24. execsql/metacommands/reference.py +1 -1
  25. execsql/metacommands/upsert.py +10 -1
  26. execsql/utils/auth.py +21 -14
  27. execsql/utils/errors.py +1 -1
  28. execsql/utils/paths.py +31 -0
  29. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/README.md +9 -8
  30. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/md_compare.sql +53 -29
  31. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/md_glossary.sql +20 -20
  32. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/md_upsert.sql +59 -66
  33. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/pg_compare.sql +19 -22
  34. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/pg_glossary.sql +13 -17
  35. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/script_template.sql +302 -299
  36. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/ss_compare.sql +39 -29
  37. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/ss_glossary.sql +13 -17
  38. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/ss_upsert.sql +56 -66
  39. {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/METADATA +17 -41
  40. {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/RECORD +47 -46
  41. execsql2-2.30.0.data/data/execsql2_extras/pg_upsert.sql +0 -2791
  42. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/config_settings.sqlite +0 -0
  43. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
  44. {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/make_config_db.sql +0 -0
  45. {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/WHEEL +0 -0
  46. {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/entry_points.txt +0 -0
  47. {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/licenses/LICENSE.txt +0 -0
  48. {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/licenses/NOTICE +0 -0
execsql/api.py CHANGED
@@ -297,6 +297,7 @@ def _connect_from_dsn(dsn: str, new_db: bool = False) -> Any:
297
297
  user=user,
298
298
  password=password,
299
299
  port=port,
300
+ trust_server_certificate=params["trust_server_certificate"],
300
301
  )
301
302
  elif db_type == "o":
302
303
  return db_Oracle(
@@ -282,7 +282,7 @@ def _online_help_callback(value: bool) -> None:
282
282
  if value:
283
283
  import webbrowser
284
284
 
285
- webbrowser.open("https://execsql2.readthedocs.io/en/latest/", new=2, autoraise=True)
285
+ webbrowser.open("https://execsql2.readthedocs.io/en/stable/", new=2, autoraise=True)
286
286
  raise typer.Exit()
287
287
 
288
288
 
@@ -143,7 +143,7 @@ class LintFormat(str, Enum):
143
143
  "Every issue names its rule code, which --select and --ignore accept, as a full "
144
144
  "code (V001) or a prefix (V). Parse errors (P001) are always reported. Exits 1 "
145
145
  "when any error is found; warnings alone exit 0 unless --strict is given.\n\n"
146
- "Rules: https://execsql2.readthedocs.io/en/latest/reference/lint/"
146
+ "Rules: https://execsql2.readthedocs.io/en/stable/reference/lint/"
147
147
  ),
148
148
  )
149
149
  def lint_cmd(
execsql/cli/dsn.py CHANGED
@@ -40,7 +40,9 @@ def _parse_connection_string(dsn: str) -> dict:
40
40
  written ``%40``, ``%3A``, ``%2F``, ``%3F``, ``%23``, ``%25``.
41
41
 
42
42
  Returns a dict with keys: ``db_type``, ``server``, ``db``, ``db_file``,
43
- ``user``, ``password``, ``port``. Absent components are ``None``.
43
+ ``user``, ``password``, ``port`` and ``trust_server_certificate`` (from
44
+ ``?trust_server_certificate=yes`` on an ``mssql://`` URL). Absent
45
+ components are ``None``.
44
46
 
45
47
  Raises :class:`~execsql.exceptions.ConfigError` for an unrecognised
46
48
  URL scheme or a completely un-parseable string.
@@ -86,6 +88,8 @@ def _parse_connection_string(dsn: str) -> dict:
86
88
  # Remove leading "/"
87
89
  db = raw_path.lstrip("/") or None
88
90
 
91
+ trust_server_certificate = _query_options(parsed.query, db_type)
92
+
89
93
  return {
90
94
  "db_type": db_type,
91
95
  "server": server,
@@ -94,4 +98,34 @@ def _parse_connection_string(dsn: str) -> dict:
94
98
  "user": user,
95
99
  "password": password,
96
100
  "port": port,
101
+ "trust_server_certificate": trust_server_certificate,
97
102
  }
103
+
104
+
105
+ _TRUE = ("yes", "true", "on", "1")
106
+ _FALSE = ("no", "false", "off", "0")
107
+
108
+
109
+ def _query_options(query: str, db_type: str) -> bool | None:
110
+ """Return ``trust_server_certificate`` from a DSN's query string, or None if absent.
111
+
112
+ It is the only query parameter execsql reads, and only for SQL Server.
113
+ Anything else is an error rather than ignored, so a misspelt option
114
+ cannot silently leave a connection without the setting it asked for.
115
+ """
116
+ from urllib.parse import parse_qsl
117
+
118
+ trust: bool | None = None
119
+ for key, value in parse_qsl(query, keep_blank_values=True):
120
+ if key.lower() != "trust_server_certificate" or db_type != "s":
121
+ raise ConfigError(
122
+ f"Unrecognised connection-string parameter {key!r}. "
123
+ "The only one execsql reads is trust_server_certificate, for mssql:// URLs.",
124
+ )
125
+ if value.lower() in _TRUE:
126
+ trust = True
127
+ elif value.lower() in _FALSE:
128
+ trust = False
129
+ else:
130
+ raise ConfigError(f"trust_server_certificate must be yes or no, not {value!r}.")
131
+ return trust
execsql/cli/lint.py CHANGED
@@ -56,6 +56,7 @@ from collections.abc import Iterable
56
56
  from pathlib import Path
57
57
  from typing import TYPE_CHECKING, NamedTuple
58
58
 
59
+ from execsql.utils.paths import is_network_path
59
60
  from execsql.exceptions import ErrInfo
60
61
  from execsql.script.ast import (
61
62
  BatchBlock,
@@ -461,6 +462,8 @@ def _read_ini_vars(
461
462
  language server; a file that is unreadable or not valid INI defines
462
463
  nothing.
463
464
  """
465
+ if is_network_path(ini_file): # see _check_include_path
466
+ return
464
467
  p = Path(ini_file)
465
468
  if not p.is_absolute() and script_dir is not None:
466
469
  p = script_dir / p
@@ -582,7 +585,13 @@ def _check_include_path(
582
585
  line_no: int,
583
586
  issues: list[_Issue],
584
587
  ) -> None:
585
- """Warn if the INCLUDE target does not exist on disk."""
588
+ """Warn if the INCLUDE target does not exist on disk.
589
+
590
+ A target written as a network path is not checked: on Windows, even
591
+ asking whether it exists contacts the host it names.
592
+ """
593
+ if is_network_path(raw_path):
594
+ return
586
595
  p = Path(raw_path)
587
596
  if not p.is_absolute() and script_dir is not None:
588
597
  p = script_dir / p
execsql/cli/run.py CHANGED
@@ -281,6 +281,7 @@ def _load_config(
281
281
  subvars: SubVarSet,
282
282
  config_file: str | None,
283
283
  start_dir: str | None = None,
284
+ local_only: bool = False,
284
285
  ) -> ConfigData:
285
286
  """Load and merge configuration files for the given script.
286
287
 
@@ -291,7 +292,7 @@ def _load_config(
291
292
  script_path = str(Path(script_name).resolve().parent)
292
293
  else: # resolved, as os.getcwd() is, so the start directory's execsql.conf is not read twice
293
294
  script_path = str(Path(start_dir).resolve()) if start_dir else os.getcwd()
294
- return ConfigData(script_path, subvars, config_file=config_file, start_dir=start_dir)
295
+ return ConfigData(script_path, subvars, config_file=config_file, start_dir=start_dir, local_only=local_only)
295
296
 
296
297
 
297
298
  def _tool_config(config_file: str | None, start_dir: str | None = None) -> ConfigData:
@@ -301,9 +302,11 @@ def _tool_config(config_file: str | None, start_dir: str | None = None) -> Confi
301
302
  locations plus *config_file* — not per script, so a script directory's
302
303
  own ``execsql.conf`` applies only when passed with ``--config``. The
303
304
  language server passes a workspace folder as *start_dir*: the settings
304
- those commands would read when run from it.
305
+ those commands would read when run from it. Config chains written as
306
+ network paths are skipped untouched (``local_only``): these tools read
307
+ config from repositories other people write.
305
308
  """
306
- return _load_config(None, _seed_early_subvars(start_dir), config_file, start_dir)
309
+ return _load_config(None, _seed_early_subvars(start_dir), config_file, start_dir, local_only=True)
307
310
 
308
311
 
309
312
  def _configured_script_encoding(conf: ConfigData) -> str | None:
@@ -366,6 +369,8 @@ def _apply_dsn(dsn: str, conf: ConfigData, db_type: str | None) -> tuple[str | N
366
369
  conf.db = parsed["db"]
367
370
  if parsed["db_file"]:
368
371
  conf.db_file = parsed["db_file"]
372
+ if parsed["trust_server_certificate"] is not None:
373
+ conf.trust_server_certificate = parsed["trust_server_certificate"]
369
374
  user = parsed["user"] # may be None
370
375
  if parsed["password"]:
371
376
  conf.db_password = parsed["password"]
execsql/config.py CHANGED
@@ -23,6 +23,7 @@ from configparser import ConfigParser
23
23
  from pathlib import Path
24
24
  from typing import Protocol
25
25
 
26
+ from execsql.utils.paths import is_network_path
26
27
  from execsql.exceptions import ConfigError, ConfigFileError
27
28
  from execsql.utils.crypto import Encrypt
28
29
 
@@ -281,6 +282,7 @@ class ConfigData:
281
282
  self._get_bool(cp, self._CONNECT_SECTION, "password_prompt", "passwd_prompt")
282
283
  self._get_bool(cp, self._CONNECT_SECTION, "use_keyring", "use_keyring")
283
284
  self._get_bool(cp, self._CONNECT_SECTION, "new_db", "new_db")
285
+ self._get_bool(cp, self._CONNECT_SECTION, "trust_server_certificate", "trust_server_certificate")
284
286
  # --- [encoding] ---
285
287
  self._get_str(cp, self._ENCODING_SECTION, "database", "db_encoding")
286
288
  self._get_str(cp, self._ENCODING_SECTION, "script", "script_encoding", required=True)
@@ -402,6 +404,7 @@ class ConfigData:
402
404
  *,
403
405
  config_file: str | None = None,
404
406
  start_dir: str | None = None,
407
+ local_only: bool = False,
405
408
  ) -> None:
406
409
  """Load and merge all discoverable execsql.conf files for the given script path.
407
410
 
@@ -420,8 +423,13 @@ class ConfigData:
420
423
  ``[config]`` chains) resolve against it. Defaults to the
421
424
  working directory. The language server passes each
422
425
  workspace folder, so it never changes the process's directory.
426
+ local_only: Skip a ``config_file`` chain or ``[include_*]`` path
427
+ written as a network path (:func:`~execsql.utils.paths.is_network_path`)
428
+ without touching it. The static tools pass it: they read
429
+ config files a cloned repository may supply.
423
430
  """
424
431
  self._start_dir = start_dir
432
+ self._local_only = local_only
425
433
  self._set_defaults()
426
434
  current_script = str(Path(sys.argv[0]).resolve())
427
435
  from collections import deque
@@ -464,6 +472,8 @@ class ConfigData:
464
472
 
465
473
  #: Where relative paths resolve; ``None`` is the working directory.
466
474
  _start_dir: str | None = None
475
+ #: Whether paths written as network paths are skipped (see ``__init__``).
476
+ _local_only: bool = False
467
477
 
468
478
  #: Guard against circular config_file references.
469
479
  _MAX_CONFIG_CHAIN = 20
@@ -481,6 +491,7 @@ class ConfigData:
481
491
  self.use_keyring = True
482
492
  self.db_file: str | None = None
483
493
  self.new_db = False
494
+ self.trust_server_certificate = False
484
495
  self.user_logfile = False
485
496
  self.db_encoding: str | None = None
486
497
  self.script_encoding = "utf8"
@@ -602,12 +613,15 @@ class ConfigData:
602
613
  *,
603
614
  expand_home: bool,
604
615
  start_dir: str | None = None,
616
+ local_only: bool = False,
605
617
  ) -> str:
606
618
  """Resolve a ``config_file``-style value to the file it names.
607
619
 
608
620
  A leading ``~`` is expanded when *expand_home* is true, substitution
609
621
  variables are replaced, and a directory means the ``execsql.conf``
610
622
  inside it. The result may not exist; callers decide what that means.
623
+ With *local_only*, a value that is a network path once substituted
624
+ gives ``""`` without being touched.
611
625
  """
612
626
  conffile = value
613
627
  if expand_home and conffile and conffile[0] == "~":
@@ -616,12 +630,18 @@ class ConfigData:
616
630
  elif len(conffile) > 1 and conffile[1] == os.sep:
617
631
  conffile = str(Path("~").expanduser() / conffile[2:])
618
632
  conffile = variable_pool.substitute(conffile)[0]
633
+ if local_only and is_network_path(conffile):
634
+ return ""
619
635
  if start_dir and not Path(conffile).is_absolute():
620
636
  conffile = str(Path(start_dir) / conffile)
621
637
  if not Path(conffile).is_file():
622
638
  conffile = str(Path(conffile) / cls.config_file_name)
623
639
  return conffile
624
640
 
641
+ def _skips(self, path: str) -> bool:
642
+ """Whether *path* is left untouched: a network path when ``local_only``."""
643
+ return self._local_only and is_network_path(path)
644
+
625
645
  def _apply_file(self, cp: ConfigParser, variable_pool: _VariablePool, current_script: str) -> list[str]:
626
646
  """Apply one parsed config file to ``self``; return the files it chains to.
627
647
 
@@ -672,8 +692,9 @@ class ConfigData:
672
692
  variable_pool,
673
693
  expand_home=os.name == "posix",
674
694
  start_dir=self._start_dir,
695
+ local_only=self._local_only,
675
696
  )
676
- if Path(conffile).is_file():
697
+ if conffile and Path(conffile).is_file():
677
698
  # Silently ignore a non-existent file, for cross-OS compatibility.
678
699
  chained.append(conffile)
679
700
  # OS-specific additional config files.
@@ -690,8 +711,9 @@ class ConfigData:
690
711
  variable_pool,
691
712
  expand_home=True,
692
713
  start_dir=self._start_dir,
714
+ local_only=self._local_only,
693
715
  )
694
- if Path(conffile).is_file():
716
+ if conffile and Path(conffile).is_file():
695
717
  chained.append(conffile)
696
718
  # dao_flush_delay_secs has a specific error message — keep inline
697
719
  if cp.has_option(self._CONFIG_SECTION, "dao_flush_delay_secs"):
@@ -727,7 +749,7 @@ class ConfigData:
727
749
  if cp.has_section(self._INCLUDE_REQ_SECTION):
728
750
  imp_items = cp.items(self._INCLUDE_REQ_SECTION)
729
751
  ord_items = sorted([(int(i[0]), i[1]) for i in imp_items], key=lambda x: x[0])
730
- newfiles = [str(Path(f[1]).resolve()) for f in ord_items]
752
+ newfiles = [str(Path(f[1]).resolve()) for f in ord_items if not self._skips(f[1])]
731
753
  u_files = []
732
754
  for f in newfiles:
733
755
  if not (f in u_files or f in self.include_req or f in self.include_opt) and f != current_script:
@@ -738,7 +760,7 @@ class ConfigData:
738
760
  if cp.has_section(self._INCLUDE_OPT_SECTION):
739
761
  imp_items = cp.items(self._INCLUDE_OPT_SECTION)
740
762
  ord_items = sorted([(int(i[0]), i[1]) for i in imp_items], key=lambda x: x[0])
741
- newfiles = [str(Path(f[1]).resolve()) for f in ord_items]
763
+ newfiles = [str(Path(f[1]).resolve()) for f in ord_items if not self._skips(f[1])]
742
764
  u_files = []
743
765
  for f in newfiles:
744
766
  if (
@@ -1,6 +1,6 @@
1
1
  # execsql.conf — Configuration file for execsql
2
2
  #
3
- # Documentation: https://execsql2.readthedocs.io/en/latest/reference/configuration/
3
+ # Documentation: https://execsql2.readthedocs.io/en/stable/reference/configuration/
4
4
  #
5
5
  # This file uses INI format. Section names are case-sensitive (lowercase).
6
6
  # Property names are not case-sensitive. Lines starting with # are comments.
@@ -54,6 +54,12 @@
54
54
  # Values: Yes or No. Default: No.
55
55
  #new_db=No
56
56
 
57
+ # SQL Server only: accept the server's TLS certificate without verifying it,
58
+ # for a server with a self-signed certificate. ODBC Driver 18 otherwise
59
+ # refuses one it cannot verify. Applies to CONNECT as well.
60
+ # Values: Yes or No. Default: No.
61
+ #trust_server_certificate=No
62
+
57
63
 
58
64
  [encoding]
59
65
  # Character encoding for text input and output.
@@ -279,7 +285,7 @@
279
285
  #message_css=
280
286
 
281
287
  # Obfuscated password (XOR, not cryptographically secure — use keyring instead).
282
- # See: https://execsql2.readthedocs.io/en/latest/reference/security/#credentials
288
+ # See: https://execsql2.readthedocs.io/en/stable/reference/security/#credentials
283
289
  #enc_password=
284
290
 
285
291
 
execsql/db/access.py CHANGED
@@ -15,6 +15,7 @@ from pathlib import Path
15
15
  from typing import Any, cast
16
16
 
17
17
  from execsql.db.base import Database, _fetched
18
+ from execsql.db.odbc import odbc_value
18
19
  from execsql.db.tiers import SupportTier
19
20
  from execsql.exceptions import ErrInfo
20
21
  from execsql.utils.errors import exception_desc, fatal_error, install_hint
@@ -119,10 +120,11 @@ class AccessDatabase(Database):
119
120
  def _try_odbc_drivers():
120
121
  db_name = str(Path(self.db_name).resolve())
121
122
  for cs, jet4flag in self.connection_strings:
123
+ connstr = cs % odbc_value(db_name)
124
+ logstr = connstr
122
125
  if self.need_passwd:
123
- connstr = f"{cs % db_name} Uid={self.user}; Pwd={self.password};"
124
- else:
125
- connstr = cs % db_name
126
+ connstr += f" Uid={odbc_value(self.user)}; Pwd={odbc_value(self.password)};"
127
+ logstr += f" Uid={odbc_value(self.user)}; Pwd=***;"
126
128
  try:
127
129
  # Access is file-based but a UNC path on an
128
130
  # unreachable share can still block forever; cap
@@ -133,12 +135,12 @@ class AccessDatabase(Database):
133
135
  errors.append(e)
134
136
  if _state.exec_log is not None:
135
137
  _state.exec_log.log_status_info(
136
- f"Could not connect via ODBC using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
138
+ f"Could not connect via ODBC using: {logstr}",
137
139
  )
138
140
  else:
139
141
  if _state.exec_log is not None:
140
142
  _state.exec_log.log_status_info(
141
- f"Connected via ODBC using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
143
+ f"Connected via ODBC using: {logstr}",
142
144
  )
143
145
  self.jet4 = jet4flag
144
146
  return True
execsql/db/base.py CHANGED
@@ -641,6 +641,24 @@ class Database(ABC):
641
641
 
642
642
  raise DatabaseNotImplementedError(self.name(), "role_exists")
643
643
 
644
+ def drop_table_for_replacement(self, schema_name: str | None, table_name: str) -> None:
645
+ """Drop the table a ``TO REPLACEMENT`` import or copy is about to recreate, if it is there.
646
+
647
+ Not every DBMS has ``DROP TABLE IF EXISTS``: SQL Server, Oracle,
648
+ Firebird and Access refuse to drop a table that does not exist. The
649
+ drop is tried first, and its error is ignored only when the table is
650
+ confirmed absent, so a drop that would have succeeded always runs.
651
+ """
652
+ try:
653
+ self.drop_table(self.schema_qualified_table_name(schema_name, table_name))
654
+ except Exception:
655
+ try:
656
+ there = self.table_exists_where_created(table_name, schema_name)
657
+ except Exception:
658
+ there = True
659
+ if there:
660
+ raise
661
+
644
662
  def drop_table(self, tablename: str) -> None:
645
663
  """Drop *tablename* if it exists; *tablename* must already be schema-qualified and quoted."""
646
664
  # The 'tablename' argument should be schema-qualified and quoted as necessary.
execsql/db/factory.py CHANGED
@@ -106,9 +106,19 @@ def db_SqlServer(
106
106
  port: int | None = None,
107
107
  encoding: str | None = None,
108
108
  password: str | None = None,
109
+ trust_server_certificate: bool | None = None,
109
110
  ) -> SqlServerDatabase:
110
111
  """Open a Microsoft SQL Server connection via pyodbc."""
111
- return SqlServerDatabase(server_name, database_name, user, pw_needed, port, encoding, password=password)
112
+ return SqlServerDatabase(
113
+ server_name,
114
+ database_name,
115
+ user,
116
+ pw_needed,
117
+ port,
118
+ encoding,
119
+ password=password,
120
+ trust_server_certificate=trust_server_certificate,
121
+ )
112
122
 
113
123
 
114
124
  def db_MySQL(
execsql/db/mysql.py CHANGED
@@ -7,7 +7,9 @@ Implements :class:`MySQLDatabase`, which connects to MySQL and MariaDB
7
7
  servers via ``pymysql``. Corresponds to ``-t m`` on the CLI.
8
8
  """
9
9
 
10
+ import os
10
11
  import re
12
+ import tempfile
11
13
  from typing import Any, cast
12
14
 
13
15
  from execsql.db.base import Database
@@ -53,6 +55,49 @@ def _crlf_terminated(filename: str) -> bool:
53
55
  return i > 0 and chunk[i - 1 : i] == b"\r"
54
56
 
55
57
 
58
+ def _data_length(filename: str) -> int | None:
59
+ """Return the file's length without its trailing blank lines, or None if it has none.
60
+
61
+ LOAD DATA reads each blank line at the end of a file as a row of NULLs;
62
+ every other import path drops them. The file is read backwards from the
63
+ end, as bytes, so no decoding can hide a line ending.
64
+ """
65
+ try:
66
+ with open(filename, "rb") as f:
67
+ size = f.seek(0, 2)
68
+ pos = size
69
+ content_end = 0
70
+ while pos > 0:
71
+ start = max(0, pos - 65536)
72
+ f.seek(start)
73
+ block = f.read(pos - start)
74
+ stripped = len(block.rstrip(b"\r\n"))
75
+ if stripped:
76
+ content_end = start + stripped
77
+ break
78
+ pos = start
79
+ f.seek(content_end)
80
+ ending = f.read(2)
81
+ except OSError:
82
+ return None
83
+ # Keep the line ending of the last line of data.
84
+ keep = content_end + (2 if ending == b"\r\n" else 1 if ending[:1] in (b"\r", b"\n") else 0)
85
+ return keep if keep < size else None
86
+
87
+
88
+ def _copy_head(filename: str, length: int) -> str:
89
+ """Copy the first *length* bytes of *filename* to a temporary file; return its path."""
90
+ with open(filename, "rb") as src, tempfile.NamedTemporaryFile("wb", suffix=".csv", delete=False) as dst:
91
+ remaining = length
92
+ while remaining > 0:
93
+ chunk = src.read(min(remaining, 1 << 20))
94
+ if not chunk:
95
+ break
96
+ dst.write(chunk)
97
+ remaining -= len(chunk)
98
+ return dst.name
99
+
100
+
56
101
  class MySQLDatabase(Database):
57
102
  """MySQL and MariaDB adapter using the pymysql package."""
58
103
 
@@ -353,7 +398,11 @@ class MySQLDatabase(Database):
353
398
  # String literals in this session treat a backslash as an escape,
354
399
  # so the file name, delimiter and quote character are quoted with
355
400
  # quote_literal, which doubles it.
356
- import_sql = f"load data local infile {self.quote_literal(csv_file_obj.csvfname)} into table {sq_name}"
401
+ load_file = csv_file_obj.csvfname
402
+ data_length = _data_length(load_file)
403
+ if data_length is not None:
404
+ load_file = _copy_head(load_file, data_length)
405
+ import_sql = f"load data local infile {self.quote_literal(load_file)} into table {sq_name}"
357
406
  if csv_file_obj.encoding:
358
407
  charset = _PYTHON_TO_MYSQL_CHARSET.get(csv_file_obj.encoding.lower(), csv_file_obj.encoding)
359
408
  import_sql = f"{import_sql} character set {charset}"
@@ -385,7 +434,11 @@ class MySQLDatabase(Database):
385
434
  _state.exec_log.log_status_info(
386
435
  f"IMPORTing {csv_file_obj.csvfname} using the DBMS' fast file reading routine",
387
436
  )
388
- self.execute(import_sql)
437
+ try:
438
+ self.execute(import_sql)
439
+ finally:
440
+ if load_file != csv_file_obj.csvfname:
441
+ os.unlink(load_file)
389
442
  else:
390
443
  data_indexes = [csv_file_cols.index(col) for col in import_cols]
391
444
  paramspec = ",".join(["%s"] * len(import_cols))
@@ -450,7 +503,7 @@ class MySQLDatabase(Database):
450
503
  if not _state.conf.empty_strings and line[i].strip() == "":
451
504
  line[i] = None
452
505
  # Pad short line with nulls
453
- line.extend([None] * (len(import_cols) - len(line)))
506
+ line.extend([None] * (len(csv_file_cols) - len(line)))
454
507
  linedata = [line[ix] for ix in data_indexes]
455
508
  add_line = True
456
509
  if not _state.conf.empty_rows:
execsql/db/odbc.py ADDED
@@ -0,0 +1,45 @@
1
+ """
2
+ ODBC connection strings for the SQL Server and Access adapters.
3
+
4
+ An ODBC connection string is a ``;``-separated list of ``KEY=value``
5
+ attributes, so a value containing ``;`` (a password, a file path, a name
6
+ taken from a substitution variable) would end its attribute early and
7
+ start another. :func:`odbc_value` brace-quotes such values;
8
+ :func:`connection_string` joins the attributes, and :func:`loggable`
9
+ gives the same string with the password masked for the log.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ __all__ = ["connection_string", "loggable", "odbc_value"]
15
+
16
+ _SECRET_KEYS = frozenset({"pwd", "password"})
17
+
18
+
19
+ def odbc_value(value: object) -> str:
20
+ """Return *value* as an ODBC attribute value, brace-quoted when it needs it.
21
+
22
+ A ``;`` ends an attribute and a leading ``{`` opens a quoted value, so a
23
+ value with either is wrapped in ``{…}`` with each ``}`` doubled. Other
24
+ values are written as they are, because older drivers do not all accept
25
+ braces around every attribute.
26
+ """
27
+ text = str(value)
28
+ if ";" in text or text.startswith("{"):
29
+ return "{" + text.replace("}", "}}") + "}"
30
+ return text
31
+
32
+
33
+ def connection_string(driver: str, attributes: list[tuple[str, object]]) -> str:
34
+ """Return ``DRIVER={driver};KEY=value;…`` with every value quoted as needed."""
35
+ parts = [f"DRIVER={{{driver}}}"]
36
+ parts.extend(f"{key}={odbc_value(value)}" for key, value in attributes)
37
+ return ";".join(parts) + ";"
38
+
39
+
40
+ def loggable(driver: str, attributes: list[tuple[str, object]]) -> str:
41
+ """Return the connection string with the password replaced by ``***``."""
42
+ return connection_string(
43
+ driver,
44
+ [(key, "***" if key.lower() in _SECRET_KEYS else value) for key, value in attributes],
45
+ )
execsql/db/postgres.py CHANGED
@@ -462,7 +462,7 @@ class PostgresDatabase(Database):
462
462
  if not _state.conf.empty_strings and line[i].strip() == "":
463
463
  line[i] = None
464
464
  # Pad short line with nulls
465
- line.extend([None] * (len(import_cols) - len(line)))
465
+ line.extend([None] * (len(csv_file_cols) - len(line)))
466
466
  linedata = [line[ix] for ix in data_indexes]
467
467
  add_line = True
468
468
  if not _state.conf.empty_rows:
execsql/db/sqlserver.py CHANGED
@@ -7,9 +7,8 @@ Implements :class:`SqlServerDatabase`, which connects to Microsoft SQL
7
7
  Server via ``pyodbc``. Corresponds to ``-t s`` on the CLI.
8
8
  """
9
9
 
10
- import re
11
-
12
10
  from execsql.db.base import Database
11
+ from execsql.db.odbc import connection_string, loggable
13
12
  from execsql.exceptions import ErrInfo
14
13
  from execsql.utils.errors import fatal_error, install_hint
15
14
  from execsql.utils.auth import (
@@ -34,9 +33,10 @@ class SqlServerDatabase(Database):
34
33
  db_name: str,
35
34
  user_name: str | None,
36
35
  need_passwd: bool = False,
37
- port: int | None = 1433,
36
+ port: int | None = None,
38
37
  encoding: str | None = "latin1",
39
38
  password: str | None = None,
39
+ trust_server_certificate: bool | None = None,
40
40
  ) -> None:
41
41
  try:
42
42
  import pyodbc # noqa: F401
@@ -51,6 +51,15 @@ class SqlServerDatabase(Database):
51
51
  need_passwd=need_passwd,
52
52
  port=port if port else 1433,
53
53
  )
54
+ # Only a port that was asked for goes into the connection string: a
55
+ # named instance (``host\instance``) is found through SQL Server
56
+ # Browser, which an explicit port bypasses.
57
+ self.explicit_port = port or None
58
+ # None: the [connect] trust_server_certificate setting, which also
59
+ # covers connections a script opens with CONNECT.
60
+ if trust_server_certificate is None:
61
+ trust_server_certificate = bool(getattr(_state.conf, "trust_server_certificate", False))
62
+ self.trust_server_certificate = trust_server_certificate
54
63
  self.type = dbt_sqlserver
55
64
  self.password = password
56
65
  self.encoding = encoding or "latin1" # Default on installation of SQL Server
@@ -79,6 +88,12 @@ class SqlServerDatabase(Database):
79
88
  """
80
89
  return "[" + identifier.replace("]", "]]") + "]"
81
90
 
91
+ def _server_attribute(self) -> str:
92
+ """Return the ``SERVER`` value: ``host,port`` when a port was given."""
93
+ if self.explicit_port and "," not in self.server_name:
94
+ return f"{self.server_name},{self.explicit_port}"
95
+ return str(self.server_name)
96
+
82
97
  def open_db(self) -> None:
83
98
  """Open a connection to the SQL Server database via pyodbc."""
84
99
  import pyodbc
@@ -95,6 +110,7 @@ class SqlServerDatabase(Database):
95
110
  )
96
111
  # Use pyodbc to connect. Try different driver versions from newest to oldest.
97
112
  ssdrivers = (
113
+ "ODBC Driver 18 for SQL Server",
98
114
  "ODBC Driver 17 for SQL Server",
99
115
  "ODBC Driver 13.1 for SQL Server",
100
116
  "ODBC Driver 13 for SQL Server",
@@ -108,23 +124,23 @@ class SqlServerDatabase(Database):
108
124
  errors: list[BaseException] = []
109
125
 
110
126
  def _try_drivers():
127
+ attributes: list[tuple[str, object]] = [
128
+ ("SERVER", self._server_attribute()),
129
+ ("MARS_Connection", "Yes"),
130
+ ("DATABASE", self.db_name),
131
+ ]
132
+ if self.user:
133
+ attributes.append(("Uid", self.user))
134
+ if self.password:
135
+ attributes.append(("Pwd", self.password))
136
+ else:
137
+ attributes.append(("Trusted_Connection", "yes"))
138
+ if self.trust_server_certificate:
139
+ # Driver 18 encrypts and verifies the server's certificate
140
+ # by default; this accepts a self-signed one.
141
+ attributes.append(("TrustServerCertificate", "yes"))
111
142
  for drv in ssdrivers:
112
- if self.user:
113
- if self.password:
114
- connstr = (
115
- f"DRIVER={{{drv}}};SERVER={self.server_name};MARS_Connection=Yes; "
116
- f"DATABASE={self.db_name};Uid={self.user};Pwd={self.password}"
117
- )
118
- else:
119
- connstr = (
120
- f"DRIVER={{{drv}}};SERVER={self.server_name};MARS_Connection=Yes; "
121
- f"DATABASE={self.db_name};Uid={self.user}"
122
- )
123
- else:
124
- connstr = (
125
- f"DRIVER={{{drv}}};SERVER={self.server_name};MARS_Connection=Yes; "
126
- f"DATABASE={self.db_name};Trusted_Connection=yes"
127
- )
143
+ connstr = connection_string(drv, attributes)
128
144
  try:
129
145
  # 30 s connect timeout matches the Postgres adapter
130
146
  # default; pyodbc treats `timeout` as the login-and-
@@ -134,12 +150,12 @@ class SqlServerDatabase(Database):
134
150
  errors.append(e)
135
151
  if _state.exec_log is not None:
136
152
  _state.exec_log.log_status_info(
137
- f"Could not connect using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
153
+ f"Could not connect using: {loggable(drv, attributes)}",
138
154
  )
139
155
  else:
140
156
  if _state.exec_log is not None:
141
157
  _state.exec_log.log_status_info(
142
- f"Connected using: {re.sub(r'Pwd=[^;]*', 'Pwd=***', connstr)}",
158
+ f"Connected using: {loggable(drv, attributes)}",
143
159
  )
144
160
  return True
145
161
  return False