execsql2 2.30.0__py3-none-any.whl → 2.31.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +1 -0
- execsql/cli/application.py +1 -1
- execsql/cli/commands/lint.py +1 -1
- execsql/cli/dsn.py +35 -1
- execsql/cli/lint.py +10 -1
- execsql/cli/run.py +8 -3
- execsql/config.py +26 -4
- execsql/data/execsql.conf.template +8 -2
- execsql/db/access.py +7 -5
- execsql/db/base.py +18 -0
- execsql/db/factory.py +11 -1
- execsql/db/mysql.py +56 -3
- execsql/db/odbc.py +45 -0
- execsql/db/postgres.py +1 -1
- execsql/db/sqlserver.py +37 -21
- execsql/exporters/ods.py +146 -41
- execsql/exporters/values.py +166 -9
- execsql/importers/base.py +1 -1
- execsql/importers/csv.py +1 -1
- execsql/importers/ods.py +2 -0
- execsql/lsp/diagnostics.py +1 -1
- execsql/lsp/document.py +3 -0
- execsql/metacommands/io_fileops.py +11 -5
- execsql/metacommands/reference.py +1 -1
- execsql/metacommands/upsert.py +10 -1
- execsql/utils/auth.py +21 -14
- execsql/utils/errors.py +1 -1
- execsql/utils/paths.py +31 -0
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/README.md +9 -8
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/md_compare.sql +53 -29
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/md_glossary.sql +20 -20
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/md_upsert.sql +59 -66
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/pg_compare.sql +19 -22
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/pg_glossary.sql +13 -17
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/script_template.sql +302 -299
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/ss_compare.sql +39 -29
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/ss_glossary.sql +13 -17
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/ss_upsert.sql +56 -66
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/METADATA +17 -41
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/RECORD +47 -46
- execsql2-2.30.0.data/data/execsql2_extras/pg_upsert.sql +0 -2791
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.30.0.data → execsql2-2.31.1.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/WHEEL +0 -0
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/entry_points.txt +0 -0
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.30.0.dist-info → execsql2-2.31.1.dist-info}/licenses/NOTICE +0 -0
execsql/exporters/ods.py
CHANGED
|
@@ -13,6 +13,7 @@ import datetime
|
|
|
13
13
|
import decimal
|
|
14
14
|
import getpass
|
|
15
15
|
import os
|
|
16
|
+
import re
|
|
16
17
|
from pathlib import Path
|
|
17
18
|
from typing import Any
|
|
18
19
|
|
|
@@ -184,42 +185,25 @@ class OdsFile:
|
|
|
184
185
|
if not sheet:
|
|
185
186
|
raise OdsFileError(f"There is no sheet named {sheetname}")
|
|
186
187
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
if len(pval) == 0:
|
|
207
|
-
for _rr in range(int(repeat)):
|
|
208
|
-
p_content.append(None)
|
|
209
|
-
else:
|
|
210
|
-
for _rr in range(int(repeat)):
|
|
211
|
-
p_content.append(pval)
|
|
212
|
-
if len(p_content) == 0:
|
|
213
|
-
for _rr in range(int(repeat)):
|
|
214
|
-
rowdata.append(None)
|
|
215
|
-
elif p_content[0] != "#":
|
|
216
|
-
rowdata.extend(p_content)
|
|
217
|
-
return rowdata
|
|
218
|
-
|
|
219
|
-
rows = sheet.getElementsByType(of.table.TableRow)
|
|
220
|
-
if junk_header_rows > 0:
|
|
221
|
-
rows = rows[junk_header_rows:]
|
|
222
|
-
return [row_data(r) for r in rows]
|
|
188
|
+
rows: list[list | None] = []
|
|
189
|
+
pending_empty = 0
|
|
190
|
+
for sheetrow in sheet.getElementsByType(of.table.TableRow):
|
|
191
|
+
data = _row_values(sheetrow)
|
|
192
|
+
repeat = int(sheetrow.getAttribute("numberrowsrepeated") or 1)
|
|
193
|
+
if not data:
|
|
194
|
+
# Empty rows are kept only when data follows them, so the
|
|
195
|
+
# filler rows a spreadsheet writes below the data are dropped.
|
|
196
|
+
pending_empty += repeat
|
|
197
|
+
continue
|
|
198
|
+
rows.extend(None for _ in range(pending_empty))
|
|
199
|
+
pending_empty = 0
|
|
200
|
+
rows.extend(list(data) for _ in range(repeat))
|
|
201
|
+
rows = rows[junk_header_rows:]
|
|
202
|
+
# Every row is as wide as the widest, so a blank trailing header is
|
|
203
|
+
# still seen, and an empty row inside the data is a row of NULLs that
|
|
204
|
+
# EMPTY_ROWS keeps or drops.
|
|
205
|
+
width = max((len(r) for r in rows if r is not None), default=0)
|
|
206
|
+
return [(r or []) + [None] * (width - len(r or [])) for r in rows]
|
|
223
207
|
|
|
224
208
|
def new_sheet(self, sheetname: str) -> Any:
|
|
225
209
|
"""Create and return a detached sheet object that can later be added to the workbook."""
|
|
@@ -239,7 +223,11 @@ class OdsFile:
|
|
|
239
223
|
for item in datarow:
|
|
240
224
|
if isinstance(item, bool):
|
|
241
225
|
# Booleans must be evaluated before numbers.
|
|
242
|
-
tc = of.table.TableCell(
|
|
226
|
+
tc = of.table.TableCell(
|
|
227
|
+
valuetype="boolean",
|
|
228
|
+
booleanvalue="true" if item else "false",
|
|
229
|
+
stylename=style_name,
|
|
230
|
+
)
|
|
243
231
|
elif isinstance(item, float | int | decimal.Decimal):
|
|
244
232
|
# Decimal belongs here: PostgreSQL, MySQL and DuckDB all return it
|
|
245
233
|
# for numeric/decimal columns. Without it the value fell to the
|
|
@@ -269,8 +257,11 @@ class OdsFile:
|
|
|
269
257
|
item.microsecond,
|
|
270
258
|
item.tzinfo,
|
|
271
259
|
)
|
|
272
|
-
tc = of.table.TableCell(
|
|
273
|
-
|
|
260
|
+
tc = of.table.TableCell(
|
|
261
|
+
valuetype="time",
|
|
262
|
+
timevalue=timeval.strftime("PT%HH%MM%S.%fS"),
|
|
263
|
+
stylename="iso_datetime",
|
|
264
|
+
)
|
|
274
265
|
elif isinstance(item, str):
|
|
275
266
|
# An XML parser normalises a newline inside an attribute to a
|
|
276
267
|
# space, so office:string-value cannot carry one. ODF's answer
|
|
@@ -280,16 +271,23 @@ class OdsFile:
|
|
|
280
271
|
tc = of.table.TableCell(valuetype="string", stylename=style_name)
|
|
281
272
|
else:
|
|
282
273
|
tc = of.table.TableCell(valuetype="string", stringvalue=item, stylename=style_name)
|
|
274
|
+
elif item is None:
|
|
275
|
+
# An empty cell: no value type, no value, no text.
|
|
276
|
+
tc = of.table.TableCell(stylename=style_name)
|
|
283
277
|
else:
|
|
284
|
-
tc = of.table.TableCell(
|
|
278
|
+
tc = of.table.TableCell(valuetype="string", stringvalue=str(item), stylename=style_name)
|
|
285
279
|
if item is not None:
|
|
286
280
|
if isinstance(item, str) and ("\n" in item or "\r" in item):
|
|
287
281
|
# Previously these were flattened to spaces, silently changing
|
|
288
282
|
# the exported value.
|
|
289
283
|
for line in item.replace("\r\n", "\n").replace("\r", "\n").split("\n"):
|
|
290
284
|
tc.addElement(of.text.P(text=line))
|
|
285
|
+
elif isinstance(item, datetime.time):
|
|
286
|
+
tc.addElement(of.text.P(text=item.strftime("%H:%M:%S.%f")))
|
|
291
287
|
else:
|
|
292
|
-
|
|
288
|
+
# str(): odfpy writes no text for a falsy value, so 0 and
|
|
289
|
+
# False showed as blank cells.
|
|
290
|
+
tc.addElement(of.text.P(text=str(item)))
|
|
293
291
|
tr.addElement(tc)
|
|
294
292
|
|
|
295
293
|
def add_sheet(self, of_table: Any) -> None:
|
|
@@ -311,6 +309,113 @@ class OdsFile:
|
|
|
311
309
|
self.wbk = None
|
|
312
310
|
|
|
313
311
|
|
|
312
|
+
def _paragraph_text(node: Any) -> str:
|
|
313
|
+
"""Return the text of a ``<text:p>``, expanding ODF's space, tab and line-break elements."""
|
|
314
|
+
parts: list[str] = []
|
|
315
|
+
for child in node.childNodes:
|
|
316
|
+
if child.nodeType == child.TEXT_NODE:
|
|
317
|
+
parts.append(str(child))
|
|
318
|
+
elif child.qname == (of.text.TEXTNS, "s"):
|
|
319
|
+
parts.append(" " * int(child.getAttribute("c") or 1))
|
|
320
|
+
elif child.qname == (of.text.TEXTNS, "tab"):
|
|
321
|
+
parts.append("\t")
|
|
322
|
+
elif child.qname == (of.text.TEXTNS, "line-break"):
|
|
323
|
+
parts.append("\n")
|
|
324
|
+
else:
|
|
325
|
+
parts.append(_paragraph_text(child))
|
|
326
|
+
return "".join(parts)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _time_text(duration: str) -> str:
|
|
330
|
+
"""Convert an ODF time value (``PT13H05M09.5S``) to ``13:05:09.5``."""
|
|
331
|
+
m = re.fullmatch(r"-?PT(\d+)H(\d+)M(\d+)(\.\d*)?S", duration)
|
|
332
|
+
if not m:
|
|
333
|
+
return duration
|
|
334
|
+
hours, minutes, seconds, fraction = m.groups()
|
|
335
|
+
fraction = (fraction or "").rstrip("0").rstrip(".")
|
|
336
|
+
return f"{int(hours):02d}:{int(minutes):02d}:{int(seconds):02d}{fraction}"
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _cell_value(cell: Any) -> str | None:
|
|
340
|
+
"""Return a cell's stored value as text, or ``None`` for an empty cell.
|
|
341
|
+
|
|
342
|
+
The value comes from the cell's ``office:value-type`` and value
|
|
343
|
+
attribute, not from its displayed text, so a number formatted to two
|
|
344
|
+
places, a percentage or a date keeps its full value.
|
|
345
|
+
"""
|
|
346
|
+
vtype = cell.getAttribute("valuetype")
|
|
347
|
+
value: str | None
|
|
348
|
+
if vtype in ("float", "percentage", "currency"):
|
|
349
|
+
value = cell.getAttribute("value")
|
|
350
|
+
elif vtype == "date":
|
|
351
|
+
value = cell.getAttribute("datevalue")
|
|
352
|
+
elif vtype == "time":
|
|
353
|
+
value = cell.getAttribute("timevalue")
|
|
354
|
+
value = _time_text(value) if value else value
|
|
355
|
+
elif vtype == "boolean":
|
|
356
|
+
value = cell.getAttribute("booleanvalue")
|
|
357
|
+
if not value:
|
|
358
|
+
# Files from execsql2 2.30 and earlier wrote office:value="1"/"0".
|
|
359
|
+
flag = cell.getAttribute("value")
|
|
360
|
+
value = {"1": "true", "0": "false"}.get(flag or "", flag)
|
|
361
|
+
else:
|
|
362
|
+
value = None
|
|
363
|
+
if value:
|
|
364
|
+
return value
|
|
365
|
+
if vtype == "string":
|
|
366
|
+
value = cell.getAttribute("stringvalue")
|
|
367
|
+
if value:
|
|
368
|
+
return value
|
|
369
|
+
# A string cell, a cell with no type, or a typed cell missing its value
|
|
370
|
+
# attribute: the paragraphs are the value, one line each.
|
|
371
|
+
paragraphs = [
|
|
372
|
+
_paragraph_text(child) for child in cell.childNodes if getattr(child, "qname", None) == (of.text.TEXTNS, "p")
|
|
373
|
+
]
|
|
374
|
+
text = "\n".join(paragraphs)
|
|
375
|
+
return text or None
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _row_values(sheetrow: Any) -> list:
|
|
379
|
+
"""Return one value per column of a ``<table:table-row>``, without trailing empty cells."""
|
|
380
|
+
values: list = []
|
|
381
|
+
pending_empty = 0
|
|
382
|
+
covered_by_span = 0
|
|
383
|
+
for cell in sheetrow.childNodes:
|
|
384
|
+
qname = getattr(cell, "qname", None)
|
|
385
|
+
if qname == (of.table.TABLENS, "table-cell"):
|
|
386
|
+
covered = False
|
|
387
|
+
elif qname == (of.table.TABLENS, "covered-table-cell"):
|
|
388
|
+
covered = True
|
|
389
|
+
else:
|
|
390
|
+
continue
|
|
391
|
+
repeat = int(cell.getAttribute("numbercolumnsrepeated") or 1)
|
|
392
|
+
if covered:
|
|
393
|
+
# A horizontal merge repeats its value across the columns it
|
|
394
|
+
# covers (below); covered cells left over belong to a merge from
|
|
395
|
+
# a row above and are empty in this row.
|
|
396
|
+
skip = min(repeat, covered_by_span)
|
|
397
|
+
covered_by_span -= skip
|
|
398
|
+
repeat -= skip
|
|
399
|
+
value = None
|
|
400
|
+
else:
|
|
401
|
+
value = _cell_value(cell)
|
|
402
|
+
spanned = int(cell.getAttribute("numbercolumnsspanned") or 1)
|
|
403
|
+
if spanned > 1:
|
|
404
|
+
covered_by_span = spanned - 1
|
|
405
|
+
repeat *= spanned
|
|
406
|
+
if repeat <= 0:
|
|
407
|
+
continue
|
|
408
|
+
if value is None:
|
|
409
|
+
# Counted rather than appended, so a trailing filler cell
|
|
410
|
+
# repeated across the whole sheet width costs nothing.
|
|
411
|
+
pending_empty += repeat
|
|
412
|
+
continue
|
|
413
|
+
values.extend([None] * pending_empty)
|
|
414
|
+
pending_empty = 0
|
|
415
|
+
values.extend([value] * repeat)
|
|
416
|
+
return values
|
|
417
|
+
|
|
418
|
+
|
|
314
419
|
def export_ods(
|
|
315
420
|
outfile: str,
|
|
316
421
|
hdrs: list[str],
|
execsql/exporters/values.py
CHANGED
|
@@ -9,6 +9,7 @@ for loading data into a database from a plain SQL file.
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import decimal
|
|
12
|
+
import re
|
|
12
13
|
from typing import Any
|
|
13
14
|
|
|
14
15
|
import execsql.state as _state
|
|
@@ -20,11 +21,146 @@ from execsql.utils.fileio import filewriter_close
|
|
|
20
21
|
__all__ = ["export_values", "write_query_to_values"]
|
|
21
22
|
|
|
22
23
|
|
|
23
|
-
|
|
24
|
+
# How each DBMS joins strings and writes a character by its code. A text
|
|
25
|
+
# value that execsql would misread when a script runs the file with INCLUDE (see
|
|
26
|
+
# _text_literal) is written as pieces joined this way. (template, joiner, char)
|
|
27
|
+
_CONCAT: dict[str, tuple[str, str, str]] = {
|
|
28
|
+
"MySQL": ("CONCAT({})", ", ", "CHAR({} USING utf8mb4)"),
|
|
29
|
+
"SQL Server": ("({})", " + ", "CHAR({})"),
|
|
30
|
+
"Access": ("({})", " & ", "Chr({})"),
|
|
31
|
+
"SQLite": ("({})", " || ", "char({})"),
|
|
32
|
+
"Firebird": ("({})", " || ", "ASCII_CHAR({})"),
|
|
33
|
+
}
|
|
34
|
+
# PostgreSQL, DuckDB, Oracle, and an ODBC DSN.
|
|
35
|
+
_CONCAT_DEFAULT = ("({})", " || ", "CHR({})")
|
|
36
|
+
|
|
37
|
+
# What execsql acts on when it runs the file: a line break (a line of the
|
|
38
|
+
# value could end the statement with ';' or start a '-- !x!' metacommand)
|
|
39
|
+
# and the start of a substitution variable (!!x!!, !'!x!'!, !"!x!"!, !{x}!).
|
|
40
|
+
_RX_UNSAFE = re.compile(r"\r\n|\r|\n|!(?=!|'!|\"!|\{)")
|
|
41
|
+
_LINE_BREAK_CODES = {"\r\n": (13, 10), "\r": (13,), "\n": (10,)}
|
|
42
|
+
|
|
43
|
+
# Column names written quoted: PostgreSQL's reserved key words, which cover
|
|
44
|
+
# the names other DBMSs also reject bare, plus KEY and INDEX (MySQL).
|
|
45
|
+
_RESERVED = frozenset(
|
|
46
|
+
[
|
|
47
|
+
"ALL",
|
|
48
|
+
"ANALYSE",
|
|
49
|
+
"ANALYZE",
|
|
50
|
+
"AND",
|
|
51
|
+
"ANY",
|
|
52
|
+
"ARRAY",
|
|
53
|
+
"AS",
|
|
54
|
+
"ASC",
|
|
55
|
+
"ASYMMETRIC",
|
|
56
|
+
"BOTH",
|
|
57
|
+
"CASE",
|
|
58
|
+
"CAST",
|
|
59
|
+
"CHECK",
|
|
60
|
+
"COLLATE",
|
|
61
|
+
"COLUMN",
|
|
62
|
+
"CONSTRAINT",
|
|
63
|
+
"CREATE",
|
|
64
|
+
"CURRENT_CATALOG",
|
|
65
|
+
"CURRENT_DATE",
|
|
66
|
+
"CURRENT_ROLE",
|
|
67
|
+
"CURRENT_TIME",
|
|
68
|
+
"CURRENT_TIMESTAMP",
|
|
69
|
+
"CURRENT_USER",
|
|
70
|
+
"DEFAULT",
|
|
71
|
+
"DEFERRABLE",
|
|
72
|
+
"DESC",
|
|
73
|
+
"DISTINCT",
|
|
74
|
+
"DO",
|
|
75
|
+
"ELSE",
|
|
76
|
+
"END",
|
|
77
|
+
"EXCEPT",
|
|
78
|
+
"FALSE",
|
|
79
|
+
"FETCH",
|
|
80
|
+
"FOR",
|
|
81
|
+
"FOREIGN",
|
|
82
|
+
"FROM",
|
|
83
|
+
"GRANT",
|
|
84
|
+
"GROUP",
|
|
85
|
+
"HAVING",
|
|
86
|
+
"IN",
|
|
87
|
+
"INDEX",
|
|
88
|
+
"INITIALLY",
|
|
89
|
+
"INTERSECT",
|
|
90
|
+
"INTO",
|
|
91
|
+
"KEY",
|
|
92
|
+
"LATERAL",
|
|
93
|
+
"LEADING",
|
|
94
|
+
"LIMIT",
|
|
95
|
+
"LOCALTIME",
|
|
96
|
+
"LOCALTIMESTAMP",
|
|
97
|
+
"NOT",
|
|
98
|
+
"NULL",
|
|
99
|
+
"OFFSET",
|
|
100
|
+
"ON",
|
|
101
|
+
"ONLY",
|
|
102
|
+
"OR",
|
|
103
|
+
"ORDER",
|
|
104
|
+
"PLACING",
|
|
105
|
+
"PRIMARY",
|
|
106
|
+
"REFERENCES",
|
|
107
|
+
"RETURNING",
|
|
108
|
+
"SELECT",
|
|
109
|
+
"SESSION_USER",
|
|
110
|
+
"SOME",
|
|
111
|
+
"SYMMETRIC",
|
|
112
|
+
"SYSTEM_USER",
|
|
113
|
+
"TABLE",
|
|
114
|
+
"THEN",
|
|
115
|
+
"TO",
|
|
116
|
+
"TRAILING",
|
|
117
|
+
"TRUE",
|
|
118
|
+
"UNION",
|
|
119
|
+
"UNIQUE",
|
|
120
|
+
"USER",
|
|
121
|
+
"USING",
|
|
122
|
+
"VARIADIC",
|
|
123
|
+
"WHEN",
|
|
124
|
+
"WHERE",
|
|
125
|
+
"WINDOW",
|
|
126
|
+
"WITH",
|
|
127
|
+
],
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _text_literal(text: str, dbms: str | None) -> str:
|
|
132
|
+
"""*text* as a SQL string that execsql reads back unchanged when a script runs the file with INCLUDE.
|
|
133
|
+
|
|
134
|
+
execsql reads a script line by line, substituting variables and treating
|
|
135
|
+
a ``-- !x!`` line as a metacommand and a line ending in ``;`` as the end of
|
|
136
|
+
a statement, inside a string literal too. So a value holding a line
|
|
137
|
+
break or the start of a variable reference is written as pieces, each
|
|
138
|
+
free of both, joined in *dbms*'s syntax: ``('a' || CHR(10) || 'b')``.
|
|
139
|
+
"""
|
|
140
|
+
if not _RX_UNSAFE.search(text):
|
|
141
|
+
return "'" + text.replace("'", "''") + "'"
|
|
142
|
+
template, joiner, char = _CONCAT.get(dbms or "", _CONCAT_DEFAULT)
|
|
143
|
+
pieces: list[str] = []
|
|
144
|
+
start = 0
|
|
145
|
+
for m in _RX_UNSAFE.finditer(text):
|
|
146
|
+
if m.group() == "!": # end a piece after the '!', so no piece holds the whole marker
|
|
147
|
+
pieces.append("'" + text[start : m.end()].replace("'", "''") + "'")
|
|
148
|
+
else:
|
|
149
|
+
if m.start() > start:
|
|
150
|
+
pieces.append("'" + text[start : m.start()].replace("'", "''") + "'")
|
|
151
|
+
pieces.extend(char.format(code) for code in _LINE_BREAK_CODES[m.group()])
|
|
152
|
+
start = m.end()
|
|
153
|
+
if start < len(text):
|
|
154
|
+
pieces.append("'" + text[start:].replace("'", "''") + "'")
|
|
155
|
+
return template.format(joiner.join(pieces))
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _sql_literal(value: Any, dbms: str | None = None) -> str:
|
|
24
159
|
"""Render a value as a SQL literal for an INSERT ... VALUES list.
|
|
25
160
|
|
|
26
161
|
Only numbers are emitted bare. Everything else is quoted, with embedded
|
|
27
|
-
single quotes doubled — the SQL standard escape
|
|
162
|
+
single quotes doubled — the SQL standard escape — and text execsql would
|
|
163
|
+
misread when running the file is split into pieces (:func:`_text_literal`).
|
|
28
164
|
|
|
29
165
|
Quoting is the safe default rather than the exception. Previously anything
|
|
30
166
|
that was not a string went out through ``str()`` unquoted, so a date became
|
|
@@ -40,7 +176,17 @@ def _sql_literal(value: Any) -> str:
|
|
|
40
176
|
return "TRUE" if value else "FALSE"
|
|
41
177
|
if isinstance(value, int | float | decimal.Decimal):
|
|
42
178
|
return str(value)
|
|
43
|
-
return
|
|
179
|
+
return _text_literal(str(value), dbms)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _column_name(name: str, dbtype: Any) -> str:
|
|
183
|
+
"""*name* as written in the column list: quoted if it is not a plain, unreserved identifier."""
|
|
184
|
+
if re.fullmatch(r"[A-Za-z_]\w*", name) and name.upper() not in _RESERVED:
|
|
185
|
+
return name
|
|
186
|
+
opening, closing = getattr(dbtype, "quotechars", ('"', '"'))
|
|
187
|
+
if opening == closing:
|
|
188
|
+
name = name.replace(opening, opening * 2)
|
|
189
|
+
return opening + name + closing
|
|
44
190
|
|
|
45
191
|
|
|
46
192
|
def export_values(
|
|
@@ -50,9 +196,17 @@ def export_values(
|
|
|
50
196
|
append: bool = False,
|
|
51
197
|
desc: str | None = None,
|
|
52
198
|
zipfile: str | None = None,
|
|
199
|
+
dbtype: Any = None,
|
|
53
200
|
) -> None:
|
|
54
|
-
"""Write pre-fetched rows as SQL INSERT … VALUES statements to a file or ZIP archive.
|
|
201
|
+
"""Write pre-fetched rows as SQL INSERT … VALUES statements to a file or ZIP archive.
|
|
202
|
+
|
|
203
|
+
*dbtype* is the exporting database's :class:`~execsql.types.DbType`: its
|
|
204
|
+
identifier quotes and its string syntax are used where a name or a
|
|
205
|
+
value needs them. With no rows, the file holds a comment instead of an
|
|
206
|
+
INSERT, which would be invalid SQL.
|
|
207
|
+
"""
|
|
55
208
|
conf = _state.conf
|
|
209
|
+
dbms = getattr(dbtype, "dbms_id", None)
|
|
56
210
|
if outfile.lower() == "stdout":
|
|
57
211
|
f = _state.output
|
|
58
212
|
else:
|
|
@@ -70,17 +224,20 @@ def export_values(
|
|
|
70
224
|
try:
|
|
71
225
|
if desc is not None:
|
|
72
226
|
f.write(f"-- {desc}\n")
|
|
73
|
-
f.write(f"INSERT INTO !!target_table!!\n ({', '.join(hdrs)})\n")
|
|
74
|
-
f.write("VALUES\n")
|
|
75
227
|
firstrow = True
|
|
76
228
|
for r in rows:
|
|
77
229
|
if firstrow:
|
|
78
230
|
firstrow = False
|
|
231
|
+
columns = ", ".join(_column_name(h, dbtype) for h in hdrs)
|
|
232
|
+
f.write(f"INSERT INTO !!target_table!!\n ({columns})\nVALUES\n")
|
|
79
233
|
else:
|
|
80
234
|
f.write(",\n")
|
|
81
|
-
quoted_row = [_sql_literal(v) for v in r]
|
|
235
|
+
quoted_row = [_sql_literal(v, dbms) for v in r]
|
|
82
236
|
f.write(f" ({', '.join(quoted_row)})")
|
|
83
|
-
|
|
237
|
+
if firstrow:
|
|
238
|
+
f.write("-- The query returned no rows: nothing to insert.\n")
|
|
239
|
+
else:
|
|
240
|
+
f.write("\n ;\n")
|
|
84
241
|
finally:
|
|
85
242
|
if outfile.lower() != "stdout":
|
|
86
243
|
f.close()
|
|
@@ -101,4 +258,4 @@ def write_query_to_values(
|
|
|
101
258
|
raise
|
|
102
259
|
except Exception as e:
|
|
103
260
|
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
104
|
-
export_values(outfile, hdrs, rows, append, desc, zipfile=zipfile)
|
|
261
|
+
export_values(outfile, hdrs, rows, append, desc, zipfile=zipfile, dbtype=getattr(db, "type", None))
|
execsql/importers/base.py
CHANGED
|
@@ -107,7 +107,7 @@ def import_data_table(
|
|
|
107
107
|
if is_new == 2:
|
|
108
108
|
tblspec = db.schema_qualified_table_name(schemaname, tablename)
|
|
109
109
|
try:
|
|
110
|
-
db.
|
|
110
|
+
db.drop_table_for_replacement(schemaname, tablename)
|
|
111
111
|
except ErrInfo:
|
|
112
112
|
raise
|
|
113
113
|
except Exception as e:
|
execsql/importers/csv.py
CHANGED
|
@@ -55,7 +55,7 @@ def importtable(
|
|
|
55
55
|
sql = inf.create_table(db.type, schemaname, tablename)
|
|
56
56
|
if is_new == 2:
|
|
57
57
|
try:
|
|
58
|
-
db.
|
|
58
|
+
db.drop_table_for_replacement(schemaname, tablename)
|
|
59
59
|
except ErrInfo:
|
|
60
60
|
raise
|
|
61
61
|
except Exception as e:
|
execsql/importers/ods.py
CHANGED
|
@@ -39,6 +39,8 @@ def ods_data(
|
|
|
39
39
|
alldata = wbk.sheet_data(sheetname, junk_header_rows)
|
|
40
40
|
except Exception as e:
|
|
41
41
|
raise ErrInfo(type="cmd", other_msg=f"{sheetname} is not a worksheet in {filename}.") from e
|
|
42
|
+
if not alldata:
|
|
43
|
+
raise ErrInfo(type="error", other_msg=f"The input file {filename}, sheet {sheetname} contains no data.")
|
|
42
44
|
colhdrs = alldata[0]
|
|
43
45
|
if any(x is None or len(x.strip()) == 0 for x in colhdrs):
|
|
44
46
|
if conf.del_empty_cols:
|
execsql/lsp/diagnostics.py
CHANGED
|
@@ -8,7 +8,7 @@ from execsql.cli.lint import Issue, filter_issues, lint_source
|
|
|
8
8
|
|
|
9
9
|
__all__ = ["LINT_DOCS_URL", "diagnostics", "to_diagnostic", "too_deep_diagnostic"]
|
|
10
10
|
|
|
11
|
-
LINT_DOCS_URL = "https://execsql2.readthedocs.io/en/
|
|
11
|
+
LINT_DOCS_URL = "https://execsql2.readthedocs.io/en/stable/reference/lint/"
|
|
12
12
|
|
|
13
13
|
_SEVERITY = {
|
|
14
14
|
"error": types.DiagnosticSeverity.Error,
|
execsql/lsp/document.py
CHANGED
|
@@ -18,6 +18,7 @@ from pygls.uris import from_fs_path
|
|
|
18
18
|
|
|
19
19
|
from execsql.cli.lint import variables_defined_by
|
|
20
20
|
from execsql.lsp._util import RX_METACOMMAND
|
|
21
|
+
from execsql.utils.paths import is_network_path
|
|
21
22
|
from execsql.script.ast import IncludeDirective, MetaCommandStatement, ScriptBlock
|
|
22
23
|
from execsql.script.parser import parse_string
|
|
23
24
|
|
|
@@ -195,6 +196,8 @@ def file_facts(source: str, path: str | None, base_dir: str | None = None) -> Fi
|
|
|
195
196
|
steps.append(_ScriptCall(Location(path, row, start, end), node.target.lower()))
|
|
196
197
|
elif isinstance(node, IncludeDirective):
|
|
197
198
|
target = node.target.strip().strip("\"'")
|
|
199
|
+
if is_network_path(target):
|
|
200
|
+
continue # never followed: checking it would contact the named host
|
|
198
201
|
start, end = _span_of(target, text) if target else (0, len(text))
|
|
199
202
|
resolved = Path(target) if Path(target).is_absolute() or script_dir is None else script_dir / target
|
|
200
203
|
steps.append(_Include(Location(path, row, start, end), target, str(resolved)))
|
|
@@ -128,7 +128,7 @@ def x_copy(**kwargs: Any) -> None:
|
|
|
128
128
|
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
129
129
|
if new_tbl2 == "replacement":
|
|
130
130
|
try:
|
|
131
|
-
db2.
|
|
131
|
+
db2.drop_table_for_replacement(schema2, table2)
|
|
132
132
|
except ErrInfo:
|
|
133
133
|
raise
|
|
134
134
|
except Exception as e:
|
|
@@ -215,7 +215,7 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
215
215
|
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
216
216
|
if new_tbl2 == "replacement":
|
|
217
217
|
try:
|
|
218
|
-
db2.
|
|
218
|
+
db2.drop_table_for_replacement(schema2, table2)
|
|
219
219
|
except ErrInfo:
|
|
220
220
|
raise
|
|
221
221
|
except Exception as e:
|
|
@@ -370,8 +370,14 @@ def x_serve(**kwargs: Any) -> None:
|
|
|
370
370
|
contenttype = "application/zip"
|
|
371
371
|
else:
|
|
372
372
|
contenttype = "application/octet-stream"
|
|
373
|
-
print(f"Content-Type: {contenttype}")
|
|
374
373
|
safe_fname = fname.replace("\r", "").replace("\n", "").replace('"', '\\"')
|
|
375
|
-
|
|
374
|
+
headers = f'Content-Type: {contenttype}\nContent-Disposition: attachment; filename="{safe_fname}"\n\n'
|
|
375
|
+
# Headers and body go to the same binary stream, after anything already
|
|
376
|
+
# written as text: under CGI stdout is a pipe, and text printed into the
|
|
377
|
+
# text layer's buffer reached it after the body.
|
|
378
|
+
sys.stdout.flush()
|
|
379
|
+
out = sys.stdout.buffer
|
|
380
|
+
out.write(headers.encode(sys.stdout.encoding or "utf-8"))
|
|
376
381
|
with open(infname, "rb") as f:
|
|
377
|
-
copyfileobj(f,
|
|
382
|
+
copyfileobj(f, out)
|
|
383
|
+
out.flush()
|
execsql/metacommands/upsert.py
CHANGED
|
@@ -421,7 +421,7 @@ def _export_failures_if_requested(
|
|
|
421
421
|
|
|
422
422
|
def _handle_pg_upsert_errors(fn: Any, metacommandline: str | None) -> Any:
|
|
423
423
|
"""Run *fn*, translating pg-upsert exceptions to ErrInfo."""
|
|
424
|
-
from pg_upsert import UserCancelledError
|
|
424
|
+
from pg_upsert import PgUpsertError, UserCancelledError
|
|
425
425
|
|
|
426
426
|
try:
|
|
427
427
|
return fn()
|
|
@@ -433,6 +433,15 @@ def _handle_pg_upsert_errors(fn: Any, metacommandline: str | None) -> Any:
|
|
|
433
433
|
) from exc
|
|
434
434
|
except ErrInfo:
|
|
435
435
|
raise
|
|
436
|
+
except PgUpsertError as exc:
|
|
437
|
+
# The run was rolled back. The variables describe this failed run,
|
|
438
|
+
# not whatever an earlier PG_UPSERT left in them.
|
|
439
|
+
_set_subvars(exc.result)
|
|
440
|
+
raise ErrInfo(
|
|
441
|
+
"cmd",
|
|
442
|
+
command_text=metacommandline,
|
|
443
|
+
other_msg=f"PG_UPSERT rolled back after a database error: {exc.original}",
|
|
444
|
+
) from exc
|
|
436
445
|
except Exception as exc:
|
|
437
446
|
raise ErrInfo(
|
|
438
447
|
"exception",
|
execsql/utils/auth.py
CHANGED
|
@@ -88,22 +88,30 @@ _DEFAULT_PORTS = {"PostgreSQL": 5432, "MySQL": 3306, "SQL Server": 1433, "Oracle
|
|
|
88
88
|
_plaintext_warned: bool = False
|
|
89
89
|
|
|
90
90
|
|
|
91
|
+
def _storing_backend(backend: Any) -> Any:
|
|
92
|
+
"""The backend a password saved through *backend* lands in.
|
|
93
|
+
|
|
94
|
+
keyring's ``ChainerBackend`` saves to the first of its backends (highest
|
|
95
|
+
priority first) that accepts it, such as the macOS Keychain ahead of a
|
|
96
|
+
``keyrings.alt`` file.
|
|
97
|
+
"""
|
|
98
|
+
chained = getattr(backend, "backends", None)
|
|
99
|
+
return chained[0] if chained else backend
|
|
100
|
+
|
|
101
|
+
|
|
91
102
|
def is_plaintext_keyring() -> bool:
|
|
92
|
-
"""Return True if
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
path so callers can warn the user instead of pretending secrets
|
|
99
|
-
are encrypted.
|
|
103
|
+
"""Return True if a password stored now would be written in cleartext.
|
|
104
|
+
|
|
105
|
+
That is ``keyrings.alt``'s ``PlaintextKeyring``: on headless Linux with no
|
|
106
|
+
Secret Service, keyring falls back to it silently. Its encrypted
|
|
107
|
+
sibling (a master password protects the file), the Windows backends and
|
|
108
|
+
keyring's ``fail`` placeholder (which stores nothing) are not.
|
|
100
109
|
"""
|
|
101
110
|
try:
|
|
102
111
|
import keyring
|
|
103
112
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
return "keyrings.alt" in module or "fail" in module.lower()
|
|
113
|
+
cls = type(_storing_backend(keyring.get_keyring()))
|
|
114
|
+
return cls.__module__.startswith("keyrings.alt") and cls.__name__ == "PlaintextKeyring"
|
|
107
115
|
except Exception:
|
|
108
116
|
return False
|
|
109
117
|
|
|
@@ -118,11 +126,10 @@ def _warn_if_plaintext_keyring() -> None:
|
|
|
118
126
|
import keyring
|
|
119
127
|
import sys
|
|
120
128
|
|
|
121
|
-
backend_name = type(keyring.get_keyring()).__name__
|
|
129
|
+
backend_name = type(_storing_backend(keyring.get_keyring())).__name__
|
|
122
130
|
print(
|
|
123
131
|
f"WARNING: active keyring backend ({backend_name}) stores secrets in "
|
|
124
|
-
f"cleartext
|
|
125
|
-
f"meaningfully protected.",
|
|
132
|
+
f"cleartext. Stored passwords are not meaningfully protected.",
|
|
126
133
|
file=sys.stderr,
|
|
127
134
|
)
|
|
128
135
|
except Exception:
|
execsql/utils/errors.py
CHANGED
|
@@ -37,7 +37,7 @@ __all__ = [
|
|
|
37
37
|
"install_hint",
|
|
38
38
|
]
|
|
39
39
|
|
|
40
|
-
_ADDING_AN_EXTRA = "https://execsql2.readthedocs.io/en/
|
|
40
|
+
_ADDING_AN_EXTRA = "https://execsql2.readthedocs.io/en/stable/getting-started/installation/#adding-an-extra-later"
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
def install_hint(extra: str) -> str:
|