execsql2 2.0.1__py3-none-any.whl → 2.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/cli.py +322 -108
- execsql/config.py +134 -114
- execsql/db/access.py +89 -65
- execsql/db/base.py +97 -68
- execsql/db/dsn.py +45 -29
- execsql/db/duckdb.py +4 -5
- execsql/db/factory.py +27 -27
- execsql/db/firebird.py +30 -18
- execsql/db/mysql.py +38 -14
- execsql/db/oracle.py +58 -33
- execsql/db/postgres.py +68 -28
- execsql/db/sqlite.py +36 -27
- execsql/db/sqlserver.py +45 -30
- execsql/exceptions.py +68 -64
- execsql/exporters/__init__.py +1 -1
- execsql/exporters/base.py +42 -17
- execsql/exporters/delimited.py +60 -59
- execsql/exporters/duckdb.py +8 -12
- execsql/exporters/feather.py +32 -24
- execsql/exporters/html.py +33 -30
- execsql/exporters/json.py +18 -17
- execsql/exporters/latex.py +11 -13
- execsql/exporters/ods.py +50 -46
- execsql/exporters/parquet.py +32 -0
- execsql/exporters/pretty.py +16 -15
- execsql/exporters/raw.py +9 -11
- execsql/exporters/sqlite.py +38 -38
- execsql/exporters/templates.py +15 -72
- execsql/exporters/values.py +13 -12
- execsql/exporters/xls.py +26 -26
- execsql/exporters/xml.py +12 -12
- execsql/exporters/zip.py +0 -3
- execsql/gui/__init__.py +2 -2
- execsql/gui/console.py +0 -1
- execsql/gui/desktop.py +6 -7
- execsql/gui/tui.py +8 -14
- execsql/importers/base.py +6 -9
- execsql/importers/csv.py +10 -17
- execsql/importers/feather.py +16 -22
- execsql/importers/ods.py +3 -4
- execsql/importers/xls.py +5 -6
- execsql/metacommands/__init__.py +8 -8
- execsql/metacommands/conditions.py +41 -33
- execsql/metacommands/connect.py +113 -99
- execsql/metacommands/control.py +38 -26
- execsql/metacommands/data.py +35 -33
- execsql/metacommands/debug.py +13 -9
- execsql/metacommands/io.py +288 -229
- execsql/metacommands/prompt.py +179 -157
- execsql/metacommands/script_ext.py +11 -9
- execsql/metacommands/system.py +44 -25
- execsql/models.py +9 -16
- execsql/parser.py +10 -10
- execsql/script.py +183 -157
- execsql/state.py +170 -208
- execsql/types.py +46 -81
- execsql/utils/auth.py +114 -14
- execsql/utils/crypto.py +31 -4
- execsql/utils/datetime.py +7 -7
- execsql/utils/errors.py +34 -29
- execsql/utils/fileio.py +90 -55
- execsql/utils/gui.py +22 -23
- execsql/utils/mail.py +15 -17
- execsql/utils/numeric.py +2 -3
- execsql/utils/regex.py +9 -12
- execsql/utils/strings.py +10 -12
- execsql/utils/timer.py +0 -2
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/execsql.conf +1 -1
- execsql2-2.1.2.dist-info/METADATA +300 -0
- execsql2-2.1.2.dist-info/RECORD +96 -0
- execsql2-2.0.1.dist-info/METADATA +0 -406
- execsql2-2.0.1.dist-info/RECORD +0 -95
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/READ_ME.rst +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/pg_upsert.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.0.1.data → execsql2-2.1.2.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.0.1.dist-info → execsql2-2.1.2.dist-info}/WHEEL +0 -0
- {execsql2-2.0.1.dist-info → execsql2-2.1.2.dist-info}/entry_points.txt +0 -0
- {execsql2-2.0.1.dist-info → execsql2-2.1.2.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.0.1.dist-info → execsql2-2.1.2.dist-info}/licenses/NOTICE +0 -0
execsql/exporters/delimited.py
CHANGED
|
@@ -16,21 +16,22 @@ Provides:
|
|
|
16
16
|
"""
|
|
17
17
|
|
|
18
18
|
import copy
|
|
19
|
-
import csv
|
|
20
|
-
import codecs
|
|
21
|
-
import io
|
|
22
|
-
import os
|
|
23
19
|
import re
|
|
24
20
|
import sys
|
|
25
|
-
from typing import Any
|
|
21
|
+
from typing import Any
|
|
26
22
|
|
|
27
23
|
from execsql.utils.fileio import EncodedFile
|
|
28
|
-
from execsql.exporters.zip import
|
|
24
|
+
from execsql.exporters.zip import ZipWriter
|
|
29
25
|
import execsql.state as _state
|
|
26
|
+
from execsql.exceptions import ErrInfo
|
|
27
|
+
from execsql.models import DataTable
|
|
28
|
+
from execsql.utils.errors import exception_desc
|
|
29
|
+
from execsql.utils.fileio import filewriter_close
|
|
30
|
+
from execsql.utils.strings import clean_words, fold_words
|
|
30
31
|
|
|
31
32
|
|
|
32
33
|
class LineDelimiter:
|
|
33
|
-
def __init__(self, delim:
|
|
34
|
+
def __init__(self, delim: str | None, quote: str | None, escchar: str | None) -> None:
|
|
34
35
|
self.delimiter = delim
|
|
35
36
|
self.joinchar = delim if delim else ""
|
|
36
37
|
self.quotechar = quote
|
|
@@ -78,7 +79,7 @@ class LineDelimiter:
|
|
|
78
79
|
|
|
79
80
|
|
|
80
81
|
class DelimitedWriter:
|
|
81
|
-
def __init__(self, outfile: Any, delim:
|
|
82
|
+
def __init__(self, outfile: Any, delim: str | None, quote: str | None, escchar: str | None) -> None:
|
|
82
83
|
self.outfile = outfile
|
|
83
84
|
self.line_delimiter = LineDelimiter(delim, quote, escchar)
|
|
84
85
|
|
|
@@ -98,16 +99,16 @@ class CsvWriter:
|
|
|
98
99
|
self,
|
|
99
100
|
filename: str,
|
|
100
101
|
file_encoding: str,
|
|
101
|
-
delim:
|
|
102
|
-
quote:
|
|
103
|
-
escchar:
|
|
102
|
+
delim: str | None,
|
|
103
|
+
quote: str | None,
|
|
104
|
+
escchar: str | None,
|
|
104
105
|
append: bool = False,
|
|
105
106
|
) -> None:
|
|
106
107
|
mode = "wt" if not append else "at"
|
|
107
108
|
if filename.lower() == "stdout":
|
|
108
109
|
self.output = sys.stdout
|
|
109
110
|
else:
|
|
110
|
-
|
|
111
|
+
filewriter_close(filename)
|
|
111
112
|
self.output = EncodedFile(filename, file_encoding).open(mode)
|
|
112
113
|
self.dwriter = DelimitedWriter(self.output, delim, quote, escchar)
|
|
113
114
|
|
|
@@ -144,11 +145,11 @@ class CsvFile(EncodedFile):
|
|
|
144
145
|
def openclean(self, mode: str) -> Any:
|
|
145
146
|
# Returns an opened file object with junk headers stripped.
|
|
146
147
|
f = self.open(mode)
|
|
147
|
-
for
|
|
148
|
+
for _ in range(self.junk_header_lines):
|
|
148
149
|
f.readline()
|
|
149
150
|
return f
|
|
150
151
|
|
|
151
|
-
def lineformat(self, delimiter:
|
|
152
|
+
def lineformat(self, delimiter: str | None, quotechar: str | None, escapechar: str | None) -> None:
|
|
152
153
|
# Specifies the format of a line.
|
|
153
154
|
self.delimiter = delimiter
|
|
154
155
|
self.quotechar = quotechar
|
|
@@ -166,9 +167,10 @@ class CsvFile(EncodedFile):
|
|
|
166
167
|
def __str__(self) -> str:
|
|
167
168
|
return "; ".join(
|
|
168
169
|
[
|
|
169
|
-
"Text:
|
|
170
|
-
"Delimiter counts:
|
|
171
|
-
|
|
170
|
+
f"Text: <<{self.text}>>",
|
|
171
|
+
"Delimiter counts: <<{}>>".format(
|
|
172
|
+
", ".join([f"{k}: {self.delim_counts[k]}" for k in self.delim_counts]),
|
|
173
|
+
),
|
|
172
174
|
],
|
|
173
175
|
)
|
|
174
176
|
|
|
@@ -212,7 +214,7 @@ class CsvFile(EncodedFile):
|
|
|
212
214
|
def record_format_error(self, pos_no: int, errmsg: str) -> None:
|
|
213
215
|
self.item_errors.append(f"{errmsg} in position {pos_no}.")
|
|
214
216
|
|
|
215
|
-
def items(self, delim:
|
|
217
|
+
def items(self, delim: str | None, qchar: str | None) -> Any:
|
|
216
218
|
# Parses the line into a list of items, breaking it at delimiters that are not
|
|
217
219
|
# within quoted stretches.
|
|
218
220
|
self.item_errors = []
|
|
@@ -308,26 +310,26 @@ class CsvFile(EncodedFile):
|
|
|
308
310
|
|
|
309
311
|
exec_vector = [in_quoted, escaped, quote_in_quoted, in_unquoted, between, delimited]
|
|
310
312
|
state = _BETWEEN
|
|
311
|
-
for i, c in enumerate(self.text):
|
|
313
|
+
for i, c in enumerate(self.text): # noqa: B007
|
|
312
314
|
state = exec_vector[state]()
|
|
313
315
|
if len(esc_buf[0]) > 0:
|
|
314
316
|
current_element[0] += esc_buf[0]
|
|
315
317
|
if len(current_element[0]) > 0:
|
|
316
318
|
elements.append(current_element[0])
|
|
317
319
|
if len(self.item_errors) > 0:
|
|
318
|
-
raise
|
|
320
|
+
raise ErrInfo("error", other_msg=", ".join(self.item_errors))
|
|
319
321
|
return elements
|
|
320
322
|
|
|
321
|
-
def well_quoted_line(self, delim:
|
|
323
|
+
def well_quoted_line(self, delim: str | None, qchar: str | None):
|
|
322
324
|
# Returns a tuple of boolean, int, and boolean
|
|
323
325
|
wq = [self._well_quoted(el, qchar) for el in self.items(delim, qchar)]
|
|
324
|
-
return (all(
|
|
326
|
+
return (all(b[0] for b in wq), sum([b[1] for b in wq]), any(b[2] for b in wq))
|
|
325
327
|
|
|
326
328
|
def diagnose_delim(
|
|
327
329
|
self,
|
|
328
330
|
linestream: Any,
|
|
329
|
-
possible_delimiters:
|
|
330
|
-
possible_quotechars:
|
|
331
|
+
possible_delimiters: list[str] | None = None,
|
|
332
|
+
possible_quotechars: list[str] | None = None,
|
|
331
333
|
):
|
|
332
334
|
# Returns a tuple consisting of the delimiter, quote character, and escape
|
|
333
335
|
# character for quote characters within elements of a line. All may be None.
|
|
@@ -337,7 +339,7 @@ class CsvFile(EncodedFile):
|
|
|
337
339
|
if not possible_quotechars:
|
|
338
340
|
possible_quotechars = ['"', "'"]
|
|
339
341
|
lines = []
|
|
340
|
-
for
|
|
342
|
+
for _i in range(conf.scan_lines if conf.scan_lines and conf.scan_lines > 0 else 1000000):
|
|
341
343
|
try:
|
|
342
344
|
ln = next(linestream)
|
|
343
345
|
except StopIteration:
|
|
@@ -349,7 +351,7 @@ class CsvFile(EncodedFile):
|
|
|
349
351
|
if len(ln) > 0:
|
|
350
352
|
lines.append(self.CsvLine(ln))
|
|
351
353
|
if len(lines) == 0:
|
|
352
|
-
raise
|
|
354
|
+
raise ErrInfo(type="error", other_msg="CSV diagnosis error: no lines read")
|
|
353
355
|
for ln in lines:
|
|
354
356
|
for d in possible_delimiters:
|
|
355
357
|
ln.count_delim(d)
|
|
@@ -366,11 +368,11 @@ class CsvFile(EncodedFile):
|
|
|
366
368
|
del delim_stats[k]
|
|
367
369
|
|
|
368
370
|
def all_well_quoted(delim, qchar):
|
|
369
|
-
wq = [
|
|
371
|
+
wq = [ln.well_quoted_line(delim, qchar) for ln in lines]
|
|
370
372
|
return (
|
|
371
|
-
all(
|
|
373
|
+
all(b[0] for b in wq),
|
|
372
374
|
sum([b[1] for b in wq]),
|
|
373
|
-
self.CsvLine.escchar if any(
|
|
375
|
+
self.CsvLine.escchar if any(b[2] for b in wq) else None,
|
|
374
376
|
)
|
|
375
377
|
|
|
376
378
|
def eval_quotes(delim):
|
|
@@ -385,20 +387,19 @@ class CsvFile(EncodedFile):
|
|
|
385
387
|
max_use = max([v[0] for v in ok_quotes.values()])
|
|
386
388
|
if max_use == 0:
|
|
387
389
|
return (delim, None, None)
|
|
388
|
-
for q in ok_quotes
|
|
390
|
+
for q in ok_quotes:
|
|
389
391
|
if ok_quotes[q][0] == max_use:
|
|
390
392
|
return (delim, q, ok_quotes[q][1])
|
|
391
393
|
|
|
392
394
|
if len(delim_stats) == 0:
|
|
393
395
|
return eval_quotes(None)
|
|
394
396
|
else:
|
|
395
|
-
if len(delim_stats) > 1:
|
|
396
|
-
|
|
397
|
-
del delim_stats[" "]
|
|
397
|
+
if len(delim_stats) > 1 and " " in delim_stats:
|
|
398
|
+
del delim_stats[" "]
|
|
398
399
|
if len(delim_stats) == 1:
|
|
399
400
|
return eval_quotes(list(delim_stats)[0])
|
|
400
401
|
delim_wts = {}
|
|
401
|
-
for d in delim_stats
|
|
402
|
+
for d in delim_stats:
|
|
402
403
|
delim_wts[d] = delim_stats[d][0] ** 2 * delim_stats[d][1]
|
|
403
404
|
delim_order = sorted(delim_wts, key=delim_wts.get, reverse=True)
|
|
404
405
|
for d in delim_order:
|
|
@@ -406,7 +407,7 @@ class CsvFile(EncodedFile):
|
|
|
406
407
|
if quote_check[0] and quote_check[1]:
|
|
407
408
|
return quote_check
|
|
408
409
|
return (delim_order[0], None, None)
|
|
409
|
-
raise
|
|
410
|
+
raise ErrInfo(
|
|
410
411
|
type="error",
|
|
411
412
|
other_msg="CSV diagnosis coding error: an untested set of conditions are present",
|
|
412
413
|
)
|
|
@@ -420,7 +421,7 @@ class CsvFile(EncodedFile):
|
|
|
420
421
|
def _record_format_error(self, pos_no: int, errmsg: str) -> None:
|
|
421
422
|
self.parse_errors.append(f"{errmsg} in position {pos_no}")
|
|
422
423
|
|
|
423
|
-
def read_and_parse_line(self, f: Any) ->
|
|
424
|
+
def read_and_parse_line(self, f: Any) -> list:
|
|
424
425
|
# Returns a list of line elements, parsed according to the established delimiter and quotechar.
|
|
425
426
|
elements = []
|
|
426
427
|
eat_multiple_delims = self.delimiter == " "
|
|
@@ -562,7 +563,7 @@ class CsvFile(EncodedFile):
|
|
|
562
563
|
state = exec_vector[state]()
|
|
563
564
|
end()
|
|
564
565
|
if len(self.parse_errors) > 0:
|
|
565
|
-
raise
|
|
566
|
+
raise ErrInfo("error", other_msg=", ".join(self.parse_errors))
|
|
566
567
|
return elements
|
|
567
568
|
|
|
568
569
|
def reader(self) -> Any:
|
|
@@ -574,8 +575,8 @@ class CsvFile(EncodedFile):
|
|
|
574
575
|
line_no += 1
|
|
575
576
|
try:
|
|
576
577
|
elements = self.read_and_parse_line(f)
|
|
577
|
-
except
|
|
578
|
-
raise
|
|
578
|
+
except ErrInfo as e:
|
|
579
|
+
raise ErrInfo("error", other_msg=f"{e.other} on line {line_no}.") from e
|
|
579
580
|
except:
|
|
580
581
|
raise
|
|
581
582
|
if len(elements) > 0:
|
|
@@ -592,20 +593,20 @@ class CsvFile(EncodedFile):
|
|
|
592
593
|
def writer(self, append: bool = False) -> CsvWriter:
|
|
593
594
|
return CsvWriter(self.filename, self.encoding, self.delimiter, self.quotechar, self.escapechar, append)
|
|
594
595
|
|
|
595
|
-
def _colhdrs(self, inf: Any) ->
|
|
596
|
+
def _colhdrs(self, inf: Any) -> list[str]:
|
|
596
597
|
conf = _state.conf
|
|
597
598
|
try:
|
|
598
599
|
colnames = next(inf)
|
|
599
|
-
except
|
|
600
|
+
except ErrInfo as e:
|
|
600
601
|
e.other = f"Can't read column header line from {self.filename}. {e.other or ''}"
|
|
601
|
-
raise
|
|
602
|
-
except:
|
|
603
|
-
raise
|
|
602
|
+
raise
|
|
603
|
+
except Exception:
|
|
604
|
+
raise ErrInfo(
|
|
604
605
|
type="exception",
|
|
605
|
-
exception_msg=
|
|
606
|
+
exception_msg=exception_desc(),
|
|
606
607
|
other_msg=f"Can't read column header line from {self.filename}",
|
|
607
608
|
)
|
|
608
|
-
if any(
|
|
609
|
+
if any(x is None or len(x) == 0 for x in colnames):
|
|
609
610
|
if conf.del_empty_cols:
|
|
610
611
|
self.blank_cols = [
|
|
611
612
|
i for i in range(len(colnames)) if colnames[i] is None or len(colnames[i].strip()) == 0
|
|
@@ -620,19 +621,19 @@ class CsvFile(EncodedFile):
|
|
|
620
621
|
if colnames[i] is None or len(colnames[i]) == 0:
|
|
621
622
|
colnames[i] = f"Col{i + 1}"
|
|
622
623
|
else:
|
|
623
|
-
raise
|
|
624
|
+
raise ErrInfo(
|
|
624
625
|
type="error",
|
|
625
626
|
other_msg=f"The input file {self.csvfname} has missing column headers.",
|
|
626
627
|
)
|
|
627
628
|
if conf.clean_col_hdrs:
|
|
628
|
-
colnames =
|
|
629
|
+
colnames = clean_words(colnames)
|
|
629
630
|
if conf.fold_col_hdrs != "no":
|
|
630
|
-
colnames =
|
|
631
|
+
colnames = fold_words(colnames, conf.fold_col_hdrs)
|
|
631
632
|
if conf.dedup_col_hdrs:
|
|
632
633
|
colnames = _state.dedup_words(colnames)
|
|
633
634
|
return colnames
|
|
634
635
|
|
|
635
|
-
def column_headers(self) ->
|
|
636
|
+
def column_headers(self) -> list[str]:
|
|
636
637
|
if not self.lineformat_set:
|
|
637
638
|
self.evaluate_line_format()
|
|
638
639
|
inf = self.reader()
|
|
@@ -648,20 +649,20 @@ class CsvFile(EncodedFile):
|
|
|
648
649
|
self.evaluate_line_format()
|
|
649
650
|
inf = self.reader()
|
|
650
651
|
colnames = self._colhdrs(inf)
|
|
651
|
-
self.table_data =
|
|
652
|
+
self.table_data = DataTable(colnames, inf)
|
|
652
653
|
|
|
653
|
-
def create_table(self, database_type: Any, schemaname:
|
|
654
|
+
def create_table(self, database_type: Any, schemaname: str | None, tablename: str, pretty: bool = False) -> str:
|
|
654
655
|
return self.table_data.create_table(database_type, schemaname, tablename, pretty)
|
|
655
656
|
|
|
656
657
|
|
|
657
658
|
def write_delimited_file(
|
|
658
659
|
outfile: str,
|
|
659
660
|
filefmt: str,
|
|
660
|
-
column_headers:
|
|
661
|
+
column_headers: list[str],
|
|
661
662
|
rowsource: Any,
|
|
662
663
|
file_encoding: str = "utf8",
|
|
663
664
|
append: bool = False,
|
|
664
|
-
zipfile:
|
|
665
|
+
zipfile: str | None = None,
|
|
665
666
|
) -> None:
|
|
666
667
|
delim = None
|
|
667
668
|
quote = None
|
|
@@ -696,7 +697,7 @@ def write_delimited_file(
|
|
|
696
697
|
fdesc = f"{outfile} in {zipfile}"
|
|
697
698
|
else:
|
|
698
699
|
fmode = "w" if not append else "a"
|
|
699
|
-
|
|
700
|
+
filewriter_close(outfile)
|
|
700
701
|
ofile = EncodedFile(outfile, file_encoding).open(mode=fmode)
|
|
701
702
|
fdesc = outfile
|
|
702
703
|
if not (filefmt.lower() == "plain" or (append and zipfile is None)):
|
|
@@ -706,11 +707,11 @@ def write_delimited_file(
|
|
|
706
707
|
try:
|
|
707
708
|
datarow = line_delimiter.delimited(rec)
|
|
708
709
|
ofile.write(datarow)
|
|
709
|
-
except
|
|
710
|
+
except ErrInfo:
|
|
710
711
|
raise
|
|
711
|
-
except:
|
|
712
|
-
raise
|
|
712
|
+
except Exception:
|
|
713
|
+
raise ErrInfo(
|
|
713
714
|
"exception",
|
|
714
|
-
exception_msg=
|
|
715
|
+
exception_msg=exception_desc(),
|
|
715
716
|
other_msg=f"Can't write output to file {fdesc}.",
|
|
716
717
|
)
|
execsql/exporters/duckdb.py
CHANGED
|
@@ -9,16 +9,16 @@ Requires the ``execsql2[duckdb]`` extra.
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import math
|
|
12
|
-
import
|
|
13
|
-
from typing import Any
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
14
|
|
|
15
15
|
from execsql.exceptions import ErrInfo
|
|
16
|
-
|
|
16
|
+
from execsql.types import dbt_duckdb
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def export_duckdb(
|
|
20
20
|
outfile: str,
|
|
21
|
-
hdrs:
|
|
21
|
+
hdrs: list[str],
|
|
22
22
|
rows: Any,
|
|
23
23
|
append: bool,
|
|
24
24
|
tablename: str,
|
|
@@ -32,13 +32,12 @@ def export_duckdb(
|
|
|
32
32
|
return
|
|
33
33
|
|
|
34
34
|
from execsql.models import DataTable
|
|
35
|
-
from execsql.utils.errors import exception_info
|
|
36
35
|
|
|
37
36
|
chunksize = 10000
|
|
38
|
-
pre_exist =
|
|
37
|
+
pre_exist = Path(outfile).is_file()
|
|
39
38
|
ddb = duckdb.connect(outfile, read_only=False)
|
|
40
39
|
if pre_exist:
|
|
41
|
-
catalog =
|
|
40
|
+
catalog = Path(outfile).stem
|
|
42
41
|
curs = ddb.cursor()
|
|
43
42
|
res = curs.execute(
|
|
44
43
|
f"select count(*) as rows from information_schema.tables "
|
|
@@ -49,14 +48,11 @@ def export_duckdb(
|
|
|
49
48
|
if append:
|
|
50
49
|
raise ErrInfo(type="error", other_msg=f"The table {tablename} already exists in {outfile}.")
|
|
51
50
|
else:
|
|
52
|
-
from execsql.utils.fileio import Logger
|
|
53
|
-
|
|
54
51
|
curs.execute(f"drop table {tablename};")
|
|
55
52
|
curs.close()
|
|
56
53
|
# Construct and run the CREATE TABLE statement
|
|
57
54
|
rowdata = list(rows)
|
|
58
55
|
tablespec = DataTable(hdrs, rowdata)
|
|
59
|
-
dbt_duckdb = _state.dbt_duckdb
|
|
60
56
|
sql = tablespec.create_table(dbt_duckdb, schemaname=None, tablename=tablename)
|
|
61
57
|
curs = ddb.cursor()
|
|
62
58
|
curs.execute(sql)
|
|
@@ -83,12 +79,12 @@ def write_query_to_duckdb(
|
|
|
83
79
|
append: bool,
|
|
84
80
|
tablename: str,
|
|
85
81
|
) -> None:
|
|
86
|
-
from execsql.utils.errors import
|
|
82
|
+
from execsql.utils.errors import exception_desc
|
|
87
83
|
|
|
88
84
|
try:
|
|
89
85
|
hdrs, rows = db.select_rowsource(select_stmt)
|
|
90
86
|
except ErrInfo:
|
|
91
87
|
raise
|
|
92
88
|
except Exception:
|
|
93
|
-
raise ErrInfo("db", select_stmt, exception_msg=
|
|
89
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc())
|
|
94
90
|
export_duckdb(outfile, hdrs, rows, append, tablename)
|
execsql/exporters/feather.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
from execsql.exceptions import ErrInfo
|
|
2
3
|
|
|
3
4
|
"""
|
|
4
5
|
Apache Feather and HDF5 export for execsql.
|
|
@@ -9,24 +10,31 @@ and ``tables``). Used by ``EXPORT … FORMAT feather`` and
|
|
|
9
10
|
``FORMAT hdf5``. Both packages are optional dependencies.
|
|
10
11
|
"""
|
|
11
12
|
|
|
12
|
-
from typing import Any
|
|
13
|
+
from typing import Any
|
|
13
14
|
|
|
14
15
|
import execsql.state as _state
|
|
16
|
+
from execsql.models import DataTable
|
|
17
|
+
from execsql.types import DT_Boolean, DT_Date, DT_Timestamp, DT_TimestampTZ
|
|
18
|
+
from execsql.utils.errors import exception_desc
|
|
19
|
+
from execsql.utils.fileio import filewriter_close
|
|
15
20
|
|
|
16
21
|
|
|
17
|
-
def write_query_to_feather(outfile: str, headers:
|
|
22
|
+
def write_query_to_feather(outfile: str, headers: list[str], rows: Any) -> None:
|
|
18
23
|
try:
|
|
19
|
-
import
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
raise _state.ErrInfo(
|
|
24
|
+
import polars as pl
|
|
25
|
+
except ImportError:
|
|
26
|
+
raise ErrInfo(
|
|
23
27
|
"exception",
|
|
24
|
-
exception_msg=
|
|
25
|
-
other_msg="The
|
|
28
|
+
exception_msg=exception_desc(),
|
|
29
|
+
other_msg="The polars Python package must be installed to export data to the feather format.",
|
|
26
30
|
)
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
31
|
+
rows_list = list(rows)
|
|
32
|
+
if rows_list:
|
|
33
|
+
df = pl.DataFrame(rows_list, schema=headers, orient="row")
|
|
34
|
+
else:
|
|
35
|
+
df = pl.DataFrame({h: [] for h in headers})
|
|
36
|
+
filewriter_close(outfile)
|
|
37
|
+
df.write_ipc(outfile)
|
|
30
38
|
|
|
31
39
|
|
|
32
40
|
def write_query_to_hdf5(
|
|
@@ -35,22 +43,22 @@ def write_query_to_hdf5(
|
|
|
35
43
|
db: Any,
|
|
36
44
|
outfile: str,
|
|
37
45
|
append: bool = False,
|
|
38
|
-
desc:
|
|
46
|
+
desc: str | None = None,
|
|
39
47
|
) -> None:
|
|
40
48
|
try:
|
|
41
49
|
import tables
|
|
42
|
-
except:
|
|
43
|
-
raise
|
|
50
|
+
except ImportError:
|
|
51
|
+
raise ErrInfo(
|
|
44
52
|
"exception",
|
|
45
|
-
exception_msg=
|
|
53
|
+
exception_msg=exception_desc(),
|
|
46
54
|
other_msg="The tables Python library must be installed to export data to the HDF5 format.",
|
|
47
55
|
)
|
|
48
56
|
try:
|
|
49
57
|
hdrs, rows = db.select_rowsource(select_stmt)
|
|
50
|
-
except
|
|
58
|
+
except ErrInfo:
|
|
51
59
|
raise
|
|
52
|
-
except:
|
|
53
|
-
raise
|
|
60
|
+
except Exception:
|
|
61
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc())
|
|
54
62
|
|
|
55
63
|
def h5type(datatype, size):
|
|
56
64
|
if datatype in (_state.DT_Varchar, _state.DT_Text):
|
|
@@ -65,29 +73,29 @@ def write_query_to_hdf5(
|
|
|
65
73
|
elif datatype in (_state.DT_Float, _state.DT_Decimal):
|
|
66
74
|
t = tables.Float64Col()
|
|
67
75
|
do_cast = False
|
|
68
|
-
elif datatype ==
|
|
76
|
+
elif datatype == DT_Boolean:
|
|
69
77
|
t = tables.BoolCol()
|
|
70
78
|
do_cast = False
|
|
71
|
-
elif datatype in (
|
|
79
|
+
elif datatype in (DT_TimestampTZ, DT_Timestamp, DT_Date, _state.DT_Time):
|
|
72
80
|
t = tables.StringCol(50)
|
|
73
81
|
do_cast = True
|
|
74
82
|
else:
|
|
75
|
-
raise
|
|
83
|
+
raise ErrInfo("error", other_msg=f"Invalid data type for export to HDF5: {repr(datatype)}")
|
|
76
84
|
return t, do_cast
|
|
77
85
|
|
|
78
86
|
# Create a dictionary of column names with the HDF5 data types
|
|
79
|
-
tbl_desc =
|
|
87
|
+
tbl_desc = DataTable(hdrs, rows)
|
|
80
88
|
h5type_dict = {}
|
|
81
89
|
cast_flags = []
|
|
82
90
|
# Iterate over hdrs instead of tbl_desc.cols to preserve column order.
|
|
83
|
-
for
|
|
91
|
+
for h in hdrs:
|
|
84
92
|
dt = [col for col in tbl_desc.cols if col.name == h][0].dt
|
|
85
93
|
# dt is a tuple of: 0: the column name; 1: the data type class; 2: the maximum length or None if NA; other info.
|
|
86
94
|
h5typ, as_str = h5type(dt[1], dt[2])
|
|
87
95
|
h5type_dict[h] = h5typ
|
|
88
96
|
cast_flags.append(as_str)
|
|
89
97
|
# Open the HDF5 table
|
|
90
|
-
|
|
98
|
+
filewriter_close(outfile)
|
|
91
99
|
h5file_mode = "a" if append else "w"
|
|
92
100
|
h5file = tables.open_file(outfile, mode=h5file_mode)
|
|
93
101
|
h5grp = h5file.create_group("/", table_name, title=desc)
|
execsql/exporters/html.py
CHANGED
|
@@ -11,25 +11,28 @@ CSS styling.
|
|
|
11
11
|
|
|
12
12
|
import datetime
|
|
13
13
|
import getpass
|
|
14
|
-
import io
|
|
15
14
|
import os
|
|
16
|
-
import re
|
|
17
15
|
import sys
|
|
18
16
|
import tempfile
|
|
19
|
-
from
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
20
19
|
|
|
21
20
|
import execsql.state as _state
|
|
22
21
|
from execsql.exporters.zip import ZipWriter
|
|
22
|
+
from execsql.exceptions import ErrInfo
|
|
23
|
+
from execsql.script import current_script_line
|
|
24
|
+
from execsql.utils.errors import exception_desc
|
|
25
|
+
from execsql.utils.fileio import filewriter_close
|
|
23
26
|
|
|
24
27
|
|
|
25
28
|
def export_html(
|
|
26
29
|
outfile: str,
|
|
27
|
-
hdrs:
|
|
30
|
+
hdrs: list[str],
|
|
28
31
|
rows: Any,
|
|
29
32
|
append: bool = False,
|
|
30
|
-
querytext:
|
|
31
|
-
desc:
|
|
32
|
-
zipfile:
|
|
33
|
+
querytext: str | None = None,
|
|
34
|
+
desc: str | None = None,
|
|
35
|
+
zipfile: str | None = None,
|
|
33
36
|
) -> None:
|
|
34
37
|
conf = _state.conf
|
|
35
38
|
|
|
@@ -48,7 +51,7 @@ def export_html(
|
|
|
48
51
|
f.write("</tr>\n")
|
|
49
52
|
f.write("</tbody>\n</table>\n")
|
|
50
53
|
|
|
51
|
-
script, lno =
|
|
54
|
+
script, lno = current_script_line()
|
|
52
55
|
# If not append, write a complete HTML document with header and table.
|
|
53
56
|
# If append and the file does not exist, write just the table.
|
|
54
57
|
# If append and the file exists, R/W up to the </body> tag, write the table, write the remainder of the input.
|
|
@@ -57,7 +60,7 @@ def export_html(
|
|
|
57
60
|
if outfile.lower() == "stdout":
|
|
58
61
|
f = sys.stdout
|
|
59
62
|
else:
|
|
60
|
-
|
|
63
|
+
filewriter_close(outfile)
|
|
61
64
|
from execsql.utils.fileio import EncodedFile
|
|
62
65
|
|
|
63
66
|
ef = EncodedFile(outfile, conf.output_encoding)
|
|
@@ -66,9 +69,9 @@ def export_html(
|
|
|
66
69
|
f = ZipWriter(zipfile, outfile, append)
|
|
67
70
|
f.write('<!DOCTYPE html>\n<html>\n<head>\n<meta charset="utf-8" />\n')
|
|
68
71
|
if querytext:
|
|
69
|
-
descrip = f"Source: [{querytext}] with database {_state.dbs.current().name()} in script {
|
|
72
|
+
descrip = f"Source: [{querytext}] with database {_state.dbs.current().name()} in script {str(Path(script).resolve())}, line {lno}"
|
|
70
73
|
else:
|
|
71
|
-
descrip = f"From database {_state.dbs.current().name()} in script {
|
|
74
|
+
descrip = f"From database {_state.dbs.current().name()} in script {str(Path(script).resolve())}, line {lno}"
|
|
72
75
|
f.write(f'<meta name="description" content="{descrip}" />\n')
|
|
73
76
|
datecontent = datetime.datetime.now().strftime("%Y-%m-%d")
|
|
74
77
|
f.write(f'<meta name="created" content="{datecontent}" />\n')
|
|
@@ -104,7 +107,7 @@ def export_html(
|
|
|
104
107
|
if outfile.lower() == "stdout":
|
|
105
108
|
f = sys.stdout
|
|
106
109
|
write_table(f)
|
|
107
|
-
elif not
|
|
110
|
+
elif not Path(outfile).is_file():
|
|
108
111
|
from execsql.utils.fileio import EncodedFile
|
|
109
112
|
|
|
110
113
|
ef = EncodedFile(outfile, conf.output_encoding)
|
|
@@ -112,7 +115,7 @@ def export_html(
|
|
|
112
115
|
write_table(f)
|
|
113
116
|
f.close()
|
|
114
117
|
else:
|
|
115
|
-
|
|
118
|
+
filewriter_close(outfile)
|
|
116
119
|
from execsql.utils.fileio import EncodedFile
|
|
117
120
|
|
|
118
121
|
ef = EncodedFile(outfile, conf.output_encoding)
|
|
@@ -144,12 +147,12 @@ def export_html(
|
|
|
144
147
|
|
|
145
148
|
def export_cgi_html(
|
|
146
149
|
outfile: str,
|
|
147
|
-
hdrs:
|
|
150
|
+
hdrs: list[str],
|
|
148
151
|
rows: Any,
|
|
149
152
|
append: bool = False,
|
|
150
|
-
querytext:
|
|
151
|
-
desc:
|
|
152
|
-
zipfile:
|
|
153
|
+
querytext: str | None = None,
|
|
154
|
+
desc: str | None = None,
|
|
155
|
+
zipfile: str | None = None,
|
|
153
156
|
) -> None:
|
|
154
157
|
conf = _state.conf
|
|
155
158
|
|
|
@@ -168,13 +171,13 @@ def export_cgi_html(
|
|
|
168
171
|
f.write("</tr>\n")
|
|
169
172
|
f.write("</tbody>\n</table>\n")
|
|
170
173
|
|
|
171
|
-
script, lno =
|
|
172
|
-
if zipfile or not append or (append and not
|
|
174
|
+
script, lno = current_script_line()
|
|
175
|
+
if zipfile or not append or (append and not Path(outfile).is_file()):
|
|
173
176
|
if zipfile is None:
|
|
174
177
|
if outfile.lower() == "stdout":
|
|
175
178
|
f = sys.stdout
|
|
176
179
|
else:
|
|
177
|
-
|
|
180
|
+
filewriter_close(outfile)
|
|
178
181
|
from execsql.utils.fileio import EncodedFile
|
|
179
182
|
|
|
180
183
|
ef = EncodedFile(outfile, conf.output_encoding)
|
|
@@ -203,15 +206,15 @@ def write_query_to_html(
|
|
|
203
206
|
db: Any,
|
|
204
207
|
outfile: str,
|
|
205
208
|
append: bool = False,
|
|
206
|
-
desc:
|
|
207
|
-
zipfile:
|
|
209
|
+
desc: str | None = None,
|
|
210
|
+
zipfile: str | None = None,
|
|
208
211
|
) -> None:
|
|
209
212
|
try:
|
|
210
213
|
hdrs, rows = db.select_rowsource(select_stmt)
|
|
211
|
-
except
|
|
214
|
+
except ErrInfo:
|
|
212
215
|
raise
|
|
213
|
-
except:
|
|
214
|
-
raise
|
|
216
|
+
except Exception:
|
|
217
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc())
|
|
215
218
|
export_html(outfile, hdrs, rows, append, select_stmt, desc, zipfile=zipfile)
|
|
216
219
|
|
|
217
220
|
|
|
@@ -220,13 +223,13 @@ def write_query_to_cgi_html(
|
|
|
220
223
|
db: Any,
|
|
221
224
|
outfile: str,
|
|
222
225
|
append: bool = False,
|
|
223
|
-
desc:
|
|
224
|
-
zipfile:
|
|
226
|
+
desc: str | None = None,
|
|
227
|
+
zipfile: str | None = None,
|
|
225
228
|
) -> None:
|
|
226
229
|
try:
|
|
227
230
|
hdrs, rows = db.select_rowsource(select_stmt)
|
|
228
|
-
except
|
|
231
|
+
except ErrInfo:
|
|
229
232
|
raise
|
|
230
|
-
except:
|
|
231
|
-
raise
|
|
233
|
+
except Exception:
|
|
234
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc())
|
|
232
235
|
export_cgi_html(outfile, hdrs, rows, append, select_stmt, desc, zipfile=zipfile)
|