execsql2 2.1.2__py3-none-any.whl → 2.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/cli/__init__.py +436 -0
- execsql/cli/dsn.py +86 -0
- execsql/cli/help.py +140 -0
- execsql/{cli.py → cli/run.py} +14 -589
- execsql/config.py +13 -1
- execsql/db/access.py +16 -12
- execsql/db/base.py +158 -90
- execsql/db/dsn.py +6 -5
- execsql/db/duckdb.py +2 -2
- execsql/db/firebird.py +23 -19
- execsql/db/mysql.py +8 -7
- execsql/db/oracle.py +11 -11
- execsql/db/postgres.py +28 -16
- execsql/db/sqlite.py +12 -11
- execsql/db/sqlserver.py +5 -3
- execsql/exceptions.py +7 -7
- execsql/exporters/base.py +6 -1
- execsql/exporters/delimited.py +44 -35
- execsql/exporters/duckdb.py +2 -2
- execsql/exporters/feather.py +6 -6
- execsql/exporters/html.py +83 -69
- execsql/exporters/json.py +50 -42
- execsql/exporters/latex.py +33 -27
- execsql/exporters/ods.py +4 -4
- execsql/exporters/parquet.py +2 -2
- execsql/exporters/pretty.py +11 -9
- execsql/exporters/raw.py +17 -13
- execsql/exporters/sqlite.py +2 -2
- execsql/exporters/templates.py +23 -15
- execsql/exporters/values.py +22 -20
- execsql/exporters/xls.py +4 -4
- execsql/exporters/xml.py +28 -13
- execsql/importers/base.py +4 -4
- execsql/importers/csv.py +6 -6
- execsql/importers/feather.py +4 -4
- execsql/importers/ods.py +4 -4
- execsql/importers/xls.py +4 -4
- execsql/metacommands/__init__.py +518 -67
- execsql/metacommands/conditions.py +101 -27
- execsql/metacommands/control.py +8 -4
- execsql/metacommands/data.py +6 -6
- execsql/metacommands/debug.py +6 -2
- execsql/metacommands/io.py +67 -1310
- execsql/metacommands/io_export.py +442 -0
- execsql/metacommands/io_fileops.py +287 -0
- execsql/metacommands/io_import.py +398 -0
- execsql/metacommands/io_write.py +248 -0
- execsql/metacommands/prompt.py +22 -66
- execsql/metacommands/system.py +7 -2
- execsql/py.typed +0 -0
- execsql/script.py +49 -5
- execsql/types.py +20 -20
- execsql/utils/fileio.py +15 -8
- {execsql2-2.1.2.dist-info → execsql2-2.2.1.dist-info}/METADATA +6 -6
- execsql2-2.2.1.dist-info/RECORD +104 -0
- execsql2-2.1.2.dist-info/RECORD +0 -96
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/READ_ME.rst +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/execsql.conf +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/pg_upsert.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.1.2.data → execsql2-2.2.1.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.1.2.dist-info → execsql2-2.2.1.dist-info}/WHEEL +0 -0
- {execsql2-2.1.2.dist-info → execsql2-2.2.1.dist-info}/entry_points.txt +0 -0
- {execsql2-2.1.2.dist-info → execsql2-2.2.1.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.1.2.dist-info → execsql2-2.2.1.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
"""File and system operation metacommand handlers.
|
|
2
|
+
|
|
3
|
+
Implements ``x_include``, ``x_copy``, ``x_copy_query``, ``x_zip``,
|
|
4
|
+
``x_zip_buffer_mb``, ``x_rm_file``, ``x_make_export_dirs``, ``x_cd``,
|
|
5
|
+
``x_scan_lines``, ``x_hdf5_text_len``, and ``x_serve``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from shutil import copyfileobj
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import execsql.state as _state
|
|
17
|
+
from execsql.exceptions import ErrInfo
|
|
18
|
+
from execsql.models import DataTable
|
|
19
|
+
from execsql.script import current_script_line, read_sqlfile
|
|
20
|
+
from execsql.types import dbt_firebird
|
|
21
|
+
from execsql.utils.errors import exception_desc
|
|
22
|
+
from execsql.utils.fileio import filewriter_close
|
|
23
|
+
from execsql.utils.strings import unquoted
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def x_include(**kwargs: Any) -> None:
|
|
27
|
+
filename = kwargs["filename"]
|
|
28
|
+
if len(filename) > 1 and filename[0] == "~" and filename[1] == os.sep:
|
|
29
|
+
filename = str(Path.home() / filename[2:])
|
|
30
|
+
exists = kwargs["exists"]
|
|
31
|
+
if exists is not None:
|
|
32
|
+
if Path(filename).is_file():
|
|
33
|
+
read_sqlfile(filename)
|
|
34
|
+
else:
|
|
35
|
+
if not Path(filename).is_file():
|
|
36
|
+
raise ErrInfo(type="error", other_msg=f"File {filename} does not exist.")
|
|
37
|
+
read_sqlfile(filename)
|
|
38
|
+
return None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def x_copy(**kwargs: Any) -> None:
|
|
42
|
+
alias1 = kwargs["alias1"].lower()
|
|
43
|
+
schema1 = kwargs["schema1"]
|
|
44
|
+
table1 = kwargs["table1"]
|
|
45
|
+
new = kwargs["new"]
|
|
46
|
+
new_tbl2 = new.lower() if new else None
|
|
47
|
+
alias2 = kwargs["alias2"].lower()
|
|
48
|
+
schema2 = kwargs["schema2"]
|
|
49
|
+
table2 = kwargs["table2"]
|
|
50
|
+
if alias1 not in _state.dbs.aliases():
|
|
51
|
+
raise ErrInfo(
|
|
52
|
+
type="cmd",
|
|
53
|
+
command_text=kwargs["metacommandline"],
|
|
54
|
+
other_msg=f"Unrecognized database alias: {alias1}.",
|
|
55
|
+
)
|
|
56
|
+
if alias2 not in _state.dbs.aliases():
|
|
57
|
+
raise ErrInfo(
|
|
58
|
+
type="cmd",
|
|
59
|
+
command_text=kwargs["metacommandline"],
|
|
60
|
+
other_msg=f"Unrecognized database alias: {alias2}.",
|
|
61
|
+
)
|
|
62
|
+
db1 = _state.dbs.aliased_as(alias1)
|
|
63
|
+
db2 = _state.dbs.aliased_as(alias2)
|
|
64
|
+
tbl1 = db1.schema_qualified_table_name(schema1, table1)
|
|
65
|
+
tbl2 = db2.schema_qualified_table_name(schema2, table2)
|
|
66
|
+
try:
|
|
67
|
+
if not db1.table_exists(table1, schema1):
|
|
68
|
+
raise ErrInfo(
|
|
69
|
+
type="cmd",
|
|
70
|
+
command_text=kwargs["metacommandline"],
|
|
71
|
+
other_msg=f"Table {tbl1} does not exist",
|
|
72
|
+
)
|
|
73
|
+
except Exception:
|
|
74
|
+
pass # Best-effort check; some adapters lack information_schema.
|
|
75
|
+
if new_tbl2 and new_tbl2 == "new":
|
|
76
|
+
try:
|
|
77
|
+
if db2.table_exists(table2, schema2):
|
|
78
|
+
raise ErrInfo(
|
|
79
|
+
type="cmd",
|
|
80
|
+
command_text=kwargs["metacommandline"],
|
|
81
|
+
other_msg=f"Table {tbl2} already exists",
|
|
82
|
+
)
|
|
83
|
+
except Exception:
|
|
84
|
+
pass # Best-effort check; some adapters lack information_schema.
|
|
85
|
+
select_stmt = f"select * from {tbl1};"
|
|
86
|
+
|
|
87
|
+
def get_ts() -> DataTable:
|
|
88
|
+
if get_ts.tablespec is None:
|
|
89
|
+
hdrs, rows = db1.select_rowsource(select_stmt)
|
|
90
|
+
get_ts.tablespec = DataTable(hdrs, rows)
|
|
91
|
+
return get_ts.tablespec
|
|
92
|
+
|
|
93
|
+
get_ts.tablespec = None
|
|
94
|
+
|
|
95
|
+
if new_tbl2:
|
|
96
|
+
tbl_desc = get_ts()
|
|
97
|
+
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
98
|
+
if new_tbl2 == "replacement":
|
|
99
|
+
try:
|
|
100
|
+
db2.drop_table(tbl2)
|
|
101
|
+
except Exception:
|
|
102
|
+
_state.exec_log.log_status_info(f"Could not drop existing table ({tbl2}) for COPY metacommand")
|
|
103
|
+
db2.execute(create_tbl)
|
|
104
|
+
if db2.type == dbt_firebird:
|
|
105
|
+
db2.execute("COMMIT;")
|
|
106
|
+
try:
|
|
107
|
+
hdrs, rows = db1.select_rowsource(select_stmt)
|
|
108
|
+
except ErrInfo:
|
|
109
|
+
raise
|
|
110
|
+
except Exception as e:
|
|
111
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
112
|
+
try:
|
|
113
|
+
db2.populate_table(schema2, table2, rows, hdrs, get_ts)
|
|
114
|
+
db2.commit()
|
|
115
|
+
except ErrInfo:
|
|
116
|
+
raise
|
|
117
|
+
except Exception as e:
|
|
118
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def x_copy_query(**kwargs: Any) -> None:
|
|
122
|
+
alias1 = kwargs["alias1"].lower()
|
|
123
|
+
select_stmt = kwargs["query"]
|
|
124
|
+
new = kwargs["new"]
|
|
125
|
+
new_tbl2 = new.lower() if new else None
|
|
126
|
+
alias2 = kwargs["alias2"].lower()
|
|
127
|
+
schema2 = kwargs["schema"]
|
|
128
|
+
table2 = kwargs["table"]
|
|
129
|
+
if alias1 not in _state.dbs.aliases():
|
|
130
|
+
raise ErrInfo(
|
|
131
|
+
type="cmd",
|
|
132
|
+
command_text=kwargs["metacommandline"],
|
|
133
|
+
other_msg=f"Unrecognized database alias: {alias1}.",
|
|
134
|
+
)
|
|
135
|
+
if alias2 not in _state.dbs.aliases():
|
|
136
|
+
raise ErrInfo(
|
|
137
|
+
type="cmd",
|
|
138
|
+
command_text=kwargs["metacommandline"],
|
|
139
|
+
other_msg=f"Unrecognized database alias: {alias2}.",
|
|
140
|
+
)
|
|
141
|
+
db1 = _state.dbs.aliased_as(alias1)
|
|
142
|
+
db2 = _state.dbs.aliased_as(alias2)
|
|
143
|
+
tbl2 = db2.schema_qualified_table_name(schema2, table2)
|
|
144
|
+
if new_tbl2 and new_tbl2 == "new":
|
|
145
|
+
try:
|
|
146
|
+
if db2.table_exists(table2, schema2):
|
|
147
|
+
raise ErrInfo(
|
|
148
|
+
type="cmd",
|
|
149
|
+
command_text=kwargs["metacommandline"],
|
|
150
|
+
other_msg=f"Table {tbl2} already exists",
|
|
151
|
+
)
|
|
152
|
+
except Exception:
|
|
153
|
+
pass # Best-effort check; some adapters lack information_schema.
|
|
154
|
+
|
|
155
|
+
def get_ts() -> DataTable:
|
|
156
|
+
if not get_ts.tablespec:
|
|
157
|
+
hdrs, rows = db1.select_rowsource(select_stmt)
|
|
158
|
+
get_ts.tablespec = DataTable(hdrs, rows)
|
|
159
|
+
return get_ts.tablespec
|
|
160
|
+
|
|
161
|
+
get_ts.tablespec = None
|
|
162
|
+
|
|
163
|
+
if new_tbl2:
|
|
164
|
+
try:
|
|
165
|
+
hdrs, rows = db1.select_rowsource(select_stmt)
|
|
166
|
+
except ErrInfo:
|
|
167
|
+
raise
|
|
168
|
+
except Exception as e:
|
|
169
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
170
|
+
get_ts.tablespec = DataTable(hdrs, rows)
|
|
171
|
+
tbl_desc = get_ts.tablespec
|
|
172
|
+
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
173
|
+
if new_tbl2 == "replacement":
|
|
174
|
+
try:
|
|
175
|
+
db2.drop_table(tbl2)
|
|
176
|
+
except Exception:
|
|
177
|
+
_state.exec_log.log_status_info(f"Could not drop existing table ({tbl2}) for COPY metacommand")
|
|
178
|
+
db2.execute(create_tbl)
|
|
179
|
+
if db2.type == dbt_firebird:
|
|
180
|
+
db2.execute("COMMIT;")
|
|
181
|
+
try:
|
|
182
|
+
hdrs, rows = db1.select_rowsource(select_stmt)
|
|
183
|
+
except ErrInfo:
|
|
184
|
+
raise
|
|
185
|
+
except Exception as e:
|
|
186
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
187
|
+
try:
|
|
188
|
+
db2.populate_table(schema2, table2, rows, hdrs, get_ts)
|
|
189
|
+
db2.commit()
|
|
190
|
+
except ErrInfo:
|
|
191
|
+
raise
|
|
192
|
+
except Exception as e:
|
|
193
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def x_zip(**kwargs: Any) -> None:
|
|
197
|
+
import zipfile as _zipfile
|
|
198
|
+
import glob as _glob
|
|
199
|
+
|
|
200
|
+
files = kwargs["filename"].strip(' "')
|
|
201
|
+
zipfile_name = kwargs["zipfilename"].strip(' "')
|
|
202
|
+
append = kwargs["append"]
|
|
203
|
+
zmode = "a" if append is not None else "w"
|
|
204
|
+
zf = _zipfile.ZipFile(zipfile_name, mode=zmode, compression=_zipfile.ZIP_BZIP2, compresslevel=9)
|
|
205
|
+
fnlist = _glob.glob(files)
|
|
206
|
+
for f in fnlist:
|
|
207
|
+
if Path(f).is_file():
|
|
208
|
+
zf.write(f)
|
|
209
|
+
zf.close()
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def x_zip_buffer_mb(**kwargs: Any) -> None:
|
|
213
|
+
size_mb = kwargs["size"]
|
|
214
|
+
_state.conf.zip_buffer_mb = int(size_mb)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def x_rm_file(**kwargs: Any) -> None:
|
|
218
|
+
import glob as _glob
|
|
219
|
+
|
|
220
|
+
fn = kwargs["filename"].strip(' "')
|
|
221
|
+
fnlist = _glob.glob(fn)
|
|
222
|
+
for f in fnlist:
|
|
223
|
+
if Path(f).is_file():
|
|
224
|
+
filewriter_close(f)
|
|
225
|
+
os.unlink(f)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def x_make_export_dirs(**kwargs: Any) -> None:
|
|
229
|
+
setting = kwargs["setting"].lower()
|
|
230
|
+
_state.conf.make_export_dirs = setting in ("yes", "on", "true", "1")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def x_cd(**kwargs: Any) -> None:
|
|
234
|
+
new_dir = unquoted(kwargs["dir"])
|
|
235
|
+
if not Path(new_dir).is_dir():
|
|
236
|
+
raise ErrInfo(
|
|
237
|
+
type="cmd",
|
|
238
|
+
command_text=kwargs["metacommandline"],
|
|
239
|
+
other_msg="Directory does not exist",
|
|
240
|
+
)
|
|
241
|
+
os.chdir(new_dir)
|
|
242
|
+
script, lno = current_script_line()
|
|
243
|
+
_state.exec_log.log_status_info(f"Current directory changed to {new_dir} at line {lno} of {script}")
|
|
244
|
+
return None
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def x_scan_lines(**kwargs: Any) -> None:
|
|
248
|
+
_state.conf.scan_lines = int(kwargs["scanlines"])
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def x_hdf5_text_len(**kwargs: Any) -> None:
|
|
252
|
+
_state.conf.hdf5_text_len = int(kwargs["textlen"])
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def x_serve(**kwargs: Any) -> None:
|
|
256
|
+
infname = kwargs["filename"]
|
|
257
|
+
fmt = kwargs["format"].lower()
|
|
258
|
+
if not Path(infname).is_file():
|
|
259
|
+
raise ErrInfo(
|
|
260
|
+
type="cmd",
|
|
261
|
+
command_text=kwargs["metacommandline"],
|
|
262
|
+
other_msg=f"Input file {infname} does not exist",
|
|
263
|
+
)
|
|
264
|
+
fname = Path(infname).name
|
|
265
|
+
if fmt == "binary":
|
|
266
|
+
contenttype = "application/octet-stream"
|
|
267
|
+
elif fmt == "csv":
|
|
268
|
+
contenttype = "text/csv"
|
|
269
|
+
elif fmt in ("txt", "text"):
|
|
270
|
+
contenttype = "text/plain"
|
|
271
|
+
elif fmt == "ods":
|
|
272
|
+
contenttype = "application/vnd.oasis.opendocument.spreadsheet"
|
|
273
|
+
elif fmt == "json":
|
|
274
|
+
contenttype = "application/json"
|
|
275
|
+
elif fmt == "html":
|
|
276
|
+
contenttype = "text/html"
|
|
277
|
+
elif fmt == "pdf":
|
|
278
|
+
contenttype = "application/pdf"
|
|
279
|
+
elif fmt == "zip":
|
|
280
|
+
contenttype = "application/zip"
|
|
281
|
+
else:
|
|
282
|
+
contenttype = "application/octet-stream"
|
|
283
|
+
print(f"Content-Type: {contenttype}")
|
|
284
|
+
safe_fname = fname.replace("\r", "").replace("\n", "").replace('"', '\\"')
|
|
285
|
+
print(f'Content-Disposition: attachment; filename="{safe_fname}"\n')
|
|
286
|
+
with open(infname, "rb") as f:
|
|
287
|
+
copyfileobj(f, sys.stdout.buffer)
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
"""Import metacommand handlers.
|
|
2
|
+
|
|
3
|
+
Implements ``x_import``, ``x_import_file``, ODS/XLS/Parquet/Feather
|
|
4
|
+
import handlers, and the import row buffer setting.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import execsql.state as _state
|
|
14
|
+
from execsql.exceptions import ErrInfo
|
|
15
|
+
from execsql.importers.csv import importfile, importtable
|
|
16
|
+
from execsql.importers.feather import import_feather, import_parquet
|
|
17
|
+
from execsql.importers.ods import OdsFile, importods
|
|
18
|
+
from execsql.exporters.xls import XlsFile, XlsxFile
|
|
19
|
+
from execsql.importers.xls import importxls
|
|
20
|
+
from execsql.utils.errors import exception_desc
|
|
21
|
+
from execsql.utils.strings import clean_words, fold_words
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def x_import(**kwargs: Any) -> None:
|
|
25
|
+
newstr = kwargs["new"]
|
|
26
|
+
if newstr:
|
|
27
|
+
is_new = 1 + ["new", "replacement"].index(newstr.lower())
|
|
28
|
+
else:
|
|
29
|
+
is_new = 0
|
|
30
|
+
schemaname = kwargs["schema"]
|
|
31
|
+
tablename = kwargs["table"]
|
|
32
|
+
filename = kwargs["filename"]
|
|
33
|
+
if len(filename) > 1 and filename[0] == "~" and filename[1] == os.sep:
|
|
34
|
+
filename = str(Path.home() / filename[2:])
|
|
35
|
+
if not Path(filename).exists():
|
|
36
|
+
raise ErrInfo(
|
|
37
|
+
type="cmd",
|
|
38
|
+
command_text=kwargs["metacommandline"],
|
|
39
|
+
other_msg=f"Input file {filename} does not exist",
|
|
40
|
+
)
|
|
41
|
+
quotechar = kwargs["quotechar"]
|
|
42
|
+
if quotechar:
|
|
43
|
+
quotechar = quotechar.lower()
|
|
44
|
+
delimchar = kwargs["delimchar"]
|
|
45
|
+
if delimchar:
|
|
46
|
+
if delimchar.lower() == "tab":
|
|
47
|
+
delimchar = chr(9)
|
|
48
|
+
elif delimchar.lower() in ("unitsep", "us"):
|
|
49
|
+
delimchar = chr(31)
|
|
50
|
+
enc = kwargs["encoding"]
|
|
51
|
+
junk_hdrs = kwargs["skip"]
|
|
52
|
+
if not junk_hdrs:
|
|
53
|
+
junk_hdrs = 0
|
|
54
|
+
else:
|
|
55
|
+
junk_hdrs = int(junk_hdrs)
|
|
56
|
+
from execsql.metacommands.conditions import file_size_date
|
|
57
|
+
|
|
58
|
+
sz, dt = file_size_date(filename)
|
|
59
|
+
_state.exec_log.log_status_info(f"IMPORTing {filename} ({sz}, {dt})")
|
|
60
|
+
try:
|
|
61
|
+
importtable(
|
|
62
|
+
_state.dbs.current(),
|
|
63
|
+
schemaname,
|
|
64
|
+
tablename,
|
|
65
|
+
filename,
|
|
66
|
+
is_new,
|
|
67
|
+
skip_header_line=True,
|
|
68
|
+
quotechar=quotechar,
|
|
69
|
+
delimchar=delimchar,
|
|
70
|
+
encoding=enc,
|
|
71
|
+
junk_header_lines=junk_hdrs,
|
|
72
|
+
)
|
|
73
|
+
except ErrInfo:
|
|
74
|
+
raise
|
|
75
|
+
except Exception as e:
|
|
76
|
+
raise ErrInfo(
|
|
77
|
+
"exception",
|
|
78
|
+
exception_msg=exception_desc(),
|
|
79
|
+
other_msg=f"Can't import data from tabular text file {filename}",
|
|
80
|
+
) from e
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def x_import_file(**kwargs: Any) -> None:
|
|
85
|
+
schemaname = kwargs["schema"]
|
|
86
|
+
tablename = kwargs["table"]
|
|
87
|
+
columnname = kwargs["columnname"]
|
|
88
|
+
filename = kwargs["filename"]
|
|
89
|
+
if not Path(filename).exists():
|
|
90
|
+
raise ErrInfo(
|
|
91
|
+
type="cmd",
|
|
92
|
+
command_text=kwargs["metacommandline"],
|
|
93
|
+
other_msg=f"Input file {filename} does not exist",
|
|
94
|
+
)
|
|
95
|
+
from execsql.metacommands.conditions import file_size_date
|
|
96
|
+
|
|
97
|
+
sz, dt = file_size_date(filename)
|
|
98
|
+
_state.exec_log.log_status_info(f"IMPORTing_FILE {filename} ({sz}, {dt})")
|
|
99
|
+
try:
|
|
100
|
+
importfile(_state.dbs.current(), schemaname, tablename, columnname, filename)
|
|
101
|
+
except ErrInfo:
|
|
102
|
+
raise
|
|
103
|
+
except Exception as e:
|
|
104
|
+
raise ErrInfo(
|
|
105
|
+
"exception",
|
|
106
|
+
exception_msg=exception_desc(),
|
|
107
|
+
other_msg=f"Can't import file {filename}",
|
|
108
|
+
) from e
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def x_import_ods(**kwargs: Any) -> None:
|
|
113
|
+
newstr = kwargs["new"]
|
|
114
|
+
if newstr:
|
|
115
|
+
is_new = 1 + ["new", "replacement"].index(newstr.lower())
|
|
116
|
+
else:
|
|
117
|
+
is_new = 0
|
|
118
|
+
schemaname = kwargs["schema"]
|
|
119
|
+
tablename = kwargs["table"]
|
|
120
|
+
filename = kwargs["filename"]
|
|
121
|
+
sheetname = kwargs["sheetname"]
|
|
122
|
+
hdr_rows = kwargs["skip"]
|
|
123
|
+
if not hdr_rows:
|
|
124
|
+
hdr_rows = 0
|
|
125
|
+
else:
|
|
126
|
+
hdr_rows = int(hdr_rows)
|
|
127
|
+
if not Path(filename).exists():
|
|
128
|
+
raise ErrInfo(
|
|
129
|
+
type="cmd",
|
|
130
|
+
command_text=kwargs["metacommandline"],
|
|
131
|
+
other_msg="Input file does not exist",
|
|
132
|
+
)
|
|
133
|
+
try:
|
|
134
|
+
importods(_state.dbs.current(), schemaname, tablename, is_new, filename, sheetname, hdr_rows)
|
|
135
|
+
except ErrInfo:
|
|
136
|
+
raise
|
|
137
|
+
except Exception as e:
|
|
138
|
+
raise ErrInfo(
|
|
139
|
+
"exception",
|
|
140
|
+
exception_msg=exception_desc(),
|
|
141
|
+
other_msg=f"Can't import data from ODS file {filename}",
|
|
142
|
+
) from e
|
|
143
|
+
return None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def x_import_ods_pattern(**kwargs: Any) -> None:
|
|
147
|
+
import re
|
|
148
|
+
|
|
149
|
+
newstr = kwargs["new"]
|
|
150
|
+
if newstr:
|
|
151
|
+
is_new = 1 + ["new", "replacement"].index(newstr.lower())
|
|
152
|
+
else:
|
|
153
|
+
is_new = 0
|
|
154
|
+
schemaname = kwargs["schema"]
|
|
155
|
+
filename = kwargs["filename"]
|
|
156
|
+
rx = re.compile(kwargs["patn"], re.I)
|
|
157
|
+
hdr_rows = kwargs["skip"]
|
|
158
|
+
if not hdr_rows:
|
|
159
|
+
hdr_rows = 0
|
|
160
|
+
else:
|
|
161
|
+
hdr_rows = int(hdr_rows)
|
|
162
|
+
if not Path(filename).exists():
|
|
163
|
+
raise ErrInfo(
|
|
164
|
+
type="cmd",
|
|
165
|
+
command_text=kwargs["metacommandline"],
|
|
166
|
+
other_msg="Input file does not exist",
|
|
167
|
+
)
|
|
168
|
+
wbk = OdsFile()
|
|
169
|
+
try:
|
|
170
|
+
wbk.open(filename)
|
|
171
|
+
except Exception as e:
|
|
172
|
+
raise ErrInfo(type="cmd", other_msg=f"{filename} is not a valid OpenDocument spreadsheet.") from e
|
|
173
|
+
sheets = wbk.sheetnames()
|
|
174
|
+
impsheets = [s for s in sheets if rx.search(s)]
|
|
175
|
+
tables = list(impsheets)
|
|
176
|
+
if _state.conf.clean_col_hdrs:
|
|
177
|
+
tables = clean_words(tables)
|
|
178
|
+
if _state.conf.fold_col_hdrs != "no":
|
|
179
|
+
tables = fold_words(tables, _state.conf.fold_col_hdrs)
|
|
180
|
+
for ix in range(len(impsheets)):
|
|
181
|
+
sheetname = impsheets[ix]
|
|
182
|
+
tablename = tables[ix]
|
|
183
|
+
try:
|
|
184
|
+
importods(_state.dbs.current(), schemaname, tablename, is_new, filename, sheetname, hdr_rows)
|
|
185
|
+
except ErrInfo:
|
|
186
|
+
raise
|
|
187
|
+
except Exception as e:
|
|
188
|
+
raise ErrInfo(
|
|
189
|
+
"exception",
|
|
190
|
+
exception_msg=exception_desc(),
|
|
191
|
+
other_msg=f"Can't import data from ODS file {filename}",
|
|
192
|
+
) from e
|
|
193
|
+
_state.subvars.add_substitution("$SHEETS_IMPORTED", ",".join(impsheets))
|
|
194
|
+
_state.subvars.add_substitution("$SHEETS_TABLES", ",".join(tables))
|
|
195
|
+
if schemaname is None:
|
|
196
|
+
_state.subvars.add_substitution(
|
|
197
|
+
"$SHEETS_TABLES_VALUES",
|
|
198
|
+
",".join([f"('{t.replace(chr(39), chr(39) + chr(39))}')" for t in tables]),
|
|
199
|
+
)
|
|
200
|
+
else:
|
|
201
|
+
_state.subvars.add_substitution(
|
|
202
|
+
"$SHEETS_TABLES_VALUES",
|
|
203
|
+
",".join([f"('{schemaname}.{t.replace(chr(39), chr(39) + chr(39))}')" for t in tables]),
|
|
204
|
+
)
|
|
205
|
+
return None
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def x_import_xls(**kwargs: Any) -> None:
|
|
209
|
+
newstr = kwargs["new"]
|
|
210
|
+
if newstr:
|
|
211
|
+
is_new = 1 + ["new", "replacement"].index(newstr.lower())
|
|
212
|
+
else:
|
|
213
|
+
is_new = 0
|
|
214
|
+
schemaname = kwargs["schema"]
|
|
215
|
+
tablename = kwargs["table"]
|
|
216
|
+
filename = kwargs["filename"]
|
|
217
|
+
sheetname = kwargs["sheetname"]
|
|
218
|
+
junk_hdrs = kwargs["skip"]
|
|
219
|
+
encoding = kwargs["encoding"]
|
|
220
|
+
if not junk_hdrs:
|
|
221
|
+
junk_hdrs = 0
|
|
222
|
+
else:
|
|
223
|
+
junk_hdrs = int(junk_hdrs)
|
|
224
|
+
if not Path(filename).exists():
|
|
225
|
+
raise ErrInfo(
|
|
226
|
+
type="cmd",
|
|
227
|
+
command_text=kwargs["metacommandline"],
|
|
228
|
+
other_msg="Input file does not exist",
|
|
229
|
+
)
|
|
230
|
+
try:
|
|
231
|
+
importxls(_state.dbs.current(), schemaname, tablename, is_new, filename, sheetname, junk_hdrs, encoding)
|
|
232
|
+
except ErrInfo:
|
|
233
|
+
raise
|
|
234
|
+
except Exception as e:
|
|
235
|
+
raise ErrInfo(
|
|
236
|
+
"exception",
|
|
237
|
+
exception_msg=exception_desc(),
|
|
238
|
+
other_msg=f"Can't import data from Excel file {filename}",
|
|
239
|
+
) from e
|
|
240
|
+
return None
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def x_import_xls_pattern(**kwargs: Any) -> None:
|
|
244
|
+
import re
|
|
245
|
+
|
|
246
|
+
newstr = kwargs["new"]
|
|
247
|
+
if newstr:
|
|
248
|
+
is_new = 1 + ["new", "replacement"].index(newstr.lower())
|
|
249
|
+
else:
|
|
250
|
+
is_new = 0
|
|
251
|
+
schemaname = kwargs["schema"]
|
|
252
|
+
filename = kwargs["filename"]
|
|
253
|
+
rx = re.compile(kwargs["patn"], re.I)
|
|
254
|
+
hdr_rows = kwargs["skip"]
|
|
255
|
+
encoding = kwargs["encoding"]
|
|
256
|
+
if not hdr_rows:
|
|
257
|
+
hdr_rows = 0
|
|
258
|
+
else:
|
|
259
|
+
hdr_rows = int(hdr_rows)
|
|
260
|
+
if not Path(filename).exists():
|
|
261
|
+
raise ErrInfo(
|
|
262
|
+
type="cmd",
|
|
263
|
+
command_text=kwargs["metacommandline"],
|
|
264
|
+
other_msg="Input file does not exist",
|
|
265
|
+
)
|
|
266
|
+
if len(filename) < 4:
|
|
267
|
+
raise ErrInfo(type="cmd", other_msg=f"{filename} is not a recognizable Excel spreadsheet name.")
|
|
268
|
+
ext3 = filename[-3:].lower()
|
|
269
|
+
if ext3 == "xls":
|
|
270
|
+
wbk = XlsFile()
|
|
271
|
+
elif ext3 == "lsx":
|
|
272
|
+
wbk = XlsxFile()
|
|
273
|
+
else:
|
|
274
|
+
raise ErrInfo(type="cmd", other_msg=f"{filename} is not a recognizable Excel spreadsheet name.")
|
|
275
|
+
try:
|
|
276
|
+
wbk.open(filename, encoding, read_only=True)
|
|
277
|
+
except Exception as e:
|
|
278
|
+
raise ErrInfo(type="cmd", other_msg=f"{filename} is not a valid Excel spreadsheet.") from e
|
|
279
|
+
sheets = wbk.sheetnames()
|
|
280
|
+
impsheets = [s for s in sheets if rx.search(s)]
|
|
281
|
+
tables = list(impsheets)
|
|
282
|
+
if _state.conf.clean_col_hdrs:
|
|
283
|
+
tables = clean_words(tables)
|
|
284
|
+
if _state.conf.fold_col_hdrs != "no":
|
|
285
|
+
tables = fold_words(tables, _state.conf.fold_col_hdrs)
|
|
286
|
+
for ix in range(len(impsheets)):
|
|
287
|
+
sheetname = impsheets[ix]
|
|
288
|
+
tablename = tables[ix]
|
|
289
|
+
try:
|
|
290
|
+
importxls(
|
|
291
|
+
_state.dbs.current(),
|
|
292
|
+
schemaname,
|
|
293
|
+
tablename,
|
|
294
|
+
is_new,
|
|
295
|
+
filename,
|
|
296
|
+
sheetname,
|
|
297
|
+
hdr_rows,
|
|
298
|
+
encoding,
|
|
299
|
+
)
|
|
300
|
+
except ErrInfo:
|
|
301
|
+
raise
|
|
302
|
+
except Exception as e:
|
|
303
|
+
raise ErrInfo(
|
|
304
|
+
"exception",
|
|
305
|
+
exception_msg=exception_desc(),
|
|
306
|
+
other_msg=f"Can't import data from ODS file {filename}",
|
|
307
|
+
) from e
|
|
308
|
+
_state.subvars.add_substitution("$SHEETS_IMPORTED", ",".join(impsheets))
|
|
309
|
+
_state.subvars.add_substitution("$SHEETS_TABLES", ",".join(tables))
|
|
310
|
+
if schemaname is None:
|
|
311
|
+
_state.subvars.add_substitution(
|
|
312
|
+
"$SHEETS_TABLES_VALUES",
|
|
313
|
+
",".join([f"('{t.replace(chr(39), chr(39) + chr(39))}')" for t in tables]),
|
|
314
|
+
)
|
|
315
|
+
else:
|
|
316
|
+
_state.subvars.add_substitution(
|
|
317
|
+
"$SHEETS_TABLES_VALUES",
|
|
318
|
+
",".join([f"('{schemaname}.{t.replace(chr(39), chr(39) + chr(39))}')" for t in tables]),
|
|
319
|
+
)
|
|
320
|
+
return None
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def x_import_parquet(**kwargs: Any) -> None:
|
|
324
|
+
newstr = kwargs["new"]
|
|
325
|
+
if newstr:
|
|
326
|
+
is_new = 1 + ["new", "replacement"].index(newstr.lower())
|
|
327
|
+
else:
|
|
328
|
+
is_new = 0
|
|
329
|
+
schemaname = kwargs["schema"]
|
|
330
|
+
tablename = kwargs["table"]
|
|
331
|
+
filename = kwargs["filename"]
|
|
332
|
+
if len(filename) > 1 and filename[0] == "~" and filename[1] == os.sep:
|
|
333
|
+
filename = str(Path.home() / filename[2:])
|
|
334
|
+
if not Path(filename).exists():
|
|
335
|
+
raise ErrInfo(
|
|
336
|
+
type="cmd",
|
|
337
|
+
command_text=kwargs["metacommandline"],
|
|
338
|
+
other_msg=f"Input file {filename} does not exist",
|
|
339
|
+
)
|
|
340
|
+
from execsql.metacommands.conditions import file_size_date
|
|
341
|
+
|
|
342
|
+
sz, dt = file_size_date(filename)
|
|
343
|
+
_state.exec_log.log_status_info(f"IMPORTing from Parquet file {filename} ({sz}, {dt})")
|
|
344
|
+
try:
|
|
345
|
+
import_parquet(_state.dbs.current(), schemaname, tablename, filename, is_new)
|
|
346
|
+
except ErrInfo:
|
|
347
|
+
raise
|
|
348
|
+
except Exception as e:
|
|
349
|
+
raise ErrInfo(
|
|
350
|
+
"exception",
|
|
351
|
+
exception_msg=exception_desc(),
|
|
352
|
+
other_msg=f"Can't import data from Parquet data file {filename}",
|
|
353
|
+
) from e
|
|
354
|
+
return None
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def x_import_feather(**kwargs: Any) -> None:
|
|
358
|
+
newstr = kwargs["new"]
|
|
359
|
+
if newstr:
|
|
360
|
+
is_new = 1 + ["new", "replacement"].index(newstr.lower())
|
|
361
|
+
else:
|
|
362
|
+
is_new = 0
|
|
363
|
+
schemaname = kwargs["schema"]
|
|
364
|
+
tablename = kwargs["table"]
|
|
365
|
+
filename = kwargs["filename"]
|
|
366
|
+
if len(filename) > 1 and filename[0] == "~" and filename[1] == os.sep:
|
|
367
|
+
filename = str(Path.home() / filename[2:])
|
|
368
|
+
if not Path(filename).exists():
|
|
369
|
+
raise ErrInfo(
|
|
370
|
+
type="cmd",
|
|
371
|
+
command_text=kwargs["metacommandline"],
|
|
372
|
+
other_msg=f"Input file {filename} does not exist",
|
|
373
|
+
)
|
|
374
|
+
from execsql.metacommands.conditions import file_size_date
|
|
375
|
+
|
|
376
|
+
sz, dt = file_size_date(filename)
|
|
377
|
+
_state.exec_log.log_status_info(f"IMPORTing from Feather file {filename} ({sz}, {dt})")
|
|
378
|
+
try:
|
|
379
|
+
import_feather(_state.dbs.current(), schemaname, tablename, filename, is_new)
|
|
380
|
+
except ErrInfo:
|
|
381
|
+
raise
|
|
382
|
+
except Exception as e:
|
|
383
|
+
raise ErrInfo(
|
|
384
|
+
"exception",
|
|
385
|
+
exception_msg=exception_desc(),
|
|
386
|
+
other_msg=f"Can't import data from Feather data file {filename}",
|
|
387
|
+
) from e
|
|
388
|
+
return None
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def x_import_row_buffer(**kwargs: Any) -> None:
|
|
392
|
+
rows = kwargs["rows"]
|
|
393
|
+
_state.conf.import_row_buffer = int(rows)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def x_show_progress(**kwargs: Any) -> None:
|
|
397
|
+
setting = kwargs["setting"].lower()
|
|
398
|
+
_state.conf.show_progress = setting in ("yes", "on", "true", "1")
|