execsql2 2.40.0__py3-none-any.whl → 2.41.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/__init__.py +4 -2
- execsql/api.py +40 -27
- execsql/cli/commands/__init__.py +1 -0
- execsql/cli/commands/inspect.py +292 -0
- execsql/cli/dispatch.py +3 -3
- execsql/cli/inspection.py +276 -0
- execsql/cli/lint.py +3 -4
- execsql/cli/options.py +15 -7
- execsql/config.py +15 -2
- execsql/db/access.py +6 -5
- execsql/db/base.py +14 -0
- execsql/db/dsn.py +3 -2
- execsql/db/duckdb.py +2 -1
- execsql/db/firebird.py +3 -2
- execsql/db/mysql.py +3 -2
- execsql/db/oracle.py +3 -2
- execsql/db/postgres.py +3 -2
- execsql/db/sqlite.py +2 -1
- execsql/db/sqlserver.py +3 -2
- execsql/ddl.py +310 -0
- execsql/exporters/xls.py +18 -4
- execsql/file_inspection.py +393 -0
- execsql/importers/base.py +19 -12
- execsql/importers/feather.py +16 -21
- execsql/importers/json.py +26 -32
- execsql/importers/xls.py +9 -4
- execsql/schema.py +816 -0
- execsql/script/executor.py +4 -0
- execsql/script/variables.py +14 -2
- execsql/utils/errors.py +21 -3
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/README.md +7 -0
- execsql2-2.41.0.data/data/execsql2_extras/inspect/README.md +116 -0
- execsql2-2.41.0.data/data/execsql2_extras/inspect/make_demo.py +106 -0
- {execsql2-2.40.0.dist-info → execsql2-2.41.0.dist-info}/METADATA +84 -19
- {execsql2-2.40.0.dist-info → execsql2-2.41.0.dist-info}/RECORD +51 -44
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.40.0.data → execsql2-2.41.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.40.0.dist-info → execsql2-2.41.0.dist-info}/WHEEL +0 -0
- {execsql2-2.40.0.dist-info → execsql2-2.41.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.40.0.dist-info → execsql2-2.41.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.40.0.dist-info → execsql2-2.41.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
"""Read-only file metadata using the same readers and inference as IMPORT."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import codecs
|
|
6
|
+
import importlib
|
|
7
|
+
from contextlib import closing
|
|
8
|
+
from enum import Enum
|
|
9
|
+
from itertools import islice
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, cast
|
|
12
|
+
from collections.abc import Iterable
|
|
13
|
+
|
|
14
|
+
from execsql.exceptions import ErrInfo, ScriptExit
|
|
15
|
+
from execsql.config import check_explicit_config_file
|
|
16
|
+
from execsql.models import DataTable
|
|
17
|
+
from execsql.state import RuntimeContext, active_context
|
|
18
|
+
|
|
19
|
+
__all__ = ["FileFormat", "inspect_file"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class FileFormat(str, Enum):
|
|
23
|
+
"""Input formats understood by the built-in IMPORT readers."""
|
|
24
|
+
|
|
25
|
+
csv = "csv"
|
|
26
|
+
tsv = "tsv"
|
|
27
|
+
text = "text"
|
|
28
|
+
json = "json"
|
|
29
|
+
jsonl = "jsonl"
|
|
30
|
+
xls = "xls"
|
|
31
|
+
xlsx = "xlsx"
|
|
32
|
+
ods = "ods"
|
|
33
|
+
parquet = "parquet"
|
|
34
|
+
feather = "feather"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
_EXTENSIONS = {
|
|
38
|
+
".csv": "csv",
|
|
39
|
+
".tsv": "tsv",
|
|
40
|
+
".tab": "tsv",
|
|
41
|
+
".txt": "text",
|
|
42
|
+
".json": "json",
|
|
43
|
+
".jsonl": "jsonl",
|
|
44
|
+
".ndjson": "jsonl",
|
|
45
|
+
".xls": "xls",
|
|
46
|
+
".xlsx": "xlsx",
|
|
47
|
+
".ods": "ods",
|
|
48
|
+
".parquet": "parquet",
|
|
49
|
+
".feather": "feather",
|
|
50
|
+
".arrow": "feather",
|
|
51
|
+
".ipc": "feather",
|
|
52
|
+
}
|
|
53
|
+
_TEXT = {"csv", "tsv", "text", "json", "jsonl"}
|
|
54
|
+
_DELIMITED = {"csv", "tsv", "text"}
|
|
55
|
+
_WORKBOOK = {"xls", "xlsx", "ods"}
|
|
56
|
+
_SETTINGS = (
|
|
57
|
+
"import_encoding",
|
|
58
|
+
"enc_err_disposition",
|
|
59
|
+
"scan_lines",
|
|
60
|
+
"max_int",
|
|
61
|
+
"boolean_int",
|
|
62
|
+
"boolean_words",
|
|
63
|
+
"exact_decimals",
|
|
64
|
+
"only_strings",
|
|
65
|
+
"empty_strings",
|
|
66
|
+
"empty_rows",
|
|
67
|
+
"trim_strings",
|
|
68
|
+
"replace_newlines",
|
|
69
|
+
"del_empty_cols",
|
|
70
|
+
"create_col_hdrs",
|
|
71
|
+
"clean_col_hdrs",
|
|
72
|
+
"trim_col_hdrs",
|
|
73
|
+
"fold_col_hdrs",
|
|
74
|
+
"dedup_col_hdrs",
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _columns(table: DataTable, stored: list[str] | None = None) -> list[dict[str, Any]]:
|
|
79
|
+
columns = []
|
|
80
|
+
for index, column in enumerate(table.cols):
|
|
81
|
+
name, kind, length, _nullable, precision, scale = column.column_type()
|
|
82
|
+
inferred = kind.data_type_name
|
|
83
|
+
if length is not None:
|
|
84
|
+
inferred += f"({length})"
|
|
85
|
+
elif precision is not None and scale is not None:
|
|
86
|
+
inferred += f"({precision},{scale})"
|
|
87
|
+
columns.append(
|
|
88
|
+
{
|
|
89
|
+
"name": name,
|
|
90
|
+
"inferred_type": inferred,
|
|
91
|
+
"source_type": stored[index] if stored is not None else None,
|
|
92
|
+
"null_count": column.nullrows,
|
|
93
|
+
},
|
|
94
|
+
)
|
|
95
|
+
return columns
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _scan(
|
|
99
|
+
result: dict[str, Any],
|
|
100
|
+
headers: list[str],
|
|
101
|
+
rows: Iterable[Any],
|
|
102
|
+
sample_rows: int | None,
|
|
103
|
+
*,
|
|
104
|
+
materialized: bool,
|
|
105
|
+
stored: list[str] | None = None,
|
|
106
|
+
) -> DataTable:
|
|
107
|
+
total = len(rows) if isinstance(rows, list) else None
|
|
108
|
+
if sample_rows is None:
|
|
109
|
+
table = DataTable(headers, rows)
|
|
110
|
+
complete = True
|
|
111
|
+
else:
|
|
112
|
+
prefix = list(islice(rows, sample_rows + 1))
|
|
113
|
+
complete = len(prefix) <= sample_rows
|
|
114
|
+
table = DataTable(headers, prefix[:sample_rows])
|
|
115
|
+
result["columns"] = _columns(table, stored)
|
|
116
|
+
result["scan"] = {
|
|
117
|
+
"mode": "full" if sample_rows is None else "sample",
|
|
118
|
+
"row_limit": sample_rows,
|
|
119
|
+
"rows_examined": table.inputrows,
|
|
120
|
+
"complete": complete,
|
|
121
|
+
"total_rows": table.inputrows if complete else total,
|
|
122
|
+
"source_materialized": materialized,
|
|
123
|
+
}
|
|
124
|
+
return table
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _workbook_names(filename: str, kind: str, encoding: str | None) -> list[str]:
|
|
128
|
+
if kind == "ods":
|
|
129
|
+
from execsql.importers.ods import OdsFile, check_ods_archive
|
|
130
|
+
|
|
131
|
+
check_ods_archive(filename)
|
|
132
|
+
workbook: Any = OdsFile()
|
|
133
|
+
else:
|
|
134
|
+
from execsql.exporters.xls import XlsFile, XlsxFile
|
|
135
|
+
from execsql.utils.fileio import check_zip_decompression_ratio
|
|
136
|
+
|
|
137
|
+
if kind == "xlsx":
|
|
138
|
+
check_zip_decompression_ratio(filename)
|
|
139
|
+
workbook = XlsFile() if kind == "xls" else XlsxFile()
|
|
140
|
+
try:
|
|
141
|
+
if kind == "ods":
|
|
142
|
+
workbook.open(filename)
|
|
143
|
+
else:
|
|
144
|
+
workbook.open(filename, encoding, read_only=True)
|
|
145
|
+
if kind == "xls":
|
|
146
|
+
return [sheet.name for sheet in workbook.sheetnames()]
|
|
147
|
+
return list(workbook.sheetnames())
|
|
148
|
+
finally:
|
|
149
|
+
if kind == "xls":
|
|
150
|
+
if workbook.wbk is not None:
|
|
151
|
+
workbook.wbk.release_resources()
|
|
152
|
+
else:
|
|
153
|
+
workbook.close()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _read(
|
|
157
|
+
result: dict[str, Any],
|
|
158
|
+
filename: str,
|
|
159
|
+
kind: str,
|
|
160
|
+
conf: Any,
|
|
161
|
+
sheet: str | None,
|
|
162
|
+
sample_rows: int | None,
|
|
163
|
+
encoding: str | None,
|
|
164
|
+
delimiter: str | None,
|
|
165
|
+
quote: str | None,
|
|
166
|
+
skip_rows: int,
|
|
167
|
+
) -> DataTable | None:
|
|
168
|
+
from execsql.importers.base import prepare_import_data
|
|
169
|
+
|
|
170
|
+
enc = encoding or conf.import_encoding
|
|
171
|
+
enc_source = "explicit" if encoding else "config" if conf.sources.get("import_encoding") else "default"
|
|
172
|
+
if kind in _DELIMITED:
|
|
173
|
+
from execsql.exporters.delimited import CsvFile
|
|
174
|
+
|
|
175
|
+
source = CsvFile(filename, enc, junk_header_lines=skip_rows)
|
|
176
|
+
try:
|
|
177
|
+
if delimiter is not None:
|
|
178
|
+
source.lineformat(delimiter, None if quote == "none" else quote or '"', None)
|
|
179
|
+
else:
|
|
180
|
+
source.evaluate_line_format()
|
|
181
|
+
result.update(
|
|
182
|
+
encoding={
|
|
183
|
+
"name": codecs.lookup(source.encoding).name,
|
|
184
|
+
"source": "bom" if source.bom_length else enc_source,
|
|
185
|
+
},
|
|
186
|
+
delimiter=source.delimiter,
|
|
187
|
+
quote=source.quotechar,
|
|
188
|
+
dialect_source="explicit" if delimiter is not None else "detected",
|
|
189
|
+
)
|
|
190
|
+
with closing(source.reader()) as rows:
|
|
191
|
+
headers = source._colhdrs(rows)
|
|
192
|
+
return _scan(result, headers, rows, sample_rows, materialized=False)
|
|
193
|
+
finally:
|
|
194
|
+
source.close()
|
|
195
|
+
stored = None
|
|
196
|
+
if kind in _WORKBOOK:
|
|
197
|
+
names = _workbook_names(filename, kind, encoding)
|
|
198
|
+
result["worksheets"] = names
|
|
199
|
+
if sheet is None:
|
|
200
|
+
return None
|
|
201
|
+
if sheet.isdecimal() and 1 <= int(sheet) <= len(names):
|
|
202
|
+
result["sheet"] = names[int(sheet) - 1]
|
|
203
|
+
if kind == "ods":
|
|
204
|
+
from execsql.importers.ods import ods_data
|
|
205
|
+
|
|
206
|
+
headers, data = ods_data(filename, sheet, skip_rows)
|
|
207
|
+
else:
|
|
208
|
+
from execsql.importers.xls import xls_data
|
|
209
|
+
|
|
210
|
+
headers, data = xls_data(filename, sheet, skip_rows, encoding, file_format=kind)
|
|
211
|
+
if kind == "xls" and encoding is not None:
|
|
212
|
+
result["encoding"] = {"name": codecs.lookup(encoding).name, "source": "explicit"}
|
|
213
|
+
elif kind in ("json", "jsonl"):
|
|
214
|
+
from execsql.importers.json import json_data, json_format
|
|
215
|
+
|
|
216
|
+
headers, data = json_data(filename, enc)
|
|
217
|
+
result.update(format=json_format(filename, enc), format_source="detected")
|
|
218
|
+
result["encoding"] = {"name": codecs.lookup(enc).name, "source": enc_source}
|
|
219
|
+
else:
|
|
220
|
+
from execsql.importers.feather import columnar_data
|
|
221
|
+
|
|
222
|
+
frame = columnar_data(filename, kind)
|
|
223
|
+
headers, data = frame.columns, [list(row) for row in frame.rows()]
|
|
224
|
+
stored = [str(dtype) for dtype in frame.schema.values()]
|
|
225
|
+
if conf.del_empty_cols:
|
|
226
|
+
stored = [dtype for name, dtype in zip(headers, stored) if name and name.strip()]
|
|
227
|
+
# Spreadsheet readers already perform their first header pass; IMPORT's
|
|
228
|
+
# shared loader performs another. Keep both passes, including empty sheets.
|
|
229
|
+
headers, data = prepare_import_data(headers, data)
|
|
230
|
+
return _scan(result, headers, data, sample_rows, materialized=True, stored=stored)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def inspect_file(
|
|
234
|
+
file: str | Path,
|
|
235
|
+
*,
|
|
236
|
+
file_format: str | None = None,
|
|
237
|
+
sheet: str | None = None,
|
|
238
|
+
sample_rows: int | None = None,
|
|
239
|
+
encoding: str | None = None,
|
|
240
|
+
delimiter: str | None = None,
|
|
241
|
+
quote: str | None = None,
|
|
242
|
+
skip_rows: int = 0,
|
|
243
|
+
config_file: str | Path | None = None,
|
|
244
|
+
dialect: str | None = None,
|
|
245
|
+
table_name: str | None = None,
|
|
246
|
+
schema: str | None = None,
|
|
247
|
+
) -> dict[str, Any]:
|
|
248
|
+
"""Inspect an importable local file without connecting, loading or writing.
|
|
249
|
+
|
|
250
|
+
Full scans use IMPORT's inference and configuration. Samples are provisional;
|
|
251
|
+
eager readers can still materialize the full source. Workbooks without a sheet
|
|
252
|
+
selector list worksheets without inferring columns. Supplying *dialect* and
|
|
253
|
+
*table_name* adds a CREATE statement in ``sql``, using IMPORT's inference and
|
|
254
|
+
observed nullability. Optional *schema* names its destination namespace.
|
|
255
|
+
SQL generation requires a full scan and an explicit workbook sheet.
|
|
256
|
+
No destination connection is opened. See the file reference.
|
|
257
|
+
"""
|
|
258
|
+
from execsql.api import _LibraryConfig
|
|
259
|
+
from execsql.cli.run import _tool_config
|
|
260
|
+
from execsql.utils.spreadsheet import INPUT_LIMIT_DEFAULTS
|
|
261
|
+
from execsql.ddl import DdlError, SqlDialect, file_sql
|
|
262
|
+
|
|
263
|
+
sql_requested = any(value is not None for value in (dialect, table_name, schema))
|
|
264
|
+
if sql_requested:
|
|
265
|
+
if not isinstance(dialect, str) or dialect not in {item.value for item in SqlDialect}:
|
|
266
|
+
raise ValueError("SQL output requires an explicit supported dialect.")
|
|
267
|
+
if not isinstance(table_name, str) or not table_name or "\x00" in table_name:
|
|
268
|
+
raise ValueError("SQL output requires a nonempty table_name without NUL characters.")
|
|
269
|
+
if schema is not None and (not isinstance(schema, str) or not schema or "\x00" in schema):
|
|
270
|
+
raise ValueError("schema must be a nonempty name without NUL characters.")
|
|
271
|
+
if schema is not None and dialect in ("access", "firebird"):
|
|
272
|
+
raise ValueError("The selected dialect has no schema namespace; omit schema.")
|
|
273
|
+
if sample_rows is not None:
|
|
274
|
+
raise ValueError("SQL output requires a full scan; omit sample_rows.")
|
|
275
|
+
|
|
276
|
+
if not isinstance(file, (str, Path)):
|
|
277
|
+
raise ValueError("file must be a local path.")
|
|
278
|
+
for name, value in (
|
|
279
|
+
("file_format", file_format),
|
|
280
|
+
("sheet", sheet),
|
|
281
|
+
("encoding", encoding),
|
|
282
|
+
("delimiter", delimiter),
|
|
283
|
+
("quote", quote),
|
|
284
|
+
):
|
|
285
|
+
if value is not None and not isinstance(value, str):
|
|
286
|
+
raise ValueError(f"{name} must be text.")
|
|
287
|
+
filename = str(file)
|
|
288
|
+
if not filename or "\x00" in filename:
|
|
289
|
+
raise ValueError("file must be a nonempty path without NUL characters.")
|
|
290
|
+
if sample_rows is not None and (type(sample_rows) is not int or sample_rows < 1):
|
|
291
|
+
raise ValueError("sample_rows must be a positive integer.")
|
|
292
|
+
if type(skip_rows) is not int or skip_rows < 0:
|
|
293
|
+
raise ValueError("skip_rows must be a non-negative integer.")
|
|
294
|
+
if file_format is not None and file_format not in {item.value for item in FileFormat}:
|
|
295
|
+
raise ValueError("Unsupported file_format.")
|
|
296
|
+
kind = file_format or _EXTENSIONS.get(Path(filename).suffix.lower())
|
|
297
|
+
if encoding is not None:
|
|
298
|
+
try:
|
|
299
|
+
codecs.lookup(encoding)
|
|
300
|
+
except LookupError as exc:
|
|
301
|
+
raise ValueError("Unknown encoding.") from exc
|
|
302
|
+
if sheet is not None and (not sheet or "\x00" in sheet or kind not in _WORKBOOK):
|
|
303
|
+
raise ValueError("sheet requires a nonempty worksheet selector and a workbook format.")
|
|
304
|
+
if encoding is not None and kind not in _TEXT | {"xls"}:
|
|
305
|
+
raise ValueError("encoding applies only to text and legacy XLS files.")
|
|
306
|
+
if delimiter is not None:
|
|
307
|
+
delimiter = {"tab": "\t", "us": "\x1f", "unitsep": "\x1f", "space": " "}.get(delimiter.lower(), delimiter)
|
|
308
|
+
if kind not in _DELIMITED or len(delimiter) != 1 or delimiter in "\r\n\x00":
|
|
309
|
+
raise ValueError("delimiter requires delimited text and one non-newline character (or TAB/US/SPACE).")
|
|
310
|
+
if quote is not None and (delimiter is None or (quote != "none" and (len(quote) != 1 or quote in "\r\n\x00"))):
|
|
311
|
+
raise ValueError("quote requires delimiter and one character, or 'none'.")
|
|
312
|
+
if quote is not None and quote == delimiter:
|
|
313
|
+
raise ValueError("quote and delimiter must differ.")
|
|
314
|
+
if skip_rows and kind not in _WORKBOOK | _DELIMITED:
|
|
315
|
+
raise ValueError("skip_rows applies only to delimited text and worksheets.")
|
|
316
|
+
result: dict[str, Any] = {
|
|
317
|
+
"schema_version": 1,
|
|
318
|
+
"success": False,
|
|
319
|
+
"file": filename,
|
|
320
|
+
"format": "delimited" if kind in _DELIMITED else kind,
|
|
321
|
+
"format_source": "explicit" if file_format else "extension",
|
|
322
|
+
"encoding": None,
|
|
323
|
+
"delimiter": None,
|
|
324
|
+
"quote": None,
|
|
325
|
+
"dialect_source": None,
|
|
326
|
+
"sheet": sheet,
|
|
327
|
+
"skip_rows": skip_rows,
|
|
328
|
+
"worksheets": None,
|
|
329
|
+
"scan": None,
|
|
330
|
+
"columns": [],
|
|
331
|
+
"settings": {},
|
|
332
|
+
"config_files": [],
|
|
333
|
+
"error": None,
|
|
334
|
+
}
|
|
335
|
+
ctx = RuntimeContext()
|
|
336
|
+
if sql_requested:
|
|
337
|
+
result["sql"] = None
|
|
338
|
+
ctx.conf = cast(Any, _LibraryConfig())
|
|
339
|
+
phase = "file"
|
|
340
|
+
with active_context(ctx):
|
|
341
|
+
try:
|
|
342
|
+
if not Path(filename).is_file():
|
|
343
|
+
raise FileNotFoundError("Input is not an existing regular file.")
|
|
344
|
+
if kind is None:
|
|
345
|
+
result["error"] = {"code": "unsupported_format", "message": "Unknown file extension; supply --format."}
|
|
346
|
+
return result
|
|
347
|
+
phase = "configuration"
|
|
348
|
+
check_explicit_config_file(config_file)
|
|
349
|
+
conf = _tool_config(str(config_file) if config_file is not None else None)
|
|
350
|
+
ctx.conf = conf
|
|
351
|
+
result["settings"] = {name: getattr(conf, name) for name in (*_SETTINGS, *INPUT_LIMIT_DEFAULTS)}
|
|
352
|
+
result["config_files"] = list(conf.files_read)
|
|
353
|
+
phase = "dependency"
|
|
354
|
+
dependencies = {
|
|
355
|
+
"xls": ["xlrd"],
|
|
356
|
+
"xlsx": ["openpyxl"],
|
|
357
|
+
"ods": ["defusedxml", "odf"],
|
|
358
|
+
"parquet": ["polars"],
|
|
359
|
+
"feather": ["polars"],
|
|
360
|
+
}.get(kind, [])
|
|
361
|
+
for dependency in dependencies:
|
|
362
|
+
importlib.import_module(dependency)
|
|
363
|
+
phase = "read"
|
|
364
|
+
table = _read(result, filename, kind, conf, sheet, sample_rows, encoding, delimiter, quote, skip_rows)
|
|
365
|
+
if sql_requested:
|
|
366
|
+
phase = "sql"
|
|
367
|
+
if table is None:
|
|
368
|
+
raise DdlError("sql_error", "SQL output for a workbook requires an explicit sheet.")
|
|
369
|
+
assert dialect is not None and table_name is not None
|
|
370
|
+
result["sql"] = file_sql(table, dialect, schema, table_name)
|
|
371
|
+
result["success"] = True
|
|
372
|
+
except (Exception, ScriptExit) as exc:
|
|
373
|
+
if isinstance(exc, DdlError):
|
|
374
|
+
message = exc.message
|
|
375
|
+
elif phase == "sql":
|
|
376
|
+
message = "Cannot generate SQL for the selected dialect and inferred column types."
|
|
377
|
+
elif phase == "configuration":
|
|
378
|
+
message = "Cannot load inspection configuration; check --config and execsql.conf settings."
|
|
379
|
+
elif phase == "dependency":
|
|
380
|
+
message = "Install execsql2[formats] with its working optional readers to inspect this format."
|
|
381
|
+
elif isinstance(exc, ErrInfo):
|
|
382
|
+
message = exc.errmsg()
|
|
383
|
+
elif isinstance(exc, ScriptExit):
|
|
384
|
+
message = "File reader halted."
|
|
385
|
+
else:
|
|
386
|
+
message = str(exc) if phase == "file" else "Cannot read file; check its format and parser settings."
|
|
387
|
+
result["columns"], result["scan"] = [], None
|
|
388
|
+
result["error"] = {"code": f"{phase}_error", "message": message}
|
|
389
|
+
if sql_requested:
|
|
390
|
+
result["sql"] = None
|
|
391
|
+
if result["encoding"] is not None:
|
|
392
|
+
result["settings"]["import_encoding"] = result["encoding"]["name"]
|
|
393
|
+
return result
|
execsql/importers/base.py
CHANGED
|
@@ -18,7 +18,7 @@ from execsql.exceptions import ErrInfo
|
|
|
18
18
|
from execsql.db.base import Database
|
|
19
19
|
import execsql.state as _state
|
|
20
20
|
|
|
21
|
-
__all__ = ["import_data_table", "refuse_existing_table"]
|
|
21
|
+
__all__ = ["import_data_table", "prepare_import_data", "refuse_existing_table"]
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def refuse_existing_table(db: Database, schema_name: str | None, table_name: str) -> None:
|
|
@@ -43,17 +43,8 @@ def refuse_existing_table(db: Database, schema_name: str | None, table_name: str
|
|
|
43
43
|
)
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
def
|
|
47
|
-
|
|
48
|
-
schemaname: str | None,
|
|
49
|
-
tablename: str,
|
|
50
|
-
is_new: Any,
|
|
51
|
-
hdrs: list[str],
|
|
52
|
-
data: list[Any],
|
|
53
|
-
) -> None:
|
|
54
|
-
"""Create (if needed) and populate a database table from in-memory row data."""
|
|
55
|
-
from execsql.utils.errors import exception_desc
|
|
56
|
-
|
|
46
|
+
def prepare_import_data(hdrs: list[str], data: list[Any]) -> tuple[list[str], list[Any]]:
|
|
47
|
+
"""Apply the existing row-import header rules without touching a database."""
|
|
57
48
|
conf = _state.conf
|
|
58
49
|
if any(x is None or len(x.strip()) == 0 for x in hdrs):
|
|
59
50
|
if conf.del_empty_cols:
|
|
@@ -86,6 +77,22 @@ def import_data_table(
|
|
|
86
77
|
from execsql.utils.strings import dedup_words
|
|
87
78
|
|
|
88
79
|
hdrs = dedup_words(hdrs)
|
|
80
|
+
return hdrs, data
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def import_data_table(
|
|
84
|
+
db: Database,
|
|
85
|
+
schemaname: str | None,
|
|
86
|
+
tablename: str,
|
|
87
|
+
is_new: Any,
|
|
88
|
+
hdrs: list[str],
|
|
89
|
+
data: list[Any],
|
|
90
|
+
) -> None:
|
|
91
|
+
"""Create (if needed) and populate a database table from in-memory row data."""
|
|
92
|
+
from execsql.utils.errors import exception_desc
|
|
93
|
+
|
|
94
|
+
conf = _state.conf
|
|
95
|
+
hdrs, data = prepare_import_data(hdrs, data)
|
|
89
96
|
|
|
90
97
|
tablespec_cache: Any = None
|
|
91
98
|
|
execsql/importers/feather.py
CHANGED
|
@@ -17,14 +17,8 @@ from execsql.importers.base import import_data_table
|
|
|
17
17
|
__all__ = ["import_feather", "import_parquet"]
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
def
|
|
21
|
-
|
|
22
|
-
schemaname: str | None,
|
|
23
|
-
tablename: str,
|
|
24
|
-
filename: str,
|
|
25
|
-
is_new: Any,
|
|
26
|
-
) -> None:
|
|
27
|
-
"""Import an Apache Arrow Feather (IPC) file into a database table."""
|
|
20
|
+
def columnar_data(filename: str, kind: str) -> Any:
|
|
21
|
+
"""Read the native frame identically for IMPORT and read-only inspection."""
|
|
28
22
|
from execsql.utils.errors import exception_desc
|
|
29
23
|
|
|
30
24
|
try:
|
|
@@ -33,9 +27,20 @@ def import_feather(
|
|
|
33
27
|
raise ErrInfo(
|
|
34
28
|
"exception",
|
|
35
29
|
exception_msg=exception_desc(),
|
|
36
|
-
other_msg="The polars Python library must be installed to import data from the
|
|
30
|
+
other_msg=f"The polars Python library must be installed to import data from the {kind.title()} format.",
|
|
37
31
|
) from e
|
|
38
|
-
|
|
32
|
+
return pl.read_ipc(filename) if kind == "feather" else pl.read_parquet(filename)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def import_feather(
|
|
36
|
+
db: Database,
|
|
37
|
+
schemaname: str | None,
|
|
38
|
+
tablename: str,
|
|
39
|
+
filename: str,
|
|
40
|
+
is_new: Any,
|
|
41
|
+
) -> None:
|
|
42
|
+
"""Import an Apache Arrow Feather (IPC) file into a database table."""
|
|
43
|
+
df = columnar_data(filename, "feather")
|
|
39
44
|
hdrs = df.columns
|
|
40
45
|
data = [list(row) for row in df.rows()]
|
|
41
46
|
import_data_table(db, schemaname, tablename, is_new, hdrs, data)
|
|
@@ -49,17 +54,7 @@ def import_parquet(
|
|
|
49
54
|
is_new: Any,
|
|
50
55
|
) -> None:
|
|
51
56
|
"""Import a Parquet file into a database table."""
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
try:
|
|
55
|
-
import polars as pl
|
|
56
|
-
except Exception as e:
|
|
57
|
-
raise ErrInfo(
|
|
58
|
-
"exception",
|
|
59
|
-
exception_msg=exception_desc(),
|
|
60
|
-
other_msg="The polars Python library must be installed to import data from the Parquet format.",
|
|
61
|
-
) from e
|
|
62
|
-
df = pl.read_parquet(filename)
|
|
57
|
+
df = columnar_data(filename, "parquet")
|
|
63
58
|
hdrs = df.columns
|
|
64
59
|
data = [list(row) for row in df.rows()]
|
|
65
60
|
import_data_table(db, schemaname, tablename, is_new, hdrs, data)
|
execsql/importers/json.py
CHANGED
|
@@ -63,6 +63,17 @@ def _flatten(obj: Any, prefix: str = "", sep: str = ".") -> dict[str, Any]:
|
|
|
63
63
|
return items
|
|
64
64
|
|
|
65
65
|
|
|
66
|
+
def json_format(filename: str, encoding: str) -> str:
|
|
67
|
+
"""Identify the same array/object-stream forms IMPORT accepts, ignoring suffix."""
|
|
68
|
+
with open(filename, encoding=encoding) as fh:
|
|
69
|
+
first_char = next((ch for ch in iter(lambda: fh.read(1), "") if not ch.isspace()), None)
|
|
70
|
+
if first_char == "[":
|
|
71
|
+
return "json"
|
|
72
|
+
if first_char == "{":
|
|
73
|
+
return "jsonl"
|
|
74
|
+
raise ErrInfo(type="error", other_msg="JSON import expects a file starting with '[' (array) or '{' (object/JSONL).")
|
|
75
|
+
|
|
76
|
+
|
|
66
77
|
def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
|
|
67
78
|
"""Read a JSON file and return a list of flat dicts.
|
|
68
79
|
|
|
@@ -75,19 +86,8 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
|
|
|
75
86
|
whole file — switching to a streaming parser would require
|
|
76
87
|
``ijson`` as a dependency.
|
|
77
88
|
"""
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
with open(filename, encoding=encoding) as fh:
|
|
81
|
-
first_char: str | None = None
|
|
82
|
-
while True:
|
|
83
|
-
ch = fh.read(1)
|
|
84
|
-
if not ch:
|
|
85
|
-
break
|
|
86
|
-
if not ch.isspace():
|
|
87
|
-
first_char = ch
|
|
88
|
-
break
|
|
89
|
-
|
|
90
|
-
if first_char == "[":
|
|
89
|
+
kind = json_format(filename, encoding)
|
|
90
|
+
if kind == "json":
|
|
91
91
|
# Standard JSON array — must buffer the whole file (json.loads
|
|
92
92
|
# has no streaming mode; ijson would be needed for that).
|
|
93
93
|
text = Path(filename).read_text(encoding=encoding)
|
|
@@ -95,7 +95,7 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
|
|
|
95
95
|
if not isinstance(raw, list):
|
|
96
96
|
raise ErrInfo(type="error", other_msg="JSON file root is not an array of objects.")
|
|
97
97
|
records = raw
|
|
98
|
-
|
|
98
|
+
else:
|
|
99
99
|
# JSONL (JSON Lines) — stream the file line-by-line.
|
|
100
100
|
records = []
|
|
101
101
|
with open(filename, encoding=encoding) as fh:
|
|
@@ -116,12 +116,6 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
|
|
|
116
116
|
other_msg=f"Line {lineno} is not a JSON object.",
|
|
117
117
|
)
|
|
118
118
|
records.append(obj)
|
|
119
|
-
else:
|
|
120
|
-
raise ErrInfo(
|
|
121
|
-
type="error",
|
|
122
|
-
other_msg="JSON import expects a file starting with '[' (array) or '{' (object/JSONL).",
|
|
123
|
-
)
|
|
124
|
-
|
|
125
119
|
if not records:
|
|
126
120
|
raise ErrInfo(type="error", other_msg="JSON file contains no records.")
|
|
127
121
|
|
|
@@ -136,6 +130,17 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
|
|
|
136
130
|
return [_flatten(rec) for rec in records]
|
|
137
131
|
|
|
138
132
|
|
|
133
|
+
def json_data(filename: str, encoding: str) -> tuple[list[str], list[list[Any]]]:
|
|
134
|
+
"""Read, flatten and align JSON records using the IMPORT rules."""
|
|
135
|
+
flat_records = _parse_json_file(filename, encoding)
|
|
136
|
+
seen: dict[str, None] = {}
|
|
137
|
+
for rec in flat_records:
|
|
138
|
+
for key in rec:
|
|
139
|
+
seen[key] = None
|
|
140
|
+
hdrs = list(seen)
|
|
141
|
+
return hdrs, [[rec.get(h) for h in hdrs] for rec in flat_records]
|
|
142
|
+
|
|
143
|
+
|
|
139
144
|
def import_json(
|
|
140
145
|
db: Database,
|
|
141
146
|
schemaname: str | None,
|
|
@@ -159,7 +164,7 @@ def import_json(
|
|
|
159
164
|
enc = encoding if encoding else _state.conf.import_encoding
|
|
160
165
|
|
|
161
166
|
try:
|
|
162
|
-
|
|
167
|
+
hdrs, data = json_data(filename, enc)
|
|
163
168
|
except ErrInfo:
|
|
164
169
|
raise
|
|
165
170
|
except Exception as e:
|
|
@@ -169,15 +174,4 @@ def import_json(
|
|
|
169
174
|
other_msg=f"Can't parse JSON file {filename}",
|
|
170
175
|
) from e
|
|
171
176
|
|
|
172
|
-
# Build a union of all keys across records (preserving first-seen order).
|
|
173
|
-
seen: dict[str, None] = {}
|
|
174
|
-
for rec in flat_records:
|
|
175
|
-
for key in rec:
|
|
176
|
-
if key not in seen:
|
|
177
|
-
seen[key] = None
|
|
178
|
-
hdrs = list(seen)
|
|
179
|
-
|
|
180
|
-
# Build row data aligned to hdrs — missing keys become None.
|
|
181
|
-
data = [[rec.get(h) for h in hdrs] for rec in flat_records]
|
|
182
|
-
|
|
183
177
|
import_data_table(db, schemaname, tablename, is_new, hdrs, data)
|
execsql/importers/xls.py
CHANGED
|
@@ -24,15 +24,17 @@ def xls_data(
|
|
|
24
24
|
sheetname: str,
|
|
25
25
|
junk_header_rows: int,
|
|
26
26
|
encoding: str | None = None,
|
|
27
|
+
*,
|
|
28
|
+
file_format: str | None = None,
|
|
27
29
|
) -> tuple:
|
|
28
30
|
"""Returns the data from the specified worksheet as a list of headers and a list of lists of rows."""
|
|
29
31
|
from execsql.utils.strings import clean_words, trim_words, fold_words, dedup_words
|
|
30
32
|
|
|
31
33
|
conf = _state.conf
|
|
32
34
|
|
|
33
|
-
if len(filename) < 4:
|
|
35
|
+
if len(filename) < 4 and file_format is None:
|
|
34
36
|
raise ErrInfo(type="cmd", other_msg=f"{filename} is not a recognizable Excel spreadsheet name.")
|
|
35
|
-
ext3 = filename[-3:].lower()
|
|
37
|
+
ext3 = {"xls": "xls", "xlsx": "lsx"}.get(file_format or "", filename[-3:].lower())
|
|
36
38
|
wbk: Any
|
|
37
39
|
if ext3 == "xls":
|
|
38
40
|
# xlrd imported lazily
|
|
@@ -59,10 +61,13 @@ def xls_data(
|
|
|
59
61
|
alldata = wbk.sheet_data(sheetname, junk_header_rows)
|
|
60
62
|
except Exception as e:
|
|
61
63
|
raise ErrInfo(type="cmd", other_msg=f"Error reading worksheet {sheetname} from {filename}.") from e
|
|
64
|
+
finally:
|
|
65
|
+
if ext3 == "lsx":
|
|
66
|
+
wbk.close()
|
|
67
|
+
elif wbk.wbk is not None:
|
|
68
|
+
wbk.wbk.release_resources()
|
|
62
69
|
if len(alldata) == 0:
|
|
63
70
|
raise ErrInfo(type="cmd", other_msg=f"There are no data on worksheet {sheetname} of file {filename}.")
|
|
64
|
-
if ext3 == "lsx":
|
|
65
|
-
wbk.close()
|
|
66
71
|
if len(alldata) == 1:
|
|
67
72
|
return alldata[0], []
|
|
68
73
|
colhdrs = alldata[0]
|