execsql2 2.39.1__py3-none-any.whl → 2.41.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. execsql/__init__.py +4 -2
  2. execsql/api.py +40 -27
  3. execsql/cli/commands/__init__.py +1 -0
  4. execsql/cli/commands/inspect.py +292 -0
  5. execsql/cli/dispatch.py +3 -3
  6. execsql/cli/inspection.py +276 -0
  7. execsql/cli/lint.py +3 -4
  8. execsql/cli/options.py +15 -7
  9. execsql/config.py +15 -2
  10. execsql/db/access.py +6 -5
  11. execsql/db/base.py +14 -0
  12. execsql/db/dsn.py +3 -2
  13. execsql/db/duckdb.py +2 -1
  14. execsql/db/firebird.py +3 -2
  15. execsql/db/mysql.py +3 -2
  16. execsql/db/oracle.py +3 -2
  17. execsql/db/postgres.py +3 -2
  18. execsql/db/sqlite.py +2 -1
  19. execsql/db/sqlserver.py +3 -2
  20. execsql/ddl.py +310 -0
  21. execsql/exporters/xls.py +18 -4
  22. execsql/file_inspection.py +393 -0
  23. execsql/importers/base.py +19 -12
  24. execsql/importers/feather.py +16 -21
  25. execsql/importers/json.py +26 -32
  26. execsql/importers/xls.py +9 -4
  27. execsql/metacommands/control.py +21 -23
  28. execsql/metacommands/script_ext.py +6 -2
  29. execsql/schema.py +816 -0
  30. execsql/script/executor.py +11 -0
  31. execsql/script/parser.py +31 -0
  32. execsql/script/variables.py +14 -2
  33. execsql/utils/errors.py +21 -3
  34. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/README.md +7 -0
  35. execsql2-2.41.0.data/data/execsql2_extras/inspect/README.md +116 -0
  36. execsql2-2.41.0.data/data/execsql2_extras/inspect/make_demo.py +106 -0
  37. {execsql2-2.39.1.dist-info → execsql2-2.41.0.dist-info}/METADATA +84 -19
  38. {execsql2-2.39.1.dist-info → execsql2-2.41.0.dist-info}/RECORD +54 -47
  39. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
  40. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/example_config_prompt.sql +0 -0
  41. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
  42. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/md_compare.sql +0 -0
  43. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
  44. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
  45. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
  46. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
  47. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/script_template.sql +0 -0
  48. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
  49. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
  50. {execsql2-2.39.1.data → execsql2-2.41.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
  51. {execsql2-2.39.1.dist-info → execsql2-2.41.0.dist-info}/WHEEL +0 -0
  52. {execsql2-2.39.1.dist-info → execsql2-2.41.0.dist-info}/entry_points.txt +0 -0
  53. {execsql2-2.39.1.dist-info → execsql2-2.41.0.dist-info}/licenses/LICENSE.txt +0 -0
  54. {execsql2-2.39.1.dist-info → execsql2-2.41.0.dist-info}/licenses/NOTICE +0 -0
@@ -0,0 +1,393 @@
1
+ """Read-only file metadata using the same readers and inference as IMPORT."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import codecs
6
+ import importlib
7
+ from contextlib import closing
8
+ from enum import Enum
9
+ from itertools import islice
10
+ from pathlib import Path
11
+ from typing import Any, cast
12
+ from collections.abc import Iterable
13
+
14
+ from execsql.exceptions import ErrInfo, ScriptExit
15
+ from execsql.config import check_explicit_config_file
16
+ from execsql.models import DataTable
17
+ from execsql.state import RuntimeContext, active_context
18
+
19
+ __all__ = ["FileFormat", "inspect_file"]
20
+
21
+
22
+ class FileFormat(str, Enum):
23
+ """Input formats understood by the built-in IMPORT readers."""
24
+
25
+ csv = "csv"
26
+ tsv = "tsv"
27
+ text = "text"
28
+ json = "json"
29
+ jsonl = "jsonl"
30
+ xls = "xls"
31
+ xlsx = "xlsx"
32
+ ods = "ods"
33
+ parquet = "parquet"
34
+ feather = "feather"
35
+
36
+
37
+ _EXTENSIONS = {
38
+ ".csv": "csv",
39
+ ".tsv": "tsv",
40
+ ".tab": "tsv",
41
+ ".txt": "text",
42
+ ".json": "json",
43
+ ".jsonl": "jsonl",
44
+ ".ndjson": "jsonl",
45
+ ".xls": "xls",
46
+ ".xlsx": "xlsx",
47
+ ".ods": "ods",
48
+ ".parquet": "parquet",
49
+ ".feather": "feather",
50
+ ".arrow": "feather",
51
+ ".ipc": "feather",
52
+ }
53
+ _TEXT = {"csv", "tsv", "text", "json", "jsonl"}
54
+ _DELIMITED = {"csv", "tsv", "text"}
55
+ _WORKBOOK = {"xls", "xlsx", "ods"}
56
+ _SETTINGS = (
57
+ "import_encoding",
58
+ "enc_err_disposition",
59
+ "scan_lines",
60
+ "max_int",
61
+ "boolean_int",
62
+ "boolean_words",
63
+ "exact_decimals",
64
+ "only_strings",
65
+ "empty_strings",
66
+ "empty_rows",
67
+ "trim_strings",
68
+ "replace_newlines",
69
+ "del_empty_cols",
70
+ "create_col_hdrs",
71
+ "clean_col_hdrs",
72
+ "trim_col_hdrs",
73
+ "fold_col_hdrs",
74
+ "dedup_col_hdrs",
75
+ )
76
+
77
+
78
+ def _columns(table: DataTable, stored: list[str] | None = None) -> list[dict[str, Any]]:
79
+ columns = []
80
+ for index, column in enumerate(table.cols):
81
+ name, kind, length, _nullable, precision, scale = column.column_type()
82
+ inferred = kind.data_type_name
83
+ if length is not None:
84
+ inferred += f"({length})"
85
+ elif precision is not None and scale is not None:
86
+ inferred += f"({precision},{scale})"
87
+ columns.append(
88
+ {
89
+ "name": name,
90
+ "inferred_type": inferred,
91
+ "source_type": stored[index] if stored is not None else None,
92
+ "null_count": column.nullrows,
93
+ },
94
+ )
95
+ return columns
96
+
97
+
98
+ def _scan(
99
+ result: dict[str, Any],
100
+ headers: list[str],
101
+ rows: Iterable[Any],
102
+ sample_rows: int | None,
103
+ *,
104
+ materialized: bool,
105
+ stored: list[str] | None = None,
106
+ ) -> DataTable:
107
+ total = len(rows) if isinstance(rows, list) else None
108
+ if sample_rows is None:
109
+ table = DataTable(headers, rows)
110
+ complete = True
111
+ else:
112
+ prefix = list(islice(rows, sample_rows + 1))
113
+ complete = len(prefix) <= sample_rows
114
+ table = DataTable(headers, prefix[:sample_rows])
115
+ result["columns"] = _columns(table, stored)
116
+ result["scan"] = {
117
+ "mode": "full" if sample_rows is None else "sample",
118
+ "row_limit": sample_rows,
119
+ "rows_examined": table.inputrows,
120
+ "complete": complete,
121
+ "total_rows": table.inputrows if complete else total,
122
+ "source_materialized": materialized,
123
+ }
124
+ return table
125
+
126
+
127
+ def _workbook_names(filename: str, kind: str, encoding: str | None) -> list[str]:
128
+ if kind == "ods":
129
+ from execsql.importers.ods import OdsFile, check_ods_archive
130
+
131
+ check_ods_archive(filename)
132
+ workbook: Any = OdsFile()
133
+ else:
134
+ from execsql.exporters.xls import XlsFile, XlsxFile
135
+ from execsql.utils.fileio import check_zip_decompression_ratio
136
+
137
+ if kind == "xlsx":
138
+ check_zip_decompression_ratio(filename)
139
+ workbook = XlsFile() if kind == "xls" else XlsxFile()
140
+ try:
141
+ if kind == "ods":
142
+ workbook.open(filename)
143
+ else:
144
+ workbook.open(filename, encoding, read_only=True)
145
+ if kind == "xls":
146
+ return [sheet.name for sheet in workbook.sheetnames()]
147
+ return list(workbook.sheetnames())
148
+ finally:
149
+ if kind == "xls":
150
+ if workbook.wbk is not None:
151
+ workbook.wbk.release_resources()
152
+ else:
153
+ workbook.close()
154
+
155
+
156
+ def _read(
157
+ result: dict[str, Any],
158
+ filename: str,
159
+ kind: str,
160
+ conf: Any,
161
+ sheet: str | None,
162
+ sample_rows: int | None,
163
+ encoding: str | None,
164
+ delimiter: str | None,
165
+ quote: str | None,
166
+ skip_rows: int,
167
+ ) -> DataTable | None:
168
+ from execsql.importers.base import prepare_import_data
169
+
170
+ enc = encoding or conf.import_encoding
171
+ enc_source = "explicit" if encoding else "config" if conf.sources.get("import_encoding") else "default"
172
+ if kind in _DELIMITED:
173
+ from execsql.exporters.delimited import CsvFile
174
+
175
+ source = CsvFile(filename, enc, junk_header_lines=skip_rows)
176
+ try:
177
+ if delimiter is not None:
178
+ source.lineformat(delimiter, None if quote == "none" else quote or '"', None)
179
+ else:
180
+ source.evaluate_line_format()
181
+ result.update(
182
+ encoding={
183
+ "name": codecs.lookup(source.encoding).name,
184
+ "source": "bom" if source.bom_length else enc_source,
185
+ },
186
+ delimiter=source.delimiter,
187
+ quote=source.quotechar,
188
+ dialect_source="explicit" if delimiter is not None else "detected",
189
+ )
190
+ with closing(source.reader()) as rows:
191
+ headers = source._colhdrs(rows)
192
+ return _scan(result, headers, rows, sample_rows, materialized=False)
193
+ finally:
194
+ source.close()
195
+ stored = None
196
+ if kind in _WORKBOOK:
197
+ names = _workbook_names(filename, kind, encoding)
198
+ result["worksheets"] = names
199
+ if sheet is None:
200
+ return None
201
+ if sheet.isdecimal() and 1 <= int(sheet) <= len(names):
202
+ result["sheet"] = names[int(sheet) - 1]
203
+ if kind == "ods":
204
+ from execsql.importers.ods import ods_data
205
+
206
+ headers, data = ods_data(filename, sheet, skip_rows)
207
+ else:
208
+ from execsql.importers.xls import xls_data
209
+
210
+ headers, data = xls_data(filename, sheet, skip_rows, encoding, file_format=kind)
211
+ if kind == "xls" and encoding is not None:
212
+ result["encoding"] = {"name": codecs.lookup(encoding).name, "source": "explicit"}
213
+ elif kind in ("json", "jsonl"):
214
+ from execsql.importers.json import json_data, json_format
215
+
216
+ headers, data = json_data(filename, enc)
217
+ result.update(format=json_format(filename, enc), format_source="detected")
218
+ result["encoding"] = {"name": codecs.lookup(enc).name, "source": enc_source}
219
+ else:
220
+ from execsql.importers.feather import columnar_data
221
+
222
+ frame = columnar_data(filename, kind)
223
+ headers, data = frame.columns, [list(row) for row in frame.rows()]
224
+ stored = [str(dtype) for dtype in frame.schema.values()]
225
+ if conf.del_empty_cols:
226
+ stored = [dtype for name, dtype in zip(headers, stored) if name and name.strip()]
227
+ # Spreadsheet readers already perform their first header pass; IMPORT's
228
+ # shared loader performs another. Keep both passes, including empty sheets.
229
+ headers, data = prepare_import_data(headers, data)
230
+ return _scan(result, headers, data, sample_rows, materialized=True, stored=stored)
231
+
232
+
233
+ def inspect_file(
234
+ file: str | Path,
235
+ *,
236
+ file_format: str | None = None,
237
+ sheet: str | None = None,
238
+ sample_rows: int | None = None,
239
+ encoding: str | None = None,
240
+ delimiter: str | None = None,
241
+ quote: str | None = None,
242
+ skip_rows: int = 0,
243
+ config_file: str | Path | None = None,
244
+ dialect: str | None = None,
245
+ table_name: str | None = None,
246
+ schema: str | None = None,
247
+ ) -> dict[str, Any]:
248
+ """Inspect an importable local file without connecting, loading or writing.
249
+
250
+ Full scans use IMPORT's inference and configuration. Samples are provisional;
251
+ eager readers can still materialize the full source. Workbooks without a sheet
252
+ selector list worksheets without inferring columns. Supplying *dialect* and
253
+ *table_name* adds a CREATE statement in ``sql``, using IMPORT's inference and
254
+ observed nullability. Optional *schema* names its destination namespace.
255
+ SQL generation requires a full scan and an explicit workbook sheet.
256
+ No destination connection is opened. See the file reference.
257
+ """
258
+ from execsql.api import _LibraryConfig
259
+ from execsql.cli.run import _tool_config
260
+ from execsql.utils.spreadsheet import INPUT_LIMIT_DEFAULTS
261
+ from execsql.ddl import DdlError, SqlDialect, file_sql
262
+
263
+ sql_requested = any(value is not None for value in (dialect, table_name, schema))
264
+ if sql_requested:
265
+ if not isinstance(dialect, str) or dialect not in {item.value for item in SqlDialect}:
266
+ raise ValueError("SQL output requires an explicit supported dialect.")
267
+ if not isinstance(table_name, str) or not table_name or "\x00" in table_name:
268
+ raise ValueError("SQL output requires a nonempty table_name without NUL characters.")
269
+ if schema is not None and (not isinstance(schema, str) or not schema or "\x00" in schema):
270
+ raise ValueError("schema must be a nonempty name without NUL characters.")
271
+ if schema is not None and dialect in ("access", "firebird"):
272
+ raise ValueError("The selected dialect has no schema namespace; omit schema.")
273
+ if sample_rows is not None:
274
+ raise ValueError("SQL output requires a full scan; omit sample_rows.")
275
+
276
+ if not isinstance(file, (str, Path)):
277
+ raise ValueError("file must be a local path.")
278
+ for name, value in (
279
+ ("file_format", file_format),
280
+ ("sheet", sheet),
281
+ ("encoding", encoding),
282
+ ("delimiter", delimiter),
283
+ ("quote", quote),
284
+ ):
285
+ if value is not None and not isinstance(value, str):
286
+ raise ValueError(f"{name} must be text.")
287
+ filename = str(file)
288
+ if not filename or "\x00" in filename:
289
+ raise ValueError("file must be a nonempty path without NUL characters.")
290
+ if sample_rows is not None and (type(sample_rows) is not int or sample_rows < 1):
291
+ raise ValueError("sample_rows must be a positive integer.")
292
+ if type(skip_rows) is not int or skip_rows < 0:
293
+ raise ValueError("skip_rows must be a non-negative integer.")
294
+ if file_format is not None and file_format not in {item.value for item in FileFormat}:
295
+ raise ValueError("Unsupported file_format.")
296
+ kind = file_format or _EXTENSIONS.get(Path(filename).suffix.lower())
297
+ if encoding is not None:
298
+ try:
299
+ codecs.lookup(encoding)
300
+ except LookupError as exc:
301
+ raise ValueError("Unknown encoding.") from exc
302
+ if sheet is not None and (not sheet or "\x00" in sheet or kind not in _WORKBOOK):
303
+ raise ValueError("sheet requires a nonempty worksheet selector and a workbook format.")
304
+ if encoding is not None and kind not in _TEXT | {"xls"}:
305
+ raise ValueError("encoding applies only to text and legacy XLS files.")
306
+ if delimiter is not None:
307
+ delimiter = {"tab": "\t", "us": "\x1f", "unitsep": "\x1f", "space": " "}.get(delimiter.lower(), delimiter)
308
+ if kind not in _DELIMITED or len(delimiter) != 1 or delimiter in "\r\n\x00":
309
+ raise ValueError("delimiter requires delimited text and one non-newline character (or TAB/US/SPACE).")
310
+ if quote is not None and (delimiter is None or (quote != "none" and (len(quote) != 1 or quote in "\r\n\x00"))):
311
+ raise ValueError("quote requires delimiter and one character, or 'none'.")
312
+ if quote is not None and quote == delimiter:
313
+ raise ValueError("quote and delimiter must differ.")
314
+ if skip_rows and kind not in _WORKBOOK | _DELIMITED:
315
+ raise ValueError("skip_rows applies only to delimited text and worksheets.")
316
+ result: dict[str, Any] = {
317
+ "schema_version": 1,
318
+ "success": False,
319
+ "file": filename,
320
+ "format": "delimited" if kind in _DELIMITED else kind,
321
+ "format_source": "explicit" if file_format else "extension",
322
+ "encoding": None,
323
+ "delimiter": None,
324
+ "quote": None,
325
+ "dialect_source": None,
326
+ "sheet": sheet,
327
+ "skip_rows": skip_rows,
328
+ "worksheets": None,
329
+ "scan": None,
330
+ "columns": [],
331
+ "settings": {},
332
+ "config_files": [],
333
+ "error": None,
334
+ }
335
+ ctx = RuntimeContext()
336
+ if sql_requested:
337
+ result["sql"] = None
338
+ ctx.conf = cast(Any, _LibraryConfig())
339
+ phase = "file"
340
+ with active_context(ctx):
341
+ try:
342
+ if not Path(filename).is_file():
343
+ raise FileNotFoundError("Input is not an existing regular file.")
344
+ if kind is None:
345
+ result["error"] = {"code": "unsupported_format", "message": "Unknown file extension; supply --format."}
346
+ return result
347
+ phase = "configuration"
348
+ check_explicit_config_file(config_file)
349
+ conf = _tool_config(str(config_file) if config_file is not None else None)
350
+ ctx.conf = conf
351
+ result["settings"] = {name: getattr(conf, name) for name in (*_SETTINGS, *INPUT_LIMIT_DEFAULTS)}
352
+ result["config_files"] = list(conf.files_read)
353
+ phase = "dependency"
354
+ dependencies = {
355
+ "xls": ["xlrd"],
356
+ "xlsx": ["openpyxl"],
357
+ "ods": ["defusedxml", "odf"],
358
+ "parquet": ["polars"],
359
+ "feather": ["polars"],
360
+ }.get(kind, [])
361
+ for dependency in dependencies:
362
+ importlib.import_module(dependency)
363
+ phase = "read"
364
+ table = _read(result, filename, kind, conf, sheet, sample_rows, encoding, delimiter, quote, skip_rows)
365
+ if sql_requested:
366
+ phase = "sql"
367
+ if table is None:
368
+ raise DdlError("sql_error", "SQL output for a workbook requires an explicit sheet.")
369
+ assert dialect is not None and table_name is not None
370
+ result["sql"] = file_sql(table, dialect, schema, table_name)
371
+ result["success"] = True
372
+ except (Exception, ScriptExit) as exc:
373
+ if isinstance(exc, DdlError):
374
+ message = exc.message
375
+ elif phase == "sql":
376
+ message = "Cannot generate SQL for the selected dialect and inferred column types."
377
+ elif phase == "configuration":
378
+ message = "Cannot load inspection configuration; check --config and execsql.conf settings."
379
+ elif phase == "dependency":
380
+ message = "Install execsql2[formats] with its working optional readers to inspect this format."
381
+ elif isinstance(exc, ErrInfo):
382
+ message = exc.errmsg()
383
+ elif isinstance(exc, ScriptExit):
384
+ message = "File reader halted."
385
+ else:
386
+ message = str(exc) if phase == "file" else "Cannot read file; check its format and parser settings."
387
+ result["columns"], result["scan"] = [], None
388
+ result["error"] = {"code": f"{phase}_error", "message": message}
389
+ if sql_requested:
390
+ result["sql"] = None
391
+ if result["encoding"] is not None:
392
+ result["settings"]["import_encoding"] = result["encoding"]["name"]
393
+ return result
execsql/importers/base.py CHANGED
@@ -18,7 +18,7 @@ from execsql.exceptions import ErrInfo
18
18
  from execsql.db.base import Database
19
19
  import execsql.state as _state
20
20
 
21
- __all__ = ["import_data_table", "refuse_existing_table"]
21
+ __all__ = ["import_data_table", "prepare_import_data", "refuse_existing_table"]
22
22
 
23
23
 
24
24
  def refuse_existing_table(db: Database, schema_name: str | None, table_name: str) -> None:
@@ -43,17 +43,8 @@ def refuse_existing_table(db: Database, schema_name: str | None, table_name: str
43
43
  )
44
44
 
45
45
 
46
- def import_data_table(
47
- db: Database,
48
- schemaname: str | None,
49
- tablename: str,
50
- is_new: Any,
51
- hdrs: list[str],
52
- data: list[Any],
53
- ) -> None:
54
- """Create (if needed) and populate a database table from in-memory row data."""
55
- from execsql.utils.errors import exception_desc
56
-
46
+ def prepare_import_data(hdrs: list[str], data: list[Any]) -> tuple[list[str], list[Any]]:
47
+ """Apply the existing row-import header rules without touching a database."""
57
48
  conf = _state.conf
58
49
  if any(x is None or len(x.strip()) == 0 for x in hdrs):
59
50
  if conf.del_empty_cols:
@@ -86,6 +77,22 @@ def import_data_table(
86
77
  from execsql.utils.strings import dedup_words
87
78
 
88
79
  hdrs = dedup_words(hdrs)
80
+ return hdrs, data
81
+
82
+
83
+ def import_data_table(
84
+ db: Database,
85
+ schemaname: str | None,
86
+ tablename: str,
87
+ is_new: Any,
88
+ hdrs: list[str],
89
+ data: list[Any],
90
+ ) -> None:
91
+ """Create (if needed) and populate a database table from in-memory row data."""
92
+ from execsql.utils.errors import exception_desc
93
+
94
+ conf = _state.conf
95
+ hdrs, data = prepare_import_data(hdrs, data)
89
96
 
90
97
  tablespec_cache: Any = None
91
98
 
@@ -17,14 +17,8 @@ from execsql.importers.base import import_data_table
17
17
  __all__ = ["import_feather", "import_parquet"]
18
18
 
19
19
 
20
- def import_feather(
21
- db: Database,
22
- schemaname: str | None,
23
- tablename: str,
24
- filename: str,
25
- is_new: Any,
26
- ) -> None:
27
- """Import an Apache Arrow Feather (IPC) file into a database table."""
20
+ def columnar_data(filename: str, kind: str) -> Any:
21
+ """Read the native frame identically for IMPORT and read-only inspection."""
28
22
  from execsql.utils.errors import exception_desc
29
23
 
30
24
  try:
@@ -33,9 +27,20 @@ def import_feather(
33
27
  raise ErrInfo(
34
28
  "exception",
35
29
  exception_msg=exception_desc(),
36
- other_msg="The polars Python library must be installed to import data from the Feather format.",
30
+ other_msg=f"The polars Python library must be installed to import data from the {kind.title()} format.",
37
31
  ) from e
38
- df = pl.read_ipc(filename)
32
+ return pl.read_ipc(filename) if kind == "feather" else pl.read_parquet(filename)
33
+
34
+
35
+ def import_feather(
36
+ db: Database,
37
+ schemaname: str | None,
38
+ tablename: str,
39
+ filename: str,
40
+ is_new: Any,
41
+ ) -> None:
42
+ """Import an Apache Arrow Feather (IPC) file into a database table."""
43
+ df = columnar_data(filename, "feather")
39
44
  hdrs = df.columns
40
45
  data = [list(row) for row in df.rows()]
41
46
  import_data_table(db, schemaname, tablename, is_new, hdrs, data)
@@ -49,17 +54,7 @@ def import_parquet(
49
54
  is_new: Any,
50
55
  ) -> None:
51
56
  """Import a Parquet file into a database table."""
52
- from execsql.utils.errors import exception_desc
53
-
54
- try:
55
- import polars as pl
56
- except Exception as e:
57
- raise ErrInfo(
58
- "exception",
59
- exception_msg=exception_desc(),
60
- other_msg="The polars Python library must be installed to import data from the Parquet format.",
61
- ) from e
62
- df = pl.read_parquet(filename)
57
+ df = columnar_data(filename, "parquet")
63
58
  hdrs = df.columns
64
59
  data = [list(row) for row in df.rows()]
65
60
  import_data_table(db, schemaname, tablename, is_new, hdrs, data)
execsql/importers/json.py CHANGED
@@ -63,6 +63,17 @@ def _flatten(obj: Any, prefix: str = "", sep: str = ".") -> dict[str, Any]:
63
63
  return items
64
64
 
65
65
 
66
+ def json_format(filename: str, encoding: str) -> str:
67
+ """Identify the same array/object-stream forms IMPORT accepts, ignoring suffix."""
68
+ with open(filename, encoding=encoding) as fh:
69
+ first_char = next((ch for ch in iter(lambda: fh.read(1), "") if not ch.isspace()), None)
70
+ if first_char == "[":
71
+ return "json"
72
+ if first_char == "{":
73
+ return "jsonl"
74
+ raise ErrInfo(type="error", other_msg="JSON import expects a file starting with '[' (array) or '{' (object/JSONL).")
75
+
76
+
66
77
  def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
67
78
  """Read a JSON file and return a list of flat dicts.
68
79
 
@@ -75,19 +86,8 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
75
86
  whole file — switching to a streaming parser would require
76
87
  ``ijson`` as a dependency.
77
88
  """
78
- # Peek at the first non-whitespace character to decide which
79
- # parsing strategy to use, without slurping the entire file.
80
- with open(filename, encoding=encoding) as fh:
81
- first_char: str | None = None
82
- while True:
83
- ch = fh.read(1)
84
- if not ch:
85
- break
86
- if not ch.isspace():
87
- first_char = ch
88
- break
89
-
90
- if first_char == "[":
89
+ kind = json_format(filename, encoding)
90
+ if kind == "json":
91
91
  # Standard JSON array — must buffer the whole file (json.loads
92
92
  # has no streaming mode; ijson would be needed for that).
93
93
  text = Path(filename).read_text(encoding=encoding)
@@ -95,7 +95,7 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
95
95
  if not isinstance(raw, list):
96
96
  raise ErrInfo(type="error", other_msg="JSON file root is not an array of objects.")
97
97
  records = raw
98
- elif first_char == "{":
98
+ else:
99
99
  # JSONL (JSON Lines) — stream the file line-by-line.
100
100
  records = []
101
101
  with open(filename, encoding=encoding) as fh:
@@ -116,12 +116,6 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
116
116
  other_msg=f"Line {lineno} is not a JSON object.",
117
117
  )
118
118
  records.append(obj)
119
- else:
120
- raise ErrInfo(
121
- type="error",
122
- other_msg="JSON import expects a file starting with '[' (array) or '{' (object/JSONL).",
123
- )
124
-
125
119
  if not records:
126
120
  raise ErrInfo(type="error", other_msg="JSON file contains no records.")
127
121
 
@@ -136,6 +130,17 @@ def _parse_json_file(filename: str, encoding: str) -> list[dict[str, Any]]:
136
130
  return [_flatten(rec) for rec in records]
137
131
 
138
132
 
133
+ def json_data(filename: str, encoding: str) -> tuple[list[str], list[list[Any]]]:
134
+ """Read, flatten and align JSON records using the IMPORT rules."""
135
+ flat_records = _parse_json_file(filename, encoding)
136
+ seen: dict[str, None] = {}
137
+ for rec in flat_records:
138
+ for key in rec:
139
+ seen[key] = None
140
+ hdrs = list(seen)
141
+ return hdrs, [[rec.get(h) for h in hdrs] for rec in flat_records]
142
+
143
+
139
144
  def import_json(
140
145
  db: Database,
141
146
  schemaname: str | None,
@@ -159,7 +164,7 @@ def import_json(
159
164
  enc = encoding if encoding else _state.conf.import_encoding
160
165
 
161
166
  try:
162
- flat_records = _parse_json_file(filename, enc)
167
+ hdrs, data = json_data(filename, enc)
163
168
  except ErrInfo:
164
169
  raise
165
170
  except Exception as e:
@@ -169,15 +174,4 @@ def import_json(
169
174
  other_msg=f"Can't parse JSON file {filename}",
170
175
  ) from e
171
176
 
172
- # Build a union of all keys across records (preserving first-seen order).
173
- seen: dict[str, None] = {}
174
- for rec in flat_records:
175
- for key in rec:
176
- if key not in seen:
177
- seen[key] = None
178
- hdrs = list(seen)
179
-
180
- # Build row data aligned to hdrs — missing keys become None.
181
- data = [[rec.get(h) for h in hdrs] for rec in flat_records]
182
-
183
177
  import_data_table(db, schemaname, tablename, is_new, hdrs, data)
execsql/importers/xls.py CHANGED
@@ -24,15 +24,17 @@ def xls_data(
24
24
  sheetname: str,
25
25
  junk_header_rows: int,
26
26
  encoding: str | None = None,
27
+ *,
28
+ file_format: str | None = None,
27
29
  ) -> tuple:
28
30
  """Returns the data from the specified worksheet as a list of headers and a list of lists of rows."""
29
31
  from execsql.utils.strings import clean_words, trim_words, fold_words, dedup_words
30
32
 
31
33
  conf = _state.conf
32
34
 
33
- if len(filename) < 4:
35
+ if len(filename) < 4 and file_format is None:
34
36
  raise ErrInfo(type="cmd", other_msg=f"{filename} is not a recognizable Excel spreadsheet name.")
35
- ext3 = filename[-3:].lower()
37
+ ext3 = {"xls": "xls", "xlsx": "lsx"}.get(file_format or "", filename[-3:].lower())
36
38
  wbk: Any
37
39
  if ext3 == "xls":
38
40
  # xlrd imported lazily
@@ -59,10 +61,13 @@ def xls_data(
59
61
  alldata = wbk.sheet_data(sheetname, junk_header_rows)
60
62
  except Exception as e:
61
63
  raise ErrInfo(type="cmd", other_msg=f"Error reading worksheet {sheetname} from {filename}.") from e
64
+ finally:
65
+ if ext3 == "lsx":
66
+ wbk.close()
67
+ elif wbk.wbk is not None:
68
+ wbk.wbk.release_resources()
62
69
  if len(alldata) == 0:
63
70
  raise ErrInfo(type="cmd", other_msg=f"There are no data on worksheet {sheetname} of file {filename}.")
64
- if ext3 == "lsx":
65
- wbk.close()
66
71
  if len(alldata) == 1:
67
72
  return alldata[0], []
68
73
  colhdrs = alldata[0]