execsql2 2.35.0__py3-none-any.whl → 2.38.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- execsql/api.py +111 -53
- execsql/cli/commands/run.py +2 -2
- execsql/cli/lint.py +60 -7
- execsql/cli/run.py +10 -10
- execsql/config.py +7 -0
- execsql/data/execsql.conf.template +31 -7
- execsql/db/base.py +64 -26
- execsql/db/duckdb.py +51 -5
- execsql/db/firebird.py +8 -0
- execsql/db/mysql.py +8 -3
- execsql/db/oracle.py +5 -0
- execsql/db/postgres.py +2 -2
- execsql/db/sqlite.py +117 -10
- execsql/db/sqlserver.py +11 -7
- execsql/db/target_types.py +151 -4
- execsql/debug/repl.py +4 -3
- execsql/exceptions.py +14 -0
- execsql/exporters/html.py +26 -42
- execsql/exporters/ods.py +93 -21
- execsql/exporters/sqlite.py +82 -47
- execsql/format.py +12 -4
- execsql/gui/desktop.py +2 -1
- execsql/gui/tui.py +5 -2
- execsql/importers/base.py +9 -7
- execsql/importers/json.py +37 -13
- execsql/importers/ods.py +22 -2
- execsql/lsp/document.py +6 -1
- execsql/lsp/hover.py +2 -1
- execsql/metacommands/conditions.py +3 -1
- execsql/metacommands/connect.py +28 -12
- execsql/metacommands/dispatch.py +10 -0
- execsql/metacommands/io.py +1 -0
- execsql/metacommands/io_export.py +1 -2
- execsql/metacommands/io_fileops.py +32 -93
- execsql/metacommands/io_import.py +26 -8
- execsql/metacommands/io_write.py +3 -1
- execsql/metacommands/reference.py +7 -0
- execsql/metacommands/script_ext.py +13 -10
- execsql/metacommands/system.py +13 -6
- execsql/parser.py +14 -37
- execsql/script/ast.py +21 -0
- execsql/script/control.py +73 -3
- execsql/script/executor.py +503 -227
- execsql/script/parser.py +102 -52
- execsql/script/preparation.py +42 -0
- execsql/state.py +23 -0
- execsql/types.py +6 -5
- execsql/utils/errors.py +24 -6
- execsql/utils/fileio.py +342 -134
- execsql/utils/sigterm.py +154 -0
- execsql/utils/spreadsheet.py +74 -0
- execsql/utils/strings.py +13 -7
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/README.md +5 -5
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/example_config_prompt.sql +2 -2
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/METADATA +32 -119
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/RECORD +71 -68
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/WHEEL +1 -1
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/config_settings.sqlite +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/make_config_db.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/md_upsert.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/pg_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/script_template.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_compare.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_glossary.sql +0 -0
- {execsql2-2.35.0.data → execsql2-2.38.0.data}/data/execsql2_extras/ss_upsert.sql +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/entry_points.txt +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/licenses/LICENSE.txt +0 -0
- {execsql2-2.35.0.dist-info → execsql2-2.38.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -18,6 +18,7 @@ from shutil import copyfileobj
|
|
|
18
18
|
from typing import Any, Literal
|
|
19
19
|
|
|
20
20
|
import execsql.state as _state
|
|
21
|
+
from execsql.db.base import Database
|
|
21
22
|
from execsql.exceptions import ErrInfo
|
|
22
23
|
from execsql.importers.base import refuse_existing_table
|
|
23
24
|
from execsql.models import DataTable
|
|
@@ -67,11 +68,14 @@ def _close_rowsource(rows: Any) -> None:
|
|
|
67
68
|
|
|
68
69
|
|
|
69
70
|
def x_include(**kwargs: Any) -> None:
|
|
70
|
-
#
|
|
71
|
-
#
|
|
71
|
+
# The executor runs INCLUDE itself (an IncludeDirective) wherever the
|
|
72
|
+
# script writes it. Only a metacommand whose text a substitution variable
|
|
73
|
+
# supplies (-- !x! !!cmd!!) reaches the dispatch table. The entry also
|
|
74
|
+
# gives INCLUDE its place in the keyword list, the language server and the
|
|
75
|
+
# run manifest.
|
|
72
76
|
raise ErrInfo(
|
|
73
77
|
type="cmd",
|
|
74
|
-
other_msg="INCLUDE
|
|
78
|
+
other_msg="An INCLUDE that a substitution variable supplies cannot run; write the INCLUDE in the script.",
|
|
75
79
|
)
|
|
76
80
|
|
|
77
81
|
|
|
@@ -99,73 +103,8 @@ def x_copy(**kwargs: Any) -> None:
|
|
|
99
103
|
db1 = _state.dbs.aliased_as(alias1)
|
|
100
104
|
db2 = _state.dbs.aliased_as(alias2)
|
|
101
105
|
tbl1 = db1.schema_qualified_table_name(schema1, table1)
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
# (PostgreSQL reads Orders as orders), and the SELECT below names a missing
|
|
105
|
-
# source in the database's own words.
|
|
106
|
-
if new_tbl2 and new_tbl2 == "new":
|
|
107
|
-
refuse_existing_table(db2, schema2, table2)
|
|
108
|
-
select_stmt = f"select * from {tbl1};"
|
|
109
|
-
|
|
110
|
-
get_ts_tablespec = None
|
|
111
|
-
rows_to_close = None
|
|
112
|
-
spool: _RowSpool | None = None
|
|
113
|
-
rows_for_insert: Any
|
|
114
|
-
|
|
115
|
-
if new_tbl2:
|
|
116
|
-
try:
|
|
117
|
-
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
118
|
-
except ErrInfo:
|
|
119
|
-
raise
|
|
120
|
-
except Exception as e:
|
|
121
|
-
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
122
|
-
rows_to_close = rows
|
|
123
|
-
# Values come from a database cursor, so a text value stays text.
|
|
124
|
-
spool = _RowSpool(rows)
|
|
125
|
-
get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
|
|
126
|
-
rows_for_insert = spool.replay()
|
|
127
|
-
tbl_desc = get_ts_tablespec
|
|
128
|
-
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
129
|
-
if new_tbl2 == "replacement":
|
|
130
|
-
try:
|
|
131
|
-
db2.drop_table_for_replacement(schema2, table2)
|
|
132
|
-
except ErrInfo:
|
|
133
|
-
raise
|
|
134
|
-
except Exception as e:
|
|
135
|
-
raise ErrInfo(
|
|
136
|
-
type="db",
|
|
137
|
-
other_msg=f"Could not drop existing table ({tbl2}) for COPY metacommand",
|
|
138
|
-
exception_msg=exception_desc(),
|
|
139
|
-
) from e
|
|
140
|
-
db2.execute(create_tbl)
|
|
141
|
-
if db2.needs_explicit_commit_after_ddl():
|
|
142
|
-
db2.execute("COMMIT;")
|
|
143
|
-
else:
|
|
144
|
-
try:
|
|
145
|
-
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
146
|
-
except ErrInfo:
|
|
147
|
-
raise
|
|
148
|
-
except Exception as e:
|
|
149
|
-
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
150
|
-
rows_to_close = rows
|
|
151
|
-
rows_for_insert = rows
|
|
152
|
-
|
|
153
|
-
def get_ts() -> DataTable:
|
|
154
|
-
nonlocal get_ts_tablespec
|
|
155
|
-
if get_ts_tablespec is None:
|
|
156
|
-
ts_hdrs, ts_rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
157
|
-
get_ts_tablespec = DataTable(ts_hdrs, ts_rows, infer_strings=False)
|
|
158
|
-
return get_ts_tablespec
|
|
159
|
-
|
|
160
|
-
try:
|
|
161
|
-
db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
|
|
162
|
-
db2.commit()
|
|
163
|
-
except BaseException:
|
|
164
|
-
_close_rowsource(rows_to_close)
|
|
165
|
-
raise
|
|
166
|
-
finally:
|
|
167
|
-
if spool is not None:
|
|
168
|
-
spool.close()
|
|
106
|
+
# Let the source database resolve unquoted names and report missing tables.
|
|
107
|
+
_copy_rows(db1, db2, f"select * from {tbl1};", schema2, table2, new_tbl2)
|
|
169
108
|
|
|
170
109
|
|
|
171
110
|
def x_copy_query(**kwargs: Any) -> None:
|
|
@@ -190,29 +129,38 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
190
129
|
)
|
|
191
130
|
db1 = _state.dbs.aliased_as(alias1)
|
|
192
131
|
db2 = _state.dbs.aliased_as(alias2)
|
|
132
|
+
_copy_rows(db1, db2, select_stmt, schema2, table2, new_tbl2)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _copy_rows(
|
|
136
|
+
db1: Database,
|
|
137
|
+
db2: Database,
|
|
138
|
+
select_stmt: str,
|
|
139
|
+
schema2: str | None,
|
|
140
|
+
table2: str,
|
|
141
|
+
new_tbl2: str | None,
|
|
142
|
+
) -> None:
|
|
143
|
+
"""Copy a query's rows, sharing table creation, type inference and spool cleanup."""
|
|
193
144
|
tbl2 = db2.schema_qualified_table_name(schema2, table2)
|
|
194
|
-
if new_tbl2
|
|
145
|
+
if new_tbl2 == "new":
|
|
195
146
|
refuse_existing_table(db2, schema2, table2)
|
|
196
147
|
|
|
197
|
-
get_ts_tablespec = None
|
|
198
|
-
rows_to_close = None
|
|
148
|
+
get_ts_tablespec: DataTable | None = None
|
|
199
149
|
spool: _RowSpool | None = None
|
|
200
|
-
|
|
150
|
+
try:
|
|
151
|
+
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
152
|
+
except ErrInfo:
|
|
153
|
+
raise
|
|
154
|
+
except Exception as e:
|
|
155
|
+
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
156
|
+
rows_for_insert: Iterator[Any] = rows
|
|
201
157
|
|
|
202
158
|
if new_tbl2:
|
|
203
|
-
try:
|
|
204
|
-
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
205
|
-
except ErrInfo:
|
|
206
|
-
raise
|
|
207
|
-
except Exception as e:
|
|
208
|
-
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
209
|
-
rows_to_close = rows
|
|
210
159
|
# Values come from a database cursor, so a text value stays text.
|
|
211
160
|
spool = _RowSpool(rows)
|
|
212
161
|
get_ts_tablespec = DataTable(hdrs, spool, infer_strings=False)
|
|
213
162
|
rows_for_insert = spool.replay()
|
|
214
|
-
|
|
215
|
-
create_tbl = tbl_desc.create_table(db2.type, schema2, table2)
|
|
163
|
+
create_tbl = get_ts_tablespec.create_table(db2.type, schema2, table2)
|
|
216
164
|
if new_tbl2 == "replacement":
|
|
217
165
|
try:
|
|
218
166
|
db2.drop_table_for_replacement(schema2, table2)
|
|
@@ -227,15 +175,6 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
227
175
|
db2.execute(create_tbl)
|
|
228
176
|
if db2.needs_explicit_commit_after_ddl():
|
|
229
177
|
db2.execute("COMMIT;")
|
|
230
|
-
else:
|
|
231
|
-
try:
|
|
232
|
-
hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
|
|
233
|
-
except ErrInfo:
|
|
234
|
-
raise
|
|
235
|
-
except Exception as e:
|
|
236
|
-
raise ErrInfo("db", select_stmt, exception_msg=exception_desc()) from e
|
|
237
|
-
rows_to_close = rows
|
|
238
|
-
rows_for_insert = rows
|
|
239
178
|
|
|
240
179
|
def get_ts() -> DataTable:
|
|
241
180
|
nonlocal get_ts_tablespec
|
|
@@ -248,7 +187,7 @@ def x_copy_query(**kwargs: Any) -> None:
|
|
|
248
187
|
db2.populate_table(schema2, table2, rows_for_insert, hdrs, get_ts, existing_table=not new_tbl2)
|
|
249
188
|
db2.commit()
|
|
250
189
|
except BaseException:
|
|
251
|
-
_close_rowsource(
|
|
190
|
+
_close_rowsource(rows)
|
|
252
191
|
raise
|
|
253
192
|
finally:
|
|
254
193
|
if spool is not None:
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"""Import metacommand handlers.
|
|
2
2
|
|
|
3
|
-
- ``x_import`` — IMPORT (delimited text / CSV;
|
|
4
|
-
for .ods/.xls/.xlsx in the same handler).
|
|
3
|
+
- ``x_import`` — IMPORT (delimited text / CSV; spreadsheet forms have separate handlers).
|
|
5
4
|
- ``x_import_file`` — IMPORT_FILE (insert a binary file blob as a row).
|
|
6
|
-
- ``x_import_ods`` / ``x_import_ods_pattern`` — IMPORT … FROM
|
|
7
|
-
|
|
8
|
-
- ``x_import_xls`` / ``x_import_xls_pattern`` —
|
|
5
|
+
- ``x_import_ods`` / ``x_import_ods_pattern`` — IMPORT … FROM <file>
|
|
6
|
+
SHEET <name> / SHEETS MATCHING <pattern> (ODS).
|
|
7
|
+
- ``x_import_xls`` / ``x_import_xls_pattern`` — IMPORT … FROM EXCEL <file>
|
|
8
|
+
SHEET <name> / SHEETS MATCHING <pattern> (XLS/XLSX).
|
|
9
9
|
- ``x_import_parquet`` — IMPORT … FROM PARQUET (via polars).
|
|
10
10
|
- ``x_import_feather`` — IMPORT … FROM FEATHER (via polars).
|
|
11
11
|
- ``x_import_json`` — IMPORT … FROM JSON (array of objects or JSON Lines).
|
|
@@ -22,9 +22,10 @@ from typing import Any
|
|
|
22
22
|
import execsql.state as _state
|
|
23
23
|
from execsql.exceptions import ErrInfo
|
|
24
24
|
from execsql.importers.csv import importfile, importtable
|
|
25
|
+
from execsql.utils.spreadsheet import INPUT_LIMIT_DEFAULTS, SheetBudget
|
|
25
26
|
from execsql.importers.feather import import_feather, import_parquet
|
|
26
27
|
from execsql.importers.json import import_json
|
|
27
|
-
from execsql.importers.ods import OdsFile, importods
|
|
28
|
+
from execsql.importers.ods import OdsFile, check_ods_archive, importods
|
|
28
29
|
from execsql.exporters.xls import XlsFile, XlsxFile
|
|
29
30
|
from execsql.importers.xls import importxls
|
|
30
31
|
from execsql.utils.errors import exception_desc
|
|
@@ -185,13 +186,21 @@ def x_import_ods_pattern(**kwargs: Any) -> None:
|
|
|
185
186
|
command_text=kwargs["metacommandline"],
|
|
186
187
|
other_msg="Input file does not exist",
|
|
187
188
|
)
|
|
189
|
+
check_ods_archive(filename)
|
|
188
190
|
wbk = OdsFile()
|
|
189
191
|
try:
|
|
190
192
|
wbk.open(filename)
|
|
191
193
|
except Exception as e:
|
|
192
194
|
raise ErrInfo(type="cmd", other_msg=f"{filename} is not a valid OpenDocument spreadsheet.") from e
|
|
193
|
-
|
|
194
|
-
|
|
195
|
+
try:
|
|
196
|
+
sheets = wbk.sheetnames()
|
|
197
|
+
impsheets = [s for s in sheets if rx.search(s)]
|
|
198
|
+
# Validate each selected sheet before the first target can be changed;
|
|
199
|
+
# discard the rows between sheets instead of retaining their expansion.
|
|
200
|
+
for sheet in impsheets:
|
|
201
|
+
wbk.sheet_data(sheet, hdr_rows, budget=SheetBudget(_state.conf, filename, sheet))
|
|
202
|
+
finally:
|
|
203
|
+
wbk.close()
|
|
195
204
|
tables = list(impsheets)
|
|
196
205
|
if _state.conf.clean_col_hdrs:
|
|
197
206
|
tables = clean_words(tables)
|
|
@@ -461,3 +470,12 @@ def x_import_row_buffer(**kwargs: Any) -> None:
|
|
|
461
470
|
def x_show_progress(**kwargs: Any) -> None:
|
|
462
471
|
setting = kwargs["setting"].lower()
|
|
463
472
|
_state.conf.show_progress = setting in ("yes", "on", "true", "1")
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def x_input_limit(**kwargs: Any) -> None:
|
|
476
|
+
"""Set a non-negative spreadsheet input budget for subsequent imports."""
|
|
477
|
+
name = kwargs["name"].lower()
|
|
478
|
+
value = int(kwargs["value"])
|
|
479
|
+
if name not in INPUT_LIMIT_DEFAULTS or value < 0:
|
|
480
|
+
raise ErrInfo(type="cmd", other_msg=f"Invalid spreadsheet input limit: {name}={value}.")
|
|
481
|
+
setattr(_state.conf, name, value)
|
execsql/metacommands/io_write.py
CHANGED
|
@@ -261,11 +261,13 @@ def _render_script_nodes(nodes: Any, emit: Any) -> None:
|
|
|
261
261
|
# Preserve comment text verbatim; the parser stores it including
|
|
262
262
|
# the leading "--" / "/* ... */" delimiters.
|
|
263
263
|
emit(f"{node.text}\n")
|
|
264
|
+
elif isinstance(node, IncludeDirective) and node.command:
|
|
265
|
+
emit(f"-- !x! {node.command}\n")
|
|
264
266
|
elif isinstance(node, IncludeDirective):
|
|
265
267
|
kw = "EXECUTE SCRIPT" if node.is_execute_script else "INCLUDE"
|
|
266
268
|
if_exists = " IF EXISTS" if node.if_exists else ""
|
|
267
269
|
args = f" ({node.arguments})" if node.arguments else ""
|
|
268
|
-
loop = f"
|
|
270
|
+
loop = f" {node.loop_type} ({node.loop_condition})" if node.loop_type else ""
|
|
269
271
|
emit(f"-- !x! {kw}{if_exists} {node.target}{args}{loop}\n")
|
|
270
272
|
elif isinstance(node, IfBlock):
|
|
271
273
|
emit(f"-- !x! IF ({node.condition})\n")
|
|
@@ -260,6 +260,13 @@ _METACOMMANDS: tuple[Metacommand, ...] = (
|
|
|
260
260
|
"CONFIG GUI_LEVEL <n>",
|
|
261
261
|
"CONFIG HDF5_TEXT_LEN <n>",
|
|
262
262
|
"CONFIG IMPORT_ROW_BUFFER <n>",
|
|
263
|
+
"CONFIG ODS_MAX_XML_MB <n>",
|
|
264
|
+
"CONFIG ODS_MAX_COMPRESSION_RATIO <n>",
|
|
265
|
+
"CONFIG IMPORT_MAX_ROWS <n>",
|
|
266
|
+
"CONFIG IMPORT_MAX_COLUMNS <n>",
|
|
267
|
+
"CONFIG IMPORT_MAX_CELLS <n>",
|
|
268
|
+
"CONFIG IMPORT_MAX_CELL_CHARS <n>",
|
|
269
|
+
"CONFIG IMPORT_MAX_TEXT_CHARS <n>",
|
|
263
270
|
"CONFIG LOG_DATAVARS YES|NO",
|
|
264
271
|
"CONFIG LOG_SQL YES|NO",
|
|
265
272
|
"CONFIG LOG_WRITE_MESSAGES YES|NO",
|
|
@@ -11,10 +11,11 @@ AST executor:
|
|
|
11
11
|
an inline source).
|
|
12
12
|
- ``x_extendscript_metacommand`` — ``EXTEND SCRIPT … WITH METACOMMAND …``.
|
|
13
13
|
- ``x_extendscript_sql`` — ``EXTEND SCRIPT … WITH SQL …``.
|
|
14
|
-
- ``x_executescript`` — dispatch-table
|
|
15
|
-
/ ``RUN SCRIPT``
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
- ``x_executescript`` — dispatch-table entry. The executor runs
|
|
15
|
+
``EXECUTE SCRIPT`` / ``RUN SCRIPT`` itself wherever the script writes them
|
|
16
|
+
(a line of their own, an inline ``IF``, ``EXTEND SCRIPT … WITH
|
|
17
|
+
METACOMMAND``); this stub is reached only by one that a substitution
|
|
18
|
+
variable supplies, and reports that it cannot run.
|
|
18
19
|
|
|
19
20
|
Registration of ``BEGIN SCRIPT … END SCRIPT`` blocks themselves is
|
|
20
21
|
handled by the AST parser (block boundaries) and executor (registering
|
|
@@ -68,12 +69,12 @@ def x_extendscript(**kwargs: Any) -> None:
|
|
|
68
69
|
|
|
69
70
|
def x_extendscript_metacommand(**kwargs: Any) -> None:
|
|
70
71
|
"""Append a single metacommand line to an existing SCRIPT body."""
|
|
71
|
-
from execsql.script.
|
|
72
|
+
from execsql.script.parser import metacommand_node
|
|
72
73
|
|
|
73
74
|
block = _get_ast_script(kwargs["script"])
|
|
74
75
|
script_file, script_line_no = current_script_line()
|
|
75
76
|
span = _new_span(script_file, script_line_no or 0)
|
|
76
|
-
block.body.append(
|
|
77
|
+
block.body.append(metacommand_node(kwargs["cmd"].strip(), span))
|
|
77
78
|
|
|
78
79
|
|
|
79
80
|
def x_extendscript_sql(**kwargs: Any) -> None:
|
|
@@ -87,10 +88,12 @@ def x_extendscript_sql(**kwargs: Any) -> None:
|
|
|
87
88
|
|
|
88
89
|
|
|
89
90
|
def x_executescript(**kwargs: Any) -> None:
|
|
90
|
-
#
|
|
91
|
-
#
|
|
92
|
-
#
|
|
91
|
+
# The executor runs EXECUTE SCRIPT itself (an IncludeDirective) wherever
|
|
92
|
+
# the script writes it. Only a metacommand whose text a substitution
|
|
93
|
+
# variable supplies (-- !x! !!cmd!!) reaches the dispatch table. The entry
|
|
94
|
+
# also gives EXECUTE / RUN SCRIPT their place in the keyword list, the
|
|
95
|
+
# language server and the run manifest.
|
|
93
96
|
raise ErrInfo(
|
|
94
97
|
"cmd",
|
|
95
|
-
other_msg="EXECUTE SCRIPT
|
|
98
|
+
other_msg="An EXECUTE SCRIPT that a substitution variable supplies cannot run; write it in the script.",
|
|
96
99
|
)
|
execsql/metacommands/system.py
CHANGED
|
@@ -33,6 +33,7 @@ from execsql.utils.gui import (
|
|
|
33
33
|
gui_console_width,
|
|
34
34
|
)
|
|
35
35
|
from execsql.utils.mail import MailSpec, Mailer
|
|
36
|
+
from execsql.utils.sigterm import command
|
|
36
37
|
|
|
37
38
|
|
|
38
39
|
def x_system_cmd(**kwargs: Any) -> None:
|
|
@@ -51,19 +52,25 @@ def x_system_cmd(**kwargs: Any) -> None:
|
|
|
51
52
|
if cont is None:
|
|
52
53
|
timeout = getattr(_state.conf, "system_cmd_timeout", 0.0)
|
|
53
54
|
try:
|
|
54
|
-
if
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
55
|
+
# The command is killed if the block is left before it finishes,
|
|
56
|
+
# and by a SIGTERM at any point once it has started.
|
|
57
|
+
with command(cmdlist) as proc:
|
|
58
|
+
# communicate(), as subprocess.run() does: after Ctrl-C it
|
|
59
|
+
# gives the command a moment to handle its own SIGINT.
|
|
60
|
+
proc.communicate(timeout=timeout if timeout and timeout > 0 else None)
|
|
58
61
|
except subprocess.TimeoutExpired as e:
|
|
59
62
|
_state.subvars.add_substitution("$SYSTEM_CMD_EXIT_STATUS", "124")
|
|
63
|
+
msg = f"SYSTEM_CMD timed out after {timeout:g} seconds."
|
|
64
|
+
if proc.returncode is None:
|
|
65
|
+
# The OS refused to kill it: it runs as another user.
|
|
66
|
+
msg += f" The command could not be stopped and is still running (process {proc.pid})."
|
|
60
67
|
raise ErrInfo(
|
|
61
68
|
type="cmd",
|
|
62
69
|
command_text=kwargs.get("metacommandline", "SYSTEM_CMD"),
|
|
63
|
-
other_msg=
|
|
70
|
+
other_msg=msg,
|
|
64
71
|
exception_msg=exception_desc(),
|
|
65
72
|
) from e
|
|
66
|
-
_state.subvars.add_substitution("$SYSTEM_CMD_EXIT_STATUS", str(
|
|
73
|
+
_state.subvars.add_substitution("$SYSTEM_CMD_EXIT_STATUS", str(proc.returncode))
|
|
67
74
|
else:
|
|
68
75
|
proc = subprocess.Popen(cmdlist)
|
|
69
76
|
_state.subvars.add_substitution("$SYSTEM_CMD_PID", str(proc.pid))
|
execsql/parser.py
CHANGED
|
@@ -48,8 +48,6 @@ class SourceString:
|
|
|
48
48
|
|
|
49
49
|
def eoi(self) -> bool:
|
|
50
50
|
"""Return True if the entire source string has been consumed."""
|
|
51
|
-
# Returns True or False indicating whether or not there is any of
|
|
52
|
-
# the source string left to be consumed.
|
|
53
51
|
return self.currpos >= len(self.str)
|
|
54
52
|
|
|
55
53
|
def eat_whitespace(self) -> None:
|
|
@@ -59,55 +57,34 @@ class SourceString:
|
|
|
59
57
|
|
|
60
58
|
def match_str(self, str: str) -> str | None:
|
|
61
59
|
"""Match a string case-insensitively at the current position and advance."""
|
|
62
|
-
# Tries to match the 'str' argument at the current position in the
|
|
63
|
-
# source string. Matching is case-insensitive. If matching succeeds,
|
|
64
|
-
# the matched string is returned and the internal pointer is incremented.
|
|
65
|
-
# If matching fails, None is returned and the internal pointer is unchanged.
|
|
66
60
|
self.eat_whitespace()
|
|
67
|
-
if self.eoi():
|
|
61
|
+
if self.eoi() or not self.str.lower().startswith(str.lower(), self.currpos):
|
|
68
62
|
return None
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
matched = self.str[self.currpos : self.currpos + len(str)]
|
|
73
|
-
self.currpos += len(str)
|
|
74
|
-
return matched
|
|
75
|
-
else:
|
|
76
|
-
return None
|
|
63
|
+
matched = self.str[self.currpos : self.currpos + len(str)]
|
|
64
|
+
self.currpos += len(str)
|
|
65
|
+
return matched
|
|
77
66
|
|
|
78
67
|
def match_regex(self, regex: Any) -> dict | None:
|
|
79
68
|
"""Match a compiled regex at the current position and return named groups."""
|
|
80
|
-
# Tries to match the 'regex' argument at the current position in the
|
|
81
|
-
# source string. If it succeeds, a dictionary of all of the named
|
|
82
|
-
# groups is returned, and the internal pointer is incremented.
|
|
83
69
|
self.eat_whitespace()
|
|
84
70
|
if self.eoi():
|
|
85
71
|
return None
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
else:
|
|
92
|
-
return None
|
|
72
|
+
m = regex.match(self.str[self.currpos :])
|
|
73
|
+
if not m:
|
|
74
|
+
return None
|
|
75
|
+
self.currpos += m.end(0)
|
|
76
|
+
return m.groupdict() or {}
|
|
93
77
|
|
|
94
78
|
def match_metacommand(self, commandlist: Any) -> tuple | None:
|
|
95
79
|
"""Match a metacommand from the command list at the current position."""
|
|
96
|
-
# Tries to match text at the current position to any metacommand
|
|
97
|
-
# in the specified commandlist.
|
|
98
|
-
# If it succeeds, the return value is a tuple of the MetaCommand object
|
|
99
|
-
# and a dictionary of all of the named groups. The internal pointer is
|
|
100
|
-
# incremented past the match.
|
|
101
80
|
self.eat_whitespace()
|
|
102
81
|
if self.eoi():
|
|
103
82
|
return None
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
else:
|
|
110
|
-
return None
|
|
83
|
+
m = commandlist.get_match(self.str[self.currpos :])
|
|
84
|
+
if m is None:
|
|
85
|
+
return None
|
|
86
|
+
self.currpos += m[1].end(0)
|
|
87
|
+
return (m[0], m[1].groupdict() or {})
|
|
111
88
|
|
|
112
89
|
def remainder(self) -> str:
|
|
113
90
|
"""Return the unconsumed portion of the source string."""
|
execsql/script/ast.py
CHANGED
|
@@ -212,12 +212,15 @@ class ElseIfClause:
|
|
|
212
212
|
condition_modifiers: ANDIF/ORIF modifiers that compound the ELSEIF
|
|
213
213
|
condition, evaluated left-to-right at runtime.
|
|
214
214
|
body: Nodes executed when this condition is true.
|
|
215
|
+
command: The metacommand as written, without ``-- !x!``, for error
|
|
216
|
+
reports.
|
|
215
217
|
"""
|
|
216
218
|
|
|
217
219
|
condition: str
|
|
218
220
|
span: SourceSpan
|
|
219
221
|
condition_modifiers: list[ConditionModifier] = field(default_factory=list)
|
|
220
222
|
body: list[Node] = field(default_factory=list)
|
|
223
|
+
command: str = ""
|
|
221
224
|
|
|
222
225
|
|
|
223
226
|
@dataclass
|
|
@@ -232,6 +235,8 @@ class IfBlock(Node):
|
|
|
232
235
|
elseif_clauses: Zero or more ELSEIF branches, evaluated in order.
|
|
233
236
|
else_body: Nodes executed when the IF condition (and all ELSEIFs)
|
|
234
237
|
are false. Empty list means no ELSE branch.
|
|
238
|
+
command: The metacommand as written, without ``-- !x!``, for error
|
|
239
|
+
reports.
|
|
235
240
|
"""
|
|
236
241
|
|
|
237
242
|
condition: str
|
|
@@ -240,6 +245,7 @@ class IfBlock(Node):
|
|
|
240
245
|
elseif_clauses: list[ElseIfClause] = field(default_factory=list)
|
|
241
246
|
else_body: list[Node] = field(default_factory=list)
|
|
242
247
|
else_span: SourceSpan | None = None
|
|
248
|
+
command: str = ""
|
|
243
249
|
|
|
244
250
|
def children(self) -> Iterator[Node]:
|
|
245
251
|
yield from self.body
|
|
@@ -261,11 +267,14 @@ class LoopBlock(Node):
|
|
|
261
267
|
loop_type: Either ``"WHILE"`` or ``"UNTIL"``.
|
|
262
268
|
condition: The condition expression text.
|
|
263
269
|
body: Nodes executed on each iteration.
|
|
270
|
+
command: The metacommand as written, without ``-- !x!``, for error
|
|
271
|
+
reports.
|
|
264
272
|
"""
|
|
265
273
|
|
|
266
274
|
loop_type: str # "WHILE" or "UNTIL"
|
|
267
275
|
condition: str
|
|
268
276
|
body: list[Node] = field(default_factory=list)
|
|
277
|
+
command: str = ""
|
|
269
278
|
|
|
270
279
|
def children(self) -> Iterator[Node]:
|
|
271
280
|
yield from self.body
|
|
@@ -283,9 +292,12 @@ class BatchBlock(Node):
|
|
|
283
292
|
|
|
284
293
|
Attributes:
|
|
285
294
|
body: Nodes within the batch.
|
|
295
|
+
command: The metacommand as written, without ``-- !x!``, for error
|
|
296
|
+
reports.
|
|
286
297
|
"""
|
|
287
298
|
|
|
288
299
|
body: list[Node] = field(default_factory=list)
|
|
300
|
+
command: str = ""
|
|
289
301
|
|
|
290
302
|
def children(self) -> Iterator[Node]:
|
|
291
303
|
yield from self.body
|
|
@@ -325,12 +337,15 @@ class ScriptBlock(Node):
|
|
|
325
337
|
doc: Optional docstring extracted from comments immediately following
|
|
326
338
|
the BEGIN SCRIPT line.
|
|
327
339
|
body: Nodes within the script block.
|
|
340
|
+
command: The metacommand as written, without ``-- !x!``, for error
|
|
341
|
+
reports.
|
|
328
342
|
"""
|
|
329
343
|
|
|
330
344
|
name: str
|
|
331
345
|
param_defs: list[ParamDef] | None = None
|
|
332
346
|
doc: str | None = None
|
|
333
347
|
body: list[Node] = field(default_factory=list)
|
|
348
|
+
command: str = ""
|
|
334
349
|
|
|
335
350
|
@property
|
|
336
351
|
def param_names(self) -> list[str] | None:
|
|
@@ -358,9 +373,12 @@ class SqlBlock(Node):
|
|
|
358
373
|
Attributes:
|
|
359
374
|
body: Nodes within the SQL block (typically a single
|
|
360
375
|
:class:`SqlStatement`).
|
|
376
|
+
command: The metacommand as written, without ``-- !x!``, for error
|
|
377
|
+
reports.
|
|
361
378
|
"""
|
|
362
379
|
|
|
363
380
|
body: list[Node] = field(default_factory=list)
|
|
381
|
+
command: str = ""
|
|
364
382
|
|
|
365
383
|
def children(self) -> Iterator[Node]:
|
|
366
384
|
yield from self.body
|
|
@@ -384,6 +402,8 @@ class IncludeDirective(Node):
|
|
|
384
402
|
arguments: Optional argument expression for EXECUTE SCRIPT.
|
|
385
403
|
loop_type: Optional ``"WHILE"`` or ``"UNTIL"`` for looped execution.
|
|
386
404
|
loop_condition: The loop condition expression, if *loop_type* is set.
|
|
405
|
+
command: The metacommand as written, without ``-- !x!``, for error
|
|
406
|
+
reports.
|
|
387
407
|
"""
|
|
388
408
|
|
|
389
409
|
target: str
|
|
@@ -392,6 +412,7 @@ class IncludeDirective(Node):
|
|
|
392
412
|
arguments: str | None = None
|
|
393
413
|
loop_type: str | None = None
|
|
394
414
|
loop_condition: str | None = None
|
|
415
|
+
command: str = ""
|
|
395
416
|
|
|
396
417
|
def __repr__(self) -> str:
|
|
397
418
|
kind = "EXECUTE SCRIPT" if self.is_execute_script else "INCLUDE"
|
execsql/script/control.py
CHANGED
|
@@ -30,6 +30,8 @@ class BatchLevels:
|
|
|
30
30
|
class Batch:
|
|
31
31
|
def __init__(self) -> None:
|
|
32
32
|
self.dbs_used: list[Any] = []
|
|
33
|
+
# Why this batch can no longer commit (an error inside it), or None.
|
|
34
|
+
self.failed: str | None = None
|
|
33
35
|
|
|
34
36
|
def __init__(self) -> None:
|
|
35
37
|
self.batchlevels: list[BatchLevels.Batch] = []
|
|
@@ -38,6 +40,25 @@ class BatchLevels:
|
|
|
38
40
|
"""Return True if execution is currently inside at least one BATCH block."""
|
|
39
41
|
return len(self.batchlevels) > 0
|
|
40
42
|
|
|
43
|
+
def fail_enclosing(self, reason: str) -> None:
|
|
44
|
+
"""Mark every open batch level as failed, so its END BATCH rolls back instead of committing."""
|
|
45
|
+
for b in self.batchlevels:
|
|
46
|
+
if b.failed is None:
|
|
47
|
+
b.failed = reason
|
|
48
|
+
|
|
49
|
+
def failure(self) -> str | None:
|
|
50
|
+
"""Why the innermost batch can no longer commit, or None.
|
|
51
|
+
|
|
52
|
+
A batch inside a failed one cannot commit either: its commit would
|
|
53
|
+
commit the failed batch's later statements on a shared connection.
|
|
54
|
+
Nothing clears the mark, ROLLBACK BATCH included: once anything in a
|
|
55
|
+
batch has failed, the whole stack of batches rolls back.
|
|
56
|
+
"""
|
|
57
|
+
for b in reversed(self.batchlevels):
|
|
58
|
+
if b.failed is not None:
|
|
59
|
+
return b.failed
|
|
60
|
+
return None
|
|
61
|
+
|
|
41
62
|
def new_batch(self) -> None:
|
|
42
63
|
"""Push a new empty batch level onto the stack."""
|
|
43
64
|
self.batchlevels.append(self.Batch())
|
|
@@ -61,7 +82,56 @@ class BatchLevels:
|
|
|
61
82
|
db.rollback()
|
|
62
83
|
|
|
63
84
|
def end_batch(self) -> None:
|
|
64
|
-
"""Commit all databases in the innermost batch level and pop the stack.
|
|
85
|
+
"""Commit all databases in the innermost batch level and pop the stack.
|
|
86
|
+
|
|
87
|
+
If a commit fails, :meth:`Database.commit` rolls that database back,
|
|
88
|
+
and the ones not yet committed are rolled back too (except one with
|
|
89
|
+
autocommit off, whose transaction is the script's) before the error
|
|
90
|
+
is raised: a commit that already succeeded
|
|
91
|
+
cannot be undone, but nothing else of the batch is left pending for
|
|
92
|
+
a later statement to commit. An interrupt rolls back the database
|
|
93
|
+
it landed on as well, since it may have come before that database's
|
|
94
|
+
own rollback. The level leaves the stack last, so until then an
|
|
95
|
+
interrupt finds it there to abandon.
|
|
96
|
+
"""
|
|
97
|
+
b = self.batchlevels[-1]
|
|
98
|
+
committed = 0
|
|
99
|
+
try:
|
|
100
|
+
for db in b.dbs_used:
|
|
101
|
+
db.commit()
|
|
102
|
+
committed += 1
|
|
103
|
+
except Exception:
|
|
104
|
+
_roll_back(b.dbs_used[committed + 1 :])
|
|
105
|
+
raise
|
|
106
|
+
except BaseException:
|
|
107
|
+
_roll_back(b.dbs_used[committed:])
|
|
108
|
+
raise
|
|
109
|
+
finally:
|
|
110
|
+
self.batchlevels.pop()
|
|
111
|
+
|
|
112
|
+
def abandon_batch(self) -> None:
|
|
113
|
+
"""Pop the innermost batch level without committing it.
|
|
114
|
+
|
|
115
|
+
Inside another batch, its databases join the enclosing level, which
|
|
116
|
+
commits or rolls them back. Otherwise they are rolled back, except
|
|
117
|
+
one with autocommit off: :meth:`end_batch` would not have committed
|
|
118
|
+
it either, so its transaction stays the script's to end.
|
|
119
|
+
"""
|
|
65
120
|
b = self.batchlevels.pop()
|
|
66
|
-
|
|
67
|
-
db.
|
|
121
|
+
if self.batchlevels:
|
|
122
|
+
for db in b.dbs_used:
|
|
123
|
+
self.using_db(db)
|
|
124
|
+
return
|
|
125
|
+
_roll_back(b.dbs_used)
|
|
126
|
+
|
|
127
|
+
def abandon_all(self) -> None:
|
|
128
|
+
"""Abandon every open batch level, innermost first."""
|
|
129
|
+
while self.batchlevels:
|
|
130
|
+
self.abandon_batch()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _roll_back(dbs: list[Any]) -> None:
|
|
134
|
+
"""Roll back each of *dbs* except one with autocommit off, whose transaction is the script's."""
|
|
135
|
+
for db in dbs:
|
|
136
|
+
if db.autocommit:
|
|
137
|
+
db.rollback()
|