execsql2 2.27.0__py3-none-any.whl → 2.29.0__py3-none-any.whl

execsql/exporters/yaml.py

@@ -6,8 +6,7 @@ YAML export for execsql.
 Provides :func:`write_query_to_yaml` which serialises a query result set as a
 YAML sequence of mappings (one mapping per row).
 
-Requires the ``PyYAML`` package (``pip install PyYAML`` or
-``pip install 'execsql2[formats]'``).
+Requires the ``PyYAML`` package (the ``formats`` extra).
 """
 
 from typing import Any
@@ -15,7 +14,7 @@ from typing import Any
 import execsql.state as _state
 from execsql.exceptions import ErrInfo
 from execsql.exporters.zip import ZipWriter
-from execsql.utils.errors import exception_desc
+from execsql.utils.errors import exception_desc, install_hint
 from execsql.utils.fileio import filewriter_close
 
 __all__ = ["write_query_to_yaml"]
@@ -54,7 +53,7 @@ def write_query_to_yaml(
     except ImportError as exc:
         raise ErrInfo(
             "error",
-            other_msg=("PyYAML is required for FORMAT YAML export. Install it with: pip install PyYAML"),
+            other_msg=f"PyYAML is required for FORMAT YAML export. {install_hint('formats')}",
         ) from exc
 
     conf = _state.conf

execsql/format.py

@@ -21,11 +21,14 @@ from types import ModuleType
 __all__ = ["collect_paths", "format_file", "open_dollar_quote", "parse_keyword", "run_formatter"]
 
 
-_SQLGLOT_MISSING_MSG = (
-    "execsql format requires sqlglot for SQL reformatting.\n"
-    '  Install the formatter extra:  uv tool install "execsql2[formatter]"  (or pip install "execsql2[formatter]")\n'
-    "  Or skip SQL reformatting with the --no-sql flag."
-)
+def _sqlglot_missing_msg() -> str:
+    from execsql.utils.errors import install_hint
+
+    return (
+        "execsql format requires sqlglot for SQL reformatting.\n"
+        f"  {install_hint('formatter')}\n"
+        "  Or skip SQL reformatting with the --no-sql flag."
+    )
 
 
 def _require_sqlglot():
@@ -36,7 +39,7 @@ def _require_sqlglot():
 
         return sqlglot
     except ImportError as e:
-        raise ImportError(_SQLGLOT_MISSING_MSG) from e
+        raise ImportError(_sqlglot_missing_msg()) from e
 
 
 # ---------------------------------------------------------------------------
@@ -532,20 +535,57 @@ def _statement_sources(sqlglot: ModuleType, sql: str) -> list[str]:
     return sources
 
 
-def _layout_tokens(sqlglot: ModuleType, sql: str) -> list[str]:
-    """*sql*'s tokens with everything the formatter may change taken out.
+def _ends_with_semicolon(sqlglot: ModuleType, sql: str) -> bool:
+    """Whether the last token of *sql* is a statement-ending ``;``."""
+    tokens = sqlglot.tokenize(sql, read="postgres")
+    return bool(tokens) and tokens[-1].token_type.name == "SEMICOLON"
+
+
+def _layout_tokens(sqlglot: ModuleType, sql: str) -> list[tuple[str, str]]:
+    """*sql*'s tokens, as (type, text), with everything the formatter may change taken out.
 
-    Whitespace and comments are not tokens.  Keywords and unquoted names are
-    compared without case; strings and quoted names exactly.
+    Whitespace and comments are not tokens.  Keywords and unquoted names
+    (function names included) are compared without case; strings and quoted
+    names exactly.  The type is compared too, so a string that came back as a
+    quoted name of the same text is a change.
     """
     return [
-        token.text
-        if token.token_type.name.endswith("STRING") or token.token_type.name == "IDENTIFIER"
-        else token.text.upper()
+        (
+            token.token_type.name,
+            token.text
+            if token.token_type.name.endswith("STRING") or token.token_type.name == "IDENTIFIER"
+            else token.text.upper(),
+        )
         for token in sqlglot.tokenize(sql, read="postgres")
     ]
 
 
+# An optimizer hint (``/*+ ... */``, Oracle and MySQL) or a MySQL executable
+# comment (``/*! ... */``) is SQL the server reads, though it looks like a
+# comment.  sqlglot keeps neither: it writes them back as ``/* + ... */`` and
+# moves a hint away from its keyword, which disables it.
+_RX_HINT_COMMENT = re.compile(r"/\*[+!]")
+
+
+def _has_hint_comment(sqlglot: ModuleType, sql: str) -> bool:
+    """Whether *sql* holds a hint or executable comment outside its strings and quoted names."""
+    if not _RX_HINT_COMMENT.search(sql):
+        return False
+    tokens = [(t.start, t.end) for t in sqlglot.tokenize(sql, read="postgres")]
+    return any(not any(start <= m.start() <= end for start, end in tokens) for m in _RX_HINT_COMMENT.finditer(sql))
+
+
+def _as_written(source: str) -> str:
+    """A statement's source text to keep as written, without the space that followed the previous ``;``.
+
+    A statement that starts on its own line keeps that line's indentation.
+    """
+    source = source.rstrip()
+    body = source.lstrip()
+    lead = source[: len(source) - len(body)]
+    return source[lead.rfind("\n") + 1 :] if "\n" in lead else body
+
+
 def _sqlglot_format(
     sql_lines: list[str],
     sql_indent: int = 4,
@@ -596,9 +636,10 @@ def _sqlglot_format(
             # parse is left exactly as written.
             ast = sqlglot.parse(protected, read="postgres", error_level=sqlglot_errors.ErrorLevel.RAISE)
             nodes = [node for node in ast if node is not None]
-            sources = None if rewrite_sql else _statement_sources(sqlglot, protected)
-            if sources is not None and len(sources) != len(nodes):
+            sources = _statement_sources(sqlglot, protected)
+            if len(sources) != len(nodes) and (not rewrite_sql or _has_hint_comment(sqlglot, protected)):
                 return sql_lines
+            aligned = len(sources) == len(nodes)
             statements: list[str] = []
             for i, node in enumerate(nodes):
                 # For Command nodes (psql backslash commands, ERROR:, etc.)
@@ -614,20 +655,29 @@ def _sqlglot_format(
                         max_text_width=120,
                     )
                 generated = re.sub(r"\bINTO TEMPORARY\b(?!\s+TABLE)", "INTO TEMPORARY TABLE", generated)
-                if sources is not None and _layout_tokens(sqlglot, generated) != _layout_tokens(sqlglot, sources[i]):
-                    generated = sources[i].lstrip("\n").rstrip()  # layout only: keep it as written
+                if aligned and (
+                    _has_hint_comment(sqlglot, sources[i])
+                    or (not rewrite_sql and _layout_tokens(sqlglot, generated) != _layout_tokens(sqlglot, sources[i]))
+                ):
+                    generated = _as_written(sources[i])
                 statements.append(generated)
         stmts = [s for s in statements if s]
         if not stmts:
             return sql_lines
 
+        # Terminate the last statement only if the source did: execsql refuses
+        # a script whose last statement is unfinished, and formatting must not
+        # turn it into one that runs.
+        terminated = _ends_with_semicolon(sqlglot, protected)
+
         # Safety: if sqlglot produced more statements than the input had
-        # semicolons, it likely split a fragment (e.g. a SELECT column list)
-        # into multiple pseudo-statements.  Fall back to the original text.
-        if len(stmts) > max(input_semis, 1):
+        # semicolons (plus an unterminated last one), it likely split a
+        # fragment (e.g. a SELECT column list) into multiple pseudo-statements.
+        # Fall back to the original text.
+        if len(stmts) > max(input_semis + (0 if terminated else 1), 1):
             return sql_lines
 
-        joined = ";\n".join(stmts) + ";"
+        joined = ";\n".join(stmts) + (";" if terminated else "")
 
         # Content-loss check: sqlglot with IGNORE error level can silently
         # drop tokens it doesn't understand (e.g. ``ERROR: ...``).  If the

execsql/importers/base.py

@@ -16,15 +16,7 @@ from execsql.exceptions import ErrInfo
 from execsql.db.base import Database
 import execsql.state as _state
 
-__all__ = ["import_data_table", "refuse_existing_table", "table_exists_if_known"]
-
-
-def table_exists_if_known(db: Database, table_name: str, schema_name: str | None = None) -> bool | None:
-    """Whether the table exists, or ``None`` when the database cannot say (no catalog access, ...)."""
-    try:
-        return db.table_exists(table_name, schema_name)
-    except Exception:
-        return None
+__all__ = ["import_data_table", "refuse_existing_table"]
 
 
 def refuse_existing_table(db: Database, schema_name: str | None, table_name: str) -> None:
@@ -43,7 +35,11 @@ def refuse_existing_table(db: Database, schema_name: str | None, table_name: str
             lookup_schema = None
         if lookup_schema is None:
             return  # An unqualified lookup would search every schema.
-    if table_exists_if_known(db, table_name, lookup_schema):
+    try:
+        exists = db.table_exists_where_created(table_name, lookup_schema)
+    except Exception:
+        return  # The database cannot say; CREATE TABLE reports any conflict.
+    if exists:
         name = db.schema_qualified_table_name(schema_name, table_name)
         raise ErrInfo(
             type="cmd",

execsql/lsp/document.py

@@ -86,6 +86,8 @@ class ScriptIndex:
 # cannot make every request read a huge file or walk an endless include chain.
 MAX_INCLUDED_BYTES = 10 * 1024 * 1024
 MAX_INCLUDED_FILES = 200
+# The files an INCLUDE names, all together; past it indexing follows no more of them.
+MAX_INCLUDED_TOTAL_BYTES = 20 * 1024 * 1024
 
 
 def read_included(path: Path) -> str | None:
@@ -121,7 +123,7 @@ def index_script(
     prefers the editor's unsaved copy of a file that is open.
     """
     index = ScriptIndex(path=path)
-    _index_into(index, source, path, set(), read, top=True)
+    _index_into(index, source, path, set(), read, top=True, budget=[MAX_INCLUDED_TOTAL_BYTES])
     return index
 
 
@@ -133,6 +135,7 @@ def _index_into(
     read: Callable[[Path], str | None],
     *,
     top: bool,
+    budget: list[int],
 ) -> None:
     if path is not None:
         seen.add(str(Path(path).resolve()))
@@ -166,8 +169,9 @@ def _index_into(
                 index.includes.append((Location(path, row, start, end), str(resolved)))
             if "!" not in target and str(resolved.resolve()) not in seen and len(seen) < MAX_INCLUDED_FILES:
                 included = read(resolved)
-                if included is not None:
-                    _index_into(index, included, str(resolved), seen, read, top=False)
+                if included is not None and len(included) <= budget[0]:
+                    budget[0] -= len(included)
+                    _index_into(index, included, str(resolved), seen, read, top=False, budget=budget)
 
     for row, text in enumerate(lines):
         for m in _RX_VARIABLE.finditer(text):

execsql/metacommands/debug.py

@@ -27,7 +27,7 @@ from pathlib import Path
 from typing import Any
 
 import execsql.state as _state
-from execsql.utils.errors import fatal_error
+from execsql.utils.errors import fatal_error, install_hint
 from execsql.utils.fileio import EncodedFile, filewriter_open_as_new, filewriter_write
 
 
@@ -110,7 +110,7 @@ def x_debug_write_odbc_drivers(**kwargs: Any) -> None:
     try:
         import pyodbc
     except Exception:
-        fatal_error("The pyodbc module is required.")
+        fatal_error(f"The pyodbc module is required to list ODBC drivers. {install_hint('odbc')}")
     output_dest = kwargs["filename"]
     append = kwargs["append"]
     if output_dest is not None and output_dest != "stdout" and append is None:

execsql/metacommands/io_fileops.py

@@ -114,7 +114,7 @@ def x_copy(**kwargs: Any) -> None:
 
     if new_tbl2:
         try:
-            hdrs, rows = db1.select_rowsource(select_stmt)
+            hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
         except ErrInfo:
             raise
         except Exception as e:
@@ -142,7 +142,7 @@ def x_copy(**kwargs: Any) -> None:
             db2.execute("COMMIT;")
     else:
         try:
-            hdrs, rows = db1.select_rowsource(select_stmt)
+            hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
         except ErrInfo:
             raise
         except Exception as e:
@@ -153,7 +153,7 @@ def x_copy(**kwargs: Any) -> None:
     def get_ts() -> DataTable:
         nonlocal get_ts_tablespec
         if get_ts_tablespec is None:
-            ts_hdrs, ts_rows = db1.select_rowsource(select_stmt)
+            ts_hdrs, ts_rows = db1.select_rowsource(select_stmt, decode_bytes=False)
             get_ts_tablespec = DataTable(ts_hdrs, ts_rows, infer_strings=False)
         return get_ts_tablespec
 
@@ -201,7 +201,7 @@ def x_copy_query(**kwargs: Any) -> None:
 
     if new_tbl2:
         try:
-            hdrs, rows = db1.select_rowsource(select_stmt)
+            hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
         except ErrInfo:
             raise
         except Exception as e:
@@ -229,7 +229,7 @@ def x_copy_query(**kwargs: Any) -> None:
             db2.execute("COMMIT;")
     else:
         try:
-            hdrs, rows = db1.select_rowsource(select_stmt)
+            hdrs, rows = db1.select_rowsource(select_stmt, decode_bytes=False)
         except ErrInfo:
             raise
         except Exception as e:
@@ -240,7 +240,7 @@ def x_copy_query(**kwargs: Any) -> None:
     def get_ts() -> DataTable:
         nonlocal get_ts_tablespec
         if get_ts_tablespec is None:
-            ts_hdrs, ts_rows = db1.select_rowsource(select_stmt)
+            ts_hdrs, ts_rows = db1.select_rowsource(select_stmt, decode_bytes=False)
             get_ts_tablespec = DataTable(ts_hdrs, ts_rows, infer_strings=False)
         return get_ts_tablespec
 

execsql/metacommands/upsert.py

@@ -17,7 +17,7 @@ from typing import Any
 import execsql.state as _state
 from execsql.exceptions import ErrInfo
 from execsql.types import dbt_postgres
-from execsql.utils.errors import exception_desc
+from execsql.utils.errors import exception_desc, install_hint
 
 
 # ---------------------------------------------------------------------------
@@ -213,10 +213,7 @@ def _require_pg_upsert() -> None:
     except ImportError as exc:
         raise ErrInfo(
             "exception",
-            other_msg=(
-                'PG_UPSERT requires the pg-upsert package. Install the upsert extra: uv tool install "execsql2[upsert]"'
-                ' (or pip install "execsql2[upsert]").'
-            ),
+            other_msg=f"PG_UPSERT requires the pg-upsert package. {install_hint('upsert')}",
         ) from exc
 
 

execsql/models.py

@@ -167,8 +167,16 @@ class Column:
                 self.Accum(DT_Text()),
                 self.Accum(DT_Binary()),
             )
+        if not infer_strings:
+            # Values from a database cursor: a bytes value is binary data, so
+            # Binary is tried before the text types (which accept anything).
+            text = [ac for ac in self.accums if isinstance(ac.dt, (DT_Character, DT_Varchar, DT_Text))]
+            binary = [ac for ac in self.accums if isinstance(ac.dt, DT_Binary)]
+            self.accums = tuple(ac for ac in self.accums if ac not in text and ac not in binary) + tuple(binary + text)
         self.infer_strings = infer_strings
         self.str_accums = tuple(ac for ac in self.accums if isinstance(ac.dt, (DT_Character, DT_Varchar, DT_Text)))
+        # A bytes value is checked as binary or text only: int(b"12") is 12.
+        self.bytes_accums = tuple(ac for ac in self.accums if isinstance(ac.dt, DT_Binary)) + self.str_accums
         self.dt_eval = False
         # self.dt is a tuple of: 0: column name; 1: data type class; 2: max length or None;
         # 3: bool indicating any null values; 4: precision or None; 5: scale or None.
@@ -198,6 +206,9 @@ class Column:
         if not self.infer_strings and isinstance(column_value, str):
             for dt in self.str_accums:
                 dt.check(column_value)
+        elif not self.infer_strings and isinstance(column_value, (bytes, bytearray, memoryview)):
+            for dt in self.bytes_accums:
+                dt.check(column_value)
         else:
             for dt in self.accums:
                 dt.check(column_value)

execsql/script/variables.py

@@ -17,6 +17,42 @@ from execsql.exceptions import ErrInfo
 __all__ = ["CounterVars", "SubVarSet", "LocalSubVarSet", "ScriptArgSubVarSet"]
 
 
+def _after_escape_string_prefix(command: str, start: int) -> bool:
+    """Whether the ``!'!`` token at *start* follows an ``E`` that makes it a PostgreSQL escape string (``E!'!v!'!``).
+
+    The ``E`` must stand alone: in ``TYPE!'!v!'!`` it ends a word.
+    """
+    if start < 1 or command[start - 1] not in "Ee":
+        return False
+    return start < 2 or not (command[start - 2].isalnum() or command[start - 2] == "_")
+
+
+def _sql_string_literal(value: str, *, escape_string: bool = False) -> str:
+    """*value* as the string literal ``!'!var!'!`` substitutes, in the current connection's form.
+
+    An apostrophe is doubled everywhere; a backslash only where the DBMS
+    reads it as an escape (MySQL/MariaDB), or inside an escape string the
+    script wrote as ``E!'!var!'!`` (*escape_string*), so ``C:\\data`` is
+    stored as written in every case.  Without a connection, the standard SQL
+    form.  (Upstream doubled backslashes only on Windows, to protect
+    ``re.sub``'s replacement template, not the SQL.)
+    """
+    if escape_string:
+        return "'" + value.replace("\\", "\\\\").replace("'", "''") + "'"
+    db = None
+    try:
+        import execsql.state as _state
+
+        dbs = _state.dbs
+        if dbs is not None and dbs.current_db is not None:
+            db = dbs.current()
+    except Exception:
+        db = None
+    if db is not None:
+        return db.quote_literal(value)
+    return "'" + value.replace("'", "''") + "'"
+
+
 # ---------------------------------------------------------------------------
 # CounterVars
 # ---------------------------------------------------------------------------
@@ -253,13 +289,10 @@ class SubVarSet:
                     )
                 quote = m.group("q")
                 if quote == "'":
-                    # B07a/F002: wrap in single quotes, doubling embedded
-                    # apostrophes AND escaping backslashes so MySQL default
-                    # mode and PostgreSQL E-string literals can't end the
-                    # quoted region via ``\'``. The previous Windows-only
-                    # branch (``os.name != 'posix'``) applied the escape on
-                    # the wrong axis — host OS vs target DBMS.
-                    sub = "'" + sub.replace("\\", "\\\\").replace("'", "''") + "'"
+                    sub = _sql_string_literal(
+                        sub,
+                        escape_string=_after_escape_string_prefix(command_str, m.start()),
+                    )
                 elif quote == '"':
                     # B07a/F001: wrap in double quotes, doubling embedded
                     # ``"`` so a value containing ``"; DROP TABLE x; --``
@@ -306,13 +339,12 @@ class SubVarSet:
             if idx != -1:
                 _check_nul(varname, sub)
                 return command_str[:idx] + sub + command_str[idx + len(token) :], True
-            # Single-quote-wrapped token: !'!varname!'! — escape ``\`` and
-            # double embedded ``'`` (see substitute_one for rationale).
+            # Single-quote-wrapped token: !'!varname!'! — a string literal.
             tokenq = f"!'!{varname}!'!"
             idxq = cmd_lower.find(tokenq)
             if idxq != -1:
                 _check_nul(varname, sub)
-                wrapped = "'" + sub.replace("\\", "\\\\").replace("'", "''") + "'"
+                wrapped = _sql_string_literal(sub, escape_string=_after_escape_string_prefix(command_str, idxq))
                 return (
                     command_str[:idxq] + wrapped + command_str[idxq + len(tokenq) :],
                     True,

execsql/types.py

@@ -884,6 +884,14 @@ dbt_oracle.name_datatype(DT_TimestampTZ, "TIMESTAMP WITH TIME ZONE")
 dbt_oracle.name_datatype(DT_Timestamp, "TIMESTAMP", casting_name="TIMESTAMP")
 dbt_oracle.name_datatype(DT_Date, "DATE", casting_name="DATE")
 dbt_oracle.name_datatype(DT_Time_Oracle, "VARCHAR2", True, casting_name="VARCHAR(20)")
+# Oracle has no time-of-day type, and oracledb cannot bind a datetime.time: a
+# time column is stored as its ISO text (at most 15 characters, HH:MM:SS.ffffff).
+dbt_oracle.name_datatype(
+    DT_Time,
+    "VARCHAR2(15)",
+    casting_name="VARCHAR(20)",
+    conv_mod_fn=lambda x: x.isoformat() if x is not None else None,
+)
 dbt_oracle.name_datatype(DT_Integer, "NUMBER")
 dbt_oracle.name_datatype(DT_Long, "NUMBER")
 dbt_oracle.name_datatype(DT_Float, "FLOAT")

execsql/utils/errors.py

@@ -37,8 +37,24 @@ __all__ = [
     "stamp_errinfo",
     "write_warning",
     "file_size_date",
+    "install_hint",
 ]
 
+_ADDING_AN_EXTRA = "https://execsql2.readthedocs.io/en/latest/getting-started/installation/#adding-an-extra-later"
+
+
+def install_hint(extra: str) -> str:
+    """How to install the execsql2 *extra*, for a message about a missing optional package.
+
+    ``uv tool install`` and ``pipx install --force`` replace an install's
+    extras with the ones named, so the hint says to keep those already there.
+    """
+    return (
+        f'Install the {extra} extra along with the extras you already have: uv tool install "execsql2[...,{extra}]" '
+        f'or pipx install --force "execsql2[...,{extra}]" (in a project environment: pip install "execsql2[{extra}]"). '
+        f"See {_ADDING_AN_EXTRA}"
+    )
+
 
 def exception_info() -> tuple:
     # Returns the exception type, value, source file name, source line number, and source line text.

{execsql2-2.27.0.dist-info → execsql2-2.29.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: execsql2
-Version: 2.27.0
+Version: 2.29.0
 Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
 Project-URL: Homepage, https://execsql2.readthedocs.io
 Project-URL: Repository, https://github.com/geocoug/execsql
@@ -95,8 +95,10 @@ Requires-Dist: openpyxl>=3.1; extra == 'dev'
 Requires-Dist: polars>=1.0; extra == 'dev'
 Requires-Dist: pre-commit>=3.5.0; extra == 'dev'
 Requires-Dist: pygls<3,>=2.1; extra == 'dev'
+Requires-Dist: pytest-asyncio>=1.0; extra == 'dev'
 Requires-Dist: pytest-cov>=5.0.0; extra == 'dev'
 Requires-Dist: pytest-lsp>=1.0; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
 Requires-Dist: pyyaml>=6.0; extra == 'dev'
 Requires-Dist: ruff>=0.4; extra == 'dev'
 Requires-Dist: sqlglot<30.22,>=30.21.0; extra == 'dev'
@@ -464,7 +466,7 @@ execsql format --no-sql --in-place scripts/
 ```yaml
 repos:
   - repo: https://github.com/geocoug/execsql
-    rev: v2.27.0
+    rev: v2.29.0
     hooks:
       - id: execsql-format
       - id: execsql-lint