dbt-sqlserver 1.11.0rc1__tar.gz → 1.11.1__tar.gz

{dbt_sqlserver-1.11.0rc1 → dbt_sqlserver-1.11.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbt-sqlserver
-Version: 1.11.0rc1
+Version: 1.11.1
 Summary: A Microsoft SQL Server adapter plugin for dbt
 Author: Mikael Ene, Anders Swanson, Sam Debruyn, Cor Zuurmond, Cody Scott
 License: MIT
@@ -14,10 +14,11 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Python: >=3.11
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: dbt-core<2.0,>=1.11.0
@@ -38,8 +39,19 @@ Dynamic: license-file
 
 [dbt](https://www.getdbt.com) adapter for Microsoft SQL Server and Azure SQL services.
 
-The adapter supports dbt-core 1.10 or newer and follows the same versioning scheme.
-E.g. version 1.10.x of the adapter is compatible with dbt-core 1.10.x.
+The adapter supports dbt-core 1.11 or newer and follows the same versioning scheme.
+E.g. version 1.11.x of the adapter is compatible with dbt-core 1.11.x.
+
+## Supported Python versions
+
+The adapter is tested against:
+
+| Python version | Status |
+|---|---|
+| 3.10 | Installable (not tested in CI) |
+| 3.11 | Officially supported |
+| 3.12 | Officially supported |
+| 3.13 | Officially supported |
 
 ## Supported SQL Server versions
 

{dbt_sqlserver-1.11.0rc1 → dbt_sqlserver-1.11.1}/README.md

@@ -2,8 +2,19 @@
 
 [dbt](https://www.getdbt.com) adapter for Microsoft SQL Server and Azure SQL services.
 
-The adapter supports dbt-core 1.10 or newer and follows the same versioning scheme.
-E.g. version 1.10.x of the adapter is compatible with dbt-core 1.10.x.
+The adapter supports dbt-core 1.11 or newer and follows the same versioning scheme.
+E.g. version 1.11.x of the adapter is compatible with dbt-core 1.11.x.
+
+## Supported Python versions
+
+The adapter is tested against:
+
+| Python version | Status |
+|---|---|
+| 3.10 | Installable (not tested in CI) |
+| 3.11 | Officially supported |
+| 3.12 | Officially supported |
+| 3.13 | Officially supported |
 
 ## Supported SQL Server versions
 

dbt_sqlserver-1.11.1/dbt/adapters/sqlserver/__version__.py

@@ -0,0 +1 @@
+version = "1.11.1"

{dbt_sqlserver-1.11.0rc1 → dbt_sqlserver-1.11.1}/dbt/adapters/sqlserver/sqlserver_adapter.py

@@ -1,3 +1,4 @@
+import re
 from typing import Any, Dict, List, Optional
 
 import agate
@@ -26,16 +27,118 @@ from dbt.adapters.sqlserver.relation_configs.index import (
     index_config_changes,
     normalize_drop_unmanaged,
 )
+from dbt.adapters.sqlserver.sqlserver_auth import is_mssql_python_backend
 from dbt.adapters.sqlserver.sqlserver_column import SQLServerColumn, SQLServerColumnNative
 from dbt.adapters.sqlserver.sqlserver_configs import SQLServerConfigs
-from dbt.adapters.sqlserver.sqlserver_connections import SQLServerConnectionManager
+from dbt.adapters.sqlserver.sqlserver_connections import (
+    SQLServerConnectionManager,
+    _discard_pending_results,
+)
 from dbt.adapters.sqlserver.sqlserver_mask import ColumnMask
 from dbt.adapters.sqlserver.sqlserver_mask import mask_changes as _mask_changes
 from dbt.adapters.sqlserver.sqlserver_mask import resolve_masks as _resolve_masks
 from dbt.adapters.sqlserver.sqlserver_relation import SQLServerRelation
+from dbt.adapters.sqlserver.sqlserver_runtime import _get_pyodbc
 
 logger = AdapterLogger("SQLServer")
 
+# Mirrors sqlserver__select_starts_with_cte
+# (dbt/include/sqlserver/macros/adapters/columns.sql): a query opening with a
+# CTE cannot be neutered as ``select * from (...) where 1 = 0``, so it reaches
+# get_column_schema_from_query unwrapped and would otherwise be executed in
+# full just to read its column names.
+_SQL_COMMENT = re.compile(r"(?s)/\*.*?\*/|--[^\n]*\n")
+
+# sp_describe_first_result_set reports true SQL Server types; reading
+# ``cursor.description`` reports Python classes, which collapse whole families
+# (every integer width arrives as ``int``, every string type as ``varchar``).
+# Contract comparison comes through this method either way, so the describe
+# path is mapped back onto exactly the names the execute path yields via
+# ``data_type_code_to_name``. TestCteProbeAvoidsExecution pins the two
+# together; a type missing from this map falls back to executing rather than
+# guessing.
+#
+# The names below are pyodbc's. The class a driver picks for a column is its
+# own choice, not SQL Server's, and mssql-python decodes three of these
+# differently -- see ``_MSSQL_PYTHON_TYPE_OVERRIDES``.
+_SYSTEM_TYPE_TO_EXECUTED_NAME = {
+    "bigint": "int",
+    "int": "int",
+    "smallint": "int",
+    "tinyint": "int",
+    "bit": "bit",
+    "decimal": "decimal",
+    "numeric": "decimal",
+    "money": "decimal",
+    "smallmoney": "decimal",
+    "float": "float",
+    "real": "float",
+    "date": "date",
+    "time": "time",
+    "datetime": "datetime2(6)",
+    "smalldatetime": "datetime2(6)",
+    "datetime2": "datetime2(6)",
+    "char": "varchar",
+    "nchar": "varchar",
+    "varchar": "varchar",
+    "nvarchar": "varchar",
+    "text": "varchar",
+    "ntext": "varchar",
+    "xml": "varchar",
+    "uniqueidentifier": "varchar",
+    "binary": "varbinary",
+    "varbinary": "varbinary",
+    "image": "varbinary",
+    "timestamp": "varbinary",
+    "rowversion": "varbinary",
+    # datetimeoffset is deliberately absent *from this map*: add_query
+    # registers the -155 output converter after the execute that needed it,
+    # so pyodbc reports the column as bytearray on a connection's first query
+    # and as str on every one after. No fixed name mirrors that, so on pyodbc
+    # describing gives up and the query is executed -- which agrees with
+    # itself by construction. mssql-python needs no converter and is pinned
+    # in _MSSQL_PYTHON_TYPE_OVERRIDES.
+    "sql_variant": "varbinary",
+    "hierarchyid": "varbinary",
+    "geography": "varbinary",
+    "geometry": "varbinary",
+}
+
+# mssql-python decodes three types into richer Python objects than pyodbc
+# does: uniqueidentifier as ``uuid.UUID`` (pyodbc: ``str``), datetimeoffset
+# as ``datetime`` and sql_variant as ``str`` (pyodbc: ``bytearray`` for
+# both). Executing reports those classes, so describing has to agree.
+_MSSQL_PYTHON_TYPE_OVERRIDES = {
+    "uniqueidentifier": "uniqueidentifier",
+    "datetimeoffset": "datetime2(6)",
+    "sql_variant": "varchar",
+}
+
+
+def _executed_name_for_system_type(base_type: str, backend: Any) -> Optional[str]:
+    """The name the executed probe would report for a described column type.
+
+    None means "no confident answer" -- the caller then executes the query,
+    which is slower but cannot disagree with itself.
+    """
+    if is_mssql_python_backend(backend):
+        overridden = _MSSQL_PYTHON_TYPE_OVERRIDES.get(base_type)
+        if overridden is not None:
+            return overridden
+
+    elif base_type == "uniqueidentifier":
+        # pyodbc yields uuid.UUID or str for a GUID depending on its
+        # module-level ``native_uuid`` flag -- process-global state anything
+        # in the process can flip, so read it rather than assume a default.
+        try:
+            native_uuid = bool(_get_pyodbc().native_uuid)
+        except Exception as e:  # pragma: no cover - pyodbc is present if in use
+            logger.debug(f"Could not read pyodbc.native_uuid, executing the query: {e}")
+            return None
+        return "uniqueidentifier" if native_uuid else "varchar"
+
+    return _SYSTEM_TYPE_TO_EXECUTED_NAME.get(base_type)
+
 
 class SQLServerAdapter(SQLAdapter):
     """
@@ -77,14 +180,6 @@ class SQLServerAdapter(SQLAdapter):
     @property
     def _behavior_flags(self) -> List[BehaviorFlag]:
         return [
-            {
-                "name": "empty",
-                "default": False,
-                "description": (
-                    "When enabled, table and view materializations will be created as empty "
-                    "structures (no data)."
-                ),
-            },
             {
                 "name": "dbt_sqlserver_use_default_schema_concat",
                 "default": False,
@@ -143,18 +238,108 @@ class SQLServerAdapter(SQLAdapter):
 
     @available.parse(lambda *a, **k: [])
     def get_column_schema_from_query(self, sql: str) -> List[BaseColumn]:
-        """Get a list of the Columns with names and data types from the given sql."""
+        """Get a list of the Columns with names and data types from the given sql.
+
+        Only the result *shape* is wanted, but the query still runs, so the
+        cursor comes back holding the whole result set. Usually that set is
+        empty: dbt-core's ``get_column_schema_from_query`` macro wraps the
+        query first, and ``sqlserver__get_empty_subquery_sql`` renders that as
+        ``select * from (...) where 1 = 0``. A query that opens with a CTE
+        cannot be wrapped that way, though, and is passed through untouched
+        (dbt/include/sqlserver/macros/adapters/columns.sql), so snapshot
+        staging queries and CTE-headed contract models arrive here in full.
+
+        Either way the cursor must not be abandoned holding rows -- see
+        ``_discard_pending_results`` for what that costs.
+        """
+        if _SQL_COMMENT.sub("", sql).strip().lower().startswith("with"):
+            described = self._describe_result_set(sql)
+            if described is not None:
+                return described
+
         _, cursor = self.connections.add_select_query(sql)
 
-        columns = [
-            self.Column.create(
-                column_name, self.connections.data_type_code_to_name(column_type_code)
-            )
-            # https://peps.python.org/pep-0249/#description
-            for column_name, column_type_code, *_ in cursor.description
-        ]
+        try:
+            columns = [
+                self.Column.create(
+                    column_name, self.connections.data_type_code_to_name(column_type_code)
+                )
+                # https://peps.python.org/pep-0249/#description
+                for column_name, column_type_code, *_ in cursor.description
+            ]
+        finally:
+            _discard_pending_results(cursor)
+
         return columns
 
+    def _describe_result_set(self, sql: str) -> Optional[List[BaseColumn]]:
+        """Read a query's column shape without running it, or None to fall back.
+
+        ``sp_describe_first_result_set`` compiles the query and reports its
+        result shape, which is all this method ever wanted. It is already how
+        ``sqlserver__get_columns_in_query`` handles CTEs (#698).
+
+        Returns None -- deliberately, rather than raising -- whenever the
+        describe cannot be trusted to match what executing would have reported:
+        a query it refuses to describe (it cannot see through ``#temp`` tables,
+        where executing works), or a type this backend's driver has no known
+        executed name for. The caller then executes as before, which is
+        slower but never disagrees with itself.
+        """
+        credentials = self.connections.profile.credentials
+
+        # Inline rather than bound: mssql-python binds str as varchar and the
+        # procedure demands nvarchar(max). columns.sql:24 escapes it the same
+        # way for the same reason.
+        describe_sql = "exec sp_describe_first_result_set @tsql = N'{}'".format(
+            sql.replace("'", "''")
+        )
+
+        try:
+            _, cursor = self.connections.add_select_query(describe_sql)
+        except Exception as e:
+            logger.debug(f"Could not describe a CTE query, falling back to executing it: {e}")
+            return None
+
+        try:
+            fields = [description[0].lower() for description in cursor.description]
+            rows = cursor.fetchall()
+        except Exception as e:
+            logger.debug(f"Could not read a described result set, executing the query: {e}")
+            return None
+        finally:
+            _discard_pending_results(cursor)
+
+        try:
+            hidden, name, type_name = (
+                fields.index("is_hidden"),
+                fields.index("name"),
+                fields.index("system_type_name"),
+            )
+        except ValueError:  # pragma: no cover - shape is fixed by SQL Server
+            return None
+
+        columns = []
+        for row in rows:
+            if row[hidden]:
+                continue
+            # "varchar(10)" / "decimal(10,2)" -> "varchar" / "decimal"
+            base_type = str(row[type_name]).split("(")[0].strip().lower()
+            executed_name = _executed_name_for_system_type(base_type, credentials.backend)
+            if executed_name is None or row[name] is None:
+                logger.debug(
+                    f"Describing a CTE query reported {base_type!r}, which has no "
+                    "equivalent in the executed path; executing it instead"
+                )
+                return None
+            columns.append(self.Column.create(row[name], executed_name))
+
+        # Every select has at least one column, so nothing described means
+        # sp_describe_first_result_set could not work the shape out. Returning
+        # an empty list would read as "this query has no columns" and surface
+        # as a baffling contract mismatch; execute instead.
+        return columns or None
+
     @classmethod
     def convert_boolean_type(cls, agate_table, col_idx):
         return "bit"
@@ -332,7 +517,10 @@ class SQLServerAdapter(SQLAdapter):
         """Return the number of rows in the given relation."""
         sql = f"SELECT COUNT_BIG(*) FROM {relation}"
         _, cursor = self.connections.add_select_query(sql)
-        row = cursor.fetchone()
+        try:
+            row = cursor.fetchone()
+        finally:
+            _discard_pending_results(cursor)
         return int(row[0]) if row else 0
 
     def expand_column_types(self, goal, current, max_rows: int = 1000000):
@@ -427,6 +615,57 @@ class SQLServerAdapter(SQLAdapter):
     def parse_index(self, raw_index: Any) -> Optional[SQLServerIndexConfig]:
         return SQLServerIndexConfig.parse(raw_index)
 
+    @available
+    def index_needs_own_batch(self, raw_index: Any) -> bool:
+        """True when raw_index's build_options (ONLINE / RESUMABLE) force its
+        CREATE INDEX to run outside any transaction: SQL Server rejects
+        RESUMABLE inside a user transaction (error 574), and an ONLINE build
+        wrapped in one holds its locks until commit, negating the point."""
+        parsed = self.parse_index(raw_index)
+        if not parsed:
+            return False
+        return create_needs_own_batch(parsed.build_options)
+
+    @available
+    def commit_if_open(self) -> None:
+        """Commit the current transaction if one is open - a no-op otherwise.
+
+        dbt_sqlserver_use_dbt_transactions on (default) wraps a
+        materialization's whole build, from its first statement through its
+        own trailing ``adapter.commit()``, in one continuous ambient
+        transaction. Some statements must not share that transaction with
+        whatever runs after them: an ONLINE/RESUMABLE index build (SQL Server
+        rejects RESUMABLE inside a user transaction outright, and ONLINE
+        holds its locks until commit either way), or a full-refresh-in-
+        progress marker, which exists specifically to survive a later
+        failure and so must not roll back with it. This makes the prior
+        statement durable on its own; pair with begin_if_closed once the
+        statement(s) that must run outside a transaction are done, to leave
+        later code (more such statements, or the materialization's own
+        trailing ``adapter.commit()``, which raises if it finds nothing open)
+        working as if this call had never happened.
+
+        A no-op when no transaction is open: the caller's statement already
+        ran autocommitted on its own (e.g. via ``run_query``'s
+        ``auto_begin=false``, before anything else began one), so there is
+        nothing to flush. Also a no-op, at the SQL level, whenever
+        dbt_sqlserver_use_dbt_transactions is off: begin/commit still flip
+        dbt-core's bookkeeping (see
+        SQLServerConnectionManager.add_begin_query/add_commit_query), but
+        emit no real T-SQL, matching the driver's own autocommit.
+        """
+        connection = self.connections.get_thread_connection()
+        if connection is not None and connection.transaction_open:
+            self.connections.commit()
+
+    @available
+    def begin_if_closed(self) -> None:
+        """Begin a transaction if none is open - a no-op otherwise. See
+        commit_if_open, which this pairs with."""
+        connection = self.connections.get_thread_connection()
+        if connection is not None and not connection.transaction_open:
+            self.connections.begin()
+
     @available
     def validate_indexes(
         self, raw_indexes: Any, as_columnstore: Any = False, drop_unmanaged: Any = False

{dbt_sqlserver-1.11.0rc1 → dbt_sqlserver-1.11.1}/dbt/adapters/sqlserver/sqlserver_connections.py

@@ -61,6 +61,64 @@ from dbt.adapters.sqlserver.sqlserver_runtime import (
 
 logger = AdapterLogger("sqlserver")
 
+
+def _try_drain_nextset(cursor: Any) -> bool:
+    """Drain one additional result set from *cursor*, if supported.
+
+    Returns ``True`` if a further result set was consumed, ``False`` if the
+    cursor has no more result sets.
+    """
+    return bool(cursor.nextset())
+
+
+# Rows per round trip when shedding a result set nobody asked for. Large
+# enough that discarding even a big one costs a handful of round trips.
+_DISCARD_CHUNK_SIZE = 10000
+
+
+def _discard_pending_results(cursor: Any) -> None:
+    """Consume and close *cursor*, leaving nothing for the driver to cancel.
+
+    Closing a cursor whose result set the server is still producing makes the
+    driver cancel the request, and the cancel reaches SQL Server as an
+    *attention*. Every connection opened here runs ``SET XACT_ABORT ON``
+    (``_apply_session_settings``, for dbt-msft/dbt-sqlserver#718), and SQL
+    Server answers an attention under ``XACT_ABORT ON`` by rolling back the
+    open transaction.
+
+    None of that raises -- an attention is not an error -- so a materialization
+    that abandons a cursor part-way through a build silently loses whatever it
+    had already created inside that transaction, then fails further on against
+    relations that no longer exist. Fetching the rows first makes the close an
+    ordinary one.
+
+    Failures while discarding are logged and swallowed: the caller already has
+    what it came for, and the connection is about to be reused for real work,
+    so trouble shedding rows nobody wanted must not become the error the user
+    sees.
+    """
+    try:
+        while True:
+            # ``description`` is None for statements that return no rows at
+            # all, where fetching would raise rather than yield nothing.
+            if cursor.description is not None:
+                while cursor.fetchmany(_DISCARD_CHUNK_SIZE):
+                    pass
+            # nextset() only advances; whatever rows the set it lands on holds
+            # still have to be fetched, hence the outer loop.
+            if not _try_drain_nextset(cursor):
+                break
+    except Exception as e:
+        # AdapterLogger cannot serialize an exception as a log argument, so
+        # interpolate rather than passing ``e`` through.
+        logger.debug(f"Discarding a pending result set failed: {e}")
+
+    try:
+        cursor.close()
+    except Exception as e:
+        logger.debug(f"Closing a cursor failed: {e}")
+
+
 # Attribute used to stash the in-flight pyodbc / mssql-python cursor on a
 # Connection so cancel() can reach it from another thread. See cancel().
 _IN_FLIGHT_CURSOR_ATTR = "_dbt_sqlserver_in_flight_cursor"

{dbt_sqlserver-1.11.0rc1 → dbt_sqlserver-1.11.1}/dbt/adapters/sqlserver/sqlserver_runtime.py

@@ -45,6 +45,11 @@ class PyodbcModuleProtocol(Protocol):
     InterfaceError: type[Exception]
     DatabaseError: type[Exception]
     pooling: bool
+    # Whether pyodbc decodes a uniqueidentifier column as uuid.UUID rather
+    # than str. Read by the CTE probe, which has to report the same type
+    # executing the query would have. Module-global, so it can change under
+    # us; nothing here sets it.
+    native_uuid: bool
 
     def connect(self, *args: Any, **kwargs: Any) -> Any: ...