databricks-sql-connector 4.1.3__tar.gz → 4.2.0__tar.gz

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Release History
 
+# 4.2.0 (2025-11-14)
+- Add multi-statement transaction support (databricks/databricks-sql-python#704 by @jayantsing-db)
+- Add a workflow to parallelise the E2E tests (databricks/databricks-sql-python#697 by @msrathore-db)
+- Bring Python telemetry event model consistent with JDBC (databricks/databricks-sql-python#701 by @nikhilsuri-db)
+
+# 4.1.4 (2025-10-15)
+- Add support for Token Federation (databricks/databricks-sql-python#691 by @madhav-db)
+- Add metric view support (databricks/databricks-sql-python#688 by @shivam2680)
+- Increased time limit for long running queries (databricks/databricks-sql-python#686 by @jprakash-db)
+
 # 4.1.3 (2025-09-17)
 - Query tags integration (databricks/databricks-sql-python#663 by @sreekanth-db)
 - Add variant support (databricks/databricks-sql-python#560 by @shivam2680)

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-sql-connector
-Version: 4.1.3
+Version: 4.2.0
 Summary: Databricks SQL Connector for Python
 License: Apache-2.0
 License-File: LICENSE
@@ -102,6 +102,12 @@ or to a Databricks Runtime interactive cluster (e.g. /sql/protocolv1/o/123456789
 > to authenticate the target Databricks user account and needs to open the browser for authentication. So it 
 > can only run on the user's machine.
 
+## Transaction Support
+
+The connector supports multi-statement transactions with manual commit/rollback control. Set `connection.autocommit = False` to disable autocommit mode, then use `connection.commit()` and `connection.rollback()` to control transactions.
+
+For detailed documentation, examples, and best practices, see **[TRANSACTIONS.md](TRANSACTIONS.md)**.
+
 ## SQLAlchemy
 Starting from `databricks-sql-connector` version 4.0.0 SQLAlchemy support has been extracted to a new library `databricks-sqlalchemy`.
 

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/README.md

@@ -67,6 +67,12 @@ or to a Databricks Runtime interactive cluster (e.g. /sql/protocolv1/o/123456789
 > to authenticate the target Databricks user account and needs to open the browser for authentication. So it 
 > can only run on the user's machine.
 
+## Transaction Support
+
+The connector supports multi-statement transactions with manual commit/rollback control. Set `connection.autocommit = False` to disable autocommit mode, then use `connection.commit()` and `connection.rollback()` to control transactions.
+
+For detailed documentation, examples, and best practices, see **[TRANSACTIONS.md](TRANSACTIONS.md)**.
+
 ## SQLAlchemy
 Starting from `databricks-sql-connector` version 4.0.0 SQLAlchemy support has been extracted to a new library `databricks-sqlalchemy`.
 

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databricks-sql-connector"
-version = "4.1.3"
+version = "4.2.0"
 description = "Databricks SQL Connector for Python"
 authors = ["Databricks <databricks-sql-connector-maintainers@databricks.com>"]
 license = "Apache-2.0"
@@ -39,6 +39,7 @@ pylint = ">=2.12.0"
 black = "^22.3.0"
 pytest-dotenv = "^0.5.2"
 pytest-cov = "^4.0.0"
+pytest-xdist = "^3.0.0"
 numpy = [
     { version = ">=1.16.6", python = ">=3.8,<3.11" },
     { version = ">=1.23.4", python = ">=3.11" },

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/src/databricks/sql/__init__.py

@@ -8,6 +8,9 @@ threadsafety = 1  # Threads may share the module, but not connections.
 
 paramstyle = "named"
 
+# Transaction isolation level constants (extension to PEP 249)
+TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ = "REPEATABLE_READ"
+
 import re
 
 from typing import TYPE_CHECKING
@@ -68,7 +71,7 @@ DATETIME = DBAPITypeObject("timestamp")
 DATE = DBAPITypeObject("date")
 ROWID = DBAPITypeObject()
 
-__version__ = "4.1.3"
+__version__ = "4.2.0"
 USER_AGENT_NAME = "PyDatabricksSqlConnector"
 
 # These two functions are pyhive legacy

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/src/databricks/sql/auth/auth.py

@@ -8,13 +8,17 @@ from databricks.sql.auth.authenticators import (
     AzureServicePrincipalCredentialProvider,
 )
 from databricks.sql.auth.common import AuthType, ClientContext
+from databricks.sql.auth.token_federation import TokenFederationProvider
 
 
 def get_auth_provider(cfg: ClientContext, http_client):
+    # Determine the base auth provider
+    base_provider: Optional[AuthProvider] = None
+
     if cfg.credentials_provider:
-        return ExternalAuthProvider(cfg.credentials_provider)
+        base_provider = ExternalAuthProvider(cfg.credentials_provider)
     elif cfg.auth_type == AuthType.AZURE_SP_M2M.value:
-        return ExternalAuthProvider(
+        base_provider = ExternalAuthProvider(
             AzureServicePrincipalCredentialProvider(
                 cfg.hostname,
                 cfg.azure_client_id,
@@ -29,7 +33,7 @@ def get_auth_provider(cfg: ClientContext, http_client):
         assert cfg.oauth_client_id is not None
         assert cfg.oauth_scopes is not None
 
-        return DatabricksOAuthProvider(
+        base_provider = DatabricksOAuthProvider(
             cfg.hostname,
             cfg.oauth_persistence,
             cfg.oauth_redirect_port_range,
@@ -39,17 +43,17 @@ def get_auth_provider(cfg: ClientContext, http_client):
             cfg.auth_type,
         )
     elif cfg.access_token is not None:
-        return AccessTokenAuthProvider(cfg.access_token)
+        base_provider = AccessTokenAuthProvider(cfg.access_token)
     elif cfg.use_cert_as_auth and cfg.tls_client_cert_file:
         # no op authenticator. authentication is performed using ssl certificate outside of headers
-        return AuthProvider()
+        base_provider = AuthProvider()
     else:
         if (
             cfg.oauth_redirect_port_range is not None
             and cfg.oauth_client_id is not None
             and cfg.oauth_scopes is not None
         ):
-            return DatabricksOAuthProvider(
+            base_provider = DatabricksOAuthProvider(
                 cfg.hostname,
                 cfg.oauth_persistence,
                 cfg.oauth_redirect_port_range,
@@ -61,6 +65,17 @@ def get_auth_provider(cfg: ClientContext, http_client):
         else:
             raise RuntimeError("No valid authentication settings!")
 
+    # Always wrap with token federation (falls back gracefully if not needed)
+    if base_provider:
+        return TokenFederationProvider(
+            hostname=cfg.hostname,
+            external_provider=base_provider,
+            http_client=http_client,
+            identity_federation_client_id=cfg.identity_federation_client_id,
+        )
+
+    return base_provider
+
 
 PYSQL_OAUTH_SCOPES = ["sql", "offline_access"]
 PYSQL_OAUTH_CLIENT_ID = "databricks-sql-python"
@@ -114,5 +129,6 @@ def get_python_sql_connector_auth_provider(hostname: str, http_client, **kwargs)
         else redirect_port_range,
         oauth_persistence=kwargs.get("experimental_oauth_persistence"),
         credentials_provider=kwargs.get("credentials_provider"),
+        identity_federation_client_id=kwargs.get("identity_federation_client_id"),
     )
     return get_auth_provider(cfg, http_client)

databricks_sql_connector-4.2.0/src/databricks/sql/auth/auth_utils.py

@@ -0,0 +1,64 @@
+import logging
+import jwt
+from datetime import datetime, timedelta
+from typing import Optional, Dict, Tuple
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+
+def parse_hostname(hostname: str) -> str:
+    """
+    Normalize the hostname to include scheme and trailing slash.
+
+    Args:
+        hostname: The hostname to normalize
+
+    Returns:
+        Normalized hostname with scheme and trailing slash
+    """
+    if not hostname.startswith("http://") and not hostname.startswith("https://"):
+        hostname = f"https://{hostname}"
+    if not hostname.endswith("/"):
+        hostname = f"{hostname}/"
+    return hostname
+
+
+def decode_token(access_token: str) -> Optional[Dict]:
+    """
+    Decode a JWT token without verification to extract claims.
+
+    Args:
+        access_token: The JWT access token to decode
+
+    Returns:
+        Decoded token claims or None if decoding fails
+    """
+    try:
+        return jwt.decode(access_token, options={"verify_signature": False})
+    except Exception as e:
+        logger.debug("Failed to decode JWT token: %s", e)
+        return None
+
+
+def is_same_host(url1: str, url2: str) -> bool:
+    """
+    Check if two URLs have the same host.
+
+    Args:
+        url1: First URL
+        url2: Second URL
+
+    Returns:
+        True if hosts are the same, False otherwise
+    """
+    try:
+        host1 = urlparse(url1).netloc
+        host2 = urlparse(url2).netloc
+        # Handle port differences (e.g., example.com vs example.com:443)
+        host1_without_port = host1.split(":")[0]
+        host2_without_port = host2.split(":")[0]
+        return host1_without_port == host2_without_port
+    except Exception as e:
+        logger.debug("Failed to parse URLs: %s", e)
+        return False

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/src/databricks/sql/auth/common.py

@@ -37,6 +37,7 @@ class ClientContext:
         tls_client_cert_file: Optional[str] = None,
         oauth_persistence=None,
         credentials_provider=None,
+        identity_federation_client_id: Optional[str] = None,
         # HTTP client configuration parameters
         ssl_options=None,  # SSLOptions type
         socket_timeout: Optional[float] = None,
@@ -65,6 +66,7 @@ class ClientContext:
         self.tls_client_cert_file = tls_client_cert_file
         self.oauth_persistence = oauth_persistence
         self.credentials_provider = credentials_provider
+        self.identity_federation_client_id = identity_federation_client_id
 
         # HTTP client configuration
         self.ssl_options = ssl_options

databricks_sql_connector-4.2.0/src/databricks/sql/auth/token_federation.py

@@ -0,0 +1,206 @@
+import logging
+import json
+from datetime import datetime, timedelta
+from typing import Optional, Dict, Tuple
+from urllib.parse import urlencode
+
+from databricks.sql.auth.authenticators import AuthProvider
+from databricks.sql.auth.auth_utils import (
+    parse_hostname,
+    decode_token,
+    is_same_host,
+)
+from databricks.sql.common.http import HttpMethod
+
+logger = logging.getLogger(__name__)
+
+
+class Token:
+    """
+    Represents an OAuth token with expiration management.
+    """
+
+    def __init__(self, access_token: str, token_type: str = "Bearer"):
+        """
+        Initialize a token.
+
+        Args:
+            access_token: The access token string
+            token_type: The token type (default: Bearer)
+        """
+        self.access_token = access_token
+        self.token_type = token_type
+        self.expiry_time = self._calculate_expiry()
+
+    def _calculate_expiry(self) -> datetime:
+        """
+        Calculate the token expiry time from JWT claims.
+
+        Returns:
+            The token expiry datetime
+        """
+        decoded = decode_token(self.access_token)
+        if decoded and "exp" in decoded:
+            # Use JWT exp claim with 1 minute buffer
+            return datetime.fromtimestamp(decoded["exp"]) - timedelta(minutes=1)
+        # Default to 1 hour if no expiry info
+        return datetime.now() + timedelta(hours=1)
+
+    def is_expired(self) -> bool:
+        """
+        Check if the token is expired.
+
+        Returns:
+            True if token is expired, False otherwise
+        """
+        return datetime.now() >= self.expiry_time
+
+    def to_dict(self) -> Dict[str, str]:
+        """
+        Convert token to dictionary format.
+
+        Returns:
+            Dictionary with access_token and token_type
+        """
+        return {
+            "access_token": self.access_token,
+            "token_type": self.token_type,
+        }
+
+
+class TokenFederationProvider(AuthProvider):
+    """
+    Implementation of Token Federation for Databricks SQL Python driver.
+
+    This provider exchanges third-party access tokens for Databricks in-house tokens
+    when the token issuer is different from the Databricks host.
+    """
+
+    TOKEN_EXCHANGE_ENDPOINT = "/oidc/v1/token"
+    TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
+    TOKEN_EXCHANGE_SUBJECT_TYPE = "urn:ietf:params:oauth:token-type:jwt"
+
+    def __init__(
+        self,
+        hostname: str,
+        external_provider: AuthProvider,
+        http_client,
+        identity_federation_client_id: Optional[str] = None,
+    ):
+        """
+        Initialize the Token Federation Provider.
+
+        Args:
+            hostname: The Databricks workspace hostname
+            external_provider: The external authentication provider
+            http_client: HTTP client for making requests (required)
+            identity_federation_client_id: Optional client ID for token federation
+        """
+        if not http_client:
+            raise ValueError("http_client is required for TokenFederationProvider")
+
+        self.hostname = parse_hostname(hostname)
+        self.external_provider = external_provider
+        self.http_client = http_client
+        self.identity_federation_client_id = identity_federation_client_id
+
+        self._cached_token: Optional[Token] = None
+        self._external_headers: Dict[str, str] = {}
+
+    def add_headers(self, request_headers: Dict[str, str]):
+        """Add authentication headers to the request."""
+
+        if self._cached_token and not self._cached_token.is_expired():
+            request_headers[
+                "Authorization"
+            ] = f"{self._cached_token.token_type} {self._cached_token.access_token}"
+            return
+
+        # Get the external headers first to check if we need token federation
+        self._external_headers = {}
+        self.external_provider.add_headers(self._external_headers)
+
+        # If no Authorization header from external provider, pass through all headers
+        if "Authorization" not in self._external_headers:
+            request_headers.update(self._external_headers)
+            return
+
+        token = self._get_token()
+        request_headers["Authorization"] = f"{token.token_type} {token.access_token}"
+
+    def _get_token(self) -> Token:
+        """Get or refresh the authentication token."""
+        # Check if cached token is still valid
+        if self._cached_token and not self._cached_token.is_expired():
+            return self._cached_token
+
+        # Extract token from already-fetched headers
+        auth_header = self._external_headers.get("Authorization", "")
+        token_type, access_token = self._extract_token_from_header(auth_header)
+
+        # Check if token exchange is needed
+        if self._should_exchange_token(access_token):
+            try:
+                token = self._exchange_token(access_token)
+                self._cached_token = token
+                return token
+            except Exception as e:
+                logger.warning("Token exchange failed, using external token: %s", e)
+
+        # Use external token directly
+        token = Token(access_token, token_type)
+        self._cached_token = token
+        return token
+
+    def _should_exchange_token(self, access_token: str) -> bool:
+        """Check if the token should be exchanged based on issuer."""
+        decoded = decode_token(access_token)
+        if not decoded:
+            return False
+
+        issuer = decoded.get("iss", "")
+        # Check if issuer host is different from Databricks host
+        return not is_same_host(issuer, self.hostname)
+
+    def _exchange_token(self, access_token: str) -> Token:
+        """Exchange the external token for a Databricks token."""
+        token_url = f"{self.hostname.rstrip('/')}{self.TOKEN_EXCHANGE_ENDPOINT}"
+
+        data = {
+            "grant_type": self.TOKEN_EXCHANGE_GRANT_TYPE,
+            "subject_token": access_token,
+            "subject_token_type": self.TOKEN_EXCHANGE_SUBJECT_TYPE,
+            "scope": "sql",
+            "return_original_token_if_authenticated": "true",
+        }
+
+        if self.identity_federation_client_id:
+            data["client_id"] = self.identity_federation_client_id
+
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Accept": "*/*",
+        }
+
+        body = urlencode(data)
+
+        response = self.http_client.request(
+            HttpMethod.POST, url=token_url, body=body, headers=headers
+        )
+
+        token_response = json.loads(response.data.decode())
+
+        return Token(
+            token_response["access_token"], token_response.get("token_type", "Bearer")
+        )
+
+    def _extract_token_from_header(self, auth_header: str) -> Tuple[str, str]:
+        """Extract token type and access token from Authorization header."""
+        if not auth_header:
+            raise ValueError("Authorization header is missing")
+
+        parts = auth_header.split(" ", 1)
+        if len(parts) != 2:
+            raise ValueError("Invalid Authorization header format")
+
+        return parts[0], parts[1]

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/src/databricks/sql/client.py

@@ -9,6 +9,7 @@ except ImportError:
 import json
 import os
 import decimal
+from urllib.parse import urlparse
 from uuid import UUID
 
 from databricks.sql import __version__
@@ -20,6 +21,8 @@ from databricks.sql.exc import (
     InterfaceError,
     NotSupportedError,
     ProgrammingError,
+    TransactionError,
+    DatabaseError,
 )
 
 from databricks.sql.thrift_api.TCLIService import ttypes
@@ -86,6 +89,9 @@ DEFAULT_ARRAY_SIZE = 100000
 
 NO_NATIVE_PARAMS: List = []
 
+# Transaction isolation level constants (extension to PEP 249)
+TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ = "REPEATABLE_READ"
+
 
 class Connection:
     def __init__(
@@ -200,6 +206,17 @@ class Connection:
                     STRUCT is returned as Dict[str, Any]
                     ARRAY is returned as numpy.ndarray
                 When False, complex types are returned as a strings. These are generally deserializable as JSON.
+            :param enable_metric_view_metadata: `bool`, optional (default is False)
+                When True, enables metric view metadata support by setting the
+                spark.sql.thriftserver.metadata.metricview.enabled session configuration.
+                This allows
+                1. cursor.tables() to return METRIC_VIEW table type
+                2. cursor.columns() to return "measure" column type
+            :param fetch_autocommit_from_server: `bool`, optional (default is False)
+                When True, the connection.autocommit property queries the server for current state
+                using SET AUTOCOMMIT instead of returning cached value.
+                Set to True if autocommit might be changed by external means (e.g., external SQL commands).
+                When False (default), uses cached state for better performance.
         """
 
         # Internal arguments in **kwargs:
@@ -248,6 +265,14 @@ class Connection:
             access_token_kv = {"access_token": access_token}
             kwargs = {**kwargs, **access_token_kv}
 
+        enable_metric_view_metadata = kwargs.get("enable_metric_view_metadata", False)
+        if enable_metric_view_metadata:
+            if session_configuration is None:
+                session_configuration = {}
+            session_configuration[
+                "spark.sql.thriftserver.metadata.metricview.enabled"
+            ] = "true"
+
         self.disable_pandas = kwargs.get("_disable_pandas", False)
         self.lz4_compression = kwargs.get("enable_query_result_lz4_compression", True)
         self.use_cloud_fetch = kwargs.get("use_cloud_fetch", True)
@@ -290,6 +315,9 @@ class Connection:
             kwargs.get("use_inline_params", False)
         )
         self.staging_allowed_local_path = kwargs.get("staging_allowed_local_path", None)
+        self._fetch_autocommit_from_server = kwargs.get(
+            "fetch_autocommit_from_server", False
+        )
 
         self.force_enable_telemetry = kwargs.get("force_enable_telemetry", False)
         self.enable_telemetry = kwargs.get("enable_telemetry", False)
@@ -308,6 +336,20 @@ class Connection:
             session_id_hex=self.get_session_id_hex()
         )
 
+        # Determine proxy usage
+        use_proxy = self.http_client.using_proxy()
+        proxy_host_info = None
+        if (
+            use_proxy
+            and self.http_client.proxy_uri
+            and isinstance(self.http_client.proxy_uri, str)
+        ):
+            parsed = urlparse(self.http_client.proxy_uri)
+            proxy_host_info = HostDetails(
+                host_url=parsed.hostname or self.http_client.proxy_uri,
+                port=parsed.port or 8080,
+            )
+
         driver_connection_params = DriverConnectionParameters(
             http_path=http_path,
             mode=DatabricksClientType.SEA
@@ -317,13 +359,31 @@ class Connection:
             auth_mech=TelemetryHelper.get_auth_mechanism(self.session.auth_provider),
             auth_flow=TelemetryHelper.get_auth_flow(self.session.auth_provider),
             socket_timeout=kwargs.get("_socket_timeout", None),
+            azure_workspace_resource_id=kwargs.get("azure_workspace_resource_id", None),
+            azure_tenant_id=kwargs.get("azure_tenant_id", None),
+            use_proxy=use_proxy,
+            use_system_proxy=use_proxy,
+            proxy_host_info=proxy_host_info,
+            use_cf_proxy=False,  # CloudFlare proxy not yet supported in Python
+            cf_proxy_host_info=None,  # CloudFlare proxy not yet supported in Python
+            non_proxy_hosts=None,
+            allow_self_signed_support=kwargs.get("_tls_no_verify", False),
+            use_system_trust_store=True,  # Python uses system SSL by default
+            enable_arrow=pyarrow is not None,
+            enable_direct_results=True,  # Always enabled in Python
+            enable_sea_hybrid_results=kwargs.get("use_hybrid_disposition", False),
+            http_connection_pool_size=kwargs.get("pool_maxsize", None),
+            rows_fetched_per_block=DEFAULT_ARRAY_SIZE,
+            async_poll_interval_millis=2000,  # Default polling interval
+            support_many_parameters=True,  # Native parameters supported
+            enable_complex_datatype_support=_use_arrow_native_complex_types,
+            allowed_volume_ingestion_paths=self.staging_allowed_local_path,
         )
 
         self._telemetry_client.export_initial_telemetry_log(
             driver_connection_params=driver_connection_params,
             user_agent=self.session.useragent_header,
         )
-        self.staging_allowed_local_path = kwargs.get("staging_allowed_local_path", None)
 
     def _set_use_inline_params_with_warning(self, value: Union[bool, str]):
         """Valid values are True, False, and "silent"
@@ -459,15 +519,261 @@ class Connection:
         if self.http_client:
             self.http_client.close()
 
-    def commit(self):
-        """No-op because Databricks does not support transactions"""
-        pass
+    @property
+    def autocommit(self) -> bool:
+        """
+        Get auto-commit mode for this connection.
 
-    def rollback(self):
-        raise NotSupportedError(
-            "Transactions are not supported on Databricks",
-            session_id_hex=self.get_session_id_hex(),
-        )
+        Extension to PEP 249. Returns cached value by default.
+        If fetch_autocommit_from_server=True was set during connection,
+        queries server for current state.
+
+        Returns:
+            bool: True if auto-commit is enabled, False otherwise
+
+        Raises:
+            InterfaceError: If connection is closed
+            TransactionError: If fetch_autocommit_from_server=True and query fails
+        """
+        if not self.open:
+            raise InterfaceError(
+                "Cannot get autocommit on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        if self._fetch_autocommit_from_server:
+            return self._fetch_autocommit_state_from_server()
+
+        return self.session.get_autocommit()
+
+    @autocommit.setter
+    def autocommit(self, value: bool) -> None:
+        """
+        Set auto-commit mode for this connection.
+
+        Extension to PEP 249. Executes SET AUTOCOMMIT command on server.
+
+        Args:
+            value: True to enable auto-commit, False to disable
+
+        Raises:
+            InterfaceError: If connection is closed
+            TransactionError: If server rejects the change
+        """
+        if not self.open:
+            raise InterfaceError(
+                "Cannot set autocommit on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        # Create internal cursor for transaction control
+        cursor = None
+        try:
+            cursor = self.cursor()
+            sql = f"SET AUTOCOMMIT = {'TRUE' if value else 'FALSE'}"
+            cursor.execute(sql)
+
+            # Update cached state on success
+            self.session.set_autocommit(value)
+
+        except DatabaseError as e:
+            # Wrap in TransactionError with context
+            raise TransactionError(
+                f"Failed to set autocommit to {value}: {e.message}",
+                context={
+                    **e.context,
+                    "operation": "set_autocommit",
+                    "autocommit_value": value,
+                },
+                session_id_hex=self.get_session_id_hex(),
+            ) from e
+        finally:
+            if cursor:
+                cursor.close()
+
+    def _fetch_autocommit_state_from_server(self) -> bool:
+        """
+        Query server for current autocommit state using SET AUTOCOMMIT.
+
+        Returns:
+            bool: Server's autocommit state
+
+        Raises:
+            TransactionError: If query fails
+        """
+        cursor = None
+        try:
+            cursor = self.cursor()
+            cursor.execute("SET AUTOCOMMIT")
+
+            # Fetch result: should return row with value column
+            result = cursor.fetchone()
+            if result is None:
+                raise TransactionError(
+                    "No result returned from SET AUTOCOMMIT query",
+                    context={"operation": "fetch_autocommit"},
+                    session_id_hex=self.get_session_id_hex(),
+                )
+
+            # Parse value (first column should be "true" or "false")
+            value_str = str(result[0]).lower()
+            autocommit_state = value_str == "true"
+
+            # Update cache
+            self.session.set_autocommit(autocommit_state)
+
+            return autocommit_state
+
+        except TransactionError:
+            # Re-raise TransactionError as-is
+            raise
+        except DatabaseError as e:
+            # Wrap other DatabaseErrors
+            raise TransactionError(
+                f"Failed to fetch autocommit state from server: {e.message}",
+                context={**e.context, "operation": "fetch_autocommit"},
+                session_id_hex=self.get_session_id_hex(),
+            ) from e
+        finally:
+            if cursor:
+                cursor.close()
+
+    def commit(self) -> None:
+        """
+        Commit the current transaction.
+
+        Per PEP 249. Should be called only when autocommit is disabled.
+
+        When autocommit is False:
+        - Commits the current transaction
+        - Server automatically starts new transaction
+
+        When autocommit is True:
+        - Server may throw error if no active transaction
+
+        Raises:
+            InterfaceError: If connection is closed
+            TransactionError: If commit fails (e.g., no active transaction)
+        """
+        if not self.open:
+            raise InterfaceError(
+                "Cannot commit on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        cursor = None
+        try:
+            cursor = self.cursor()
+            cursor.execute("COMMIT")
+
+        except DatabaseError as e:
+            raise TransactionError(
+                f"Failed to commit transaction: {e.message}",
+                context={**e.context, "operation": "commit"},
+                session_id_hex=self.get_session_id_hex(),
+            ) from e
+        finally:
+            if cursor:
+                cursor.close()
+
+    def rollback(self) -> None:
+        """
+        Rollback the current transaction.
+
+        Per PEP 249. Should be called only when autocommit is disabled.
+
+        When autocommit is False:
+        - Rolls back the current transaction
+        - Server automatically starts new transaction
+
+        When autocommit is True:
+        - ROLLBACK is forgiving (no-op, doesn't throw exception)
+
+        Note: ROLLBACK is safe to call even without active transaction.
+
+        Raises:
+            InterfaceError: If connection is closed
+            TransactionError: If rollback fails
+        """
+        if not self.open:
+            raise InterfaceError(
+                "Cannot rollback on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        cursor = None
+        try:
+            cursor = self.cursor()
+            cursor.execute("ROLLBACK")
+
+        except DatabaseError as e:
+            raise TransactionError(
+                f"Failed to rollback transaction: {e.message}",
+                context={**e.context, "operation": "rollback"},
+                session_id_hex=self.get_session_id_hex(),
+            ) from e
+        finally:
+            if cursor:
+                cursor.close()
+
+    def get_transaction_isolation(self) -> str:
+        """
+        Get the transaction isolation level.
+
+        Extension to PEP 249.
+
+        Databricks supports REPEATABLE_READ isolation level (Snapshot Isolation),
+        which is the default and only supported level.
+
+        Returns:
+            str: "REPEATABLE_READ" - the transaction isolation level constant
+
+        Raises:
+            InterfaceError: If connection is closed
+        """
+        if not self.open:
+            raise InterfaceError(
+                "Cannot get transaction isolation on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        return TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ
+
+    def set_transaction_isolation(self, level: str) -> None:
+        """
+        Set transaction isolation level.
+
+        Extension to PEP 249.
+
+        Databricks supports only REPEATABLE_READ isolation level (Snapshot Isolation).
+        This method validates that the requested level is supported but does not
+        execute any SQL, as REPEATABLE_READ is the default server behavior.
+
+        Args:
+            level: Isolation level. Must be "REPEATABLE_READ" or "REPEATABLE READ"
+                   (case-insensitive, underscores and spaces are interchangeable)
+
+        Raises:
+            InterfaceError: If connection is closed
+            NotSupportedError: If isolation level not supported
+        """
+        if not self.open:
+            raise InterfaceError(
+                "Cannot set transaction isolation on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        # Normalize and validate isolation level
+        normalized_level = level.upper().replace("_", " ")
+
+        if normalized_level != TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ.replace(
+            "_", " "
+        ):
+            raise NotSupportedError(
+                f"Setting transaction isolation level '{level}' is not supported. "
+                f"Only {TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ} is supported.",
+                session_id_hex=self.get_session_id_hex(),
+            )
 
 
 class Cursor:

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/src/databricks/sql/common/unified_http_client.py

@@ -301,6 +301,11 @@ class UnifiedHttpClient:
         """Check if proxy support is available (not whether it's being used for a specific request)."""
         return self._proxy_pool_manager is not None
 
+    @property
+    def proxy_uri(self) -> Optional[str]:
+        """Get the configured proxy URI, if any."""
+        return self._proxy_uri
+
     def close(self):
         """Close the underlying connection pools."""
         if self._direct_pool_manager:

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/src/databricks/sql/exc.py

@@ -70,6 +70,23 @@ class NotSupportedError(DatabaseError):
     pass
 
 
+class TransactionError(DatabaseError):
+    """
+    Exception raised for transaction-specific errors.
+
+    This exception is used when transaction control operations fail, such as:
+    - Setting autocommit mode (AUTOCOMMIT_SET_DURING_ACTIVE_TRANSACTION)
+    - Committing a transaction (MULTI_STATEMENT_TRANSACTION_NO_ACTIVE_TRANSACTION)
+    - Rolling back a transaction
+    - Setting transaction isolation level
+
+    The exception includes context about which transaction operation failed
+    and preserves the underlying cause via exception chaining.
+    """
+
+    pass
+
+
 ### Custom error classes ###
 class InvalidServerResponseError(OperationalError):
     """Thrown if the server does not set the initial namespace correctly"""

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/src/databricks/sql/session.py

@@ -45,6 +45,9 @@ class Session:
         self.schema = schema
         self.http_path = http_path
 
+        # Initialize autocommit state (JDBC default is True)
+        self._autocommit = True
+
         user_agent_entry = kwargs.get("user_agent_entry")
         if user_agent_entry is None:
             user_agent_entry = kwargs.get("_user_agent_entry")
@@ -168,6 +171,24 @@ class Session:
         """Get the session ID in hex format"""
         return self._session_id.hex_guid
 
+    def get_autocommit(self) -> bool:
+        """
+        Get the cached autocommit state for this session.
+
+        Returns:
+            bool: True if autocommit is enabled, False otherwise
+        """
+        return self._autocommit
+
+    def set_autocommit(self, value: bool) -> None:
+        """
+        Update the cached autocommit state for this session.
+
+        Args:
+            value: True to cache autocommit as enabled, False as disabled
+        """
+        self._autocommit = value
+
     def close(self) -> None:
         """Close the underlying session."""
         logger.info("Closing session %s", self.guid_hex)

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/src/databricks/sql/telemetry/models/event.py

@@ -38,6 +38,25 @@ class DriverConnectionParameters(JsonSerializableMixin):
         auth_mech (AuthMech): The authentication mechanism used
         auth_flow (AuthFlow): The authentication flow type
         socket_timeout (int): Connection timeout in milliseconds
+        azure_workspace_resource_id (str): Azure workspace resource ID
+        azure_tenant_id (str): Azure tenant ID
+        use_proxy (bool): Whether proxy is being used
+        use_system_proxy (bool): Whether system proxy is being used
+        proxy_host_info (HostDetails): Proxy host details if configured
+        use_cf_proxy (bool): Whether CloudFlare proxy is being used
+        cf_proxy_host_info (HostDetails): CloudFlare proxy host details if configured
+        non_proxy_hosts (list): List of hosts that bypass proxy
+        allow_self_signed_support (bool): Whether self-signed certificates are allowed
+        use_system_trust_store (bool): Whether system trust store is used
+        enable_arrow (bool): Whether Arrow format is enabled
+        enable_direct_results (bool): Whether direct results are enabled
+        enable_sea_hybrid_results (bool): Whether SEA hybrid results are enabled
+        http_connection_pool_size (int): HTTP connection pool size
+        rows_fetched_per_block (int): Number of rows fetched per block
+        async_poll_interval_millis (int): Async polling interval in milliseconds
+        support_many_parameters (bool): Whether many parameters are supported
+        enable_complex_datatype_support (bool): Whether complex datatypes are supported
+        allowed_volume_ingestion_paths (str): Allowed paths for volume ingestion
     """
 
     http_path: str
@@ -46,6 +65,25 @@ class DriverConnectionParameters(JsonSerializableMixin):
     auth_mech: Optional[AuthMech] = None
     auth_flow: Optional[AuthFlow] = None
     socket_timeout: Optional[int] = None
+    azure_workspace_resource_id: Optional[str] = None
+    azure_tenant_id: Optional[str] = None
+    use_proxy: Optional[bool] = None
+    use_system_proxy: Optional[bool] = None
+    proxy_host_info: Optional[HostDetails] = None
+    use_cf_proxy: Optional[bool] = None
+    cf_proxy_host_info: Optional[HostDetails] = None
+    non_proxy_hosts: Optional[list] = None
+    allow_self_signed_support: Optional[bool] = None
+    use_system_trust_store: Optional[bool] = None
+    enable_arrow: Optional[bool] = None
+    enable_direct_results: Optional[bool] = None
+    enable_sea_hybrid_results: Optional[bool] = None
+    http_connection_pool_size: Optional[int] = None
+    rows_fetched_per_block: Optional[int] = None
+    async_poll_interval_millis: Optional[int] = None
+    support_many_parameters: Optional[bool] = None
+    enable_complex_datatype_support: Optional[bool] = None
+    allowed_volume_ingestion_paths: Optional[str] = None
 
 
 @dataclass
@@ -111,6 +149,69 @@ class DriverErrorInfo(JsonSerializableMixin):
     stack_trace: str
 
 
+@dataclass
+class ChunkDetails(JsonSerializableMixin):
+    """
+    Contains detailed metrics about chunk downloads during result fetching.
+
+    These metrics are accumulated across all chunk downloads for a single statement.
+
+    Attributes:
+        initial_chunk_latency_millis (int): Latency of the first chunk download
+        slowest_chunk_latency_millis (int): Latency of the slowest chunk download
+        total_chunks_present (int): Total number of chunks available
+        total_chunks_iterated (int): Number of chunks actually downloaded
+        sum_chunks_download_time_millis (int): Total time spent downloading all chunks
+    """
+
+    initial_chunk_latency_millis: Optional[int] = None
+    slowest_chunk_latency_millis: Optional[int] = None
+    total_chunks_present: Optional[int] = None
+    total_chunks_iterated: Optional[int] = None
+    sum_chunks_download_time_millis: Optional[int] = None
+
+
+@dataclass
+class ResultLatency(JsonSerializableMixin):
+    """
+    Contains latency metrics for different phases of query execution.
+
+    This tracks two distinct phases:
+    1. result_set_ready_latency_millis: Time from query submission until results are available (execute phase)
+       - Set when execute() completes
+    2. result_set_consumption_latency_millis: Time spent iterating/fetching results (fetch phase)
+       - Measured from first fetch call until no more rows available
+       - In Java: tracked via markResultSetConsumption(hasNext) method
+       - Records start time on first fetch, calculates total on last fetch
+
+    Attributes:
+        result_set_ready_latency_millis (int): Time until query results are ready (execution phase)
+        result_set_consumption_latency_millis (int): Time spent fetching/consuming results (fetch phase)
+
+    """
+
+    result_set_ready_latency_millis: Optional[int] = None
+    result_set_consumption_latency_millis: Optional[int] = None
+
+
+@dataclass
+class OperationDetail(JsonSerializableMixin):
+    """
+    Contains detailed information about the operation being performed.
+
+    Attributes:
+        n_operation_status_calls (int): Number of status polling calls made
+        operation_status_latency_millis (int): Total latency of all status calls
+        operation_type (str): Specific operation type (e.g., EXECUTE_STATEMENT, LIST_TABLES, CANCEL_STATEMENT)
+        is_internal_call (bool): Whether this is an internal driver operation
+    """
+
+    n_operation_status_calls: Optional[int] = None
+    operation_status_latency_millis: Optional[int] = None
+    operation_type: Optional[str] = None
+    is_internal_call: Optional[bool] = None
+
+
 @dataclass
 class SqlExecutionEvent(JsonSerializableMixin):
     """
@@ -122,7 +223,10 @@ class SqlExecutionEvent(JsonSerializableMixin):
         is_compressed (bool): Whether the result is compressed
         execution_result (ExecutionResultFormat): Format of the execution result
         retry_count (int): Number of retry attempts made
-        chunk_id (int): ID of the chunk if applicable
+        chunk_id (int): ID of the chunk if applicable (used for error tracking)
+        chunk_details (ChunkDetails): Aggregated chunk download metrics
+        result_latency (ResultLatency): Latency breakdown by execution phase
+        operation_detail (OperationDetail): Detailed operation information
     """
 
     statement_type: StatementType
@@ -130,6 +234,9 @@ class SqlExecutionEvent(JsonSerializableMixin):
     execution_result: ExecutionResultFormat
     retry_count: Optional[int]
     chunk_id: Optional[int]
+    chunk_details: Optional[ChunkDetails] = None
+    result_latency: Optional[ResultLatency] = None
+    operation_detail: Optional[OperationDetail] = None
 
 
 @dataclass

{databricks_sql_connector-4.1.3 → databricks_sql_connector-4.2.0}/src/databricks/sql/telemetry/telemetry_client.py

@@ -380,7 +380,7 @@ class TelemetryClientFactory:
     # Shared flush thread for all clients
     _flush_thread = None
     _flush_event = threading.Event()
-    _flush_interval_seconds = 90
+    _flush_interval_seconds = 300  # 5 minutes
 
     DEFAULT_BATCH_SIZE = 100