databricks-sql-connector 4.1.4__tar.gz → 4.2.1__tar.gz

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Release History
 
+# 4.2.1 (2025-11-20)
+- Ignore transactions by default (databricks/databricks-sql-python#711 by @jayantsing-db)
+
+# 4.2.0 (2025-11-14)
+- Add multi-statement transaction support (databricks/databricks-sql-python#704 by @jayantsing-db)
+- Add a workflow to parallelise the E2E tests (databricks/databricks-sql-python#697 by @msrathore-db)
+- Bring Python telemetry event model consistent with JDBC (databricks/databricks-sql-python#701 by @nikhilsuri-db)
+
 # 4.1.4 (2025-10-15)
 - Add support for Token Federation (databricks/databricks-sql-python#691 by @madhav-db)
 - Add metric view support (databricks/databricks-sql-python#688 by @shivam2680)

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-sql-connector
-Version: 4.1.4
+Version: 4.2.1
 Summary: Databricks SQL Connector for Python
 License: Apache-2.0
 License-File: LICENSE
@@ -102,6 +102,12 @@ or to a Databricks Runtime interactive cluster (e.g. /sql/protocolv1/o/123456789
 > to authenticate the target Databricks user account and needs to open the browser for authentication. So it 
 > can only run on the user's machine.
 
+## Transaction Support
+
+The connector supports multi-statement transactions with manual commit/rollback control. Set `connection.autocommit = False` to disable autocommit mode, then use `connection.commit()` and `connection.rollback()` to control transactions.
+
+For detailed documentation, examples, and best practices, see **[TRANSACTIONS.md](TRANSACTIONS.md)**.
+
 ## SQLAlchemy
 Starting from `databricks-sql-connector` version 4.0.0 SQLAlchemy support has been extracted to a new library `databricks-sqlalchemy`.
 

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/README.md

@@ -67,6 +67,12 @@ or to a Databricks Runtime interactive cluster (e.g. /sql/protocolv1/o/123456789
 > to authenticate the target Databricks user account and needs to open the browser for authentication. So it 
 > can only run on the user's machine.
 
+## Transaction Support
+
+The connector supports multi-statement transactions with manual commit/rollback control. Set `connection.autocommit = False` to disable autocommit mode, then use `connection.commit()` and `connection.rollback()` to control transactions.
+
+For detailed documentation, examples, and best practices, see **[TRANSACTIONS.md](TRANSACTIONS.md)**.
+
 ## SQLAlchemy
 Starting from `databricks-sql-connector` version 4.0.0 SQLAlchemy support has been extracted to a new library `databricks-sqlalchemy`.
 

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databricks-sql-connector"
-version = "4.1.4"
+version = "4.2.1"
 description = "Databricks SQL Connector for Python"
 authors = ["Databricks <databricks-sql-connector-maintainers@databricks.com>"]
 license = "Apache-2.0"
@@ -39,6 +39,7 @@ pylint = ">=2.12.0"
 black = "^22.3.0"
 pytest-dotenv = "^0.5.2"
 pytest-cov = "^4.0.0"
+pytest-xdist = "^3.0.0"
 numpy = [
     { version = ">=1.16.6", python = ">=3.8,<3.11" },
     { version = ">=1.23.4", python = ">=3.11" },

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/src/databricks/sql/__init__.py

@@ -8,6 +8,9 @@ threadsafety = 1  # Threads may share the module, but not connections.
 
 paramstyle = "named"
 
+# Transaction isolation level constants (extension to PEP 249)
+TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ = "REPEATABLE_READ"
+
 import re
 
 from typing import TYPE_CHECKING
@@ -68,7 +71,7 @@ DATETIME = DBAPITypeObject("timestamp")
 DATE = DBAPITypeObject("date")
 ROWID = DBAPITypeObject()
 
-__version__ = "4.1.4"
+__version__ = "4.2.1"
 USER_AGENT_NAME = "PyDatabricksSqlConnector"
 
 # These two functions are pyhive legacy

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/src/databricks/sql/client.py

@@ -9,6 +9,7 @@ except ImportError:
 import json
 import os
 import decimal
+from urllib.parse import urlparse
 from uuid import UUID
 
 from databricks.sql import __version__
@@ -20,6 +21,8 @@ from databricks.sql.exc import (
     InterfaceError,
     NotSupportedError,
     ProgrammingError,
+    TransactionError,
+    DatabaseError,
 )
 
 from databricks.sql.thrift_api.TCLIService import ttypes
@@ -86,6 +89,9 @@ DEFAULT_ARRAY_SIZE = 100000
 
 NO_NATIVE_PARAMS: List = []
 
+# Transaction isolation level constants (extension to PEP 249)
+TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ = "REPEATABLE_READ"
+
 
 class Connection:
     def __init__(
@@ -98,6 +104,7 @@ class Connection:
         catalog: Optional[str] = None,
         schema: Optional[str] = None,
         _use_arrow_native_complex_types: Optional[bool] = True,
+        ignore_transactions: bool = True,
         **kwargs,
     ) -> None:
         """
@@ -206,6 +213,17 @@ class Connection:
                 This allows
                 1. cursor.tables() to return METRIC_VIEW table type
                 2. cursor.columns() to return "measure" column type
+            :param fetch_autocommit_from_server: `bool`, optional (default is False)
+                When True, the connection.autocommit property queries the server for current state
+                using SET AUTOCOMMIT instead of returning cached value.
+                Set to True if autocommit might be changed by external means (e.g., external SQL commands).
+                When False (default), uses cached state for better performance.
+            :param ignore_transactions: `bool`, optional (default is True)
+                When True, transaction-related operations behave as follows:
+                - commit(): no-op (does nothing)
+                - rollback(): raises NotSupportedError
+                - autocommit setter: no-op (does nothing)
+                When False, transaction operations execute normally.
         """
 
         # Internal arguments in **kwargs:
@@ -304,6 +322,10 @@ class Connection:
             kwargs.get("use_inline_params", False)
         )
         self.staging_allowed_local_path = kwargs.get("staging_allowed_local_path", None)
+        self._fetch_autocommit_from_server = kwargs.get(
+            "fetch_autocommit_from_server", False
+        )
+        self.ignore_transactions = ignore_transactions
 
         self.force_enable_telemetry = kwargs.get("force_enable_telemetry", False)
         self.enable_telemetry = kwargs.get("enable_telemetry", False)
@@ -322,6 +344,20 @@ class Connection:
             session_id_hex=self.get_session_id_hex()
         )
 
+        # Determine proxy usage
+        use_proxy = self.http_client.using_proxy()
+        proxy_host_info = None
+        if (
+            use_proxy
+            and self.http_client.proxy_uri
+            and isinstance(self.http_client.proxy_uri, str)
+        ):
+            parsed = urlparse(self.http_client.proxy_uri)
+            proxy_host_info = HostDetails(
+                host_url=parsed.hostname or self.http_client.proxy_uri,
+                port=parsed.port or 8080,
+            )
+
         driver_connection_params = DriverConnectionParameters(
             http_path=http_path,
             mode=DatabricksClientType.SEA
@@ -331,13 +367,31 @@ class Connection:
             auth_mech=TelemetryHelper.get_auth_mechanism(self.session.auth_provider),
             auth_flow=TelemetryHelper.get_auth_flow(self.session.auth_provider),
             socket_timeout=kwargs.get("_socket_timeout", None),
+            azure_workspace_resource_id=kwargs.get("azure_workspace_resource_id", None),
+            azure_tenant_id=kwargs.get("azure_tenant_id", None),
+            use_proxy=use_proxy,
+            use_system_proxy=use_proxy,
+            proxy_host_info=proxy_host_info,
+            use_cf_proxy=False,  # CloudFlare proxy not yet supported in Python
+            cf_proxy_host_info=None,  # CloudFlare proxy not yet supported in Python
+            non_proxy_hosts=None,
+            allow_self_signed_support=kwargs.get("_tls_no_verify", False),
+            use_system_trust_store=True,  # Python uses system SSL by default
+            enable_arrow=pyarrow is not None,
+            enable_direct_results=True,  # Always enabled in Python
+            enable_sea_hybrid_results=kwargs.get("use_hybrid_disposition", False),
+            http_connection_pool_size=kwargs.get("pool_maxsize", None),
+            rows_fetched_per_block=DEFAULT_ARRAY_SIZE,
+            async_poll_interval_millis=2000,  # Default polling interval
+            support_many_parameters=True,  # Native parameters supported
+            enable_complex_datatype_support=_use_arrow_native_complex_types,
+            allowed_volume_ingestion_paths=self.staging_allowed_local_path,
         )
 
         self._telemetry_client.export_initial_telemetry_log(
             driver_connection_params=driver_connection_params,
             user_agent=self.session.useragent_header,
         )
-        self.staging_allowed_local_path = kwargs.get("staging_allowed_local_path", None)
 
     def _set_use_inline_params_with_warning(self, value: Union[bool, str]):
         """Valid values are True, False, and "silent"
@@ -473,15 +527,286 @@ class Connection:
         if self.http_client:
             self.http_client.close()
 
-    def commit(self):
-        """No-op because Databricks does not support transactions"""
-        pass
+    @property
+    def autocommit(self) -> bool:
+        """
+        Get auto-commit mode for this connection.
 
-    def rollback(self):
-        raise NotSupportedError(
-            "Transactions are not supported on Databricks",
-            session_id_hex=self.get_session_id_hex(),
-        )
+        Extension to PEP 249. Returns cached value by default.
+        If fetch_autocommit_from_server=True was set during connection,
+        queries server for current state.
+
+        Returns:
+            bool: True if auto-commit is enabled, False otherwise
+
+        Raises:
+            InterfaceError: If connection is closed
+            TransactionError: If fetch_autocommit_from_server=True and query fails
+        """
+        if not self.open:
+            raise InterfaceError(
+                "Cannot get autocommit on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        if self._fetch_autocommit_from_server:
+            return self._fetch_autocommit_state_from_server()
+
+        return self.session.get_autocommit()
+
+    @autocommit.setter
+    def autocommit(self, value: bool) -> None:
+        """
+        Set auto-commit mode for this connection.
+
+        Extension to PEP 249. Executes SET AUTOCOMMIT command on server.
+
+        Args:
+            value: True to enable auto-commit, False to disable
+
+        When ignore_transactions is True:
+        - This method is a no-op (does nothing)
+
+        Raises:
+            InterfaceError: If connection is closed
+            TransactionError: If server rejects the change
+        """
+        # No-op when ignore_transactions is True
+        if self.ignore_transactions:
+            return
+
+        if not self.open:
+            raise InterfaceError(
+                "Cannot set autocommit on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        # Create internal cursor for transaction control
+        cursor = None
+        try:
+            cursor = self.cursor()
+            sql = f"SET AUTOCOMMIT = {'TRUE' if value else 'FALSE'}"
+            cursor.execute(sql)
+
+            # Update cached state on success
+            self.session.set_autocommit(value)
+
+        except DatabaseError as e:
+            # Wrap in TransactionError with context
+            raise TransactionError(
+                f"Failed to set autocommit to {value}: {e.message}",
+                context={
+                    **e.context,
+                    "operation": "set_autocommit",
+                    "autocommit_value": value,
+                },
+                session_id_hex=self.get_session_id_hex(),
+            ) from e
+        finally:
+            if cursor:
+                cursor.close()
+
+    def _fetch_autocommit_state_from_server(self) -> bool:
+        """
+        Query server for current autocommit state using SET AUTOCOMMIT.
+
+        Returns:
+            bool: Server's autocommit state
+
+        Raises:
+            TransactionError: If query fails
+        """
+        cursor = None
+        try:
+            cursor = self.cursor()
+            cursor.execute("SET AUTOCOMMIT")
+
+            # Fetch result: should return row with value column
+            result = cursor.fetchone()
+            if result is None:
+                raise TransactionError(
+                    "No result returned from SET AUTOCOMMIT query",
+                    context={"operation": "fetch_autocommit"},
+                    session_id_hex=self.get_session_id_hex(),
+                )
+
+            # Parse value (first column should be "true" or "false")
+            value_str = str(result[0]).lower()
+            autocommit_state = value_str == "true"
+
+            # Update cache
+            self.session.set_autocommit(autocommit_state)
+
+            return autocommit_state
+
+        except TransactionError:
+            # Re-raise TransactionError as-is
+            raise
+        except DatabaseError as e:
+            # Wrap other DatabaseErrors
+            raise TransactionError(
+                f"Failed to fetch autocommit state from server: {e.message}",
+                context={**e.context, "operation": "fetch_autocommit"},
+                session_id_hex=self.get_session_id_hex(),
+            ) from e
+        finally:
+            if cursor:
+                cursor.close()
+
+    def commit(self) -> None:
+        """
+        Commit the current transaction.
+
+        Per PEP 249. Should be called only when autocommit is disabled.
+
+        When autocommit is False:
+        - Commits the current transaction
+        - Server automatically starts new transaction
+
+        When autocommit is True:
+        - Server may throw error if no active transaction
+
+        When ignore_transactions is True:
+        - This method is a no-op (does nothing)
+
+        Raises:
+            InterfaceError: If connection is closed
+            TransactionError: If commit fails (e.g., no active transaction)
+        """
+        # No-op when ignore_transactions is True
+        if self.ignore_transactions:
+            return
+
+        if not self.open:
+            raise InterfaceError(
+                "Cannot commit on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        cursor = None
+        try:
+            cursor = self.cursor()
+            cursor.execute("COMMIT")
+
+        except DatabaseError as e:
+            raise TransactionError(
+                f"Failed to commit transaction: {e.message}",
+                context={**e.context, "operation": "commit"},
+                session_id_hex=self.get_session_id_hex(),
+            ) from e
+        finally:
+            if cursor:
+                cursor.close()
+
+    def rollback(self) -> None:
+        """
+        Rollback the current transaction.
+
+        Per PEP 249. Should be called only when autocommit is disabled.
+
+        When autocommit is False:
+        - Rolls back the current transaction
+        - Server automatically starts new transaction
+
+        When autocommit is True:
+        - ROLLBACK is forgiving (no-op, doesn't throw exception)
+
+        When ignore_transactions is True:
+        - Raises NotSupportedError
+
+        Note: ROLLBACK is safe to call even without active transaction.
+
+        Raises:
+            InterfaceError: If connection is closed
+            NotSupportedError: If ignore_transactions is True
+            TransactionError: If rollback fails
+        """
+        # Raise NotSupportedError when ignore_transactions is True
+        if self.ignore_transactions:
+            raise NotSupportedError(
+                "Transactions are not supported on Databricks",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        if not self.open:
+            raise InterfaceError(
+                "Cannot rollback on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        cursor = None
+        try:
+            cursor = self.cursor()
+            cursor.execute("ROLLBACK")
+
+        except DatabaseError as e:
+            raise TransactionError(
+                f"Failed to rollback transaction: {e.message}",
+                context={**e.context, "operation": "rollback"},
+                session_id_hex=self.get_session_id_hex(),
+            ) from e
+        finally:
+            if cursor:
+                cursor.close()
+
+    def get_transaction_isolation(self) -> str:
+        """
+        Get the transaction isolation level.
+
+        Extension to PEP 249.
+
+        Databricks supports REPEATABLE_READ isolation level (Snapshot Isolation),
+        which is the default and only supported level.
+
+        Returns:
+            str: "REPEATABLE_READ" - the transaction isolation level constant
+
+        Raises:
+            InterfaceError: If connection is closed
+        """
+        if not self.open:
+            raise InterfaceError(
+                "Cannot get transaction isolation on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        return TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ
+
+    def set_transaction_isolation(self, level: str) -> None:
+        """
+        Set transaction isolation level.
+
+        Extension to PEP 249.
+
+        Databricks supports only REPEATABLE_READ isolation level (Snapshot Isolation).
+        This method validates that the requested level is supported but does not
+        execute any SQL, as REPEATABLE_READ is the default server behavior.
+
+        Args:
+            level: Isolation level. Must be "REPEATABLE_READ" or "REPEATABLE READ"
+                   (case-insensitive, underscores and spaces are interchangeable)
+
+        Raises:
+            InterfaceError: If connection is closed
+            NotSupportedError: If isolation level not supported
+        """
+        if not self.open:
+            raise InterfaceError(
+                "Cannot set transaction isolation on closed connection",
+                session_id_hex=self.get_session_id_hex(),
+            )
+
+        # Normalize and validate isolation level
+        normalized_level = level.upper().replace("_", " ")
+
+        if normalized_level != TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ.replace(
+            "_", " "
+        ):
+            raise NotSupportedError(
+                f"Setting transaction isolation level '{level}' is not supported. "
+                f"Only {TRANSACTION_ISOLATION_LEVEL_REPEATABLE_READ} is supported.",
+                session_id_hex=self.get_session_id_hex(),
+            )
 
 
 class Cursor:

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/src/databricks/sql/common/unified_http_client.py

@@ -301,6 +301,11 @@ class UnifiedHttpClient:
         """Check if proxy support is available (not whether it's being used for a specific request)."""
         return self._proxy_pool_manager is not None
 
+    @property
+    def proxy_uri(self) -> Optional[str]:
+        """Get the configured proxy URI, if any."""
+        return self._proxy_uri
+
     def close(self):
         """Close the underlying connection pools."""
         if self._direct_pool_manager:

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/src/databricks/sql/exc.py

@@ -70,6 +70,23 @@ class NotSupportedError(DatabaseError):
     pass
 
 
+class TransactionError(DatabaseError):
+    """
+    Exception raised for transaction-specific errors.
+
+    This exception is used when transaction control operations fail, such as:
+    - Setting autocommit mode (AUTOCOMMIT_SET_DURING_ACTIVE_TRANSACTION)
+    - Committing a transaction (MULTI_STATEMENT_TRANSACTION_NO_ACTIVE_TRANSACTION)
+    - Rolling back a transaction
+    - Setting transaction isolation level
+
+    The exception includes context about which transaction operation failed
+    and preserves the underlying cause via exception chaining.
+    """
+
+    pass
+
+
 ### Custom error classes ###
 class InvalidServerResponseError(OperationalError):
     """Thrown if the server does not set the initial namespace correctly"""

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/src/databricks/sql/session.py

@@ -45,6 +45,9 @@ class Session:
         self.schema = schema
         self.http_path = http_path
 
+        # Initialize autocommit state (JDBC default is True)
+        self._autocommit = True
+
         user_agent_entry = kwargs.get("user_agent_entry")
         if user_agent_entry is None:
             user_agent_entry = kwargs.get("_user_agent_entry")
@@ -168,6 +171,24 @@ class Session:
         """Get the session ID in hex format"""
         return self._session_id.hex_guid
 
+    def get_autocommit(self) -> bool:
+        """
+        Get the cached autocommit state for this session.
+
+        Returns:
+            bool: True if autocommit is enabled, False otherwise
+        """
+        return self._autocommit
+
+    def set_autocommit(self, value: bool) -> None:
+        """
+        Update the cached autocommit state for this session.
+
+        Args:
+            value: True to cache autocommit as enabled, False as disabled
+        """
+        self._autocommit = value
+
     def close(self) -> None:
         """Close the underlying session."""
         logger.info("Closing session %s", self.guid_hex)

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/src/databricks/sql/telemetry/models/event.py

@@ -38,6 +38,25 @@ class DriverConnectionParameters(JsonSerializableMixin):
         auth_mech (AuthMech): The authentication mechanism used
         auth_flow (AuthFlow): The authentication flow type
         socket_timeout (int): Connection timeout in milliseconds
+        azure_workspace_resource_id (str): Azure workspace resource ID
+        azure_tenant_id (str): Azure tenant ID
+        use_proxy (bool): Whether proxy is being used
+        use_system_proxy (bool): Whether system proxy is being used
+        proxy_host_info (HostDetails): Proxy host details if configured
+        use_cf_proxy (bool): Whether CloudFlare proxy is being used
+        cf_proxy_host_info (HostDetails): CloudFlare proxy host details if configured
+        non_proxy_hosts (list): List of hosts that bypass proxy
+        allow_self_signed_support (bool): Whether self-signed certificates are allowed
+        use_system_trust_store (bool): Whether system trust store is used
+        enable_arrow (bool): Whether Arrow format is enabled
+        enable_direct_results (bool): Whether direct results are enabled
+        enable_sea_hybrid_results (bool): Whether SEA hybrid results are enabled
+        http_connection_pool_size (int): HTTP connection pool size
+        rows_fetched_per_block (int): Number of rows fetched per block
+        async_poll_interval_millis (int): Async polling interval in milliseconds
+        support_many_parameters (bool): Whether many parameters are supported
+        enable_complex_datatype_support (bool): Whether complex datatypes are supported
+        allowed_volume_ingestion_paths (str): Allowed paths for volume ingestion
     """
 
     http_path: str
@@ -46,6 +65,25 @@ class DriverConnectionParameters(JsonSerializableMixin):
     auth_mech: Optional[AuthMech] = None
     auth_flow: Optional[AuthFlow] = None
     socket_timeout: Optional[int] = None
+    azure_workspace_resource_id: Optional[str] = None
+    azure_tenant_id: Optional[str] = None
+    use_proxy: Optional[bool] = None
+    use_system_proxy: Optional[bool] = None
+    proxy_host_info: Optional[HostDetails] = None
+    use_cf_proxy: Optional[bool] = None
+    cf_proxy_host_info: Optional[HostDetails] = None
+    non_proxy_hosts: Optional[list] = None
+    allow_self_signed_support: Optional[bool] = None
+    use_system_trust_store: Optional[bool] = None
+    enable_arrow: Optional[bool] = None
+    enable_direct_results: Optional[bool] = None
+    enable_sea_hybrid_results: Optional[bool] = None
+    http_connection_pool_size: Optional[int] = None
+    rows_fetched_per_block: Optional[int] = None
+    async_poll_interval_millis: Optional[int] = None
+    support_many_parameters: Optional[bool] = None
+    enable_complex_datatype_support: Optional[bool] = None
+    allowed_volume_ingestion_paths: Optional[str] = None
 
 
 @dataclass
@@ -111,6 +149,69 @@ class DriverErrorInfo(JsonSerializableMixin):
     stack_trace: str
 
 
+@dataclass
+class ChunkDetails(JsonSerializableMixin):
+    """
+    Contains detailed metrics about chunk downloads during result fetching.
+
+    These metrics are accumulated across all chunk downloads for a single statement.
+
+    Attributes:
+        initial_chunk_latency_millis (int): Latency of the first chunk download
+        slowest_chunk_latency_millis (int): Latency of the slowest chunk download
+        total_chunks_present (int): Total number of chunks available
+        total_chunks_iterated (int): Number of chunks actually downloaded
+        sum_chunks_download_time_millis (int): Total time spent downloading all chunks
+    """
+
+    initial_chunk_latency_millis: Optional[int] = None
+    slowest_chunk_latency_millis: Optional[int] = None
+    total_chunks_present: Optional[int] = None
+    total_chunks_iterated: Optional[int] = None
+    sum_chunks_download_time_millis: Optional[int] = None
+
+
+@dataclass
+class ResultLatency(JsonSerializableMixin):
+    """
+    Contains latency metrics for different phases of query execution.
+
+    This tracks two distinct phases:
+    1. result_set_ready_latency_millis: Time from query submission until results are available (execute phase)
+       - Set when execute() completes
+    2. result_set_consumption_latency_millis: Time spent iterating/fetching results (fetch phase)
+       - Measured from first fetch call until no more rows available
+       - In Java: tracked via markResultSetConsumption(hasNext) method
+       - Records start time on first fetch, calculates total on last fetch
+
+    Attributes:
+        result_set_ready_latency_millis (int): Time until query results are ready (execution phase)
+        result_set_consumption_latency_millis (int): Time spent fetching/consuming results (fetch phase)
+
+    """
+
+    result_set_ready_latency_millis: Optional[int] = None
+    result_set_consumption_latency_millis: Optional[int] = None
+
+
+@dataclass
+class OperationDetail(JsonSerializableMixin):
+    """
+    Contains detailed information about the operation being performed.
+
+    Attributes:
+        n_operation_status_calls (int): Number of status polling calls made
+        operation_status_latency_millis (int): Total latency of all status calls
+        operation_type (str): Specific operation type (e.g., EXECUTE_STATEMENT, LIST_TABLES, CANCEL_STATEMENT)
+        is_internal_call (bool): Whether this is an internal driver operation
+    """
+
+    n_operation_status_calls: Optional[int] = None
+    operation_status_latency_millis: Optional[int] = None
+    operation_type: Optional[str] = None
+    is_internal_call: Optional[bool] = None
+
+
 @dataclass
 class SqlExecutionEvent(JsonSerializableMixin):
     """
@@ -122,7 +223,10 @@ class SqlExecutionEvent(JsonSerializableMixin):
         is_compressed (bool): Whether the result is compressed
         execution_result (ExecutionResultFormat): Format of the execution result
         retry_count (int): Number of retry attempts made
-        chunk_id (int): ID of the chunk if applicable
+        chunk_id (int): ID of the chunk if applicable (used for error tracking)
+        chunk_details (ChunkDetails): Aggregated chunk download metrics
+        result_latency (ResultLatency): Latency breakdown by execution phase
+        operation_detail (OperationDetail): Detailed operation information
     """
 
     statement_type: StatementType
@@ -130,6 +234,9 @@ class SqlExecutionEvent(JsonSerializableMixin):
     execution_result: ExecutionResultFormat
     retry_count: Optional[int]
     chunk_id: Optional[int]
+    chunk_details: Optional[ChunkDetails] = None
+    result_latency: Optional[ResultLatency] = None
+    operation_detail: Optional[OperationDetail] = None
 
 
 @dataclass

{databricks_sql_connector-4.1.4 → databricks_sql_connector-4.2.1}/src/databricks/sql/telemetry/telemetry_client.py

@@ -380,7 +380,7 @@ class TelemetryClientFactory:
     # Shared flush thread for all clients
     _flush_thread = None
     _flush_event = threading.Event()
-    _flush_interval_seconds = 90
+    _flush_interval_seconds = 300  # 5 minutes
 
     DEFAULT_BATCH_SIZE = 100